From 0a7db140b1ac78c70243a3d731820289c050c29f Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 22 Jun 2026 10:20:54 +0200 Subject: [PATCH 1/7] rename PacketTrailer to FrameMetadata --- examples/demo/demo.ts | 66 +++++++++---------- examples/demo/index.html | 29 ++++---- package.json | 17 +++-- ...pt-worker.js => rollup.config.fm-worker.js | 10 +-- src/e2ee/E2eeManager.ts | 24 +++---- src/e2ee/types.ts | 10 +-- src/e2ee/worker/FrameCryptor.test.ts | 10 +-- src/e2ee/worker/FrameCryptor.ts | 50 +++++++------- src/e2ee/worker/e2ee.worker.ts | 10 +-- .../FrameMetadataManager.test.ts} | 14 ++-- .../FrameMetadataManager.ts} | 36 +++++----- .../frameMetadata.test.ts} | 2 +- .../frameMetadata.ts} | 18 ++--- src/{packetTrailer => frameMetadata}/types.ts | 18 +++-- .../utils.test.ts | 38 +++++------ src/{packetTrailer => frameMetadata}/utils.ts | 23 +++---- .../worker/frameMetadata.worker.ts} | 10 +-- .../worker/tsconfig.json | 0 src/index.ts | 15 ++++- src/options.ts | 13 ++-- src/room/PCTransport.ts | 1 + src/room/RTCEngine.test.ts | 14 ++-- src/room/RTCEngine.ts | 49 +++++++------- src/room/Room.ts | 20 +++--- src/room/participant/LocalParticipant.test.ts | 38 +++++------ src/room/participant/LocalParticipant.ts | 37 ++++++----- ...Extractor.ts => FrameMetadataExtractor.ts} | 16 ++--- src/room/track/RemoteVideoTrack.ts | 14 ++-- src/room/track/options.ts | 15 +++-- 29 files changed, 333 insertions(+), 284 deletions(-) rename rollup.config.pt-worker.js => rollup.config.fm-worker.js (61%) rename src/{packetTrailer/PacketTrailerManager.test.ts => frameMetadata/FrameMetadataManager.test.ts} (93%) rename src/{packetTrailer/PacketTrailerManager.ts => frameMetadata/FrameMetadataManager.ts} (86%) rename src/{packetTrailer/packetTrailer.test.ts => frameMetadata/frameMetadata.test.ts} (99%) rename src/{packetTrailer/packetTrailer.ts => frameMetadata/frameMetadata.ts} (94%) rename src/{packetTrailer => frameMetadata}/types.ts (70%) rename src/{packetTrailer => frameMetadata}/utils.test.ts (70%) rename src/{packetTrailer => frameMetadata}/utils.ts (59%) rename src/{packetTrailer/worker/packetTrailer.worker.ts => frameMetadata/worker/frameMetadata.worker.ts} (94%) rename src/{packetTrailer => frameMetadata}/worker/tsconfig.json (100%) rename src/room/track/{PacketTrailerExtractor.ts => FrameMetadataExtractor.ts} (58%) diff --git a/examples/demo/demo.ts b/examples/demo/demo.ts index 9d1bcb5988..9e7b783929 100644 --- a/examples/demo/demo.ts +++ b/examples/demo/demo.ts @@ -1,5 +1,7 @@ //@ts-ignore import E2EEWorker from '../../src/e2ee/worker/e2ee.worker?worker'; +//@ts-ignore +import FrameMetadataWorker from '../../src/frameMetadata/worker/frameMetadata.worker?worker'; import type { ChatMessage, LocalDataTrack, @@ -44,8 +46,6 @@ import { supportsAV1, supportsVP9, } from '../../src/index'; -//@ts-ignore -import PTWorker from '../../src/packetTrailer/worker/packetTrailer.worker?worker'; import type { DataTrackFrame } from '../../src/room/data-track/frame'; import { TrackEvent } from '../../src/room/events'; import { isSVCCodec, sleep, supportsH265 } from '../../src/room/utils'; @@ -102,38 +102,38 @@ function updateSearchParams(url: string, token: string, key: string) { window.history.replaceState(null, '', `${window.location.pathname}?${params.toString()}`); } -function getPacketTrailerPublishOptions() { - const enabled = ($('packet-trailer')).checked; - const timestamp = enabled && ($('packet-trailer-timestamp')).checked; - const frameId = enabled && ($('packet-trailer-frame-id')).checked; +function getFrameMetadataPublishOptions() { + const enabled = ($('frame-metadata')).checked; + const timestamp = enabled && ($('frame-metadata-timestamp')).checked; + const frameId = enabled && ($('frame-metadata-frame-id')).checked; return timestamp || frameId ? { timestamp, frameId } : undefined; } -function syncPacketTrailerPublishOptions(room = currentRoom) { - const packetTrailer = getPacketTrailerPublishOptions(); +function syncFrameMetadataPublishOptions(room = currentRoom) { + const frameMetadata = getFrameMetadataPublishOptions(); if (!room) { - return packetTrailer; + return frameMetadata; } - room.options.publishDefaults.packetTrailer = packetTrailer; + room.options.publishDefaults.frameMetadata = frameMetadata; room.localParticipant.trackPublications.forEach((pub) => { if (pub.kind !== Track.Kind.Video) { return; } - pub.options = { ...pub.options, packetTrailer }; + pub.options = { ...pub.options, frameMetadata }; if (pub.track && isVideoTrack(pub.track)) { - pub.track.publishOptions = { ...pub.track.publishOptions, packetTrailer }; + pub.track.publishOptions = { ...pub.track.publishOptions, frameMetadata }; } }); - return packetTrailer; + return frameMetadata; } -function syncPacketTrailerFeatureControls() { - const enabled = ($('packet-trailer')).checked; - const featureControls = $('packet-trailer-features'); - const timestamp = $('packet-trailer-timestamp'); - const frameId = $('packet-trailer-frame-id'); +function syncFrameMetadataFeatureControls() { + const enabled = ($('frame-metadata')).checked; + const featureControls = $('frame-metadata-features'); + const timestamp = $('frame-metadata-timestamp'); + const frameId = $('frame-metadata-frame-id'); featureControls.style.display = enabled ? 'block' : 'none'; timestamp.disabled = !enabled; @@ -142,20 +142,20 @@ function syncPacketTrailerFeatureControls() { timestamp.checked = false; frameId.checked = false; } - syncPacketTrailerPublishOptions(); + syncFrameMetadataPublishOptions(); } -($('packet-trailer')).addEventListener( +($('frame-metadata')).addEventListener( 'change', - syncPacketTrailerFeatureControls, + syncFrameMetadataFeatureControls, ); -($('packet-trailer-timestamp')).addEventListener('change', () => - syncPacketTrailerPublishOptions(), +($('frame-metadata-timestamp')).addEventListener('change', () => + syncFrameMetadataPublishOptions(), ); -($('packet-trailer-frame-id')).addEventListener('change', () => - syncPacketTrailerPublishOptions(), +($('frame-metadata-frame-id')).addEventListener('change', () => + syncFrameMetadataPublishOptions(), ); -syncPacketTrailerFeatureControls(); +syncFrameMetadataFeatureControls(); // handles actions from the HTML const appActions = { @@ -181,8 +181,8 @@ const appActions = { const cryptoKey = ($('crypto-key')).value; const autoSubscribe = ($('auto-subscribe')).checked; const e2eeEnabled = ($('e2ee')).checked; - const packetTrailerEnabled = ($('packet-trailer')).checked; - const packetTrailer = getPacketTrailerPublishOptions(); + const frameMetadataEnabled = ($('frame-metadata')).checked; + const frameMetadata = getFrameMetadataPublishOptions(); const audioOutputId = ($('audio-output')).value; let backupCodecPolicy: BackupCodecPolicy | undefined; if (($('multicodec-simulcast')).checked) { @@ -207,7 +207,7 @@ const appActions = { screenShareEncoding: ScreenSharePresets.h1080fps30.encoding, scalabilityMode: 'L3T3_KEY', backupCodecPolicy: backupCodecPolicy, - packetTrailer, + frameMetadata, }, videoCaptureDefaults: { resolution: VideoPresets.h720.resolution, @@ -215,7 +215,7 @@ const appActions = { encryption: e2eeEnabled ? { keyProvider: state.e2eeKeyProvider, worker: new E2EEWorker() } : undefined, - packetTrailer: packetTrailerEnabled ? { worker: new PTWorker() } : undefined, + frameMetadata: frameMetadataEnabled ? { worker: new FrameMetadataWorker() } : undefined, }; if ( roomOpts.publishDefaults?.videoCodec === 'av1' || @@ -571,7 +571,7 @@ const appActions = { // read and set current key from input const cryptoKey = ($('crypto-key')).value; state.e2eeKeyProvider.setKey(cryptoKey); - syncPacketTrailerPublishOptions(currentRoom); + syncFrameMetadataPublishOptions(currentRoom); await currentRoom.setE2EEEnabled(!currentRoom.isE2EEEnabled); }, @@ -649,9 +649,9 @@ const appActions = { } else { appendLog('enabling video'); } - const publishOptions = syncPacketTrailerPublishOptions(currentRoom); + const publishOptions = syncFrameMetadataPublishOptions(currentRoom); await currentRoom.localParticipant.setCameraEnabled(!enabled, undefined, { - packetTrailer: publishOptions, + frameMetadata: publishOptions, }); setButtonDisabled('toggle-video-button', false); renderParticipant(currentRoom.localParticipant); diff --git a/examples/demo/index.html b/examples/demo/index.html index 83f04068d2..9ec439b56f 100644 --- a/examples/demo/index.html +++ b/examples/demo/index.html @@ -76,19 +76,19 @@

Livekit Sample App

- - + +
- -
+
", "license": "Apache-2.0", "scripts": { - "build": "rollup --config --bundleConfigAsCjs && rollup --config rollup.config.worker.js --bundleConfigAsCjs && rollup --config rollup.config.pt-worker.js --bundleConfigAsCjs && pnpm downlevel-dts", + "build": "rollup --config --bundleConfigAsCjs && rollup --config rollup.config.worker.js --bundleConfigAsCjs && rollup --config rollup.config.fm-worker.js --bundleConfigAsCjs && pnpm downlevel-dts", "build:clean": "rm -rf ./dist && pnpm build", "build:watch": "rollup --watch --config --bundleConfigAsCjs", "build:worker:watch": "rollup --watch --config rollup.config.worker.js --bundleConfigAsCjs", diff --git a/rollup.config.pt-worker.js b/rollup.config.fm-worker.js similarity index 61% rename from rollup.config.pt-worker.js rename to rollup.config.fm-worker.js index c6f00d04f8..37e2c7509f 100644 --- a/rollup.config.pt-worker.js +++ b/rollup.config.fm-worker.js @@ -4,22 +4,22 @@ import packageJson from './package.json'; import { commonPlugins, kebabCaseToPascalCase } from './rollup.config'; export default { - input: 'src/packetTrailer/worker/packetTrailer.worker.ts', + input: 'src/frameMetadata/worker/frameMetadata.worker.ts', output: [ { - file: `dist/${packageJson.name}.pt.worker.mjs`, + file: `dist/${packageJson.name}.fm.worker.mjs`, format: 'es', strict: true, sourcemap: true, }, { - file: `dist/${packageJson.name}.pt.worker.js`, + file: `dist/${packageJson.name}.fm.worker.js`, format: 'umd', strict: true, sourcemap: true, - name: kebabCaseToPascalCase(packageJson.name) + '.pt.worker', + name: kebabCaseToPascalCase(packageJson.name) + '.fm.worker', plugins: [terser()], }, ], - plugins: [typescript({ tsconfig: './src/packetTrailer/worker/tsconfig.json' }), ...commonPlugins], + plugins: [typescript({ tsconfig: './src/frameMetadata/worker/tsconfig.json' }), ...commonPlugins], }; diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index 799cb81414..9b07e7139f 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -1,8 +1,8 @@ import { Encryption_Type, TrackInfo } from '@livekit/protocol'; import { EventEmitter } from 'events'; import type TypedEventEmitter from 'typed-emitter'; +import { hasFrameMetadataPublishOptions } from '../frameMetadata/utils'; import log, { LogLevel, workerLogger } from '../logger'; -import { hasPacketTrailerPublishOptions } from '../packetTrailer/utils'; import type RTCEngine from '../room/RTCEngine'; import type Room from '../room/Room'; import { ConnectionState } from '../room/Room'; @@ -230,7 +230,7 @@ export class E2EEManager } break; case 'packetTrailerMetadata': - this.handlePacketTrailerMetadata(data.trackId, data.rtpTimestamp, data.ssrc, data.metadata); + this.handleFrameMetadata(data.trackId, data.rtpTimestamp, data.ssrc, data.metadata); break; default: break; @@ -242,7 +242,7 @@ export class E2EEManager this.emit(EncryptionEvent.EncryptionError, ev.error, undefined); }; - private handlePacketTrailerMetadata( + private handleFrameMetadata( trackId: string, rtpTimestamp: number, ssrc: number, @@ -260,9 +260,9 @@ export class E2EEManager pub.track && pub.track.mediaStreamID === trackId && pub.track instanceof RemoteVideoTrack && - pub.track.packetTrailerExtractor + pub.track.frameMetadataExtractor ) { - pub.track.packetTrailerExtractor.storeMetadata(rtpTimestamp, ssrc, metadata); + pub.track.frameMetadataExtractor.storeMetadata(rtpTimestamp, ssrc, metadata); return; } } @@ -509,7 +509,9 @@ export class E2EEManager sender, track.mediaStreamID, undefined, - isVideoTrack(track) ? track.publishOptions?.packetTrailer : undefined, + isVideoTrack(track) + ? (track.publishOptions?.frameMetadata ?? track.publishOptions?.packetTrailer) + : undefined, ); } @@ -598,7 +600,7 @@ export class E2EEManager sender: RTCRtpSender, trackId: string, codec?: VideoCodec, - packetTrailer?: TrackPublishOptions['packetTrailer'], + frameMetadata?: TrackPublishOptions['frameMetadata'], ) { if (E2EE_FLAG in sender || !this.worker) { return; @@ -615,8 +617,8 @@ export class E2EEManager participantIdentity: this.room.localParticipant.identity, trackId, codec, - hasPacketTrailer: hasPacketTrailerPublishOptions(packetTrailer), - packetTrailer, + hasPacketTrailer: hasFrameMetadataPublishOptions(frameMetadata), + packetTrailer: frameMetadata, }; // @ts-ignore sender.transform = new RTCRtpScriptTransform(this.worker, options); @@ -633,8 +635,8 @@ export class E2EEManager trackId, participantIdentity: this.room.localParticipant.identity, isReuse: false, - hasPacketTrailer: hasPacketTrailerPublishOptions(packetTrailer), - packetTrailer, + hasPacketTrailer: hasFrameMetadataPublishOptions(frameMetadata), + packetTrailer: frameMetadata, }, }; this.worker.postMessage(msg, [senderStreams.readable, senderStreams.writable]); diff --git a/src/e2ee/types.ts b/src/e2ee/types.ts index 7db98ecfce..92f33b8664 100644 --- a/src/e2ee/types.ts +++ b/src/e2ee/types.ts @@ -1,6 +1,6 @@ +import type { FrameMetadataPayload } from '../frameMetadata/frameMetadata'; +import type { FrameMetadataPublishOptions } from '../frameMetadata/types'; import type { LogLevel } from '../logger'; -import type { PacketTrailerFramePayload } from '../packetTrailer/packetTrailer'; -import type { PacketTrailerPublishOptions } from '../packetTrailer/types'; import type { VideoCodec } from '../room/track/options'; import type { BaseE2EEManager } from './E2eeManager'; import type { BaseKeyProvider } from './KeyProvider'; @@ -62,7 +62,7 @@ export interface EncodeMessage extends BaseMessage { /** * Packet trailer metadata to append on published video frames. */ - packetTrailer?: PacketTrailerPublishOptions; + packetTrailer?: FrameMetadataPublishOptions; }; } @@ -165,7 +165,7 @@ export interface EncryptDataResponseMessage extends BaseMessage { export interface PTMetadataFromE2EEMessage extends BaseMessage { kind: 'packetTrailerMetadata'; - data: PacketTrailerFramePayload; + data: FrameMetadataPayload; } export type E2EEWorkerMessage = @@ -249,5 +249,5 @@ export type ScriptTransformOptions = { /** * Packet trailer metadata to append on published video frames. */ - packetTrailer?: PacketTrailerPublishOptions; + packetTrailer?: FrameMetadataPublishOptions; }; diff --git a/src/e2ee/worker/FrameCryptor.test.ts b/src/e2ee/worker/FrameCryptor.test.ts index b90a9b53b1..4d25c7e659 100644 --- a/src/e2ee/worker/FrameCryptor.test.ts +++ b/src/e2ee/worker/FrameCryptor.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vitest } from 'vitest'; -import { appendPacketTrailer, extractPacketTrailer } from '../../packetTrailer/packetTrailer'; -import type { PacketTrailerPublishOptions } from '../../packetTrailer/types'; +import { appendPacketTrailer, extractPacketTrailer } from '../../frameMetadata/frameMetadata'; +import type { FrameMetadataPublishOptions } from '../../frameMetadata/types'; import { IV_LENGTH, KEY_PROVIDER_DEFAULTS } from '../constants'; import { CryptorEvent } from '../events'; import type { KeyProviderOptions } from '../types'; @@ -69,7 +69,7 @@ function prepareParticipantTestDecoder( function prepareParticipantTestEncoder( participantIdentity: string, partialKeyProviderOptions: Partial, - packetTrailer?: PacketTrailerPublishOptions, + packetTrailer?: FrameMetadataPublishOptions, ) { return prepareParticipantTest( 'encode', @@ -83,7 +83,7 @@ function prepareParticipantTest( mode: 'encode' | 'decode', participantIdentity: string, partialKeyProviderOptions: Partial, - packetTrailer?: PacketTrailerPublishOptions, + packetTrailer?: FrameMetadataPublishOptions, ): { keys: ParticipantKeyHandler; cryptor: FrameCryptor; @@ -536,7 +536,7 @@ describe('FrameCryptor', () => { const postMessage = vitest.fn(); vitest.stubGlobal('postMessage', postMessage); encryptionEnabledMap.set(participantIdentity, false); - cryptor.setHasPacketTrailer(true); + cryptor.setHasFrameMetadata(true); const payload = Uint8Array.from([1, 2, 3, 4]); const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456n, 0); diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index ca80f5028e..486e6ab0d0 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -1,13 +1,13 @@ // TODO code inspired by https://github.com/webrtc/samples/blob/gh-pages/src/content/insertable-streams/endtoend-encryption/js/worker.js import { EventEmitter } from 'events'; import type TypedEventEmitter from 'typed-emitter'; -import { workerLogger } from '../../logger'; import { appendPacketTrailerToEncodedFrame, processPacketTrailer, -} from '../../packetTrailer/packetTrailer'; -import type { PacketTrailerPublishOptions } from '../../packetTrailer/types'; -import { hasPacketTrailerPublishOptions } from '../../packetTrailer/utils'; +} from '../../frameMetadata/frameMetadata'; +import type { FrameMetadataPublishOptions } from '../../frameMetadata/types'; +import { hasFrameMetadataPublishOptions } from '../../frameMetadata/utils'; +import { workerLogger } from '../../logger'; import { type VideoCodec, videoCodecs } from '../../room/track/options'; import { mimeTypeToVideoCodecString } from '../../room/track/utils'; import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants'; @@ -88,11 +88,11 @@ export class FrameCryptor extends BaseFrameCryptor { * When false, we skip the per-frame trailer extraction path entirely * on decode to avoid unnecessary work on tracks that don't use it. */ - private hasPacketTrailer: boolean = false; + private hasFrameMetadata: boolean = false; - private packetTrailer?: PacketTrailerPublishOptions; + private frameMetadataOpts?: FrameMetadataPublishOptions; - private packetTrailerFrameId = 0; + private frameMetadataFrameId = 0; /** * Throttling mechanism for decryption errors to prevent memory leaks @@ -212,13 +212,13 @@ export class FrameCryptor extends BaseFrameCryptor { * trailer data. When false, {@link decodeFunction} skips the per-frame * trailer extraction branch entirely. */ - setHasPacketTrailer(hasPacketTrailer: boolean) { - this.hasPacketTrailer = hasPacketTrailer; + setHasFrameMetadata(hasFrameMetadata: boolean) { + this.hasFrameMetadata = hasFrameMetadata; } - setPacketTrailer(packetTrailer?: PacketTrailerPublishOptions) { - this.packetTrailer = packetTrailer; - this.packetTrailerFrameId = 0; + setFrameMetadataOpts(frameMetadata?: FrameMetadataPublishOptions) { + this.frameMetadataOpts = frameMetadata; + this.frameMetadataFrameId = 0; } setupTransform( @@ -228,14 +228,14 @@ export class FrameCryptor extends BaseFrameCryptor { trackId: string, isReuse: boolean, codec?: VideoCodec, - packetTrailer?: PacketTrailerPublishOptions, + frameMetadata?: FrameMetadataPublishOptions, ) { if (codec) { workerLogger.info('setting codec on cryptor to', { codec }); this.videoCodec = codec; } if (operation === 'encode') { - this.setPacketTrailer(packetTrailer); + this.setFrameMetadataOpts(frameMetadata); } workerLogger.debug('Setting up frame cryptor transform', { @@ -406,7 +406,7 @@ export class FrameCryptor extends BaseFrameCryptor { } if (!this.isEnabled()) { - this.appendPacketTrailer(encodedFrame); + this.appendFrameMetadata(encodedFrame); return controller.enqueue(encodedFrame); } const keySet = this.keys.getKeySet(); @@ -475,7 +475,7 @@ export class FrameCryptor extends BaseFrameCryptor { newData.set(newDataWithoutHeader, frameHeader.byteLength); encodedFrame.data = newData.buffer; - this.appendPacketTrailer(encodedFrame); + this.appendFrameMetadata(encodedFrame); return controller.enqueue(encodedFrame); } catch (e: any) { @@ -494,16 +494,20 @@ export class FrameCryptor extends BaseFrameCryptor { } } - private appendPacketTrailer(encodedFrame: RTCEncodedVideoFrame | RTCEncodedAudioFrame) { - if (!hasPacketTrailerPublishOptions(this.packetTrailer) || !isVideoFrame(encodedFrame)) { + private appendFrameMetadata(encodedFrame: RTCEncodedVideoFrame | RTCEncodedAudioFrame) { + if (!hasFrameMetadataPublishOptions(this.frameMetadataOpts) || !isVideoFrame(encodedFrame)) { return; } - if (this.packetTrailer?.frameId) { - this.packetTrailerFrameId = - this.packetTrailerFrameId === 0xffffffff ? 1 : this.packetTrailerFrameId + 1; + if (this.frameMetadataOpts?.frameId) { + this.frameMetadataFrameId = + this.frameMetadataFrameId === 0xffffffff ? 1 : this.frameMetadataFrameId + 1; } - appendPacketTrailerToEncodedFrame(encodedFrame, this.packetTrailer, this.packetTrailerFrameId); + appendPacketTrailerToEncodedFrame( + encodedFrame, + this.frameMetadataOpts, + this.frameMetadataFrameId, + ); } /** @@ -516,7 +520,7 @@ export class FrameCryptor extends BaseFrameCryptor { encodedFrame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, controller: TransformStreamDefaultController, ) { - if (this.hasPacketTrailer && isVideoFrame(encodedFrame)) { + if (this.hasFrameMetadata && isVideoFrame(encodedFrame)) { try { const ptResult = processPacketTrailer(encodedFrame, this.trackId); if (ptResult.data) { diff --git a/src/e2ee/worker/e2ee.worker.ts b/src/e2ee/worker/e2ee.worker.ts index ad0287306c..4ab8674eeb 100644 --- a/src/e2ee/worker/e2ee.worker.ts +++ b/src/e2ee/worker/e2ee.worker.ts @@ -64,7 +64,7 @@ onmessage = (ev) => { break; case 'decode': let cryptor = getTrackCryptor(data.participantIdentity, data.trackId); - cryptor.setHasPacketTrailer(data.hasPacketTrailer); + cryptor.setHasFrameMetadata(data.hasPacketTrailer); cryptor.setupTransform( kind, data.readableStream, @@ -76,7 +76,7 @@ onmessage = (ev) => { break; case 'encode': let pubCryptor = getTrackCryptor(data.participantIdentity, data.trackId); - pubCryptor.setHasPacketTrailer(data.hasPacketTrailer); + pubCryptor.setHasFrameMetadata(data.hasPacketTrailer); pubCryptor.setupTransform( kind, data.readableStream, @@ -84,7 +84,7 @@ onmessage = (ev) => { data.trackId, data.isReuse, data.codec, - data.packetTrailer, + data.packetTrailer, // wire format still uses 'packetTrailer' field name ); break; @@ -164,7 +164,7 @@ onmessage = (ev) => { case 'updateCodec': const trackCryptor = getTrackCryptor(data.participantIdentity, data.trackId); trackCryptor.setVideoCodec(data.codec); - trackCryptor.setHasPacketTrailer(data.hasPacketTrailer); + trackCryptor.setHasFrameMetadata(data.hasPacketTrailer); workerLogger.info('updated codec', { participantIdentity: data.participantIdentity, trackId: data.trackId, @@ -343,7 +343,7 @@ if (self.RTCTransformEvent) { const { kind, participantIdentity, trackId, codec, hasPacketTrailer } = options; messageQueue.run(async () => { const cryptor = getTrackCryptor(participantIdentity, trackId); - cryptor.setHasPacketTrailer(hasPacketTrailer); + cryptor.setHasFrameMetadata(hasPacketTrailer); workerLogger.debug('onrtctransform setup', { participantIdentity, trackId, codec }); cryptor.setupTransform( kind, diff --git a/src/packetTrailer/PacketTrailerManager.test.ts b/src/frameMetadata/FrameMetadataManager.test.ts similarity index 93% rename from src/packetTrailer/PacketTrailerManager.test.ts rename to src/frameMetadata/FrameMetadataManager.test.ts index 4b9ad772e5..6577065a66 100644 --- a/src/packetTrailer/PacketTrailerManager.test.ts +++ b/src/frameMetadata/FrameMetadataManager.test.ts @@ -1,8 +1,8 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import type { TrackInfo } from '@livekit/protocol'; -import { PacketTrailerManager } from './PacketTrailerManager'; +import { FrameMetadataManager } from './FrameMetadataManager'; -describe('PacketTrailerManager', () => { +describe('FrameMetadataManager', () => { const originalRTCRtpSender = window.RTCRtpSender; const originalUserAgent = navigator.userAgent; const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown }) @@ -63,7 +63,7 @@ describe('PacketTrailerManager', () => { }); } - function setupWorkerReceiver(manager: PacketTrailerManager, receiver: RTCRtpReceiver) { + function setupWorkerReceiver(manager: FrameMetadataManager, receiver: RTCRtpReceiver) { ( manager as unknown as { setupWorkerReceiver: (receiver: RTCRtpReceiver, newTrackId: string) => void; @@ -72,7 +72,7 @@ describe('PacketTrailerManager', () => { } function setupReceiver( - manager: PacketTrailerManager, + manager: FrameMetadataManager, receiver: RTCRtpReceiver, trackId: string, trackInfo?: TrackInfo, @@ -109,7 +109,7 @@ describe('PacketTrailerManager', () => { setScriptTransform(RTCRtpScriptTransform); const worker = {} as Worker; - const manager = new PacketTrailerManager({ worker }); + const manager = new FrameMetadataManager({ worker }); const receiver = { createEncodedStreams: vi.fn(), } as unknown as RTCRtpReceiver; @@ -131,7 +131,7 @@ describe('PacketTrailerManager', () => { stubInsertableStreamsSupport(); const worker = { postMessage: vi.fn() } as unknown as Worker; - const manager = new PacketTrailerManager({ worker }); + const manager = new FrameMetadataManager({ worker }); const { receiver, readable, writable, createEncodedStreams } = makeReceiver(); setupReceiver(manager, receiver, 'track-without-trailer'); @@ -155,7 +155,7 @@ describe('PacketTrailerManager', () => { stubInsertableStreamsSupport(); const worker = { postMessage: vi.fn() } as unknown as Worker; - const manager = new PacketTrailerManager({ worker }); + const manager = new FrameMetadataManager({ worker }); const { receiver } = makeReceiver(); const trackInfo = { packetTrailerFeatures: [1] } as unknown as TrackInfo; diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/frameMetadata/FrameMetadataManager.ts similarity index 86% rename from src/packetTrailer/PacketTrailerManager.ts rename to src/frameMetadata/FrameMetadataManager.ts index 7e4438f5e5..c052c32692 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/frameMetadata/FrameMetadataManager.ts @@ -2,13 +2,13 @@ import type { TrackInfo } from '@livekit/protocol'; import log from '../logger'; import type Room from '../room/Room'; import { RoomEvent } from '../room/events'; -import { PacketTrailerExtractor } from '../room/track/PacketTrailerExtractor'; +import { FrameMetadataExtractor } from '../room/track/FrameMetadataExtractor'; import type RemoteTrack from '../room/track/RemoteTrack'; import RemoteVideoTrack from '../room/track/RemoteVideoTrack'; import type { PTDecodeMessage, PTUpdateTrackIdMessage, PTWorkerMessage } from './types'; -import { isPacketTrailerSupported, shouldUsePacketTrailerScriptTransform } from './utils'; +import { isFrameMetadataSupported, shouldUseFrameMetadataScriptTransform } from './utils'; -export interface PacketTrailerOptions { +export interface FrameMetadataOptions { /** * Dedicated worker for extracting packet trailers off the main thread. * @@ -34,12 +34,12 @@ export interface PacketTrailerOptions { * * @experimental */ -export class PacketTrailerManager { +export class FrameMetadataManager { private worker?: Worker; private room?: Room; - private extractors = new Map(); + private extractors = new Map(); /** * Tracks the trackId associated with each receiver that has had its @@ -49,7 +49,7 @@ export class PacketTrailerManager { */ private workerPipelines = new Map(); - constructor(options?: PacketTrailerOptions) { + constructor(options?: FrameMetadataOptions) { this.worker = options?.worker; } @@ -100,18 +100,18 @@ export class PacketTrailerManager { } if ( - !isPacketTrailerSupported(this.worker ? { worker: this.worker } : undefined) && + !isFrameMetadataSupported(this.worker ? { worker: this.worker } : undefined) && !this.room?.hasE2EESetup ) { - log.warn('packet trailer transform not supported; skipping extraction'); + log.warn('frame metadata transform not supported; skipping extraction'); return; } - const extractor = new PacketTrailerExtractor(); + const extractor = new FrameMetadataExtractor(); const trackId = track.mediaStreamID; this.extractors.set(trackId, extractor); - track.packetTrailerExtractor = extractor; + track.frameMetadataExtractor = extractor; if (this.room?.hasE2EESetup) { // E2EE worker strips the trailer and injects metadata directly into @@ -123,7 +123,7 @@ export class PacketTrailerManager { } private setupPassthroughReceiver(receiver: RTCRtpReceiver, trackId: string) { - if (shouldUsePacketTrailerScriptTransform()) { + if (shouldUseFrameMetadataScriptTransform()) { if ('transform' in receiver) { // @ts-ignore receiver.transform = null; @@ -133,7 +133,7 @@ export class PacketTrailerManager { if ( this.worker && - isPacketTrailerSupported({ worker: this.worker }) && + isFrameMetadataSupported({ worker: this.worker }) && !this.workerPipelines.has(receiver) ) { this.setupWorkerReceiver(receiver, trackId, false); @@ -155,7 +155,7 @@ export class PacketTrailerManager { return; } - if (shouldUsePacketTrailerScriptTransform()) { + if (shouldUseFrameMetadataScriptTransform()) { // @ts-ignore receiver.transform = new RTCRtpScriptTransform(worker, { kind: 'decode', @@ -215,7 +215,7 @@ export class PacketTrailerManager { } if (track instanceof RemoteVideoTrack) { - track.packetTrailerExtractor = undefined; + track.frameMetadataExtractor = undefined; } // The receiver pipeline is intentionally left running. If the receiver is @@ -245,6 +245,12 @@ export class PacketTrailerManager { }; private onWorkerError = (ev: ErrorEvent) => { - log.error('packet trailer worker encountered an error:', { error: ev.error }); + log.error('frame metadata worker encountered an error:', { error: ev.error }); }; } + +/** @deprecated Use {@link FrameMetadataManager} instead. */ +export const PacketTrailerManager = FrameMetadataManager; + +/** @deprecated Use {@link FrameMetadataOptions} instead. */ +export type PacketTrailerOptions = FrameMetadataOptions; diff --git a/src/packetTrailer/packetTrailer.test.ts b/src/frameMetadata/frameMetadata.test.ts similarity index 99% rename from src/packetTrailer/packetTrailer.test.ts rename to src/frameMetadata/frameMetadata.test.ts index 74f589b361..6a172f3e44 100644 --- a/src/packetTrailer/packetTrailer.test.ts +++ b/src/frameMetadata/frameMetadata.test.ts @@ -4,7 +4,7 @@ import { appendPacketTrailerToEncodedFrame, extractPacketTrailer, processPacketTrailer, -} from './packetTrailer'; +} from './frameMetadata'; describe('packetTrailer', () => { afterEach(() => { diff --git a/src/packetTrailer/packetTrailer.ts b/src/frameMetadata/frameMetadata.ts similarity index 94% rename from src/packetTrailer/packetTrailer.ts rename to src/frameMetadata/frameMetadata.ts index b9a6b11ca4..7b8afa4d84 100644 --- a/src/packetTrailer/packetTrailer.ts +++ b/src/frameMetadata/frameMetadata.ts @@ -1,5 +1,5 @@ -import type { PacketTrailerMetadata, PacketTrailerPublishOptions } from './types'; -import { hasPacketTrailerPublishOptions } from './utils'; +import type { FrameMetadata, FrameMetadataPublishOptions } from './types'; +import { hasFrameMetadataPublishOptions } from './utils'; export const PACKET_TRAILER_MAGIC = Uint8Array.from([ 'L'.charCodeAt(0), @@ -17,7 +17,7 @@ const FRAME_ID_TLV_SIZE = 6; export interface ExtractPacketTrailerResult { data: Uint8Array; - metadata?: PacketTrailerMetadata; + metadata?: FrameMetadata; } export function appendPacketTrailer( @@ -64,10 +64,10 @@ export function appendPacketTrailer( export function appendPacketTrailerToEncodedFrame( frame: RTCEncodedVideoFrame, - options: PacketTrailerPublishOptions | undefined, + options: FrameMetadataPublishOptions | undefined, frameId: number, ): boolean { - if (!hasPacketTrailerPublishOptions(options) || frame.data.byteLength === 0) { + if (!hasFrameMetadataPublishOptions(options) || frame.data.byteLength === 0) { return false; } @@ -108,7 +108,7 @@ export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPac const strippedData = bytes.subarray(0, trailerStart); let offset = trailerStart; let foundAny = false; - const metadata: PacketTrailerMetadata = { + const metadata: FrameMetadata = { userTimestamp: BigInt(0), frameId: 0, }; @@ -223,16 +223,16 @@ export function getFrameSsrc(frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame) return 0; } -export interface PacketTrailerFramePayload { +export interface FrameMetadataPayload { trackId: string; rtpTimestamp: number; ssrc: number; - metadata: PacketTrailerMetadata; + metadata: FrameMetadata; } export interface ProcessPacketTrailerResult { data?: ArrayBuffer; - payload?: PacketTrailerFramePayload; + payload?: FrameMetadataPayload; } /** diff --git a/src/packetTrailer/types.ts b/src/frameMetadata/types.ts similarity index 70% rename from src/packetTrailer/types.ts rename to src/frameMetadata/types.ts index 773731c17e..a96ec245f2 100644 --- a/src/packetTrailer/types.ts +++ b/src/frameMetadata/types.ts @@ -1,15 +1,21 @@ -import type { PacketTrailerFramePayload } from './packetTrailer'; +import type { FrameMetadataPayload } from './frameMetadata'; -export interface PacketTrailerMetadata { +export interface FrameMetadata { userTimestamp: bigint; frameId: number; } -export interface PacketTrailerPublishOptions { +/** @deprecated Use {@link FrameMetadata} instead. */ +export type PacketTrailerMetadata = FrameMetadata; + +export interface FrameMetadataPublishOptions { timestamp?: boolean; frameId?: boolean; } +/** @deprecated Use {@link FrameMetadataPublishOptions} instead. */ +export type PacketTrailerPublishOptions = FrameMetadataPublishOptions; + export interface PTBaseMessage { kind: string; data?: unknown; @@ -38,7 +44,7 @@ export interface PTEncodeMessage extends PTBaseMessage { data: { readableStream: ReadableStream; writableStream: WritableStream; - packetTrailer?: PacketTrailerPublishOptions; + packetTrailer?: FrameMetadataPublishOptions; }; } @@ -49,12 +55,12 @@ export type PTScriptTransformOptions = } | { kind: 'encode'; - packetTrailer?: PacketTrailerPublishOptions; + packetTrailer?: FrameMetadataPublishOptions; }; export interface PTMetadataMessage extends PTBaseMessage { kind: 'metadata'; - data: PacketTrailerFramePayload; + data: FrameMetadataPayload; } export interface PTUpdateTrackIdMessage extends PTBaseMessage { diff --git a/src/packetTrailer/utils.test.ts b/src/frameMetadata/utils.test.ts similarity index 70% rename from src/packetTrailer/utils.test.ts rename to src/frameMetadata/utils.test.ts index d0b35ad50c..20ef153552 100644 --- a/src/packetTrailer/utils.test.ts +++ b/src/frameMetadata/utils.test.ts @@ -1,12 +1,12 @@ import { PacketTrailerFeature } from '@livekit/protocol'; import { afterEach, describe, expect, it } from 'vitest'; import { - getPacketTrailerFeatures, - getPacketTrailerPublishOptions, - isPacketTrailerSupported, + getFrameMetadataFeatures, + getFrameMetadataPublishOptions, + isFrameMetadataSupported, } from './utils'; -describe('packet trailer support', () => { +describe('frame metadata support', () => { const originalRTCRtpSender = window.RTCRtpSender; const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown }) .RTCRtpScriptTransform; @@ -46,20 +46,20 @@ describe('packet trailer support', () => { }); } - it('supports packet trailers with RTCRtpScriptTransform on Safari', () => { + it('supports frame metadata with RTCRtpScriptTransform on Safari', () => { stubScriptTransformSupport( 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', ); - expect(isPacketTrailerSupported({ worker: {} as Worker })).toBe(true); + expect(isFrameMetadataSupported({ worker: {} as Worker })).toBe(true); }); - it('supports packet trailers with RTCRtpScriptTransform on Firefox', () => { + it('supports frame metadata with RTCRtpScriptTransform on Firefox', () => { stubScriptTransformSupport( 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14.0; rv:144.0) Gecko/20100101 Firefox/144.0', ); - expect(isPacketTrailerSupported({ worker: {} as Worker })).toBe(true); + expect(isFrameMetadataSupported({ worker: {} as Worker })).toBe(true); }); it('does not use RTCRtpScriptTransform support on Chromium-based browsers', () => { @@ -67,39 +67,39 @@ describe('packet trailer support', () => { 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', ); - expect(isPacketTrailerSupported({ worker: {} as Worker })).toBe(false); + expect(isFrameMetadataSupported({ worker: {} as Worker })).toBe(false); }); }); -describe('packet trailer publish features', () => { +describe('frame metadata publish features', () => { it('maps publish options to protocol features', () => { - expect(getPacketTrailerFeatures({ timestamp: true, frameId: true })).toEqual([ + expect(getFrameMetadataFeatures({ timestamp: true, frameId: true })).toEqual([ PacketTrailerFeature.PTF_USER_TIMESTAMP, PacketTrailerFeature.PTF_FRAME_ID, ]); - expect(getPacketTrailerFeatures({ timestamp: true })).toEqual([ + expect(getFrameMetadataFeatures({ timestamp: true })).toEqual([ PacketTrailerFeature.PTF_USER_TIMESTAMP, ]); - expect(getPacketTrailerFeatures({ frameId: true })).toEqual([ + expect(getFrameMetadataFeatures({ frameId: true })).toEqual([ PacketTrailerFeature.PTF_FRAME_ID, ]); - expect(getPacketTrailerFeatures()).toEqual([]); + expect(getFrameMetadataFeatures()).toEqual([]); }); it('maps protocol features to publish options', () => { expect( - getPacketTrailerPublishOptions([ + getFrameMetadataPublishOptions([ PacketTrailerFeature.PTF_USER_TIMESTAMP, PacketTrailerFeature.PTF_FRAME_ID, ]), ).toEqual({ timestamp: true, frameId: true }); - expect(getPacketTrailerPublishOptions([PacketTrailerFeature.PTF_USER_TIMESTAMP])).toEqual({ + expect(getFrameMetadataPublishOptions([PacketTrailerFeature.PTF_USER_TIMESTAMP])).toEqual({ timestamp: true, }); - expect(getPacketTrailerPublishOptions([PacketTrailerFeature.PTF_FRAME_ID])).toEqual({ + expect(getFrameMetadataPublishOptions([PacketTrailerFeature.PTF_FRAME_ID])).toEqual({ frameId: true, }); - expect(getPacketTrailerPublishOptions()).toBeUndefined(); - expect(getPacketTrailerPublishOptions([])).toBeUndefined(); + expect(getFrameMetadataPublishOptions()).toBeUndefined(); + expect(getFrameMetadataPublishOptions([])).toBeUndefined(); }); }); diff --git a/src/packetTrailer/utils.ts b/src/frameMetadata/utils.ts similarity index 59% rename from src/packetTrailer/utils.ts rename to src/frameMetadata/utils.ts index 92f57cdb41..4978230217 100644 --- a/src/packetTrailer/utils.ts +++ b/src/frameMetadata/utils.ts @@ -1,25 +1,26 @@ import { PacketTrailerFeature } from '@livekit/protocol'; import { isInsertableStreamSupported } from '../e2ee/utils'; import { isScriptTransformSupportedForWorker } from '../room/utils'; -import type { PacketTrailerOptions } from './PacketTrailerManager'; -import type { PacketTrailerPublishOptions } from './types'; +import type { FrameMetadataOptions } from './FrameMetadataManager'; +import type { FrameMetadataPublishOptions } from './types'; -export function shouldUsePacketTrailerScriptTransform() { +export function shouldUseFrameMetadataScriptTransform() { return isScriptTransformSupportedForWorker(); } -export function isPacketTrailerSupported(options?: PacketTrailerOptions) { +export function isFrameMetadataSupported(options?: FrameMetadataOptions) { return ( - !!options?.worker && (isInsertableStreamSupported() || shouldUsePacketTrailerScriptTransform()) + !!options?.worker && + (isInsertableStreamSupported() || shouldUseFrameMetadataScriptTransform()) ); } -export function hasPacketTrailerPublishOptions(options?: PacketTrailerPublishOptions): boolean { +export function hasFrameMetadataPublishOptions(options?: FrameMetadataPublishOptions): boolean { return !!(options?.timestamp || options?.frameId); } -export function getPacketTrailerFeatures( - options?: PacketTrailerPublishOptions, +export function getFrameMetadataFeatures( + options?: FrameMetadataPublishOptions, ): PacketTrailerFeature[] { const features: PacketTrailerFeature[] = []; if (options?.timestamp) { @@ -31,14 +32,14 @@ export function getPacketTrailerFeatures( return features; } -export function getPacketTrailerPublishOptions( +export function getFrameMetadataPublishOptions( features?: PacketTrailerFeature[], -): PacketTrailerPublishOptions | undefined { +): FrameMetadataPublishOptions | undefined { if (!features || features.length === 0) { return undefined; } - const options: PacketTrailerPublishOptions = {}; + const options: FrameMetadataPublishOptions = {}; if (features.includes(PacketTrailerFeature.PTF_USER_TIMESTAMP)) { options.timestamp = true; } diff --git a/src/packetTrailer/worker/packetTrailer.worker.ts b/src/frameMetadata/worker/frameMetadata.worker.ts similarity index 94% rename from src/packetTrailer/worker/packetTrailer.worker.ts rename to src/frameMetadata/worker/frameMetadata.worker.ts index 71d9ee3adc..21d2aa804f 100644 --- a/src/packetTrailer/worker/packetTrailer.worker.ts +++ b/src/frameMetadata/worker/frameMetadata.worker.ts @@ -1,11 +1,11 @@ -import { appendPacketTrailerToEncodedFrame, processPacketTrailer } from '../packetTrailer'; +import { appendPacketTrailerToEncodedFrame, processPacketTrailer } from '../frameMetadata'; import type { + FrameMetadataPublishOptions, PTMetadataMessage, PTScriptTransformOptions, PTWorkerMessage, - PacketTrailerPublishOptions, } from '../types'; -import { hasPacketTrailerPublishOptions } from '../utils'; +import { hasFrameMetadataPublishOptions } from '../utils'; /** * Holds the trackId currently associated with a pipeline. A mutable @@ -97,9 +97,9 @@ function setupDecodeTransform( function setupEncodeTransform( readable: ReadableStream, writable: WritableStream, - packetTrailer?: PacketTrailerPublishOptions, + packetTrailer?: FrameMetadataPublishOptions, ) { - if (!hasPacketTrailerPublishOptions(packetTrailer)) { + if (!hasFrameMetadataPublishOptions(packetTrailer)) { readable.pipeTo(writable).catch(() => {}); return; } diff --git a/src/packetTrailer/worker/tsconfig.json b/src/frameMetadata/worker/tsconfig.json similarity index 100% rename from src/packetTrailer/worker/tsconfig.json rename to src/frameMetadata/worker/tsconfig.json diff --git a/src/index.ts b/src/index.ts index d54746e7f1..1d218cd89d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -64,11 +64,22 @@ import { import { getBrowser } from './utils/browserParser'; export { RpcError, type RpcInvocationData, type PerformRpcParams } from './room/rpc'; -export type { PacketTrailerMetadata, PacketTrailerPublishOptions } from './packetTrailer/types'; +export type { + FrameMetadata, + FrameMetadataPublishOptions, + /** @deprecated Use {@link FrameMetadata} instead. */ + PacketTrailerMetadata, + /** @deprecated Use {@link FrameMetadataPublishOptions} instead. */ + PacketTrailerPublishOptions, +} from './frameMetadata/types'; export { + FrameMetadataManager, + /** @deprecated Use {@link FrameMetadataManager} instead. */ PacketTrailerManager, + type FrameMetadataOptions, + /** @deprecated Use {@link FrameMetadataOptions} instead. */ type PacketTrailerOptions, -} from './packetTrailer/PacketTrailerManager'; +} from './frameMetadata/FrameMetadataManager'; export * from './connectionHelper/ConnectionCheck'; export * from './connectionHelper/checks/Checker'; diff --git a/src/options.ts b/src/options.ts index e841faefbc..b4df30b32d 100644 --- a/src/options.ts +++ b/src/options.ts @@ -1,5 +1,5 @@ import type { E2EEOptions } from './e2ee/types'; -import type { PacketTrailerOptions } from './packetTrailer/PacketTrailerManager'; +import type { FrameMetadataOptions } from './frameMetadata/FrameMetadataManager'; import type { ReconnectPolicy } from './room/ReconnectPolicy'; import type { AudioCaptureOptions, @@ -103,10 +103,15 @@ export interface InternalRoomOptions { /** * @experimental - * Options for enabling packet trailers on video tracks. - * Packet trailers carry frame-level metadata such as user timestamps and frame IDs. + * Options for enabling frame metadata on video tracks. + * Frame metadata carries frame-level information such as user timestamps and frame IDs. */ - packetTrailer?: PacketTrailerOptions; + frameMetadata?: FrameMetadataOptions; + + /** + * @deprecated Use {@link InternalRoomOptions.frameMetadata} instead. + */ + packetTrailer?: FrameMetadataOptions; /** * will attempt to connect via single peer connection mode. diff --git a/src/room/PCTransport.ts b/src/room/PCTransport.ts index 4ca9691d3f..2c450d6975 100644 --- a/src/room/PCTransport.ts +++ b/src/room/PCTransport.ts @@ -544,6 +544,7 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter return; } this.pendingInitialOffer = undefined; + this.log.warn(`closing peer connection`); this._pc.close(); this._pc.onconnectionstatechange = null; this._pc.oniceconnectionstatechange = null; diff --git a/src/room/RTCEngine.test.ts b/src/room/RTCEngine.test.ts index 03db82e5e0..d4c510a44e 100644 --- a/src/room/RTCEngine.test.ts +++ b/src/room/RTCEngine.test.ts @@ -58,12 +58,12 @@ describe('RTCEngine', () => { ).makeRTCConfiguration(); } - function setupPacketTrailerSender(engine: RTCEngine, sender: RTCRtpSender, opts = {}) { + function setupFrameMetadataSender(engine: RTCEngine, sender: RTCRtpSender, opts = {}) { ( engine as unknown as { - setupPacketTrailerSender: (sender: RTCRtpSender, opts?: unknown) => void; + setupFrameMetadataSender: (sender: RTCRtpSender, opts?: unknown) => void; } - ).setupPacketTrailerSender(sender, opts); + ).setupFrameMetadataSender(sender, opts); } it('does not enable encoded insertable streams without E2EE or a packet trailer worker', () => { @@ -130,7 +130,7 @@ describe('RTCEngine', () => { createEncodedStreams, } as unknown as RTCRtpSender; - setupPacketTrailerSender(engine, sender); + setupFrameMetadataSender(engine, sender); expect(createEncodedStreams).not.toHaveBeenCalled(); }); @@ -147,7 +147,7 @@ describe('RTCEngine', () => { createEncodedStreams, } as unknown as RTCRtpSender; - setupPacketTrailerSender(engine, sender); + setupFrameMetadataSender(engine, sender); expect(createEncodedStreams).not.toHaveBeenCalled(); }); @@ -167,7 +167,7 @@ describe('RTCEngine', () => { createEncodedStreams, } as unknown as RTCRtpSender; - setupPacketTrailerSender(engine, sender, { packetTrailer: { timestamp: true, frameId: true } }); + setupFrameMetadataSender(engine, sender, { packetTrailer: { timestamp: true, frameId: true } }); expect(createEncodedStreams).toHaveBeenCalledTimes(1); expect(worker.postMessage).toHaveBeenCalledWith( @@ -211,7 +211,7 @@ describe('RTCEngine', () => { createEncodedStreams, } as unknown as RTCRtpSender; - setupPacketTrailerSender(engine, sender, { packetTrailer: { timestamp: true } }); + setupFrameMetadataSender(engine, sender, { packetTrailer: { timestamp: true } }); expect(RTCRtpScriptTransform).toHaveBeenCalledWith(worker, { kind: 'encode', diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index 130f39de9c..b5294310a7 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -54,13 +54,13 @@ import { } from '../api/SignalClient'; import type { BaseE2EEManager } from '../e2ee/E2eeManager'; import { asEncryptablePacket, isInsertableStreamSupported } from '../e2ee/utils'; +import { + hasFrameMetadataPublishOptions, + isFrameMetadataSupported, + shouldUseFrameMetadataScriptTransform, +} from '../frameMetadata/utils'; import log, { LoggerNames, getLogger } from '../logger'; import type { InternalRoomOptions } from '../options'; -import { - hasPacketTrailerPublishOptions, - isPacketTrailerSupported, - shouldUsePacketTrailerScriptTransform, -} from '../packetTrailer/utils'; import TypedPromise from '../utils/TypedPromise'; import { DataPacketBuffer } from '../utils/dataPacketBuffer'; import { TTLMap } from '../utils/ttlmap'; @@ -795,7 +795,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit // used; RTCRtpScriptTransform does not need the PeerConnection flag. const needsInsertableStreams = this.signalOpts?.e2eeEnabled || - (this.options.packetTrailer?.worker && !shouldUsePacketTrailerScriptTransform()); + (this.frameMetadataWorker && !shouldUseFrameMetadataScriptTransform()); if (needsInsertableStreams && isInsertableStreamSupported()) { this.log.debug('E2EE - setting up transports with insertable streams'); @@ -1079,7 +1079,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } else { throw new UnexpectedConnectionState('Required webRTC APIs not supported on this device'); } - this.setupPacketTrailerSender(sender, opts); + this.setupFrameMetadataSender(sender, opts); return sender; } @@ -1099,50 +1099,55 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit throw new UnexpectedConnectionState('Cannot stream on this device'); } if (sender) { - this.setupPacketTrailerSender(sender, opts); + this.setupFrameMetadataSender(sender, opts); } return sender; } - private setupPacketTrailerSender(sender: RTCRtpSender, opts: TrackPublishOptions = {}) { - if (!this.options.packetTrailer?.worker || this.signalOpts?.e2eeEnabled) { + private get frameMetadataWorker(): Worker | undefined { + return (this.options.frameMetadata ?? this.options.packetTrailer)?.worker; + } + + private setupFrameMetadataSender(sender: RTCRtpSender, opts: TrackPublishOptions = {}) { + const worker = this.frameMetadataWorker; + if (!worker || this.signalOpts?.e2eeEnabled) { return; } - const packetTrailer = opts.packetTrailer; - const hasPacketTrailer = hasPacketTrailerPublishOptions(packetTrailer); + const frameMetadata = opts.frameMetadata ?? opts.packetTrailer; + const hasMetadata = hasFrameMetadataPublishOptions(frameMetadata); - if (shouldUsePacketTrailerScriptTransform()) { - if (hasPacketTrailer) { + if (shouldUseFrameMetadataScriptTransform()) { + if (hasMetadata) { // @ts-ignore - sender.transform = new RTCRtpScriptTransform(this.options.packetTrailer.worker, { + sender.transform = new RTCRtpScriptTransform(worker, { kind: 'encode', - packetTrailer, + packetTrailer: frameMetadata, }); } return; } if ( - !isPacketTrailerSupported(this.options.packetTrailer) || + !isFrameMetadataSupported(this.options.frameMetadata ?? this.options.packetTrailer) || !('createEncodedStreams' in sender) ) { - if (hasPacketTrailer) { - this.log.warn('packet trailer transform not supported; skipping write', this.logContext); + if (hasMetadata) { + this.log.warn('frame metadata transform not supported; skipping write', this.logContext); } return; } // @ts-ignore const { readable, writable } = sender.createEncodedStreams(); - if (hasPacketTrailer) { - this.options.packetTrailer.worker.postMessage( + if (hasMetadata) { + worker.postMessage( { kind: 'encode', data: { readableStream: readable, writableStream: writable, - packetTrailer, + packetTrailer: frameMetadata, }, }, [readable, writable], diff --git a/src/room/Room.ts b/src/room/Room.ts index 17d321f8f6..169fb1b237 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -37,6 +37,8 @@ import type TypedEmitter from 'typed-emitter'; import { ensureTrailingSlash } from '../api/utils'; import { EncryptionEvent } from '../e2ee'; import { type BaseE2EEManager, E2EEManager } from '../e2ee/E2eeManager'; +import { FrameMetadataManager } from '../frameMetadata/FrameMetadataManager'; +import { isFrameMetadataSupported } from '../frameMetadata/utils'; import log, { LoggerNames, getLogger } from '../logger'; import type { InternalRoomConnectOptions, @@ -44,8 +46,6 @@ import type { RoomConnectOptions, RoomOptions, } from '../options'; -import { PacketTrailerManager } from '../packetTrailer/PacketTrailerManager'; -import { isPacketTrailerSupported } from '../packetTrailer/utils'; import TypedPromise from '../utils/TypedPromise'; import { getBrowser } from '../utils/browserParser'; import { CLIENT_PROTOCOL_DEFAULT } from '../version'; @@ -198,7 +198,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) private e2eeManager: BaseE2EEManager | undefined; - private packetTrailerManager: PacketTrailerManager | undefined; + private frameMetadataManager: FrameMetadataManager | undefined; private e2eeStateMutex: Mutex = new Mutex(); @@ -343,7 +343,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.rpcServerManager, ); - this.setupPacketTrailer(); + this.setupFrameMetadata(); if (this.options.e2ee || this.options.encryption) { this.setupE2EE(); @@ -514,11 +514,10 @@ class Room extends (EventEmitter as new () => TypedEmitter) } } - private setupPacketTrailer() { - // The manager is always created so tracks that advertise packet trailer - // features can be wired up when the app passes a packet trailer worker. - this.packetTrailerManager = new PacketTrailerManager(this.options.packetTrailer); - this.packetTrailerManager.setup(this); + private setupFrameMetadata() { + const opts = this.options.frameMetadata ?? this.options.packetTrailer; + this.frameMetadataManager = new FrameMetadataManager(opts); + this.frameMetadataManager.setup(this); } private get logContext() { @@ -972,7 +971,8 @@ class Room extends (EventEmitter as new () => TypedEmitter) adaptiveStream: typeof roomOptions.adaptiveStream === 'object' ? true : roomOptions.adaptiveStream, clientInfoCapabilities: - isPacketTrailerSupported(roomOptions.packetTrailer) || !!this.e2eeManager + isFrameMetadataSupported(roomOptions.frameMetadata ?? roomOptions.packetTrailer) || + !!this.e2eeManager ? [ClientInfo_Capability.CAP_PACKET_TRAILER] : undefined, maxRetries: connectOptions.maxRetries, diff --git a/src/room/participant/LocalParticipant.test.ts b/src/room/participant/LocalParticipant.test.ts index 3f4f753753..cc8adc04cf 100644 --- a/src/room/participant/LocalParticipant.test.ts +++ b/src/room/participant/LocalParticipant.test.ts @@ -5,18 +5,18 @@ import { Track } from '../track/Track'; import type { TrackPublishOptions } from '../track/options'; import LocalParticipant from './LocalParticipant'; -type PacketTrailerTestParticipant = { - canPublishPacketTrailer: () => boolean; +type FrameMetadataTestParticipant = { + canPublishFrameMetadata: () => boolean; log: { warn: ReturnType }; - normalizeRequestedPacketTrailerOptions: ( + normalizeRequestedFrameMetadataOptions: ( track: LocalTrack, opts: TrackPublishOptions, ) => PacketTrailerFeature[]; }; -function makeParticipant(canPublishPacketTrailer: boolean) { - const participant = Object.create(LocalParticipant.prototype) as PacketTrailerTestParticipant; - participant.canPublishPacketTrailer = () => canPublishPacketTrailer; +function makeParticipant(canPublishFrameMetadata: boolean) { + const participant = Object.create(LocalParticipant.prototype) as FrameMetadataTestParticipant; + participant.canPublishFrameMetadata = () => canPublishFrameMetadata; participant.log = { warn: vi.fn() }; return participant; } @@ -35,12 +35,12 @@ function makeTrack(kind: Track.Kind) { } as unknown as LocalTrack; } -describe('LocalParticipant packet trailer publish options', () => { - it('normalizes requested video packet trailer options to advertised features', () => { +describe('LocalParticipant frame metadata publish options', () => { + it('normalizes requested video frame metadata options to advertised features', () => { const participant = makeParticipant(true); - const opts: TrackPublishOptions = { packetTrailer: { timestamp: true, frameId: true } }; + const opts: TrackPublishOptions = { frameMetadata: { timestamp: true, frameId: true } }; - const features = participant.normalizeRequestedPacketTrailerOptions( + const features = participant.normalizeRequestedFrameMetadataOptions( makeTrack(Track.Kind.Video), opts, ); @@ -49,33 +49,33 @@ describe('LocalParticipant packet trailer publish options', () => { PacketTrailerFeature.PTF_USER_TIMESTAMP, PacketTrailerFeature.PTF_FRAME_ID, ]); - expect(opts.packetTrailer).toEqual({ timestamp: true, frameId: true }); + expect(opts.frameMetadata).toEqual({ timestamp: true, frameId: true }); }); - it('clears packet trailer options for non-video tracks', () => { + it('clears frame metadata options for non-video tracks', () => { const participant = makeParticipant(true); - const opts: TrackPublishOptions = { packetTrailer: { timestamp: true } }; + const opts: TrackPublishOptions = { frameMetadata: { timestamp: true } }; - const features = participant.normalizeRequestedPacketTrailerOptions( + const features = participant.normalizeRequestedFrameMetadataOptions( makeTrack(Track.Kind.Audio), opts, ); expect(features).toEqual([]); - expect(opts.packetTrailer).toBeUndefined(); + expect(opts.frameMetadata).toBeUndefined(); }); - it('clears packet trailer options when publishing packet trailers is unsupported', () => { + it('clears frame metadata options when publishing frame metadata is unsupported', () => { const participant = makeParticipant(false); - const opts: TrackPublishOptions = { packetTrailer: { frameId: true } }; + const opts: TrackPublishOptions = { frameMetadata: { frameId: true } }; - const features = participant.normalizeRequestedPacketTrailerOptions( + const features = participant.normalizeRequestedFrameMetadataOptions( makeTrack(Track.Kind.Video), opts, ); expect(features).toEqual([]); - expect(opts.packetTrailer).toBeUndefined(); + expect(opts.frameMetadata).toBeUndefined(); expect(participant.log.warn).toHaveBeenCalledOnce(); }); }); diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index 1444f18e00..6d435155e2 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -22,13 +22,13 @@ import { protoInt64, } from '@livekit/protocol'; import { SignalConnectionState } from '../../api/SignalClient'; -import type { InternalRoomOptions } from '../../options'; import { - getPacketTrailerFeatures, - getPacketTrailerPublishOptions, - hasPacketTrailerPublishOptions, - isPacketTrailerSupported, -} from '../../packetTrailer/utils'; + getFrameMetadataFeatures, + getFrameMetadataPublishOptions, + hasFrameMetadataPublishOptions, + isFrameMetadataSupported, +} from '../../frameMetadata/utils'; +import type { InternalRoomOptions } from '../../options'; import TypedPromise from '../../utils/TypedPromise'; import { PCTransportState } from '../PCTransportManager'; import type RTCEngine from '../RTCEngine'; @@ -1085,7 +1085,7 @@ export default class LocalParticipant extends Participant { audioFeatures.push(AudioTrackFeature.TF_PRECONNECT_BUFFER); } const packetTrailerFeatures: PacketTrailerFeature[] = - this.normalizeRequestedPacketTrailerOptions(track, opts); + this.normalizeRequestedFrameMetadataOptions(track, opts); // create track publication from track const req = new AddTrackRequest({ @@ -1416,31 +1416,36 @@ export default class LocalParticipant extends Participant { return publication; } - private canPublishPacketTrailer() { + private canPublishFrameMetadata() { return !!( this.roomOptions.e2ee || this.roomOptions.encryption || - isPacketTrailerSupported(this.roomOptions.packetTrailer) + isFrameMetadataSupported(this.roomOptions.frameMetadata ?? this.roomOptions.packetTrailer) ); } - private normalizeRequestedPacketTrailerOptions(track: LocalTrack, opts: TrackPublishOptions) { - if (track.kind !== Track.Kind.Video || !hasPacketTrailerPublishOptions(opts.packetTrailer)) { + private normalizeRequestedFrameMetadataOptions(track: LocalTrack, opts: TrackPublishOptions) { + const fmOpts = opts.frameMetadata ?? opts.packetTrailer; + if (track.kind !== Track.Kind.Video || !hasFrameMetadataPublishOptions(fmOpts)) { + opts.frameMetadata = undefined; opts.packetTrailer = undefined; return []; } - if (!this.canPublishPacketTrailer()) { - this.log.warn('packet trailer transform not supported; not advertising features', { + if (!this.canPublishFrameMetadata()) { + this.log.warn('frame metadata transform not supported; not advertising features', { ...this.logContext, ...getLogContextFromTrack(track), }); + opts.frameMetadata = undefined; opts.packetTrailer = undefined; return []; } - const features = getPacketTrailerFeatures(opts.packetTrailer); - opts.packetTrailer = getPacketTrailerPublishOptions(features); + const features = getFrameMetadataFeatures(fmOpts); + const normalized = getFrameMetadataPublishOptions(features); + opts.frameMetadata = normalized; + opts.packetTrailer = normalized; return features; } @@ -1496,7 +1501,7 @@ export default class LocalParticipant extends Participant { if (!simulcastTrack) { return; } - const packetTrailerFeatures = this.normalizeRequestedPacketTrailerOptions(track, opts); + const packetTrailerFeatures = this.normalizeRequestedFrameMetadataOptions(track, opts); const req = new AddTrackRequest({ cid: simulcastTrack.mediaStreamTrack.id, diff --git a/src/room/track/PacketTrailerExtractor.ts b/src/room/track/FrameMetadataExtractor.ts similarity index 58% rename from src/room/track/PacketTrailerExtractor.ts rename to src/room/track/FrameMetadataExtractor.ts index fbd55d48ae..83fd0bd59c 100644 --- a/src/room/track/PacketTrailerExtractor.ts +++ b/src/room/track/FrameMetadataExtractor.ts @@ -1,23 +1,23 @@ -import type { PacketTrailerMetadata } from '../../packetTrailer/types'; +import type { FrameMetadata } from '../../frameMetadata/types'; const MAX_ENTRIES = 300; /** - * Caches packet trailer metadata extracted from received video frames, + * Caches frame metadata extracted from received video frames, * keyed by RTP timestamp so it can be looked up when the frame is displayed. * - * Metadata is populated either by the packet trailer worker managed by - * `PacketTrailerManager` (non-E2EE) or by the E2EE FrameCryptor worker + * Metadata is populated either by the frame metadata worker managed by + * `FrameMetadataManager` (non-E2EE) or by the E2EE FrameCryptor worker * after decryption (E2EE). * * @experimental */ -export class PacketTrailerExtractor { - private metadataMap = new Map(); +export class FrameMetadataExtractor { + private metadataMap = new Map(); private activeSsrc: number = 0; - storeMetadata(rtpTimestamp: number, ssrc: number, metadata: PacketTrailerMetadata) { + storeMetadata(rtpTimestamp: number, ssrc: number, metadata: FrameMetadata) { // Simulcast layer switch: SSRC changed, flush stale entries from old layer. if (this.activeSsrc !== 0 && this.activeSsrc !== ssrc) { this.metadataMap.clear(); @@ -32,7 +32,7 @@ export class PacketTrailerExtractor { this.metadataMap.set(rtpTimestamp, metadata); } - lookupMetadata(rtpTimestamp: number): PacketTrailerMetadata | undefined { + lookupMetadata(rtpTimestamp: number): FrameMetadata | undefined { return this.metadataMap.get(rtpTimestamp); } diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts index 1f86211d62..84da794c98 100644 --- a/src/room/track/RemoteVideoTrack.ts +++ b/src/room/track/RemoteVideoTrack.ts @@ -1,4 +1,4 @@ -import type { PacketTrailerMetadata } from '../../packetTrailer/types'; +import type { FrameMetadata } from '../../frameMetadata/types'; import { debounce } from '../debounce'; import { TrackEvent } from '../events'; import type { VideoReceiverStats } from '../stats'; @@ -7,7 +7,7 @@ import CriticalTimers from '../timers'; import type { LoggerOptions } from '../types'; import type { ObservableMediaElement } from '../utils'; import { getDevicePixelRatio, getIntersectionObserver, getResizeObserver, isWeb } from '../utils'; -import type { PacketTrailerExtractor } from './PacketTrailerExtractor'; +import type { FrameMetadataExtractor } from './FrameMetadataExtractor'; import RemoteTrack from './RemoteTrack'; import { Track, attachToElement, detachTrack } from './Track'; import type { AdaptiveStreamSettings } from './types'; @@ -26,7 +26,7 @@ export default class RemoteVideoTrack extends RemoteTrack { private lastDimensions?: Track.Dimensions; /** @internal */ - packetTrailerExtractor?: PacketTrailerExtractor; + frameMetadataExtractor?: FrameMetadataExtractor; constructor( mediaTrack: MediaStreamTrack, @@ -48,16 +48,16 @@ export default class RemoteVideoTrack extends RemoteTrack { * Use with the `TrackEvent.TimeSyncUpdate` event to correlate displayed frames * with their capture-time metadata. * - * Requires the room to be configured with the `packetTrailer` worker option - * and the publishing track to have packet trailer features enabled. + * Requires the room to be configured with the `frameMetadata` worker option + * and the publishing track to have frame metadata features enabled. * */ lookupFrameMetadata({ rtpTimestamp, }: { rtpTimestamp: number; - }): PacketTrailerMetadata | undefined { - return this.packetTrailerExtractor?.lookupMetadata(rtpTimestamp); + }): FrameMetadata | undefined { + return this.frameMetadataExtractor?.lookupMetadata(rtpTimestamp); } override setStreamState(value: Track.StreamState) { diff --git a/src/room/track/options.ts b/src/room/track/options.ts index d0f1678620..d982a1c193 100644 --- a/src/room/track/options.ts +++ b/src/room/track/options.ts @@ -1,4 +1,4 @@ -import type { PacketTrailerPublishOptions } from '../../packetTrailer/types'; +import type { FrameMetadataPublishOptions } from '../../frameMetadata/types'; import type { Track } from './Track'; import type { AudioProcessorOptions, @@ -131,14 +131,19 @@ export interface TrackPublishDefaults { preConnectBuffer?: boolean; /** - * Packet trailer metadata to append to published video frames. + * Frame metadata to append to published video frames. * - * Requires either room-level packet trailer worker configuration or E2EE, - * because encoded frame transforms are used to write the trailer. + * Requires either room-level frame metadata worker configuration or E2EE, + * because encoded frame transforms are used to write the metadata. * * @experimental */ - packetTrailer?: PacketTrailerPublishOptions; + frameMetadata?: FrameMetadataPublishOptions; + + /** + * @deprecated Use {@link TrackPublishDefaults.frameMetadata} instead. + */ + packetTrailer?: FrameMetadataPublishOptions; } /** From 4eae30dac0b619c3ece3314df5e12c46a2acada5 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 22 Jun 2026 10:22:29 +0200 Subject: [PATCH 2/7] Create plenty-snails-tell.md --- .changeset/plenty-snails-tell.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/plenty-snails-tell.md diff --git a/.changeset/plenty-snails-tell.md b/.changeset/plenty-snails-tell.md new file mode 100644 index 0000000000..0e7650b2b6 --- /dev/null +++ b/.changeset/plenty-snails-tell.md @@ -0,0 +1,5 @@ +--- +"livekit-client": patch +--- + +Rename PacketTrailer to FrameMetadata From cf87d8301281999f24d0b569f4bf60703b9c27b4 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 22 Jun 2026 10:23:35 +0200 Subject: [PATCH 3/7] format --- src/frameMetadata/utils.ts | 3 +-- src/room/track/RemoteVideoTrack.ts | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/frameMetadata/utils.ts b/src/frameMetadata/utils.ts index 4978230217..402a05b03a 100644 --- a/src/frameMetadata/utils.ts +++ b/src/frameMetadata/utils.ts @@ -10,8 +10,7 @@ export function shouldUseFrameMetadataScriptTransform() { export function isFrameMetadataSupported(options?: FrameMetadataOptions) { return ( - !!options?.worker && - (isInsertableStreamSupported() || shouldUseFrameMetadataScriptTransform()) + !!options?.worker && (isInsertableStreamSupported() || shouldUseFrameMetadataScriptTransform()) ); } diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts index 84da794c98..38ad6f2223 100644 --- a/src/room/track/RemoteVideoTrack.ts +++ b/src/room/track/RemoteVideoTrack.ts @@ -52,11 +52,7 @@ export default class RemoteVideoTrack extends RemoteTrack { * and the publishing track to have frame metadata features enabled. * */ - lookupFrameMetadata({ - rtpTimestamp, - }: { - rtpTimestamp: number; - }): FrameMetadata | undefined { + lookupFrameMetadata({ rtpTimestamp }: { rtpTimestamp: number }): FrameMetadata | undefined { return this.frameMetadataExtractor?.lookupMetadata(rtpTimestamp); } From 39bd581c8fa0fb61f3b1fcad7d3dce243fe58720 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 22 Jun 2026 10:27:22 +0200 Subject: [PATCH 4/7] Update plenty-snails-tell.md --- .changeset/plenty-snails-tell.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/plenty-snails-tell.md b/.changeset/plenty-snails-tell.md index 0e7650b2b6..f7ef6da198 100644 --- a/.changeset/plenty-snails-tell.md +++ b/.changeset/plenty-snails-tell.md @@ -1,5 +1,5 @@ --- -"livekit-client": patch +"livekit-client": minor --- Rename PacketTrailer to FrameMetadata From eedea51cf8b098be67ff63f2d0cb88090a29bfdd Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 22 Jun 2026 18:19:37 +0200 Subject: [PATCH 5/7] Update src/room/PCTransport.ts --- src/room/PCTransport.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/room/PCTransport.ts b/src/room/PCTransport.ts index 2c450d6975..4ca9691d3f 100644 --- a/src/room/PCTransport.ts +++ b/src/room/PCTransport.ts @@ -544,7 +544,6 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter return; } this.pendingInitialOffer = undefined; - this.log.warn(`closing peer connection`); this._pc.close(); this._pc.onconnectionstatechange = null; this._pc.oniceconnectionstatechange = null; From 5b568c2b27f40feef38ac8e11c91895c6fe090b8 Mon Sep 17 00:00:00 2001 From: David Chen Date: Mon, 22 Jun 2026 15:00:28 -0700 Subject: [PATCH 6/7] add support for user data frame metadata trailer --- examples/demo/demo.ts | 7 +++ src/e2ee/E2eeManager.ts | 3 +- src/frameMetadata/frameMetadata.test.ts | 83 +++++++++++++++++++++++++ src/frameMetadata/frameMetadata.ts | 8 +++ src/frameMetadata/types.ts | 1 + 5 files changed, 101 insertions(+), 1 deletion(-) diff --git a/examples/demo/demo.ts b/examples/demo/demo.ts index 9e7b783929..3471049a1e 100644 --- a/examples/demo/demo.ts +++ b/examples/demo/demo.ts @@ -347,6 +347,13 @@ const appActions = { `\nReceive: ${fmt(receiveTime)}` + `\nLatency: ${latencyDisplay}`; } + if (meta.userData && meta.userData.byteLength > 0) { + const hex = Array.from(meta.userData.subarray(0, 6)) + .map((b) => b.toString(16).padStart(2, '0')) + .join(' '); + const ellipsis = meta.userData.byteLength > 6 ? ' ...' : ''; + text += `${text ? '\n' : ''}user data: ${meta.userData.byteLength} bytes [${hex}${ellipsis}]`; + } overlayElm.textContent = text; } }); diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index 9b07e7139f..994f6a9dcd 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -1,6 +1,7 @@ import { Encryption_Type, TrackInfo } from '@livekit/protocol'; import { EventEmitter } from 'events'; import type TypedEventEmitter from 'typed-emitter'; +import type { FrameMetadata } from '../frameMetadata/types'; import { hasFrameMetadataPublishOptions } from '../frameMetadata/utils'; import log, { LogLevel, workerLogger } from '../logger'; import type RTCEngine from '../room/RTCEngine'; @@ -246,7 +247,7 @@ export class E2EEManager trackId: string, rtpTimestamp: number, ssrc: number, - metadata: { userTimestamp: bigint; frameId: number }, + metadata: FrameMetadata, ) { if (!this.room) { return; diff --git a/src/frameMetadata/frameMetadata.test.ts b/src/frameMetadata/frameMetadata.test.ts index 6a172f3e44..2fe0d946de 100644 --- a/src/frameMetadata/frameMetadata.test.ts +++ b/src/frameMetadata/frameMetadata.test.ts @@ -1,11 +1,57 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { + PACKET_TRAILER_ENVELOPE_SIZE, + PACKET_TRAILER_FRAME_ID_TAG, + PACKET_TRAILER_MAGIC, + PACKET_TRAILER_TIMESTAMP_TAG, + PACKET_TRAILER_USER_DATA_TAG, appendPacketTrailer, appendPacketTrailerToEncodedFrame, extractPacketTrailer, processPacketTrailer, } from './frameMetadata'; +/** + * Builds a packet trailer (XORed TLVs + envelope) for the given fields. The + * SDK only writes timestamp/frameId, so this mirrors the native sender's TLV + * encoding to exercise user_data extraction. + */ +function buildTrailer( + payload: Uint8Array, + fields: { userTimestamp?: bigint; frameId?: number; userData?: Uint8Array }, +): Uint8Array { + const tlvs: number[] = []; + + if (fields.userTimestamp !== undefined) { + tlvs.push(PACKET_TRAILER_TIMESTAMP_TAG ^ 0xff, 8 ^ 0xff); + for (let i = 7; i >= 0; i -= 1) { + tlvs.push(Number((fields.userTimestamp >> BigInt(i * 8)) & BigInt(0xff)) ^ 0xff); + } + } + + if (fields.frameId !== undefined) { + tlvs.push(PACKET_TRAILER_FRAME_ID_TAG ^ 0xff, 4 ^ 0xff); + for (let i = 3; i >= 0; i -= 1) { + tlvs.push(((fields.frameId >> (i * 8)) & 0xff) ^ 0xff); + } + } + + if (fields.userData !== undefined) { + tlvs.push(PACKET_TRAILER_USER_DATA_TAG ^ 0xff, fields.userData.length ^ 0xff); + for (const byte of fields.userData) { + tlvs.push(byte ^ 0xff); + } + } + + const trailerLength = tlvs.length + PACKET_TRAILER_ENVELOPE_SIZE; + const result = new Uint8Array(payload.length + trailerLength); + result.set(payload, 0); + result.set(tlvs, payload.length); + result[payload.length + tlvs.length] = trailerLength ^ 0xff; + result.set(PACKET_TRAILER_MAGIC, payload.length + tlvs.length + 1); + return result; +} + describe('packetTrailer', () => { afterEach(() => { vi.useRealTimers(); @@ -47,6 +93,43 @@ describe('packetTrailer', () => { }); }); + it('extracts user_data from a trailer carrying only user_data', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const userData = Uint8Array.from([0x00, 0x01, 0xfe, 0xff, 0x42]); + const trailer = buildTrailer(payload, { userData }); + const extracted = extractPacketTrailer(trailer); + + expect(Array.from(extracted.data)).toEqual(Array.from(payload)); + expect(extracted.metadata?.userTimestamp).toBe(0n); + expect(extracted.metadata?.frameId).toBe(0); + expect(extracted.metadata?.userData).toBeInstanceOf(Uint8Array); + expect(Array.from(extracted.metadata!.userData!)).toEqual(Array.from(userData)); + }); + + it('extracts user_data alongside timestamp and frameId', () => { + const payload = Uint8Array.from([9, 8, 7, 6, 5]); + const userData = Uint8Array.from([10, 20, 30, 40]); + const trailer = buildTrailer(payload, { + userTimestamp: 1_744_249_600_123_456n, + frameId: 42, + userData, + }); + const extracted = extractPacketTrailer(trailer); + + expect(Array.from(extracted.data)).toEqual(Array.from(payload)); + expect(extracted.metadata?.userTimestamp).toBe(1_744_249_600_123_456n); + expect(extracted.metadata?.frameId).toBe(42); + expect(Array.from(extracted.metadata!.userData!)).toEqual(Array.from(userData)); + }); + + it('leaves userData undefined when no user_data tag is present', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456n, 42); + const extracted = extractPacketTrailer(trailer); + + expect(extracted.metadata?.userData).toBeUndefined(); + }); + it('returns data unchanged when both timestamp and frameId are 0', () => { const payload = Uint8Array.from([1, 2, 3, 4]); const result = appendPacketTrailer(payload, 0n, 0); diff --git a/src/frameMetadata/frameMetadata.ts b/src/frameMetadata/frameMetadata.ts index 7b8afa4d84..1fe7adca69 100644 --- a/src/frameMetadata/frameMetadata.ts +++ b/src/frameMetadata/frameMetadata.ts @@ -10,6 +10,7 @@ export const PACKET_TRAILER_MAGIC = Uint8Array.from([ export const PACKET_TRAILER_TIMESTAMP_TAG = 0x01; export const PACKET_TRAILER_FRAME_ID_TAG = 0x02; +export const PACKET_TRAILER_USER_DATA_TAG = 0x03; export const PACKET_TRAILER_ENVELOPE_SIZE = 5; const TIMESTAMP_TLV_SIZE = 10; @@ -127,6 +128,13 @@ export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPac } else if (tag === PACKET_TRAILER_FRAME_ID_TAG && length === 4) { metadata.frameId = readUint32Xor(bytes, offset, length); foundAny = true; + } else if (tag === PACKET_TRAILER_USER_DATA_TAG) { + const userData = new Uint8Array(length); + for (let index = 0; index < length; index += 1) { + userData[index] = bytes[offset + index] ^ 0xff; + } + metadata.userData = userData; + foundAny = true; } offset += length; diff --git a/src/frameMetadata/types.ts b/src/frameMetadata/types.ts index a96ec245f2..0c31843dee 100644 --- a/src/frameMetadata/types.ts +++ b/src/frameMetadata/types.ts @@ -3,6 +3,7 @@ import type { FrameMetadataPayload } from './frameMetadata'; export interface FrameMetadata { userTimestamp: bigint; frameId: number; + userData?: Uint8Array; } /** @deprecated Use {@link FrameMetadata} instead. */ From b2a382b3405f91dfbdc90f911311c12dd88bb1fc Mon Sep 17 00:00:00 2001 From: David Chen Date: Mon, 22 Jun 2026 15:16:42 -0700 Subject: [PATCH 7/7] add changeset --- .changeset/user-data-frame-metadata.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/user-data-frame-metadata.md diff --git a/.changeset/user-data-frame-metadata.md b/.changeset/user-data-frame-metadata.md new file mode 100644 index 0000000000..c53182fd35 --- /dev/null +++ b/.changeset/user-data-frame-metadata.md @@ -0,0 +1,5 @@ +--- +"livekit-client": patch +--- + +Add support for user_data frame metadata trailer type \ No newline at end of file