diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index 4e45616d88..902d318fef 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -10,6 +10,7 @@ import { ClientDetails, Codec, ConnectionQuality, + DegradationPreference, Error as Error$, GoAwayReason, ICETrickle as ICETrickle$, @@ -836,6 +837,10 @@ export interface VideoSender { * @generated from protobuf field: int32 publish_option_id = 5; */ publishOptionId: number; + /** + * @generated from protobuf field: stream.video.sfu.models.DegradationPreference degradation_preference = 6; + */ + degradationPreference: DegradationPreference; } /** * sent to users when they need to change the quality of their video @@ -1796,6 +1801,16 @@ class VideoSender$Type extends MessageType { kind: 'scalar', T: 5 /*ScalarType.INT32*/, }, + { + no: 6, + name: 'degradation_preference', + kind: 'enum', + T: () => [ + 'stream.video.sfu.models.DegradationPreference', + DegradationPreference, + 'DEGRADATION_PREFERENCE_', + ], + }, ]); } } diff --git a/packages/client/src/gen/video/sfu/models/models.ts b/packages/client/src/gen/video/sfu/models/models.ts index aa35bb2639..fa7dc7a977 100644 --- a/packages/client/src/gen/video/sfu/models/models.ts +++ b/packages/client/src/gen/video/sfu/models/models.ts @@ -307,6 +307,12 @@ export interface PublishOption { * @generated from protobuf field: repeated stream.video.sfu.models.AudioBitrate audio_bitrate_profiles = 10; */ audioBitrateProfiles: AudioBitrate[]; + /** + * The degradation preference for video encoding. + * + * @generated from protobuf field: stream.video.sfu.models.DegradationPreference degradation_preference = 11; + */ + degradationPreference: DegradationPreference; } /** * @generated from protobuf message stream.video.sfu.models.Codec @@ -1172,6 +1178,34 @@ export enum ClientCapability { */ SUBSCRIBER_VIDEO_PAUSE = 1, } +/** + * DegradationPreference represents the RTCDegradationPreference from WebRTC. + * See https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpSender/setParameters#degradationpreference + * + * @generated from protobuf enum stream.video.sfu.models.DegradationPreference + */ +export enum DegradationPreference { + /** + * @generated from protobuf enum value: DEGRADATION_PREFERENCE_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + /** + * @generated from protobuf enum value: DEGRADATION_PREFERENCE_BALANCED = 1; + */ + BALANCED = 1, + /** + * @generated from protobuf enum value: DEGRADATION_PREFERENCE_MAINTAIN_FRAMERATE = 2; + */ + MAINTAIN_FRAMERATE = 2, + /** + * @generated from protobuf enum value: DEGRADATION_PREFERENCE_MAINTAIN_RESOLUTION = 3; + */ + MAINTAIN_RESOLUTION = 3, + /** + * @generated from protobuf enum value: DEGRADATION_PREFERENCE_MAINTAIN_FRAMERATE_AND_RESOLUTION = 4; + */ + MAINTAIN_FRAMERATE_AND_RESOLUTION = 4, +} // @generated message type with reflection information, may provide speed optimized methods class CallState$Type extends MessageType { constructor() { @@ -1441,6 +1475,16 @@ class PublishOption$Type extends MessageType { repeat: 2 /*RepeatType.UNPACKED*/, T: () => AudioBitrate, }, + { + no: 11, + name: 'degradation_preference', + kind: 'enum', + T: () => [ + 'stream.video.sfu.models.DegradationPreference', + DegradationPreference, + 'DEGRADATION_PREFERENCE_', + ], + }, ]); } } diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 37bb36aa15..c0e232f1ed 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -20,6 +20,7 @@ import { toVideoLayers, } from './layers'; import { isSvcCodec } from './codecs'; +import { toRTCDegradationPreference } from './helpers/degradationPreference'; import { isAudioTrackType } from './helpers/tracks'; import { extractMid, removeCodecsExcept, setStartBitrate } from './helpers/sdp'; import { withoutConcurrency } from '../helpers/concurrency'; @@ -135,7 +136,9 @@ export class Publisher extends BasePeerConnection { }); const params = transceiver.sender.getParameters(); - params.degradationPreference = 'maintain-framerate'; + params.degradationPreference = + toRTCDegradationPreference(publishOption.degradationPreference) ?? + 'maintain-framerate'; await transceiver.sender.setParameters(params); const trackType = publishOption.trackType; @@ -344,6 +347,17 @@ export class Publisher extends BasePeerConnection { } } + const degradationPreference = toRTCDegradationPreference( + videoSender.degradationPreference, + ); + if ( + degradationPreference && + params.degradationPreference !== degradationPreference + ) { + params.degradationPreference = degradationPreference; + changed = true; + } + const activeEncoders = params.encodings.filter((e) => e.active); if (!changed) { return this.logger.info(`${tag} no change:`, activeEncoders); diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index 7657ce475c..2809e06134 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -2,6 +2,7 @@ import './mocks/webrtc.mocks'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { anyString } from 'vitest-mock-extended'; +import { fromPartial } from '@total-typescript/shoehorn'; import { NegotiationError } from '../NegotiationError'; import { Publisher } from '../Publisher'; import { ReconnectReason } from '../types'; @@ -9,6 +10,7 @@ import { CallState } from '../../store'; import { StreamSfuClient } from '../../StreamSfuClient'; import { DispatchableMessage, Dispatcher } from '../Dispatcher'; import { + DegradationPreference, ErrorCode, PeerType, PublishOption, @@ -82,6 +84,7 @@ describe('Publisher', () => { fps: 30, maxTemporalLayers: 3, maxSpatialLayers: 3, + degradationPreference: DegradationPreference.UNSPECIFIED, }, ], ); @@ -168,16 +171,19 @@ describe('Publisher', () => { changePublishQuality: { audioSenders: [], videoSenders: [ - { + fromPartial({ publishOptionId: 1, trackType: TrackType.VIDEO, layers: [], - }, - { + degradationPreference: DegradationPreference.BALANCED, + }), + fromPartial({ publishOptionId: 2, trackType: TrackType.SCREEN_SHARE, layers: [], - }, + degradationPreference: + DegradationPreference.MAINTAIN_RESOLUTION, + }), ], }, }, @@ -657,6 +663,7 @@ describe('Publisher', () => { await publisher['changePublishQuality']({ publishOptionId: 1, trackType: TrackType.VIDEO, + degradationPreference: DegradationPreference.UNSPECIFIED, layers: [ { name: 'q', @@ -733,6 +740,7 @@ describe('Publisher', () => { await publisher['changePublishQuality']({ publishOptionId: 1, trackType: TrackType.VIDEO, + degradationPreference: DegradationPreference.UNSPECIFIED, layers: [ { name: 'q', @@ -796,6 +804,7 @@ describe('Publisher', () => { await publisher['changePublishQuality']({ publishOptionId: 1, trackType: TrackType.VIDEO, + degradationPreference: DegradationPreference.UNSPECIFIED, layers: [ { name: 'q', @@ -855,6 +864,7 @@ describe('Publisher', () => { await publisher['changePublishQuality']({ publishOptionId: 1, trackType: TrackType.VIDEO, + degradationPreference: DegradationPreference.UNSPECIFIED, layers: [ { name: 'q', @@ -879,6 +889,93 @@ describe('Publisher', () => { }, ]); }); + + it('applies degradationPreference from the SFU event', async () => { + const transceiver = new RTCRtpTransceiver(); + const setParametersSpy = vi + .spyOn(transceiver.sender, 'setParameters') + .mockResolvedValue(); + vi.spyOn(transceiver.sender, 'getParameters').mockReturnValue({ + // @ts-expect-error incomplete data + codecs: [{ mimeType: 'video/VP8' }], + encodings: [{ rid: 'q', active: true }], + degradationPreference: 'maintain-framerate', + }); + + publisher['transceiverCache'].add({ + // @ts-expect-error incomplete data + publishOption: { trackType: TrackType.VIDEO, id: 1 }, + transceiver, + options: {}, + }); + + await publisher['changePublishQuality']({ + publishOptionId: 1, + trackType: TrackType.VIDEO, + degradationPreference: DegradationPreference.BALANCED, + layers: [ + { + name: 'q', + active: true, + maxBitrate: 100, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: '', + }, + ], + }); + + expect(setParametersSpy).toHaveBeenCalled(); + expect(setParametersSpy.mock.calls[0][0].degradationPreference).toBe( + 'balanced', + ); + }); + + it('does not call setParameters when nothing changes and degradationPreference is UNSPECIFIED', async () => { + const transceiver = new RTCRtpTransceiver(); + const setParametersSpy = vi + .spyOn(transceiver.sender, 'setParameters') + .mockResolvedValue(); + vi.spyOn(transceiver.sender, 'getParameters').mockReturnValue({ + // @ts-expect-error incomplete data + codecs: [{ mimeType: 'video/VP8' }], + encodings: [ + { + rid: 'q', + active: true, + maxBitrate: 100, + scaleResolutionDownBy: 1, + maxFramerate: 30, + }, + ], + degradationPreference: 'maintain-framerate', + }); + + publisher['transceiverCache'].add({ + // @ts-expect-error incomplete data + publishOption: { trackType: TrackType.VIDEO, id: 1 }, + transceiver, + options: {}, + }); + + await publisher['changePublishQuality']({ + publishOptionId: 1, + trackType: TrackType.VIDEO, + degradationPreference: DegradationPreference.UNSPECIFIED, + layers: [ + { + name: 'q', + active: true, + maxBitrate: 100, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: '', + }, + ], + }); + + expect(setParametersSpy).not.toHaveBeenCalled(); + }); }); describe('changePublishOptions', () => { @@ -893,12 +990,27 @@ describe('Publisher', () => { vi.spyOn(publisher, 'negotiate').mockResolvedValue(); publisher['publishOptions'] = [ - // @ts-expect-error incomplete data - { trackType: TrackType.VIDEO, id: 0, codec: { name: 'vp8' } }, - // @ts-expect-error incomplete data - { trackType: TrackType.VIDEO, id: 1, codec: { name: 'av1' } }, - // @ts-expect-error incomplete data - { trackType: TrackType.VIDEO, id: 2, codec: { name: 'vp9' } }, + { + trackType: TrackType.VIDEO, + id: 0, + // @ts-expect-error incomplete data + codec: { name: 'vp8' }, + degradationPreference: DegradationPreference.UNSPECIFIED, + }, + { + trackType: TrackType.VIDEO, + id: 1, + // @ts-expect-error incomplete data + codec: { name: 'av1' }, + degradationPreference: DegradationPreference.UNSPECIFIED, + }, + { + trackType: TrackType.VIDEO, + id: 2, + // @ts-expect-error incomplete data + codec: { name: 'vp9' }, + degradationPreference: DegradationPreference.UNSPECIFIED, + }, ]; publisher['transceiverCache'].add({ diff --git a/packages/client/src/rtc/helpers/__tests__/degradationPreference.test.ts b/packages/client/src/rtc/helpers/__tests__/degradationPreference.test.ts new file mode 100644 index 0000000000..eb463ee9e1 --- /dev/null +++ b/packages/client/src/rtc/helpers/__tests__/degradationPreference.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; +import { DegradationPreference } from '../../../gen/video/sfu/models/models'; +import { toRTCDegradationPreference } from '../degradationPreference'; + +describe('toRTCDegradationPreference', () => { + it.each([ + [DegradationPreference.BALANCED, 'balanced'], + [DegradationPreference.MAINTAIN_FRAMERATE, 'maintain-framerate'], + [DegradationPreference.MAINTAIN_RESOLUTION, 'maintain-resolution'], + [ + DegradationPreference.MAINTAIN_FRAMERATE_AND_RESOLUTION, + 'maintain-framerate-and-resolution', + ], + ])('maps %s to "%s"', (preference, expected) => { + expect(toRTCDegradationPreference(preference)).toBe(expected); + }); + + it('returns undefined for UNSPECIFIED', () => { + expect( + toRTCDegradationPreference(DegradationPreference.UNSPECIFIED), + ).toBeUndefined(); + }); +}); diff --git a/packages/client/src/rtc/helpers/degradationPreference.ts b/packages/client/src/rtc/helpers/degradationPreference.ts new file mode 100644 index 0000000000..7a69d69895 --- /dev/null +++ b/packages/client/src/rtc/helpers/degradationPreference.ts @@ -0,0 +1,22 @@ +import { DegradationPreference } from '../../gen/video/sfu/models/models'; +import { ensureExhausted } from '../../helpers/ensureExhausted'; + +export const toRTCDegradationPreference = ( + preference: DegradationPreference, +): RTCDegradationPreference | undefined => { + switch (preference) { + case DegradationPreference.BALANCED: + return 'balanced'; + case DegradationPreference.MAINTAIN_FRAMERATE: + return 'maintain-framerate'; + case DegradationPreference.MAINTAIN_RESOLUTION: + return 'maintain-resolution'; + case DegradationPreference.MAINTAIN_FRAMERATE_AND_RESOLUTION: + // @ts-expect-error not in the typedefs yet + return 'maintain-framerate-and-resolution'; + case DegradationPreference.UNSPECIFIED: + return undefined; + default: + ensureExhausted(preference, 'Unknown degradation preference'); + } +};