Skip to content

Commit f42ec3a

Browse files
davidliudavidliu
authored
Fix VirtualBackgroundVideoProcessor not responding to changing backgroundImage (#752)
* Fix VirtualBackgroundVideoProcessor not responding to changes in backgroundImage * lint fixes * spotless
1 parent 3c5325a commit f42ec3a

43 files changed

Lines changed: 335 additions & 174 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.changeset/olive-bottles-repeat.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"client-sdk-android": patch
3+
---
4+
5+
Fix VirtualBackgroundVideoProcessor not responding to changes in backgroundImage

‎examples/virtual-background/src/main/java/io/livekit/android/selfie/MainActivity.kt‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,17 @@ class MainActivity : AppCompatActivity() {
4545
enableButton.setText(if (state) "Disable" else "Enable")
4646
}
4747

48+
val enableBackgroundButton = findViewById<Button>(R.id.buttonBackground)
49+
enableBackgroundButton.setOnClickListener {
50+
val state = viewModel.toggleVirtualBackground()
51+
enableBackgroundButton.setText(if (state) "Disable Background" else "Enable Background")
52+
}
53+
54+
val flipCameraButton = findViewById<Button>(R.id.buttonFlip)
55+
flipCameraButton.setOnClickListener {
56+
viewModel.flipCamera()
57+
}
58+
4859
val renderer = findViewById<TextureViewRenderer>(R.id.renderer)
4960
viewModel.room.initVideoRenderer(renderer)
5061
viewModel.track.observe(this) { track ->
@@ -87,7 +98,7 @@ fun ComponentActivity.requestNeededPermissions(onPermissionsGranted: (() -> Unit
8798
}
8899
}
89100

90-
val neededPermissions = listOf(Manifest.permission.RECORD_AUDIO, Manifest.permission.CAMERA)
101+
val neededPermissions = listOf(Manifest.permission.CAMERA)
91102
.filter { ContextCompat.checkSelfPermission(this, it) == PackageManager.PERMISSION_DENIED }
92103
.toTypedArray()
93104

‎examples/virtual-background/src/main/java/io/livekit/android/selfie/MainViewModel.kt‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,12 @@ class MainViewModel(application: Application) : AndroidViewModel(application) {
5454
eglBase = eglBase,
5555
),
5656
)
57+
58+
private val virtualBackground = (AppCompatResources.getDrawable(application, R.drawable.background) as BitmapDrawable).bitmap
59+
5760
private var blur = 16f
5861
private val processor = VirtualBackgroundVideoProcessor(eglBase, Dispatchers.IO, initialBlurRadius = blur).apply {
59-
val drawable = AppCompatResources.getDrawable(application, R.drawable.background) as BitmapDrawable
60-
backgroundImage = drawable.bitmap
62+
backgroundImage = virtualBackground
6163
}
6264

6365
private var cameraProvider: CameraCapturerUtils.CameraProvider? = null
@@ -119,4 +121,24 @@ class MainViewModel(application: Application) : AndroidViewModel(application) {
119121
blur += 5
120122
processor.updateBlurRadius(blur)
121123
}
124+
125+
fun toggleVirtualBackground(): Boolean {
126+
if (processor.backgroundImage != virtualBackground) {
127+
processor.backgroundImage = virtualBackground
128+
return true
129+
} else {
130+
processor.backgroundImage = null
131+
return false
132+
}
133+
}
134+
135+
fun flipCamera() {
136+
val videoTrack = track.value ?: return
137+
val newPosition = when (videoTrack.options.position) {
138+
CameraPosition.FRONT -> CameraPosition.BACK
139+
CameraPosition.BACK -> CameraPosition.FRONT
140+
else -> CameraPosition.FRONT
141+
}
142+
videoTrack.switchCamera(position = newPosition)
143+
}
122144
}

‎examples/virtual-background/src/main/res/layout/activity_main.xml‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,32 @@
1010
android:layout_width="match_parent"
1111
android:layout_height="match_parent" />
1212

13-
<Button
14-
android:id="@+id/button"
13+
<LinearLayout
1514
android:layout_width="wrap_content"
1615
android:layout_height="wrap_content"
17-
android:layout_margin="10dp"
18-
android:text="Disable" />
16+
android:orientation="vertical">
17+
18+
<Button
19+
android:id="@+id/button"
20+
android:layout_width="wrap_content"
21+
android:layout_height="wrap_content"
22+
android:layout_margin="10dp"
23+
android:text="Disable" />
24+
25+
<Button
26+
android:id="@+id/buttonBackground"
27+
android:layout_width="wrap_content"
28+
android:layout_height="wrap_content"
29+
android:layout_margin="10dp"
30+
android:text="Disable Background" />
31+
32+
<Button
33+
android:id="@+id/buttonFlip"
34+
android:layout_width="wrap_content"
35+
android:layout_height="wrap_content"
36+
android:layout_margin="10dp"
37+
android:text="Flip Camera" />
38+
</LinearLayout>
1939

2040
<Button
2141
android:id="@+id/buttonIncreaseBlur"

‎livekit-android-camerax/src/main/java/livekit/org/webrtc/CameraXSession.kt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import android.hardware.camera2.CameraMetadata.LENS_OPTICAL_STABILIZATION_MODE_O
2626
import android.hardware.camera2.CameraMetadata.LENS_OPTICAL_STABILIZATION_MODE_ON
2727
import android.hardware.camera2.CaptureRequest
2828
import android.os.Build
29-
import android.os.Build.VERSION
3029
import android.os.Handler
3130
import android.util.Range
3231
import android.util.Size
@@ -345,7 +344,7 @@ internal constructor(
345344
if (id == deviceId) return CameraDeviceId(id, null)
346345

347346
// Then check if deviceId is a physical camera ID in a logical camera
348-
if (VERSION.SDK_INT >= Build.VERSION_CODES.P) {
347+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
349348
val characteristic = cameraManager.getCameraCharacteristics(id)
350349

351350
for (physicalId in characteristic.physicalCameraIds) {

‎livekit-android-sdk/src/main/java/io/livekit/android/audio/AudioSwitchHandler.kt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,12 @@ import android.os.Build
2323
import android.os.Handler
2424
import android.os.HandlerThread
2525
import android.os.Looper
26-
import com.twilio.audioswitch.*
26+
import com.twilio.audioswitch.AbstractAudioSwitch
27+
import com.twilio.audioswitch.AudioDevice
28+
import com.twilio.audioswitch.AudioDeviceChangeListener
29+
import com.twilio.audioswitch.AudioSwitch
30+
import com.twilio.audioswitch.LegacyAudioSwitch
31+
import io.livekit.android.room.Room
2732
import io.livekit.android.util.LKLog
2833
import javax.inject.Inject
2934
import javax.inject.Singleton

‎livekit-android-sdk/src/main/java/io/livekit/android/audio/ScreenAudioCapturer.kt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,6 @@ constructor(
234234
AudioFormat.ENCODING_PCM_8BIT -> 1
235235
AudioFormat.ENCODING_PCM_16BIT, AudioFormat.ENCODING_IEC61937, AudioFormat.ENCODING_DEFAULT -> 2
236236
AudioFormat.ENCODING_PCM_FLOAT -> 4
237-
AudioFormat.ENCODING_INVALID -> throw IllegalArgumentException("Bad audio format $audioFormat")
238237
else -> throw IllegalArgumentException("Bad audio format $audioFormat")
239238
}
240239
}

‎livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEManager.kt‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2023-2024 LiveKit, Inc.
2+
* Copyright 2023-2025 LiveKit, Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -58,7 +58,7 @@ constructor(
5858
this.peerConnectionFactory = peerConnectionFactory
5959
}
6060

61-
public fun keyProvider(): KeyProvider {
61+
fun keyProvider(): KeyProvider {
6262
return this.keyProvider
6363
}
6464

@@ -70,16 +70,16 @@ constructor(
7070
this.enabled = true
7171
this.room = room
7272
this.emitEvent = emitEvent
73-
this.room?.localParticipant?.trackPublications?.forEach() { item ->
73+
this.room?.localParticipant?.trackPublications?.forEach { item ->
7474
var participant = this.room!!.localParticipant
7575
var publication = item.value
7676
if (publication.track != null) {
7777
addPublishedTrack(publication.track!!, publication, participant, room)
7878
}
7979
}
80-
this.room?.remoteParticipants?.forEach() { item ->
80+
this.room?.remoteParticipants?.forEach { item ->
8181
var participant = item.value
82-
participant.trackPublications.forEach() { item ->
82+
participant.trackPublications.forEach { item ->
8383
var publication = item.value
8484
if (publication.track != null) {
8585
addSubscribedTrack(publication.track!!, publication, participant, room)
@@ -88,7 +88,7 @@ constructor(
8888
}
8989
}
9090

91-
public fun addSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
91+
fun addSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
9292
var rtpReceiver: RtpReceiver? = when (publication.track!!) {
9393
is RemoteAudioTrack -> (publication.track!! as RemoteAudioTrack).receiver
9494
is RemoteVideoTrack -> (publication.track!! as RemoteVideoTrack).receiver
@@ -111,7 +111,7 @@ constructor(
111111
}
112112
}
113113

114-
public fun removeSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
114+
fun removeSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
115115
var trackId = publication.sid
116116
var participantId = participant.identity
117117
var frameCryptor = frameCryptors.get(trackId to participantId)
@@ -122,7 +122,7 @@ constructor(
122122
}
123123
}
124124

125-
public fun addPublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
125+
fun addPublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
126126
var rtpSender: RtpSender? = when (publication.track!!) {
127127
is LocalAudioTrack -> (publication.track!! as LocalAudioTrack)?.sender
128128
is LocalVideoTrack -> (publication.track!! as LocalVideoTrack)?.sender
@@ -146,7 +146,7 @@ constructor(
146146
}
147147
}
148148

149-
public fun removePublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
149+
fun removePublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
150150
var trackId = publication.sid
151151
var participantId = participant.identity
152152
var frameCryptor = frameCryptors.get(trackId to participantId)
@@ -202,7 +202,7 @@ constructor(
202202
* Enable or disable E2EE
203203
* @param enabled
204204
*/
205-
public fun enableE2EE(enabled: Boolean) {
205+
fun enableE2EE(enabled: Boolean) {
206206
this.enabled = enabled
207207
for (item in frameCryptors.entries) {
208208
var frameCryptor = item.value

‎livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEOptions.kt‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2023-2024 LiveKit, Inc.
2+
* Copyright 2023-2025 LiveKit, Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -25,9 +25,8 @@ internal const val defaultFailureTolerance = -1
2525
internal const val defaultKeyRingSize = 16
2626
internal const val defaultDiscardFrameWhenCryptorNotReady = false
2727

28-
class E2EEOptions
29-
constructor(
30-
keyProvider: KeyProvider = BaseKeyProvider(
28+
class E2EEOptions(
29+
var keyProvider: KeyProvider = BaseKeyProvider(
3130
defaultRatchetSalt,
3231
defaultMagicBytes,
3332
defaultRatchetWindowSize,
@@ -38,11 +37,9 @@ constructor(
3837
),
3938
encryptionType: Encryption.Type = Encryption.Type.GCM,
4039
) {
41-
var keyProvider: KeyProvider
4240
var encryptionType: Encryption.Type = Encryption.Type.NONE
4341

4442
init {
45-
this.keyProvider = keyProvider
4643
this.encryptionType = encryptionType
4744
}
4845
}

‎livekit-android-sdk/src/main/java/io/livekit/android/e2ee/KeyProvider.kt‎

Lines changed: 24 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2023-2024 LiveKit, Inc.
2+
* Copyright 2023-2025 LiveKit, Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -20,14 +20,13 @@ import io.livekit.android.util.LKLog
2020
import livekit.org.webrtc.FrameCryptorFactory
2121
import livekit.org.webrtc.FrameCryptorKeyProvider
2222

23-
class KeyInfo
24-
constructor(var participantId: String, var keyIndex: Int, var key: String) {
23+
class KeyInfo(var participantId: String, var keyIndex: Int, var key: String) {
2524
override fun toString(): String {
2625
return "KeyInfo(participantId='$participantId', keyIndex=$keyIndex)"
2726
}
2827
}
2928

30-
public interface KeyProvider {
29+
interface KeyProvider {
3130
fun setSharedKey(key: String, keyIndex: Int? = 0): Boolean
3231
fun ratchetSharedKey(keyIndex: Int? = 0): ByteArray
3332
fun exportSharedKey(keyIndex: Int? = 0): ByteArray
@@ -41,17 +40,30 @@ public interface KeyProvider {
4140
var enableSharedKey: Boolean
4241
}
4342

44-
class BaseKeyProvider
45-
constructor(
46-
private var ratchetSalt: String = defaultRatchetSalt,
47-
private var uncryptedMagicBytes: String = defaultMagicBytes,
48-
private var ratchetWindowSize: Int = defaultRatchetWindowSize,
43+
class BaseKeyProvider(
44+
ratchetSalt: String = defaultRatchetSalt,
45+
uncryptedMagicBytes: String = defaultMagicBytes,
46+
ratchetWindowSize: Int = defaultRatchetWindowSize,
4947
override var enableSharedKey: Boolean = true,
50-
private var failureTolerance: Int = defaultFailureTolerance,
51-
private var keyRingSize: Int = defaultKeyRingSize,
52-
private var discardFrameWhenCryptorNotReady: Boolean = defaultDiscardFrameWhenCryptorNotReady,
48+
failureTolerance: Int = defaultFailureTolerance,
49+
keyRingSize: Int = defaultKeyRingSize,
50+
discardFrameWhenCryptorNotReady: Boolean = defaultDiscardFrameWhenCryptorNotReady,
5351
) : KeyProvider {
52+
override val rtcKeyProvider: FrameCryptorKeyProvider
53+
5454
private var keys: MutableMap<String, MutableMap<Int, String>> = mutableMapOf()
55+
56+
init {
57+
this.rtcKeyProvider = FrameCryptorFactory.createFrameCryptorKeyProvider(
58+
enableSharedKey,
59+
ratchetSalt.toByteArray(),
60+
ratchetWindowSize,
61+
uncryptedMagicBytes.toByteArray(),
62+
failureTolerance,
63+
keyRingSize,
64+
discardFrameWhenCryptorNotReady,
65+
)
66+
}
5567
override fun setSharedKey(key: String, keyIndex: Int?): Boolean {
5668
return rtcKeyProvider.setSharedKey(keyIndex ?: 0, key.toByteArray())
5769
}
@@ -100,18 +112,4 @@ constructor(
100112
override fun setSifTrailer(trailer: ByteArray) {
101113
rtcKeyProvider.setSifTrailer(trailer)
102114
}
103-
104-
override val rtcKeyProvider: FrameCryptorKeyProvider
105-
106-
init {
107-
this.rtcKeyProvider = FrameCryptorFactory.createFrameCryptorKeyProvider(
108-
enableSharedKey,
109-
ratchetSalt.toByteArray(),
110-
ratchetWindowSize,
111-
uncryptedMagicBytes.toByteArray(),
112-
failureTolerance,
113-
keyRingSize,
114-
discardFrameWhenCryptorNotReady,
115-
)
116-
}
117115
}

0 commit comments

Comments
 (0)