From 3f52c36e4a501f07cfe687e788538d8302d3d16c Mon Sep 17 00:00:00 2001
From: Denis Lyazgin <lyazgindenis@gmail.com>
Date: Mon, 2 Feb 2026 18:03:29 +0700
Subject: [PATCH] Pass rotation to selfie segmenter

---
 .changeset/friendly-forks-exist.md                          | 5 +++++
 .../processing/video/VirtualBackgroundVideoProcessor.kt     | 6 ++----
 .../track/processing/video/shader/CompositeShader.kt        | 5 +++--
 .../track/processing/video/shader/DefaultVertexShader.kt    | 4 +++-
 4 files changed, 13 insertions(+), 7 deletions(-)
 create mode 100644 .changeset/friendly-forks-exist.md

diff --git a/.changeset/friendly-forks-exist.md b/.changeset/friendly-forks-exist.md
new file mode 100644
index 000000000..b7469e44a
--- /dev/null
+++ b/.changeset/friendly-forks-exist.md
@@ -0,0 +1,5 @@
+---
+"client-sdk-android": patch
+---
+
+Make selfie segmenter work better
diff --git a/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/VirtualBackgroundVideoProcessor.kt b/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/VirtualBackgroundVideoProcessor.kt
index e8e0f2376..b667f9f29 100644
--- a/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/VirtualBackgroundVideoProcessor.kt
+++ b/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/VirtualBackgroundVideoProcessor.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2024-2025 LiveKit, Inc.
+ * Copyright 2024-2026 LiveKit, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -122,9 +122,7 @@ class VirtualBackgroundVideoProcessor(
             val image = imageProxy.image
 
             if (enabled && image != null) {
-                // Put 0 for rotation degrees
-                // We'll rotate it together with the original video frame in the shader.
-                val inputImage = InputImage.fromMediaImage(image, 0)
+                val inputImage = InputImage.fromMediaImage(image, imageProxy.imageInfo.rotationDegrees)
                 latch.acquire()
                 val task = segmenter.process(inputImage)
                 task.addOnSuccessListener { mask ->
diff --git a/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/shader/CompositeShader.kt b/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/shader/CompositeShader.kt
index 319a2bb37..6efa94db0 100644
--- a/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/shader/CompositeShader.kt
+++ b/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/shader/CompositeShader.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2025 LiveKit, Inc.
+ * Copyright 2025-2026 LiveKit, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,13 +29,14 @@ uniform sampler2D background;
 uniform samplerExternalOES frame;
 uniform sampler2D mask;
 out vec4 fragColor;
+in vec2 vMaskCoords;
 
 void main() {
 
     vec4 frameTex = texture(frame, texCoords);
     vec4 bgTex = texture(background, texCoords);
 
-    float maskVal = texture(mask, texCoords).r;
+    float maskVal = texture(mask, vMaskCoords).r;
 
     // Compute screen-space gradient to detect edge sharpness
     float grad = length(vec2(dFdx(maskVal), dFdy(maskVal)));
diff --git a/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/shader/DefaultVertexShader.kt b/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/shader/DefaultVertexShader.kt
index 396b81fda..7ec39c328 100644
--- a/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/shader/DefaultVertexShader.kt
+++ b/livekit-android-track-processors/src/main/java/io/livekit/android/track/processing/video/shader/DefaultVertexShader.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2025 LiveKit, Inc.
+ * Copyright 2025-2026 LiveKit, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,12 +18,14 @@ package io.livekit.android.track.processing.video.shader
 
 internal const val DEFAULT_VERTEX_SHADER_SOURCE = """#version 300 es
 out vec2 texCoords;
+out vec2 vMaskCoords;
 in vec4 in_pos;
 in vec4 in_tc;
 uniform mat4 tex_mat;
 void main() {
     gl_Position = in_pos;
     texCoords = (tex_mat * in_tc).xy;
+    vMaskCoords = vec2(in_tc.x, 1.0 - in_tc.y);
 }
 """