Fix pixel density in AdaptiveStream

.changes/adaptive-stream-manual-quality-merge

@@ -0,0 +1 @@
+patch type="improved" "Allow manual video quality selection with adaptive stream enabled"

.changes/adaptive-stream-pixel-density

@@ -0,0 +1 @@
+patch type="fixed" "Fix adaptive stream dimensions on high-density displays"

lib/src/publication/remote.dart

@@ -33,6 +33,7 @@ import '../types/other.dart';
 import '../types/video_dimensions.dart';
 import '../utils.dart';
 import 'track_publication.dart';
+import 'track_settings.dart';
 
 /// Represents a track publication from a RemoteParticipant. Provides methods to
 /// control if we should subscribe to the track, and its quality (for video).
@@ -41,18 +42,31 @@ class RemoteTrackPublication<T extends RemoteTrack> extends TrackPublication<T>
   @override
   final RemoteParticipant participant;
 
-  bool get enabled => _enabled;
-  bool _enabled = true;
+  bool get enabled => _enabledPreference != TrackEnabledPreference.disabled;
+
+  /// The user's explicit enable/disable request via [enable] / [disable].
+  /// [TrackEnabledPreference.unset] means no explicit request, in which case
+  /// adaptive-stream visibility decides. An explicit request takes precedence
+  /// over visibility.
+  TrackEnabledPreference _enabledPreference = TrackEnabledPreference.unset;
 
   /// The current desired FPS of the track. This is only available for video tracks that support SVC.
   int? _fps;
   int get fps => _fps ?? 0;
 
-  VideoQuality? _videoQuality = VideoQuality.HIGH;
-  VideoQuality get videoQuality => _videoQuality ?? VideoQuality.HIGH;
+  // Manual settings (set by user via setVideoQuality / setVideoDimensions)
+  VideoSettings? _userPreference;
+
+  // Adaptive stream state (set automatically by visibility observer)
+  VideoDimensions? _adaptiveStreamDimensions;
+  // Whether adaptive stream is active for this publication (room option on +
+  // remote video track). When false, view visibility never gates `disabled`.
+  bool _adaptiveStreamActive = false;
+  // Whether at least one view of this track is currently visible/sized.
+  bool _adaptiveStreamVisible = true;
 
-  VideoDimensions? _videoDimensions;
-  VideoDimensions? get videoDimensions => _videoDimensions;
+  VideoQuality get videoQuality => _userPreference?.quality ?? VideoQuality.HIGH;
+  VideoDimensions? get videoDimensions => _userPreference?.dimensions;
 
   /// The server may pause the track when they are bandwidth limitations and resume
   /// when there is more capacity. This property will be updated when the track is
@@ -144,32 +158,42 @@ class RemoteTrackPublication<T extends RemoteTrack> extends TrackPublication<T>
 
     final videoTrack = track as VideoTrack;
 
-    final settings = lk_rtc.UpdateTrackSettings(
-      trackSids: [sid],
-      disabled: true,
-    );
-
-    // filter visible build contexts
-    final viewSizes = videoTrack.viewKeys
-        .map((e) => e.currentContext)
-        .nonNulls
-        .map((e) => e.findRenderObject() as RenderBox?)
+    // Filter visible build contexts and scale each view's logical size by its
+    // own pixel density, so the server is asked for physical-pixel dimensions
+    // (retina-aware). Each view's density is configured on its VideoTrackRenderer
+    // and resolved per-view; with AdaptiveStreamPixelDensity.auto the actual
+    // device pixel ratio is read from that view via MediaQuery. The largest
+    // resulting size across all of the track's views is requested.
+    final viewSizes = videoTrack.viewRegistrations
+        .map((registration) {
+          final context = registration.key.currentContext;
+          if (context == null) return null;
+          final renderBox = context.findRenderObject() as RenderBox?;
+          if (renderBox == null || !renderBox.hasSize) return null;
+          final density = registration.pixelDensity.resolve(
+            MediaQuery.maybeDevicePixelRatioOf(context) ?? 1.0,
+          );
+          return renderBox.size * density;
+        })
         .nonNulls
-        .where((e) => e.hasSize)
-        .map((e) => e.size);
+        .toList();
 
     logger.finer('[Visibility] ${track?.sid} watching ${viewSizes.length} views...');
 
     if (viewSizes.isNotEmpty) {
-      // compute largest size
-      final largestSize = viewSizes.reduce((value, element) => maxOfSizes(value, element));
-
-      settings
-        ..disabled = false
-        ..width = largestSize.width.ceil()
-        ..height = largestSize.height.ceil();
+      final largestSize = viewSizes.reduce(maxOfSizes);
+      _adaptiveStreamDimensions = VideoDimensions(
+        largestSize.width.ceil(),
+        largestSize.height.ceil(),
+      );
+      _adaptiveStreamVisible = true;
+    } else {
+      _adaptiveStreamDimensions = null;
+      _adaptiveStreamVisible = false;
     }
 
+    final settings = _buildTrackSettings();
+
     // Only send new settings to server if it changed
     if (settings != _lastSentTrackSettings) {
       _lastSentTrackSettings = settings;
@@ -182,7 +206,13 @@ class RemoteTrackPublication<T extends RemoteTrack> extends TrackPublication<T>
     }
   }
 
-  void _sendPendingTrackSettingsUpdateRequest(lk_rtc.UpdateTrackSettings settings) {
+  void _sendPendingTrackSettingsUpdateRequest(lk_rtc.UpdateTrackSettings _) {
+    // Re-build from the current state at fire time instead of replaying the
+    // snapshot captured when the debounce was scheduled. Otherwise a stale
+    // snapshot could be sent after newer state (e.g. a manual setVideoQuality)
+    // has already been applied, clobbering it.
+    final settings = _buildTrackSettings();
+    _lastSentTrackSettings = settings;
     logger.fine('[Visibility] Sending... ${settings.toProto3Json()}');
     participant.room.engine.signalClient.sendUpdateTrackSettings(settings);
   }
@@ -200,20 +230,23 @@ class RemoteTrackPublication<T extends RemoteTrack> extends TrackPublication<T>
 
       final roomOptions = participant.room.roomOptions;
       if (roomOptions.adaptiveStream && newValue is RemoteVideoTrack) {
+        _adaptiveStreamActive = true;
         // Start monitoring visibility
         _visibilityTimer = Timer.periodic(
           const Duration(milliseconds: 300),
           (_) => _computeVideoViewVisibility(),
         );
 
-        newValue.onVideoViewBuild = (_) {
+        newValue.onVideoViewBuild = () {
           logger.finer('[Visibility] VideoView did build');
           if (_lastSentTrackSettings?.disabled == true) {
             // quick enable
             _cancelPendingTrackSettingsUpdateRequest?.call();
             _computeVideoViewVisibility(quick: true);
           }
         };
+      } else {
+        _adaptiveStreamActive = false;
       }
 
       if (newValue != null) {
@@ -229,7 +262,7 @@ class RemoteTrackPublication<T extends RemoteTrack> extends TrackPublication<T>
     return didUpdate;
   }
 
-  bool _canUpdateManualVideoSettings() {
+  bool _isManualOperationAllowed() {
     if (kind != TrackType.VIDEO) {
       logger.warning('Manual video setting updates are only supported for video tracks');
       return false;
@@ -240,55 +273,59 @@ class RemoteTrackPublication<T extends RemoteTrack> extends TrackPublication<T>
       return false;
     }
 
-    if (participant.room.roomOptions.adaptiveStream) {
-      logger.warning('Manual video setting update ignored because adaptive stream is enabled');
-      return false;
-    }
-
     return true;
   }
 
+  /// For tracks that support simulcasting, adjust subscribed quality.
+  ///
+  /// This indicates the highest quality the client can accept. If network
+  /// bandwidth does not allow, the server will automatically reduce quality to
+  /// optimize for uninterrupted video.
+  ///
+  /// When adaptive stream is active, this preference is merged client-side with
+  /// the dimensions computed from the visible views, and the smaller (more
+  /// conservative) of the two is sent to the server.
   Future<void> setVideoQuality(VideoQuality newValue) async {
-    if (newValue == _videoQuality) return;
-    if (!_canUpdateManualVideoSettings()) return;
-    _videoQuality = newValue;
-    _videoDimensions = null;
-    sendUpdateTrackSettings();
+    if (newValue == _userPreference?.quality) return;
+    if (!_isManualOperationAllowed()) return;
+    _userPreference = VideoSettings.quality(newValue);
+    _emitTrackUpdate();
   }
 
   /// Set preferred video dimensions for this track.
   ///
   /// Server will choose the appropriate layer based on these dimensions.
   /// Will override previous calls to [setVideoQuality].
+  ///
+  /// When adaptive stream is active, this preference is merged client-side with
+  /// the dimensions computed from the visible views, and the smaller (more
+  /// conservative) of the two is sent to the server.
   Future<void> setVideoDimensions(VideoDimensions newValue) async {
-    if (newValue.width == _videoDimensions?.width && newValue.height == _videoDimensions?.height) {
-      return;
-    }
-    if (!_canUpdateManualVideoSettings()) return;
-    _videoDimensions = newValue;
-    _videoQuality = null;
-    sendUpdateTrackSettings();
+    if (newValue == _userPreference?.dimensions) return;
+    if (!_isManualOperationAllowed()) return;
+    _userPreference = VideoSettings.dimensions(newValue);
+    _emitTrackUpdate();
   }
 
   /// Set desired FPS, server will do its best to return FPS close to this.
   /// It's only supported for video codecs that support SVC currently.
   Future<void> setVideoFPS(int newValue) async {
     if (newValue == _fps) return;
-    if (!_canUpdateManualVideoSettings()) return;
+    if (!_isManualOperationAllowed()) return;
     _fps = newValue;
-    sendUpdateTrackSettings();
+    _emitTrackUpdate();
   }
 
   Future<void> enable() async {
-    if (_enabled) return;
-    _enabled = true;
-    sendUpdateTrackSettings();
+    if (_enabledPreference == TrackEnabledPreference.enabled) return;
+    _enabledPreference = TrackEnabledPreference.enabled;
+    _emitTrackUpdate();
   }
 
   Future<void> disable() async {
-    if (!_enabled) return;
-    _enabled = false;
-    sendUpdateTrackSettings();
+    if (_enabledPreference == TrackEnabledPreference.disabled) return;
+    _enabledPreference = TrackEnabledPreference.disabled;
+    _emitTrackUpdate();
   }
 
   Future<void> subscribe() async {
@@ -333,26 +370,49 @@ class RemoteTrackPublication<T extends RemoteTrack> extends TrackPublication<T>
     participant.room.engine.signalClient.sendUpdateSubscription(subscription);
   }
 
-  @internal
-  void sendUpdateTrackSettings() {
-    final settings = lk_rtc.UpdateTrackSettings(
-      trackSids: [sid],
-      disabled: !_enabled,
+  lk_rtc.UpdateTrackSettings _buildTrackSettings() {
+    final isDisabled = resolveDisabled(
+      enabledPreference: _enabledPreference,
+      adaptiveStreamActive: _adaptiveStreamActive,
+      adaptiveStreamVisible: _adaptiveStreamVisible,
     );
-    if (kind == TrackType.VIDEO) {
-      if (_videoDimensions != null) {
-        settings.width = _videoDimensions!.width;
-        settings.height = _videoDimensions!.height;
-      } else if (_videoQuality != null) {
-        settings.quality = _videoQuality!.toPBType();
-      } else {
-        settings.quality = VideoQuality.HIGH.toPBType();
-      }
-      if (_fps != null) settings.fps = _fps!;
+
+    if (kind != TrackType.VIDEO) {
+      return buildUpdateTrackSettings(sid: sid, disabled: isDisabled);
     }
+
+    final resolved = resolveVideoSettings(
+      adaptiveStreamDimensions: _adaptiveStreamDimensions,
+      userPreference: _userPreference,
+      layerDimensionsForQuality: (quality) {
+        final pbQuality = quality.toPBType();
+        final layer = latestInfo?.layers.where((l) => l.quality == pbQuality).firstOrNull;
+        if (layer == null) return null;
+        return VideoDimensions(layer.width, layer.height);
+      },
+    );
+
+    return buildUpdateTrackSettings(
+      sid: sid,
+      disabled: isDisabled,
+      dimensions: resolved.dimensions,
+      quality: resolved.quality?.toPBType(),
+      fps: _fps,
+    );
+  }
+
+  void _emitTrackUpdate() {
+    // Cancel any pending debounced visibility update so its (now potentially
+    // stale) snapshot cannot fire after — and clobber — this immediate update.
+    _cancelPendingTrackSettingsUpdateRequest?.call();
+    final settings = _buildTrackSettings();
+    _lastSentTrackSettings = settings;
     participant.room.engine.signalClient.sendUpdateTrackSettings(settings);
   }
 
+  @internal
+  void sendUpdateTrackSettings() => _emitTrackUpdate();
+
   @internal
   // Update internal var and return true if changed
   Future<bool> updateSubscriptionAllowed(bool allowed) async {