Merge pull request #21299 from wireapp/feat/offering-vga-and-hd-WPB-25235

feat: offering automated resolution changes [WPB-25235]

Author: EnricoSchw

apps/webapp/src/script/components/calling/VideoControls/videoBackgroundPerformancePanel/videoBackgroundPerformancePanel.tsx

@@ -49,7 +49,7 @@ type PerformancePanelProps = {
   backgroundEffectsHandler: BackgroundEffectsHandler;
 };
 
-const QUALITY_OPTIONS: readonly QualityMode[] = ['auto', 'superhigh', 'high', 'medium', 'low', 'bypass'];
+const QUALITY_OPTIONS: readonly QualityMode[] = ['auto', 'fhd', 'hd', 'qhd', 'nhd', 'bypass'];
 
 const formatMs = (value?: number | null): string => {
   return typeof value === 'number' ? `${value.toFixed(1)} ms` : '-';
@@ -66,7 +66,7 @@ const formatValue = (value?: string | number | null): string => {
 const getMetricRows = (renderMetrics: RenderMetrics) => {
   return Maybe.of(renderMetrics)
     .map(metrics => [
-      {label: 'Quality', value: formatValue(metrics.tier)},
+      {label: 'Quality', value: formatValue(metrics.tier).toUpperCase()},
       {label: 'Total', value: formatMs(metrics.avgTotalMs)},
       {label: 'Segmentation', value: formatMs(metrics.avgSegmentationMs)},
       {label: 'GPU', value: formatMs(metrics.avgGpuMs)},
@@ -135,7 +135,7 @@ const MetricsDisplay = ({capabilityInfo}: MetricsDisplayProps) => {
 };
 
 const qualitySelectOptions = QUALITY_OPTIONS.map(option => ({
-  label: option,
+  label: option.toUpperCase(),
   value: option,
 }));
 

apps/webapp/src/script/repositories/media/backgroundEffects/backgroundEffectsController.ts

@@ -17,16 +17,20 @@
  *
  */
 
+import {PerformanceSample} from 'Repositories/media/backgroundEffects/helper/samples';
+import {
+  getBestMatchingQualityTier,
+  QualityMode,
+  QualityTier,
+  Resolution,
+  resolutionIsGreaterThanOrEqualTo,
+  TIER_DEFINITIONS,
+} from 'Repositories/media/backgroundEffects/quality/definitions';
+import {QUALITY_TIERS, QualityController} from 'Repositories/media/backgroundEffects/quality/qualityController';
 import {BackgroundSource} from 'Repositories/media/VideoBackgroundEffects';
 import {getLogger, Logger} from 'Util/logger';
 
-import {
-  type CapabilityInfo,
-  type EffectMode,
-  type Metrics,
-  QualityMode,
-  type QualityTier,
-} from './backgroundEffectsWorkerTypes';
+import {type CapabilityInfo, type EffectMode, type Metrics, Mode} from './backgroundEffectsWorkerTypes';
 import {detectCapabilities} from './helper/capability';
 import {
   defaultOpts,
@@ -41,10 +45,15 @@ import {runSegmenter, updateSegmenterOptions} from './pipe/segmenter';
 // The shader's blur radius is 30 px, so a sigma in the ~10–20 px range gives
 // visually useful blur.  Multiply by this factor to get from the 0–1 range.
 const BLUR_SIGMA_SCALE = 15;
+const DEFAULT_FRAME_RATE = 15;
+const MAX_WIDTH = 256;
+const MAX_HEIGHT = 144;
 
 export class BackgroundEffectsController {
   private readonly logger: Logger;
 
+  private inputTrack: MediaStreamTrack | null = null;
+
   private worker: Worker | null = null;
   private options: ProcessVideoTrackOptions = defaultOpts;
 
@@ -57,7 +66,10 @@ export class BackgroundEffectsController {
     requestVideoFrameCallback: false,
   };
 
-  private maxQualityTier: QualityTier = 'superhigh';
+  private qualityController: QualityController | null = null;
+  private qualitySampleQueue = Promise.resolve();
+  private maxResolution: Resolution = TIER_DEFINITIONS.hd.resolution;
+  private maxQualityTier: QualityTier = 'hd';
   private refcount = 0;
 
   /**
@@ -79,8 +91,21 @@ export class BackgroundEffectsController {
     const trackCapabilities = inputTrack.getCapabilities();
     const trackSettings = inputTrack.getSettings();
     const trackConstraints = inputTrack.getConstraints();
-    const {width, height, frameRate} = trackSettings;
+    let {width, height, frameRate} = trackSettings;
+    if (!width) {
+      width = TIER_DEFINITIONS[QUALITY_TIERS.HD].resolution.width;
+    }
+    if (!height) {
+      height = TIER_DEFINITIONS[QUALITY_TIERS.HD].resolution.height;
+    }
+    if (!frameRate) {
+      frameRate = DEFAULT_FRAME_RATE;
+    }
+    this.maxResolution = {width, height};
+    this.maxQualityTier = getBestMatchingQualityTier(this.maxResolution).tier;
+    this.qualityController = new QualityController(frameRate, this.maxQualityTier);
 
+    this.inputTrack = inputTrack;
     this.logger.info(`start: ${width}x${height} ${frameRate}fps`, {
       trackCapabilities,
       trackSettings,
@@ -90,6 +115,8 @@ export class BackgroundEffectsController {
     const {readable} = new TrackProcessor({track: inputTrack});
 
     const canvas = document.createElement('canvas');
+    canvas.width = MAX_WIDTH;
+    canvas.height = MAX_HEIGHT;
     const outputTrack = canvas.captureStream(frameRate).getVideoTracks()[0];
     const offscreen = canvas.transferControlToOffscreen();
 
@@ -109,16 +136,30 @@ export class BackgroundEffectsController {
         transferables,
       );
       onWorkerMessage = ({data}: MessageEvent) => {
-        const {name, stats} = data as {name: string; stats: Metrics};
+        const {name} = data as {name: string};
         if (name === 'stats' && this.onMetrics) {
+          const {stats} = data as {stats: Metrics};
           this.onMetrics(stats);
         }
+
+        if (name === 'performanceSample' && this.qualityController !== null) {
+          const {sample, mode} = data as {sample: PerformanceSample; mode: Mode};
+          this.enqueuePerformanceSample(sample, mode);
+        }
       };
 
       this.worker.addEventListener('message', onWorkerMessage);
     } else {
       const {options: workerOptions} = getWorkerOptions(resolved);
-      await runSegmenter(offscreen, readable, workerOptions, stats => this.onMetrics?.(stats));
+      await runSegmenter(
+        offscreen,
+        readable,
+        workerOptions,
+        stats => this.onMetrics?.(stats),
+        (sample: PerformanceSample, mode: Mode) => {
+          this.enqueuePerformanceSample(sample, mode);
+        },
+      );
     }
 
     outputTrack.stop = () => {
@@ -176,9 +217,15 @@ export class BackgroundEffectsController {
     }
   }
 
-  public setQuality(quality: QualityMode): void {
+  public async setQuality(quality: QualityMode): Promise<void> {
     this.logger.info('setQuality', quality);
-    this.options = {...this.options, quality};
+
+    let requestedQuality = quality;
+    if (quality !== 'auto') {
+      requestedQuality = await this.changeResolution(quality);
+    }
+
+    this.options = {...this.options, quality: requestedQuality};
     this.pushOptionsUpdate();
   }
 
@@ -235,6 +282,64 @@ export class BackgroundEffectsController {
       updateSegmenterOptions(finalOptions);
     }
   }
+
+  private async changeResolution(quality: QualityMode): Promise<QualityMode> {
+    if (!this.inputTrack || quality === 'auto') {
+      return quality;
+    }
+
+    const mediaConstraints = this.inputTrack.getConstraints();
+
+    let newResolution: Resolution;
+    let newQualityTier: QualityTier;
+
+    if (quality === 'bypass') {
+      newResolution = this.maxResolution;
+      newQualityTier = quality;
+    } else {
+      const requestedResolution = TIER_DEFINITIONS[quality].resolution;
+
+      if (this.maxResolution === null || resolutionIsGreaterThanOrEqualTo(this.maxResolution, requestedResolution)) {
+        newResolution = requestedResolution;
+        newQualityTier = quality;
+      } else {
+        newResolution = this.maxResolution;
+        newQualityTier = this.maxQualityTier;
+      }
+    }
+
+    mediaConstraints.width = {ideal: newResolution.width};
+    mediaConstraints.height = {ideal: newResolution.height};
+
+    try {
+      await this.inputTrack.applyConstraints(mediaConstraints);
+      return newQualityTier;
+    } catch (error: unknown) {
+      this.logger.warn('Failed to change resolution', error);
+      return this.maxQualityTier;
+    }
+  }
+
+  private enqueuePerformanceSample(sample: PerformanceSample, mode: Mode): void {
+    this.qualitySampleQueue = this.qualitySampleQueue
+      .then(() => this.onPerformanceSample(sample, mode))
+      .catch((error: unknown) => {
+        this.logger?.error?.('onPerformanceSample failed', {error});
+      });
+  }
+
+  private async onPerformanceSample(sample: PerformanceSample, mode: Mode): Promise<void> {
+    if (this.qualityController === null) {
+      this.logger.warn('onPerformanceSample: qualityController is null');
+      return;
+    }
+    const tier = this.qualityController.update(sample, mode);
+
+    if (tier.tier === this.qualityController.getCurrentTier()) {
+      return;
+    }
+    return this.setQuality(tier.tier);
+  }
 }
 
 // ---------------------------------------------------------------------------

apps/webapp/src/script/repositories/media/backgroundEffects/backgroundEffectsWorkerTypes.ts

@@ -24,6 +24,8 @@
  * including effect modes, quality settings, worker messages, and configuration options.
  */
 
+import {QualityMode, QualityTier} from 'Repositories/media/backgroundEffects/quality/definitions';
+
 /**
  * Effect mode for background processing.
  * - 'blur': Applies blur effect to the background
@@ -49,15 +51,6 @@ export type Mode = Exclude<EffectMode, 'passthrough'>;
  */
 export type DebugMode = 'off' | 'maskOverlay' | 'maskOnly' | 'edgeOnly' | 'classOverlay' | 'classOnly';
 
-export type QualityTier = 'superhigh' | 'high' | 'medium' | 'low' | 'bypass';
-
-/**
- * Quality mode for rendering performance.
- * - 'auto': Adaptive quality based on performance metrics
- * -  QualityTier
- */
-export type QualityMode = 'auto' | QualityTier;
-
 /**
  * Optional mapping of quality tiers to segmentation model paths.
  */
@@ -91,85 +84,6 @@ export type QualityPolicyResolver = (capabilities: CapabilityInfo) => QualityPol
  */
 export type PipelineType = 'worker-webgl2' | 'main-webgl2' | 'canvas2d' | 'passthrough';
 
-/**
- * Quality tier parameters that control rendering performance and visual quality.
- *
- * These parameters are generated by QualityController and used by WebGlRenderer
- * to configure the rendering pipeline. They balance visual quality against
- * performance by adjusting segmentation resolution, processing cadence, and
- * post-processing effects.
- */
-export interface QualityTierParams {
-  /** Quality tier identifier */
-  tier: QualityTier;
-  /** Segmentation mask width in pixels. Lower values reduce CPU/ML cost. */
-  segmentationWidth: number;
-  /** Segmentation mask height in pixels. Lower values reduce CPU/ML cost. */
-  segmentationHeight: number;
-  /** Segmentation cadence (process every Nth frame). Higher values reduce CPU/ML cost. */
-  segmentationCadence: number;
-  /** Scale factor for mask refinement pass (0-1). Lower values reduce GPU cost. */
-  maskRefineScale: number;
-  /** Scale factor for blur downsampling (0-1). Lower values reduce GPU cost. */
-  blurDownsampleScale: number;
-  /** Blur radius in pixels. Lower values reduce GPU cost. */
-  blurRadius: number;
-  /** Joint bilateral filter radius in pixels. Lower values reduce GPU cost. */
-  bilateralRadius: number;
-  /** Spatial sigma for joint bilateral filter. Controls spatial smoothing. */
-  bilateralSpatialSigma: number;
-  /** Range sigma for joint bilateral filter. Controls edge preservation. */
-  bilateralRangeSigma: number;
-  /** Lower threshold for soft matte edge (0-1). Controls where soft edges begin. */
-  softLow: number;
-  /** Upper threshold for soft matte edge (0-1). Controls where soft edges end. */
-  softHigh: number;
-  /** Lower threshold for matte cutoff (0-1). Pixels below this are considered background. */
-  matteLow: number;
-  /** Upper threshold for matte cutoff (0-1). Pixels above this are considered foreground. */
-  matteHigh: number;
-  /** Hysteresis value for matte thresholds to prevent flickering (0-1). */
-  matteHysteresis: number;
-  /** Temporal smoothing alpha (0-1). Higher values increase temporal stability. */
-  temporalAlpha: number;
-  /** If true, bypass all processing and pass through original frames. */
-  bypass: boolean;
-}
-
-/**
- * Background source for virtual background mode (static image).
- *
- * Used internally by BackgroundEffectsController to store and manage
- * background images for virtual background replacement.
- */
-export interface BackgroundSourceImage {
-  /** Discriminator for type narrowing. */
-  type: 'image';
-  /** Image bitmap data (transferred to worker if using worker pipeline). */
-  bitmap: ImageBitmap;
-  /** Image width in pixels. */
-  width: number;
-  /** Image height in pixels. */
-  height: number;
-}
-
-/**
- * Background source for virtual background mode (video frame).
- *
- * Used internally by BackgroundEffectsController to store and manage
- * background video frames for virtual background replacement.
- */
-export interface BackgroundSourceVideoFrame {
-  /** Discriminator for type narrowing. */
-  type: 'video';
-  /** Video frame bitmap data (transferred to worker if using worker pipeline). */
-  bitmap: ImageBitmap;
-  /** Frame width in pixels. */
-  width: number;
-  /** Frame height in pixels. */
-  height: number;
-}
-
 /**
  * Performance metrics tracked during frame processing.
  *

apps/webapp/src/script/repositories/media/backgroundEffects/helper/samples.ts

@@ -1,6 +1,6 @@
 /*
  * Wire
- * Copyright (C) 2025 Wire Swiss GmbH
+ * Copyright (C) 2026 Wire Swiss GmbH
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

apps/webapp/src/script/repositories/media/backgroundEffects/index.ts

@@ -74,12 +74,12 @@ export type {EffectMode} from './backgroundEffectsWorkerTypes';
  */
 export type {DebugMode} from './backgroundEffectsWorkerTypes';
 
-export type {QualityTier} from './backgroundEffectsWorkerTypes';
+export type {QualityTier} from './quality/definitions';
 
 /**
  * Quality mode type ('auto' or fixed QualityTier).
  */
-export type {QualityMode} from './backgroundEffectsWorkerTypes';
+export type {QualityMode} from './quality/definitions';
 
 /**
  * BackgroundEffectsRenderingPipeline selection type.

apps/webapp/src/script/repositories/media/backgroundEffects/pipe/options.ts

@@ -99,12 +99,12 @@ export const defaultOpts = {
 
   // Segmenter options.
   borderSmooth: 0,
-  smoothing: 0.8,
+  smoothing: 1,
   smoothstepMin: 0.6,
   smoothstepMax: 0.9,
   restartEvery: 0,
   bgBlur: 0,
-  bgBlurRadius: 30,
+  bgBlurRadius: 5,
 
   // Filter options.
   enableFilters: false,

apps/webapp/src/script/repositories/media/backgroundEffects/pipe/renderer.ts

@@ -441,11 +441,8 @@ export class WebGLRenderer {
     }
     const {gl, fbo, storedStateTextures, stateUpdateProgram, stateUpdateLocations, blendProgram, blendLocations} = this;
 
-    const {displayWidth: width, displayHeight: height} = videoFrame;
-    if (this.canvas.width !== width || this.canvas.height !== height) {
-      this.canvas.width = width;
-      this.canvas.height = height;
-    }
+    const width = this.canvas.width;
+    const height = this.canvas.height;
 
     if (!categoryTexture || !confidenceTexture) {
       gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);