Add camera facing mode detection (livekit#738)

Author: Ocupe

.changeset/small-tools-shout.md

@@ -0,0 +1,5 @@
+---
+"livekit-client": patch
+---
+
+Add helper function to detect camera `facingMode`.

src/room/track/utils.test.ts

@@ -1,5 +1,5 @@
 import { AudioCaptureOptions, VideoCaptureOptions, VideoPresets } from './options';
-import { constraintsForOptions, mergeDefaultOptions } from './utils';
+import { constraintsForOptions, facingModeFromDeviceLabel, mergeDefaultOptions } from './utils';
 
 describe('mergeDefaultOptions', () => {
   const audioDefaults: AudioCaptureOptions = {
@@ -108,3 +108,32 @@ describe('constraintsForOptions', () => {
     expect(videoOpts.aspectRatio).toEqual(VideoPresets.h720.resolution.aspectRatio);
   });
 });
+
+describe('Test facingMode detection', () => {
+  test('OBS virtual camera should be detected.', () => {
+    const result = facingModeFromDeviceLabel('OBS Virtual Camera');
+    expect(result?.facingMode).toEqual('environment');
+    expect(result?.confidence).toEqual('medium');
+  });
+
+  test.each([
+    ['Peter’s iPhone Camera', { facingMode: 'environment', confidence: 'medium' }],
+    ['iPhone de Théo Camera', { facingMode: 'environment', confidence: 'medium' }],
+  ])(
+    'Device labels that contain "iphone" should return facingMode "environment".',
+    (label, expected) => {
+      const result = facingModeFromDeviceLabel(label);
+      expect(result?.facingMode).toEqual(expected.facingMode);
+      expect(result?.confidence).toEqual(expected.confidence);
+    },
+  );
+
+  test.each([
+    ['Peter’s iPad Camera', { facingMode: 'environment', confidence: 'medium' }],
+    ['iPad de Théo Camera', { facingMode: 'environment', confidence: 'medium' }],
+  ])('Device label that contain "ipad" should detect.', (label, expected) => {
+    const result = facingModeFromDeviceLabel(label);
+    expect(result?.facingMode).toEqual(expected.facingMode);
+    expect(result?.confidence).toEqual(expected.confidence);
+  });
+});

src/room/track/utils.ts

@@ -1,4 +1,6 @@
 import { sleep } from '../utils';
+import log from './../../logger';
+import LocalTrack from './LocalTrack';
 import type { AudioCaptureOptions, CreateLocalTracksOptions, VideoCaptureOptions } from './options';
 import type { AudioTrack } from './types';
 
@@ -112,3 +114,103 @@ export function getNewAudioContext(): AudioContext | void {
     return new AudioContext({ latencyHint: 'interactive' });
   }
 }
+
+type FacingMode = NonNullable<VideoCaptureOptions['facingMode']>;
+type FacingModeFromLocalTrackOptions = {
+  /**
+   * If no facing mode can be determined, this value will be used.
+   * @defaultValue 'user'
+   */
+  defaultFacingMode?: FacingMode;
+};
+type FacingModeFromLocalTrackReturnValue = {
+  /**
+   * The (probable) facingMode of the track.
+   * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackConstraints/facingMode | MDN docs on facingMode}
+   */
+  facingMode: FacingMode;
+  /**
+   * The confidence that the returned facingMode is correct.
+   */
+  confidence: 'high' | 'medium' | 'low';
+};
+
+/**
+ * Try to analyze the local track to determine the facing mode of a track.
+ *
+ * @remarks
+ * There is no property supported by all browsers to detect whether a video track originated from a user- or environment-facing camera device.
+ * For this reason, we use the `facingMode` property when available, but will fall back on a string-based analysis of the device label to determine the facing mode.
+ * If both methods fail, the default facing mode will be used.
+ *
+ * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackConstraints/facingMode | MDN docs on facingMode}
+ * @experimental
+ */
+export function facingModeFromLocalTrack(
+  localTrack: LocalTrack | MediaStreamTrack,
+  options: FacingModeFromLocalTrackOptions = {},
+): FacingModeFromLocalTrackReturnValue {
+  const track = localTrack instanceof LocalTrack ? localTrack.mediaStreamTrack : localTrack;
+  const trackSettings = track.getSettings();
+  let result: FacingModeFromLocalTrackReturnValue = {
+    facingMode: options.defaultFacingMode ?? 'user',
+    confidence: 'low',
+  };
+
+  // 1. Try to get facingMode from track settings.
+  if ('facingMode' in trackSettings) {
+    const rawFacingMode = trackSettings.facingMode;
+    log.debug('rawFacingMode', { rawFacingMode });
+    if (rawFacingMode && typeof rawFacingMode === 'string' && isFacingModeValue(rawFacingMode)) {
+      result = { facingMode: rawFacingMode, confidence: 'high' };
+    }
+  }
+
+  // 2. If we don't have a high confidence we try to get the facing mode from the device label.
+  if (['low', 'medium'].includes(result.confidence)) {
+    log.debug(`Try to get facing mode from device label: (${track.label})`);
+    const labelAnalysisResult = facingModeFromDeviceLabel(track.label);
+    if (labelAnalysisResult !== undefined) {
+      result = labelAnalysisResult;
+    }
+  }
+
+  return result;
+}
+
+const knownDeviceLabels = new Map<string, FacingModeFromLocalTrackReturnValue>([
+  ['obs virtual camera', { facingMode: 'environment', confidence: 'medium' }],
+]);
+const knownDeviceLabelSections = new Map<string, FacingModeFromLocalTrackReturnValue>([
+  ['iphone', { facingMode: 'environment', confidence: 'medium' }],
+  ['ipad', { facingMode: 'environment', confidence: 'medium' }],
+]);
+/**
+ * Attempt to analyze the device label to determine the facing mode.
+ *
+ * @experimental
+ */
+export function facingModeFromDeviceLabel(
+  deviceLabel: string,
+): FacingModeFromLocalTrackReturnValue | undefined {
+  const label = deviceLabel.trim().toLowerCase();
+  // Empty string is a valid device label but we can't infer anything from it.
+  if (label === '') {
+    return undefined;
+  }
+
+  // Can we match against widely known device labels.
+  if (knownDeviceLabels.has(label)) {
+    return knownDeviceLabels.get(label);
+  }
+
+  // Can we match against sections of the device label.
+  return Array.from(knownDeviceLabelSections.entries()).find(([section]) =>
+    label.includes(section),
+  )?.[1];
+}
+
+function isFacingModeValue(item: string): item is FacingMode {
+  const allowedValues: FacingMode[] = ['user', 'environment', 'left', 'right'];
+  return item === undefined || allowedValues.includes(item as FacingMode);
+}