Video calibration

Author: jabberrock

gui/electron/main/index.ts

@@ -32,13 +32,12 @@ import { writeFileSync } from 'node:fs';
 import { spawn } from 'node:child_process';
 import { discordPresence } from './presence';
 import { options } from './cli';
-import { ServerStatusEvent } from 'electron/preload/interface';
+import { ServerStatusEvent, WebcamOfferRequest } from 'electron/preload/interface';
 import { mkdir } from 'node:fs/promises';
 import { MenuItem } from 'electron/main';
 
-
-app.setPath('userData', getGuiDataFolder())
-app.setPath('sessionData', join(getGuiDataFolder(), 'electron'))
+app.setPath('userData', getGuiDataFolder());
+app.setPath('sessionData', join(getGuiDataFolder(), 'electron'));
 
 // Register custom protocol to handle asset paths with leading slashes
 protocol.registerSchemesAsPrivileged([
@@ -55,6 +54,13 @@ protocol.registerSchemesAsPrivileged([
 
 let mainWindow: BrowserWindow | null = null;
 
+function buildWebcamOfferUrl(host: string, port: number) {
+  const normalizedHost =
+    host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;
+
+  return `http://${normalizedHost}:${port}/offer`;
+}
+
 handleIpc(IPC_CHANNELS.GH_FETCH, async (e, options) => {
   if (options.type === 'fw-releases') {
     return fetch(
@@ -153,6 +159,31 @@ handleIpc(IPC_CHANNELS.DISCORD_PRESENCE, async (e, options) => {
   }
 });
 
+handleIpc(IPC_CHANNELS.WEBCAM_OFFER, async (e, request: WebcamOfferRequest) => {
+  const response = await fetch(buildWebcamOfferUrl(request.host, request.port), {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      sdp: request.sdp,
+    }),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Offer request failed with status ${response.status}`);
+  }
+
+  const body = (await response.json()) as { sdp?: unknown };
+  if (typeof body.sdp !== 'string' || body.sdp.length === 0) {
+    throw new Error('Webcam response did not contain an SDP answer');
+  }
+
+  return {
+    sdp: body.sdp,
+  };
+});
+
 handleIpc(IPC_CHANNELS.OPEN_FILE, (e, folder) => {
   const requestedPath = path.resolve(folder);
 
@@ -339,8 +370,7 @@ function createWindow() {
     menu.append(new MenuItem({ label: 'Copy', role: 'copy' }));
     menu.append(new MenuItem({ label: 'Paste', role: 'paste' }));
 
-    if (mainWindow)
-      menu.popup({ window: mainWindow });
+    if (mainWindow) menu.popup({ window: mainWindow });
   });
 }
 

gui/electron/preload/index.ts

@@ -31,9 +31,18 @@ contextBridge.exposeInMainWorld('electronAPI', {
   setTranslations: () => {},
   openDialog: (options) => ipcRenderer.invoke(IPC_CHANNELS.OPEN_DIALOG, options),
   saveDialog: (options) => ipcRenderer.invoke(IPC_CHANNELS.SAVE_DIALOG, options),
-  openConfigFolder: async () => ipcRenderer.invoke(IPC_CHANNELS.OPEN_FILE, await ipcRenderer.invoke(IPC_CHANNELS.GET_FOLDER, 'config')),
-  openLogsFolder: async () => ipcRenderer.invoke(IPC_CHANNELS.OPEN_FILE, await ipcRenderer.invoke(IPC_CHANNELS.GET_FOLDER, 'logs')),
+  openConfigFolder: async () =>
+    ipcRenderer.invoke(
+      IPC_CHANNELS.OPEN_FILE,
+      await ipcRenderer.invoke(IPC_CHANNELS.GET_FOLDER, 'config')
+    ),
+  openLogsFolder: async () =>
+    ipcRenderer.invoke(
+      IPC_CHANNELS.OPEN_FILE,
+      await ipcRenderer.invoke(IPC_CHANNELS.GET_FOLDER, 'logs')
+    ),
   openFile: (path) => ipcRenderer.invoke(IPC_CHANNELS.OPEN_FILE, path),
   ghGet: (req) => ipcRenderer.invoke(IPC_CHANNELS.GH_FETCH, req),
-  setPresence: (options) => ipcRenderer.invoke(IPC_CHANNELS.DISCORD_PRESENCE, options)
+  setPresence: (options) => ipcRenderer.invoke(IPC_CHANNELS.DISCORD_PRESENCE, options),
+  webcamOffer: (request) => ipcRenderer.invoke(IPC_CHANNELS.WEBCAM_OFFER, request),
 } satisfies IElectronAPI);

gui/electron/preload/interface.d.ts

@@ -34,7 +34,15 @@ export type GHReturn = {
     | null;
 };
 
-export type DiscordPresence = { enable: false } | { enable: true, activity: string }
+export type DiscordPresence = { enable: false } | { enable: true; activity: string };
+export type WebcamOfferRequest = {
+  host: string;
+  port: number;
+  sdp: string;
+};
+export type WebcamOfferResponse = {
+  sdp: string;
+};
 
 export interface IElectronAPI {
   onServerStatus: (cb: (data: ServerStatusEvent) => void) => () => void;
@@ -55,6 +63,7 @@ export interface IElectronAPI {
   openFile: (path: string) => void;
   ghGet: <T extends GHGet>(options: T) => Promise<GHReturn[T['type']]>;
   setPresence: (options: DiscordPresence) => void;
+  webcamOffer: (request: WebcamOfferRequest) => Promise<WebcamOfferResponse>;
 }
 
 declare global {

gui/electron/shared.ts

@@ -4,7 +4,14 @@ import {
   SaveDialogOptions,
   SaveDialogReturnValue,
 } from 'electron';
-import { DiscordPresence, GHGet, GHReturn, OSStats } from './preload/interface';
+import {
+  DiscordPresence,
+  GHGet,
+  GHReturn,
+  OSStats,
+  WebcamOfferRequest,
+  WebcamOfferResponse,
+} from './preload/interface';
 
 export const IPC_CHANNELS = {
   SERVER_STATUS: 'server-status',
@@ -19,7 +26,8 @@ export const IPC_CHANNELS = {
   OPEN_FILE: 'open-file',
   GET_FOLDER: 'get-folder',
   GH_FETCH: 'gh-fetch',
-  DISCORD_PRESENCE: 'discord-presence'
+  DISCORD_PRESENCE: 'discord-presence',
+  WEBCAM_OFFER: 'webcam-offer',
 } as const;
 
 export interface IpcInvokeMap {
@@ -46,4 +54,7 @@ export interface IpcInvokeMap {
     options: T
   ) => Promise<GHReturn[T['type']]>;
   [IPC_CHANNELS.DISCORD_PRESENCE]: (options: DiscordPresence) => void;
+  [IPC_CHANNELS.WEBCAM_OFFER]: (
+    request: WebcamOfferRequest
+  ) => Promise<WebcamOfferResponse>;
 }

gui/package.json

@@ -17,6 +17,7 @@
   },
   "scripts": {
     "start": "vite --force",
+    "dev:clean": "rd /s /q \"node_modules/.vite\" && pnpm run gui",
     "gui": "electron-vite dev --config electron.vite.config.ts --watch",
     "build": "electron-vite build --config electron.vite.config.ts",
     "package": "electron-builder",

gui/public/i18n/en/translation.ftl

@@ -273,10 +273,44 @@ navbar-home = Home
 navbar-body_proportions = Body Proportions
 navbar-trackers_assign = Tracker Assignment
 navbar-mounting = Mounting Calibration
+navbar-video_calibration = Video Calibration
 navbar-onboarding = Setup Wizard
 navbar-settings = Settings
 navbar-connect_trackers = Connect Trackers
 
+## Video calibration
+video-calibration-loading = Looking for a webcam...
+video-calibration-connecting = Connecting to the webcam video stream...
+video-calibration-no_webcam = No webcam is currently available.
+video-calibration-error = Could not start the webcam video stream.
+video-calibration-sidebar-title = Video Calibration
+video-calibration-sidebar-description = Start calibration and monitor its progress here.
+video-calibration-sidebar-description-active = Current progress is shown under Status below.
+video-calibration-start = Start Calibration
+video-calibration-status-label = Status
+video-calibration-status-idle = Waiting to start calibration.
+video-calibration-status-starting = Waiting for calibration progress...
+video-calibration-status-calibrate_camera = Calibrating camera
+video-calibration-status-capture_forward_pose = Capture forward pose
+video-calibration-status-capture_bent_over_pose = Capture bent-over pose
+video-calibration-status-calibrate_trackers = Calibrating trackers
+video-calibration-status-calibrate_skeleton_offsets = Calibrating skeleton offsets
+video-calibration-status-done = Calibration complete
+video-calibration-instruction-calibrate_camera = Move your right controller in a ∞ loop
+video-calibration-instruction-capture_forward_pose = Stand straight and face forward
+video-calibration-instruction-capture_bent_over_pose = Carefully lean forward
+video-calibration-instruction-calibrate_trackers = Walk around in a small circle
+video-calibration-camera-label = Camera
+video-calibration-camera-available = Camera data available
+video-calibration-camera-unavailable = Camera data is not available yet.
+video-calibration-camera-resolution = Resolution
+video-calibration-camera-focal_length = Focal length
+video-calibration-camera-principal_point = Principal point
+video-calibration-done-trackers = Done trackers
+video-calibration-pending-trackers = Pending trackers
+video-calibration-none = None
+video-calibration-error-label = Error
+
 ## Biovision hierarchy recording
 bvh-start_recording = Record BVH
 bvh-stop_recording = Save BVH recording

gui/src/App.tsx

@@ -53,6 +53,7 @@ import { ChecklistPage } from './components/tracking-checklist/TrackingChecklist
 import { ElectronContextC, provideElectron } from './hooks/electron';
 import { AppLocalizationProvider } from './i18n/config';
 import { openUrl } from './hooks/crossplatform';
+import { VideoCalibrationPage } from './components/video-calibration/VideoCalibrationPage';
 
 export const GH_REPO = 'SlimeVR/SlimeVR-Server';
 export const VersionContext = createContext('');
@@ -104,6 +105,10 @@ function Layout() {
               </MainLayout>
             }
           />
+          <Route
+            path="/video-calibration"
+            element={<VideoCalibrationPage isMobile={isMobile} />}
+          />
           <Route
             path="/tracker/:trackernum/:deviceid"
             element={

gui/src/components/MainLayout.scss

@@ -17,6 +17,13 @@
       / var(--navbar-w) calc(100% - var(--navbar-w) - var(--right-section-w)) var(--right-section-w);
   }
 
+  &.full.no-toolbar {
+    grid-template:
+      't t t' var(--topbar-h)
+      'n c s' calc(100% - var(--topbar-h))
+      / var(--navbar-w) calc(100% - var(--navbar-w) - var(--right-section-w)) var(--right-section-w);
+  }
+
   @screen nsm {
     --right-section-w: 40%;
   }
@@ -58,6 +65,15 @@
         / 100%;
     }
 
+    &.full.no-toolbar {
+      grid-template:
+        't' var(--topbar-h)
+        'l' var(--checklist-h)
+        'c' calc(100% - var(--topbar-h) - var(--checklist-h) - var(--navbar-h))
+        'n' calc(var(--navbar-h))
+        / 100%;
+    }
+
     grid-template:
       't' var(--topbar-h)
       'c' calc(100% - var(--topbar-h) - var(--navbar-h))

gui/src/components/MainLayout.tsx

@@ -20,12 +20,17 @@ export function MainLayout({
   background = true,
   full = false,
   isMobile = undefined,
+  rightSidebar,
+  showToolbar = true,
+  scrollContent = true,
 }: {
   children: ReactNode;
   background?: boolean;
   isMobile?: boolean;
-  showToolbarSettings?: boolean;
   full?: boolean;
+  rightSidebar?: ReactNode;
+  showToolbar?: boolean;
+  scrollContent?: boolean;
 }) {
   const { completion } = useTrackingChecklist();
   const { sendRPCPacket } = useWebsocketAPI();
@@ -65,6 +70,7 @@ export function MainLayout({
   return (
     <div
       className={classNames('main-layout w-full h-screen', full && 'full', {
+        'no-toolbar': full && !showToolbar,
         'checklist-ok': completion === 'complete',
       })}
     >
@@ -78,7 +84,8 @@ export function MainLayout({
       <div
         style={{ gridArea: 'c' }}
         className={classNames(
-          'overflow-y-auto mr-2 my-2 mobile:m-0',
+          scrollContent ? 'overflow-y-auto' : 'overflow-hidden',
+          'mr-2 my-2 mobile:m-0',
           'flex flex-col rounded-md',
           background && 'bg-background-70',
           { 'rounded-t-none': !isMobile && full }
@@ -89,14 +96,14 @@ export function MainLayout({
       {full && isMobile && completion !== 'complete' && (
         <TrackingChecklistMobile />
       )}
-      {full && (
+      {full && showToolbar && (
         <div style={{ gridArea: 'b' }}>
           <Toolbar />
         </div>
       )}
       {!isMobile && full && (
         <div style={{ gridArea: 's' }} className="mr-2">
-          <Sidebar />
+          {rightSidebar || <Sidebar />}
         </div>
       )}
     </div>

gui/src/components/Navbar.tsx

@@ -9,6 +9,7 @@ import { useBreakpoint } from '@/hooks/breakpoint';
 import { HomeIcon } from './commons/icon/HomeIcon';
 import { SkiIcon } from './commons/icon/SkiIcon';
 import { WifiIcon } from './commons/icon/WifiIcon';
+import { EyeIcon } from './commons/icon/EyeIcon';
 
 export function NavButton({
   to,
@@ -94,6 +95,9 @@ export function MainLinks() {
       >
         {l10n.getString('navbar-body_proportions')}
       </NavButton>
+      <NavButton to="/video-calibration" icon={<EyeIcon />}>
+        {l10n.getString('navbar-video_calibration')}
+      </NavButton>
       <NavButton
         to="/onboarding/wifi-creds"
         icon={<WifiIcon value={1} disabled variant="navbar" />}