feat: ship multimodal avatar runtime upgrades

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: LessUp

CHANGELOG.md

@@ -9,6 +9,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### ✨ Runtime Capabilities
+
+- Added structured dialogue request metadata so language preference, speech configuration, and recent vision context travel together with each chat turn
+- Persisted speech preferences and recent vision context in `digitalHumanStore`, and localized the voice interaction panel for multi-language TTS flows
+- Added custom avatar upload, replacement, fallback-to-built-in handling, and avatar management controls in the settings drawer
+- Added endpoint discovery with primary/backup failover for health checks, standard chat requests, and streaming chat requests
+- Added touch/WebXR readiness detection and surfaced an `AR Ready` platform indicator in the HUD
+- Added immersive WebXR AR session entry/exit flow, persisted immersive session state, and a viewer bridge that forwards the active XR session to the Three.js renderer
+- Added operator-facing endpoint routing diagnostics so the HUD now surfaces the active service endpoint and failover count
+- Added a shared avatar source adapter so page composition, settings UI, and controller fallback logic reuse the same source/status/object-URL decisions
+
 ### 🧹 Repository Simplification
 
 - Removed repository-scoped AI workflow frameworks and generated automation from `.trellis/`, `.claude/`, and `.opencode/`

README.md

@@ -354,9 +354,11 @@ See [CHANGELOG.md](CHANGELOG.md) for released features and [GitHub Projects](htt
 - [x] Voice interaction (TTS/ASR)
 - [x] Visual perception (MediaPipe)
 - [x] Streaming dialogue
-- [ ] Mobile AR support
-- [ ] Custom avatar upload
-- [ ] Multi-language TTS
+- [x] Service discovery and endpoint failover
+- [x] Mobile AR readiness detection
+- [x] Mobile AR support
+- [x] Custom avatar upload
+- [x] Multi-language TTS
 
 ---
 

README.zh-CN.md

@@ -354,9 +354,11 @@ npm run test:ui          # Vitest UI 模式
 - [x] 语音交互（TTS/ASR）
 - [x] 视觉感知（MediaPipe）
 - [x] 流式对话
-- [ ] 移动端 AR 支持
-- [ ] 自定义形象上传
-- [ ] 多语言 TTS
+- [x] 服务发现与端点故障切换
+- [x] 移动端 AR readiness 检测
+- [x] 移动端 AR 支持
+- [x] 自定义形象上传
+- [x] 多语言 TTS
 
 ---
 

src/__tests__/AdvancedDigitalHumanPage.test.tsx

@@ -4,13 +4,21 @@ import AdvancedDigitalHumanPage from '@/pages/AdvancedDigitalHumanPage';
 import { useDigitalHumanStore } from '@/store/digitalHumanStore';
 
 const useChatStreamMock = vi.fn();
+const digitalHumanViewerMock = vi.fn();
+const topHUDMock = vi.fn();
 
 vi.mock('@/components/viewer', () => ({
-  DigitalHumanViewer: () => <div data-testid="viewer" />,
+  DigitalHumanViewer: (props: unknown) => {
+    digitalHumanViewerMock(props);
+    return <div data-testid="viewer" />;
+  },
 }));
 
 vi.mock('@/components/TopHUD', () => ({
-  default: () => <div data-testid="top-hud" />,
+  default: (props: unknown) => {
+    topHUDMock(props);
+    return <div data-testid="top-hud" />;
+  },
 }));
 
 vi.mock('@/components/SettingsDrawer', () => ({
@@ -32,6 +40,7 @@ vi.mock('@/hooks/useAdvancedDigitalHumanController', () => ({
     activeTab: 'basic',
     autoRotate: false,
     closeSettings: vi.fn(),
+    handleAvatarUpload: vi.fn(),
     handleBehaviorChange: vi.fn(),
     handleEmotionChange: vi.fn(),
     handleExpressionChange: vi.fn(),
@@ -40,7 +49,9 @@ vi.mock('@/hooks/useAdvancedDigitalHumanController', () => ({
     handleNewSession: vi.fn(),
     handlePlayPause: vi.fn(),
     handleReset: vi.fn(),
+    handleToggleImmersiveAr: vi.fn(),
     handleToggleRecording: vi.fn(),
+    handleUseBuiltInAvatar: vi.fn(),
     handleVoiceCommand: vi.fn(),
     setActiveTab: vi.fn(),
     showSettings: false,
@@ -61,6 +72,8 @@ vi.mock('@/hooks/useChatStream', () => ({
 describe('AdvancedDigitalHumanPage', () => {
   beforeEach(() => {
     useChatStreamMock.mockReset();
+    digitalHumanViewerMock.mockReset();
+    topHUDMock.mockReset();
     useChatStreamMock.mockReturnValue({
       chatInput: '',
       setChatInput: vi.fn(),
@@ -82,4 +95,33 @@ describe('AdvancedDigitalHumanPage', () => {
       }),
     );
   });
+
+  it('passes the active custom avatar model url into the viewer', () => {
+    useDigitalHumanStore.setState({
+      ...(useDigitalHumanStore.getState() as unknown as Record<string, unknown>),
+      avatarSource: {
+        kind: 'custom',
+        fileName: 'avatar.glb',
+        modelUrl: 'blob:avatar-1',
+      },
+    } as unknown as Partial<ReturnType<typeof useDigitalHumanStore.getState>>);
+
+    render(<AdvancedDigitalHumanPage />);
+
+    expect(digitalHumanViewerMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        modelUrl: 'blob:avatar-1',
+      }),
+    );
+  });
+
+  it('passes the immersive ar toggle handler into the HUD', () => {
+    render(<AdvancedDigitalHumanPage />);
+
+    expect(topHUDMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        onToggleImmersiveAr: expect.any(Function),
+      }),
+    );
+  });
 });

src/__tests__/App.test.tsx

@@ -1,11 +1,17 @@
 import { render, screen, waitFor } from '@testing-library/react';
+import type { ReactNode } from 'react';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import App from '@/App';
 
-const createServicesMock = vi.fn();
+const { servicesProviderRenderSpy } = vi.hoisted(() => ({
+  servicesProviderRenderSpy: vi.fn(),
+}));
 
-vi.mock('@/core/createServices', () => ({
-  createServices: () => createServicesMock(),
+vi.mock('@/core/services', () => ({
+  ServicesProvider: ({ children }: { children: ReactNode }) => {
+    servicesProviderRenderSpy();
+    return <div data-testid="services-provider">{children}</div>;
+  },
 }));
 
 vi.mock('@/pages/LandingPage', () => ({
@@ -16,35 +22,26 @@ vi.mock('@/pages/AdvancedDigitalHumanPage', () => ({
   default: () => <div data-testid="advanced-page" />,
 }));
 
-function buildServices() {
-  return {
-    engine: { dispose: vi.fn() },
-    tts: { dispose: vi.fn() },
-    asr: { dispose: vi.fn() },
-    dialogue: { reset: vi.fn() },
-  };
-}
-
 describe('App routing', () => {
   beforeEach(() => {
-    createServicesMock.mockReset();
-    createServicesMock.mockReturnValue(buildServices());
+    servicesProviderRenderSpy.mockReset();
     window.location.hash = '#/';
   });
 
-  it('does not create app services for the landing route', async () => {
+  it('does not render the app services provider for the landing route', async () => {
     render(<App />);
 
     expect(await screen.findByTestId('landing-page')).toBeInTheDocument();
-    await waitFor(() => expect(createServicesMock).not.toHaveBeenCalled());
+    await waitFor(() => expect(servicesProviderRenderSpy).not.toHaveBeenCalled());
   });
 
-  it('creates app services for the app route', async () => {
+  it('renders the app services provider for the app route', async () => {
     window.location.hash = '#/app';
 
     render(<App />);
 
     expect(await screen.findByTestId('advanced-page')).toBeInTheDocument();
-    await waitFor(() => expect(createServicesMock).toHaveBeenCalledTimes(1));
+    expect(await screen.findByTestId('services-provider')).toBeInTheDocument();
+    await waitFor(() => expect(servicesProviderRenderSpy).toHaveBeenCalledTimes(1));
   });
 });

src/__tests__/DigitalHumanViewer.test.tsx

@@ -0,0 +1,59 @@
+import { render } from '@testing-library/react';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import DigitalHumanViewer from '@/components/viewer/DigitalHumanViewer';
+import { useSystemStore } from '@/store/systemStore';
+
+const immersiveSessionBridgeMock = vi.fn();
+
+vi.mock('@react-three/fiber', () => ({
+  Canvas: ({ children }: { children: React.ReactNode }) => (
+    <div data-testid="canvas">{children}</div>
+  ),
+}));
+
+vi.mock('@react-three/drei', () => ({
+  Html: ({ children }: { children: React.ReactNode }) => <div>{children}</div>,
+}));
+
+vi.mock('@/components/viewer/Scene', () => ({
+  Scene: () => <div data-testid="scene" />,
+}));
+
+vi.mock('@/components/viewer/PerformanceTracker', () => ({
+  PerformanceTracker: () => <div data-testid="performance-tracker" />,
+}));
+
+vi.mock('@/components/viewer/ImmersiveSessionBridge', () => ({
+  ImmersiveSessionBridge: (props: unknown) => {
+    immersiveSessionBridgeMock(props);
+    return <div data-testid="immersive-session-bridge" />;
+  },
+}));
+
+describe('DigitalHumanViewer', () => {
+  beforeEach(() => {
+    immersiveSessionBridgeMock.mockReset();
+    useSystemStore.setState({
+      immersiveMode: 'standard',
+      immersiveSession: null,
+      immersiveError: null,
+    });
+  });
+
+  it('passes the active immersive session into the bridge', () => {
+    const session = { kind: 'immersive-session' } as unknown as XRSession;
+    useSystemStore.setState({
+      immersiveMode: 'ar-active',
+      immersiveSession: session,
+      immersiveError: null,
+    });
+
+    render(<DigitalHumanViewer showControls={false} />);
+
+    expect(immersiveSessionBridgeMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        session,
+      }),
+    );
+  });
+});

src/__tests__/ImmersiveSessionBridge.test.tsx

@@ -0,0 +1,43 @@
+import { render } from '@testing-library/react';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { ImmersiveSessionBridge } from '@/components/viewer/ImmersiveSessionBridge';
+
+const setSessionMock = vi.fn();
+const glMock = {
+  xr: {
+    enabled: false,
+    setSession: setSessionMock,
+  },
+};
+
+vi.mock('@react-three/fiber', () => ({
+  useThree: () => ({
+    gl: glMock,
+  }),
+}));
+
+describe('ImmersiveSessionBridge', () => {
+  beforeEach(() => {
+    glMock.xr.enabled = false;
+    setSessionMock.mockReset();
+    setSessionMock.mockResolvedValue(undefined);
+  });
+
+  it('enables xr on the renderer and forwards the active session', () => {
+    const session = { kind: 'immersive-session' } as unknown as XRSession;
+
+    render(<ImmersiveSessionBridge session={session} />);
+
+    expect(glMock.xr.enabled).toBe(true);
+    expect(setSessionMock).toHaveBeenCalledWith(session);
+  });
+
+  it('disables xr when no immersive session is active', () => {
+    glMock.xr.enabled = true;
+
+    render(<ImmersiveSessionBridge session={null} />);
+
+    expect(glMock.xr.enabled).toBe(false);
+    expect(setSessionMock).not.toHaveBeenCalled();
+  });
+});