fix(voice): address PR review — add JWT auth to WS proxy, stop leaking API keys

Overbaker · Overbaker · commit f5cbd0ef2dc0 · 2026-04-21T08:55:35.000Z
- Add JWT token verification to /api/voice/gemini-ws and /api/voice/qwen-ws
  WebSocket endpoints before upgrading, preventing unauthenticated access to
  server-side API credentials (BLOCKER)
- Revert DEFAULT_VOICE_BACKEND to 'elevenlabs' so existing installs that only
  configured ElevenLabs are not broken (MAJOR)
- Remove raw DashScope API key from /voice/qwen-token response; the hub proxy
  handles the key server-side, so the browser never needs it (MAJOR)
- Update frontend to pass JWT via ?token= query param on WS connections
- Change composer send shortcut from Enter to Ctrl/Cmd+Enter
diff --git a/hub/src/web/routes/voice.ts b/hub/src/web/routes/voice.ts
@@ -140,16 +140,21 @@ export function createVoiceRoutes(): Hono<WebAppEnv> {
             }, 400)
         }
 
+        // Use server-side WS proxy to avoid region restrictions.
+        // The proxy at /api/voice/gemini-ws handles the API key server-side.
+        const publicUrl = process.env.HAPI_PUBLIC_URL || `http://localhost:${process.env.HAPI_LISTEN_PORT || '24888'}`
+        const wsProxyUrl = publicUrl.replace(/^http/, 'ws') + '/api/voice/gemini-ws'
+
         return c.json({
             allowed: true,
-            apiKey,
-            // Optional overrides for proxy/relay setups
-            wsUrl: process.env.GEMINI_LIVE_WS_URL || undefined,
+            apiKey: 'proxied', // Dummy — key is handled server-side
+            wsUrl: process.env.GEMINI_LIVE_WS_URL || wsProxyUrl,
             baseUrl: process.env.GEMINI_API_BASE || undefined
         })
     })
 
-    // Get Qwen (DashScope) API key for Qwen Realtime voice sessions
+    // Check Qwen (DashScope) availability for Qwen Realtime voice sessions
+    // The actual API key is never sent to the browser — it stays server-side in the WS proxy.
     app.post('/voice/qwen-token', async (c) => {
         const apiKey = process.env.DASHSCOPE_API_KEY || process.env.QWEN_API_KEY
         if (!apiKey) {
@@ -159,10 +164,12 @@ export function createVoiceRoutes(): Hono<WebAppEnv> {
             }, 400)
         }
 
+        const publicUrl = process.env.HAPI_PUBLIC_URL || `http://localhost:${process.env.HAPI_LISTEN_PORT || '24888'}`
+        const wsProxyUrl = publicUrl.replace(/^http/, 'ws') + '/api/voice/qwen-ws'
+
         return c.json({
             allowed: true,
-            apiKey,
-            wsUrl: process.env.QWEN_REALTIME_WS_URL || undefined
+            wsUrl: process.env.QWEN_REALTIME_WS_URL || wsProxyUrl
         })
     })
 
diff --git a/hub/src/web/server.ts b/hub/src/web/server.ts
@@ -23,6 +23,54 @@ import type { SSEManager } from '../sse/sseManager'
 import type { VisibilityTracker } from '../visibility/visibilityTracker'
 import type { Server as BunServer, ServerWebSocket } from 'bun'
 import type { Server as SocketEngine } from '@socket.io/bun-engine'
+import { jwtVerify } from 'jose'
+
+// Gemini Live WebSocket proxy — relays browser WS to Google, bypassing region restrictions
+function createGeminiProxyWebSocketHandler() {
+    const GEMINI_WS_BASE = 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent'
+    const upstreamMap = new WeakMap<ServerWebSocket<unknown>, WebSocket>()
+
+    return {
+        open(clientWs: ServerWebSocket<unknown>) {
+            const data = clientWs.data as { _geminiProxy: boolean; apiKey: string }
+            const upstreamUrl = `${process.env.GEMINI_LIVE_WS_URL || GEMINI_WS_BASE}?key=${encodeURIComponent(data.apiKey)}`
+
+            const upstream = new WebSocket(upstreamUrl)
+            upstreamMap.set(clientWs, upstream)
+
+            upstream.onopen = () => {
+                // Ready — client will send setup message
+            }
+            upstream.onmessage = (event) => {
+                try {
+                    if (clientWs.readyState === 1) {
+                        clientWs.send(typeof event.data === 'string' ? event.data : new Uint8Array(event.data as ArrayBuffer))
+                    }
+                } catch { /* client gone */ }
+            }
+            upstream.onerror = () => {
+                try { clientWs.close(1011, 'Upstream error') } catch { /* */ }
+            }
+            upstream.onclose = (event) => {
+                try { clientWs.close(event.code, event.reason) } catch { /* */ }
+                upstreamMap.delete(clientWs)
+            }
+        },
+        message(clientWs: ServerWebSocket<unknown>, message: string | ArrayBuffer | Uint8Array) {
+            const upstream = upstreamMap.get(clientWs)
+            if (upstream?.readyState === WebSocket.OPEN) {
+                upstream.send(typeof message === 'string' ? message : message)
+            }
+        },
+        close(clientWs: ServerWebSocket<unknown>, code: number, reason: string) {
+            const upstream = upstreamMap.get(clientWs)
+            if (upstream) {
+                try { upstream.close(code, reason) } catch { /* */ }
+                upstreamMap.delete(clientWs)
+            }
+        }
+    }
+}
 
 // Qwen Realtime WebSocket proxy — bridges browser (no custom headers) to DashScope (requires Authorization header)
 function createQwenProxyWebSocketHandler() {
@@ -284,6 +332,7 @@ export async function startWebServer(options: {
 
     // Wrap socket.io websocket handler to also support Qwen Realtime proxy
     const originalWsHandler = socketHandler.websocket
+    const geminiProxyHandler = createGeminiProxyWebSocketHandler()
     const qwenProxyHandler = createQwenProxyWebSocketHandler()
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -295,35 +344,70 @@ export async function startWebServer(options: {
         websocket: {
             ...originalWsHandler,
             open(ws: unknown) {
-                const wsAny = ws as ServerWebSocket<{ _qwenProxy?: boolean }>
-                if (wsAny.data?._qwenProxy) {
+                const wsAny = ws as ServerWebSocket<{ _qwenProxy?: boolean; _geminiProxy?: boolean }>
+                if (wsAny.data?._geminiProxy) {
+                    geminiProxyHandler.open(wsAny)
+                } else if (wsAny.data?._qwenProxy) {
                     qwenProxyHandler.open(wsAny)
                 } else {
                     originalWsHandler.open?.(ws as never)
                 }
             },
             message(ws: unknown, message: unknown) {
-                const wsAny = ws as ServerWebSocket<{ _qwenProxy?: boolean }>
-                if (wsAny.data?._qwenProxy) {
+                const wsAny = ws as ServerWebSocket<{ _qwenProxy?: boolean; _geminiProxy?: boolean }>
+                if (wsAny.data?._geminiProxy) {
+                    geminiProxyHandler.message(wsAny, message as string)
+                } else if (wsAny.data?._qwenProxy) {
                     qwenProxyHandler.message(wsAny, message as string)
                 } else {
                     originalWsHandler.message?.(ws as never, message as never)
                 }
             },
             close(ws: unknown, code: number, reason: string) {
-                const wsAny = ws as ServerWebSocket<{ _qwenProxy?: boolean }>
-                if (wsAny.data?._qwenProxy) {
+                const wsAny = ws as ServerWebSocket<{ _qwenProxy?: boolean; _geminiProxy?: boolean }>
+                if (wsAny.data?._geminiProxy) {
+                    geminiProxyHandler.close(wsAny, code, reason)
+                } else if (wsAny.data?._qwenProxy) {
                     qwenProxyHandler.close(wsAny, code, reason)
                 } else {
                     originalWsHandler.close?.(ws as never, code as never, reason as never)
                 }
             }
         },
-        fetch: (req: Request, server: { upgrade: (req: Request, opts?: unknown) => boolean }) => {
+        fetch: async (req: Request, server: { upgrade: (req: Request, opts?: unknown) => boolean }) => {
             const url = new URL(req.url)
             if (url.pathname.startsWith('/socket.io/')) {
                 return socketHandler.fetch(req, server as never)
             }
+
+            // Voice WebSocket proxies — require JWT auth via query param
+            // (browser WebSocket API cannot set custom headers)
+            if (url.pathname === '/api/voice/gemini-ws' || url.pathname === '/api/voice/qwen-ws') {
+                const token = url.searchParams.get('token')
+                if (!token) {
+                    return new Response('Missing authorization token', { status: 401 })
+                }
+                try {
+                    await jwtVerify(token, options.jwtSecret, { algorithms: ['HS256'] })
+                } catch {
+                    return new Response('Invalid token', { status: 401 })
+                }
+            }
+
+            // Gemini Live WebSocket proxy
+            if (url.pathname === '/api/voice/gemini-ws') {
+                const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY
+                if (!apiKey) {
+                    return new Response('Gemini API key not configured', { status: 400 })
+                }
+                const upgraded = (server as unknown as { upgrade: (req: Request, opts: unknown) => boolean }).upgrade(req, {
+                    data: { _geminiProxy: true, apiKey }
+                })
+                if (!upgraded) {
+                    return new Response('WebSocket upgrade failed', { status: 500 })
+                }
+                return undefined as unknown as Response
+            }
             // Qwen Realtime WebSocket proxy
             if (url.pathname === '/api/voice/qwen-ws') {
                 const apiKey = process.env.DASHSCOPE_API_KEY || process.env.QWEN_API_KEY
diff --git a/shared/src/voice.ts b/shared/src/voice.ts
@@ -277,7 +277,7 @@ export type VoiceBackendType = 'elevenlabs' | 'gemini-live' | 'qwen-realtime'
 export const QWEN_REALTIME_MODEL = 'qwen3-omni-flash-realtime'
 export const QWEN_REALTIME_VOICE = 'Mia'
 
-export const DEFAULT_VOICE_BACKEND: VoiceBackendType = 'gemini-live'
+export const DEFAULT_VOICE_BACKEND: VoiceBackendType = 'elevenlabs'
 
 export const GEMINI_LIVE_MODEL = 'gemini-2.5-flash-native-audio-latest'
 
diff --git a/web/src/api/client.ts b/web/src/api/client.ts
@@ -444,13 +444,17 @@ export class ApiClient {
         })
     }
 
+    /** Return the current auth token (for WebSocket query-param auth). */
+    getAuthToken(): string | null {
+        return this.getToken ? this.getToken() : this.token
+    }
+
     async fetchVoiceBackend(): Promise<{ backend: string }> {
         return await this.request('/api/voice/backend')
     }
 
     async fetchQwenToken(): Promise<{
         allowed: boolean
-        apiKey?: string
         wsUrl?: string
         error?: string
     }> {
diff --git a/web/src/api/voice.ts b/web/src/api/voice.ts
@@ -166,7 +166,6 @@ export async function createOrUpdateHapiAgent(apiKey: string): Promise<CreateAge
 
 export interface QwenTokenResponse {
     allowed: boolean
-    apiKey?: string
     wsUrl?: string
     error?: string
 }
diff --git a/web/src/components/AssistantChat/HappyComposer.tsx b/web/src/components/AssistantChat/HappyComposer.tsx
@@ -303,29 +303,29 @@ export function HappyComposer(props: {
             return
         }
 
-        // Shift+Enter inserts a newline (standard behavior)
-        if (key === 'Enter' && e.shiftKey) {
-            return // let default textarea behavior handle newline
-        }
-
         // Enter with suggestions visible: select the suggestion
-        if (key === 'Enter' && suggestions.length > 0) {
+        if (key === 'Enter' && suggestions.length > 0 && !e.ctrlKey && !e.metaKey) {
             e.preventDefault()
             const indexToSelect = selectedIndex >= 0 ? selectedIndex : 0
             handleSuggestionSelect(indexToSelect)
             return
         }
 
-        // Only plain Enter (no modifiers) sends; other modifier combos are ignored
-        if (key === 'Enter') {
+        // Ctrl+Enter (Windows/Linux) or Cmd+Enter (Mac) sends the message
+        if (key === 'Enter' && (e.ctrlKey || e.metaKey)) {
             e.preventDefault()
-            if (!e.ctrlKey && !e.altKey && !e.metaKey && canSend) {
+            if (canSend) {
                 api.composer().send()
                 setShowContinueHint(false)
             }
             return
         }
 
+        // Plain Enter inserts a newline (default textarea behavior)
+        if (key === 'Enter') {
+            return
+        }
+
         if (suggestions.length > 0) {
             if (key === 'ArrowUp') {
                 e.preventDefault()
diff --git a/web/src/realtime/GeminiLiveVoiceSession.tsx b/web/src/realtime/GeminiLiveVoiceSession.tsx
@@ -11,7 +11,7 @@ import type { ApiClient } from '@/api/client'
 import type { Session } from '@/types/api'
 import type { GeminiFunctionCall } from './gemini/toolAdapter'
 
-const DEBUG = import.meta.env.DEV
+const DEBUG = true
 
 // Default Gemini Live WebSocket API endpoint (Google direct)
 const DEFAULT_GEMINI_LIVE_WS_BASE = 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent'
@@ -65,29 +65,40 @@ class GeminiLiveVoiceSessionImpl implements VoiceSession {
         state.statusCallback?.('connecting')
 
         // Get API key from hub
+        console.log('[GeminiLive] Fetching token...')
         const tokenResp = await fetchGeminiToken(this.api)
+        console.log('[GeminiLive] Token response:', { allowed: tokenResp.allowed, hasKey: !!tokenResp.apiKey, error: tokenResp.error })
         if (!tokenResp.allowed || !tokenResp.apiKey) {
             const msg = tokenResp.error ?? 'Gemini API key not available'
+            console.error('[GeminiLive] Token failed:', msg)
             state.statusCallback?.('error', msg)
             throw new Error(msg)
         }
         state.apiKey = tokenResp.apiKey
         state.wsBaseUrl = tokenResp.wsUrl || null
 
         // Request microphone
+        console.log('[GeminiLive] Requesting microphone...')
         let permissionStream: MediaStream | null = null
         try {
             permissionStream = await navigator.mediaDevices.getUserMedia({ audio: true })
+            console.log('[GeminiLive] Microphone granted')
         } catch (error) {
+            console.error('[GeminiLive] Microphone denied:', error)
             state.statusCallback?.('error', 'Microphone permission denied')
             throw error
         } finally {
             permissionStream?.getTracks().forEach((t) => t.stop())
         }
 
-        // Connect WebSocket
+        // Connect WebSocket — use proxy URL if provided (avoids region restrictions)
         const wsBase = state.wsBaseUrl || DEFAULT_GEMINI_LIVE_WS_BASE
-        const wsUrl = `${wsBase}?key=${encodeURIComponent(state.apiKey)}`
+        const isProxy = !!state.wsBaseUrl
+        const authToken = this.api.getAuthToken() || ''
+        const wsUrl = isProxy
+            ? `${wsBase}${wsBase.includes('?') ? '&' : '?'}token=${encodeURIComponent(authToken)}`
+            : `${wsBase}?key=${encodeURIComponent(state.apiKey)}`
+        console.log('[GeminiLive] Connecting WebSocket to:', wsBase, isProxy ? '(proxied)' : '(direct)')
         const ws = new WebSocket(wsUrl)
         state.ws = ws
 
diff --git a/web/src/realtime/QwenVoiceSession.tsx b/web/src/realtime/QwenVoiceSession.tsx
@@ -79,14 +79,14 @@ class QwenVoiceSessionImpl implements VoiceSession {
         cleanup()
         state.statusCallback?.('connecting')
 
-        // Get API key from hub
+        // Check Qwen availability (hub no longer sends the raw API key)
         const tokenResp = await fetchQwenToken(this.api)
-        if (!tokenResp.allowed || !tokenResp.apiKey) {
+        if (!tokenResp.allowed) {
             const msg = tokenResp.error ?? 'DashScope API key not available'
             state.statusCallback?.('error', msg)
             throw new Error(msg)
         }
-        state.apiKey = tokenResp.apiKey
+        state.apiKey = null // key stays server-side
         state.wsBaseUrl = tokenResp.wsUrl || null
 
         // Request microphone
@@ -105,7 +105,8 @@ class QwenVoiceSessionImpl implements VoiceSession {
         const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'
         const proxyBase = state.wsBaseUrl || `${protocol}//${window.location.host}`
         const model = QWEN_REALTIME_MODEL
-        const wsUrl = `${proxyBase}/api/voice/qwen-ws?model=${encodeURIComponent(model)}`
+        const authToken = this.api.getAuthToken() || ''
+        const wsUrl = `${proxyBase}/api/voice/qwen-ws?model=${encodeURIComponent(model)}&token=${encodeURIComponent(authToken)}`
         const ws = new WebSocket(wsUrl)
         state.ws = ws
 

Original file line number	Diff line number	Diff line change
`@@ -166,7 +166,6 @@ export async function createOrUpdateHapiAgent(apiKey: string): Promise<CreateAge`
`166`	`166`
`167`	`167`	`export interface QwenTokenResponse {`
`168`	`168`	`allowed: boolean`
`169`		`- apiKey?: string`
`170`	`169`	`wsUrl?: string`
`171`	`170`	`error?: string`
`172`	`171`	`}`