From 79bf7ddf38ac7900a110fcfd8fa0753b65d2d386 Mon Sep 17 00:00:00 2001 From: Colin Grant Date: Wed, 21 Jan 2026 14:36:34 -0700 Subject: [PATCH 1/2] [ui] HTTP RPC Engine attempts reconnection until disposal --- ui/src/trace_processor/http_rpc_engine.ts | 86 +++++++++++++++++------ 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/ui/src/trace_processor/http_rpc_engine.ts b/ui/src/trace_processor/http_rpc_engine.ts index 1e0425c917a..362a088d29e 100644 --- a/ui/src/trace_processor/http_rpc_engine.ts +++ b/ui/src/trace_processor/http_rpc_engine.ts @@ -18,6 +18,9 @@ import {assertExists, reportError} from '../base/logging'; import {EngineBase} from '../trace_processor/engine'; const RPC_CONNECT_TIMEOUT_MS = 2000; +const INITIAL_RETRY_DELAY_MS = 100; +const MAX_RETRY_DELAY_MS = 30000; +const BACKOFF_MULTIPLIER = 2; export interface HttpRpcState { connected: boolean; @@ -34,6 +37,8 @@ export class HttpRpcEngine extends EngineBase { private disposed = false; private queue: Blob[] = []; private isProcessingQueue = false; + private retryDelayMs = INITIAL_RETRY_DELAY_MS; + private retryTimeoutId?: ReturnType; // Can be changed by frontend/index.ts when passing ?rpc_port=1234 . static defaultRpcPort = '9001'; @@ -47,18 +52,8 @@ export class HttpRpcEngine extends EngineBase { } rpcSendRequestBytes(data: Uint8Array): void { - if (this.websocket === undefined) { - if (this.disposed) return; - const wsUrl = `ws://${HttpRpcEngine.getHostAndPort(this.port)}/websocket`; - this.websocket = new WebSocket(wsUrl); - this.websocket.onopen = () => this.onWebsocketConnected(); - this.websocket.onmessage = (e) => this.onWebsocketMessage(e); - this.websocket.onclose = (e) => this.onWebsocketClosed(e); - this.websocket.onerror = (e) => - super.fail( - `WebSocket error rs=${(e.target as WebSocket)?.readyState} (ERR:ws)`, - ); - } + if (this.disposed) return; + this.websocket ??= this.initWebSocket(); if (this.connected) { this.websocket.send(data); @@ -67,27 +62,67 @@ export class HttpRpcEngine extends EngineBase { } } + private initWebSocket(): WebSocket { + const wsUrl = `ws://${HttpRpcEngine.getHostAndPort(this.port)}/websocket`; + this.websocket = new WebSocket(wsUrl); + this.websocket.onopen = () => this.onWebsocketConnected(); + this.websocket.onmessage = (e) => this.onWebsocketMessage(e); + this.websocket.onclose = (e) => this.onWebsocketClosed(e); + this.websocket.onerror = (e) => this.onWebsocketError(e); + return this.websocket; + } + + private onWebsocketError(e: Event): void { + if (this.disposed) return; + const readyState = (e.target as WebSocket)?.readyState; + console.warn(`WebSocket error rs=${readyState}, will retry with backoff`); + // The close event will fire after this, which will trigger the retry logic + } + + private scheduleReconnect(): void { + if (this.disposed) return; + + console.debug( + `Scheduling WebSocket reconnection in ${this.retryDelayMs}ms`, + ); + + this.retryTimeoutId = setTimeout(() => { + if (this.disposed) return; + console.debug('Attempting WebSocket reconnection...'); + this.initWebSocket(); + }, this.retryDelayMs); + + // Exponential backoff with cap + this.retryDelayMs = Math.min( + this.retryDelayMs * BACKOFF_MULTIPLIER, + MAX_RETRY_DELAY_MS, + ); + } + private onWebsocketConnected() { + // Reset retry delay on successful connection + this.retryDelayMs = INITIAL_RETRY_DELAY_MS; + for (;;) { const queuedMsg = this.requestQueue.shift(); if (queuedMsg === undefined) break; assertExists(this.websocket).send(queuedMsg); } + console.debug('WebSocket (re)connected on port', this.port); this.connected = true; } private onWebsocketClosed(e: CloseEvent) { if (this.disposed) return; - if (e.code === 1006 && this.connected) { - // On macbooks the act of closing the lid / suspending often causes socket - // disconnections. Try to gracefully re-connect. - console.log('Websocket closed, reconnecting'); - this.websocket = undefined; - this.connected = false; - this.rpcSendRequestBytes(new Uint8Array()); // Triggers a reconnection. - } else { - super.fail(`Websocket closed (${e.code}: ${e.reason}) (ERR:ws)`); - } + + // Always attempt to reconnect with backoff, regardless of close code + console.debug( + `WebSocket closed (code=${e.code}, reason=${e.reason || 'none'}, wasConnected=${this.connected}), scheduling reconnect`, + ); + + this.websocket = undefined; + this.connected = false; + this.scheduleReconnect(); } private onWebsocketMessage(e: MessageEvent) { @@ -147,6 +182,13 @@ export class HttpRpcEngine extends EngineBase { [Symbol.dispose]() { this.disposed = true; this.connected = false; + + // Clear any pending retry timeout + if (this.retryTimeoutId !== undefined) { + clearTimeout(this.retryTimeoutId); + this.retryTimeoutId = undefined; + } + const websocket = this.websocket; this.websocket = undefined; websocket?.close(); From 02f940c80f74477b33ca3ae7e0afca09b1bd5ff1 Mon Sep 17 00:00:00 2001 From: Colin Grant Date: Fri, 23 Jan 2026 09:20:02 -0700 Subject: [PATCH 2/2] Better handling of websocket lifecycle --- ui/src/trace_processor/http_rpc_engine.ts | 44 +++++++++++++++++++---- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/ui/src/trace_processor/http_rpc_engine.ts b/ui/src/trace_processor/http_rpc_engine.ts index 362a088d29e..cd827b82435 100644 --- a/ui/src/trace_processor/http_rpc_engine.ts +++ b/ui/src/trace_processor/http_rpc_engine.ts @@ -53,16 +53,32 @@ export class HttpRpcEngine extends EngineBase { rpcSendRequestBytes(data: Uint8Array): void { if (this.disposed) return; - this.websocket ??= this.initWebSocket(); + const websocket = this.getOrCreateWebSocket(); if (this.connected) { - this.websocket.send(data); + websocket.send(data); } else { this.requestQueue.push(data); // onWebsocketConnected() will flush this. } } - private initWebSocket(): WebSocket { + /** + * Returns the existing WebSocket if one exists and is not closed, + * otherwise creates a new one (closing any stale socket first). + */ + private getOrCreateWebSocket(): WebSocket { + // If we have an active websocket that's not closed/closing, reuse it + if ( + this.websocket !== undefined && + this.websocket.readyState !== WebSocket.CLOSED && + this.websocket.readyState !== WebSocket.CLOSING + ) { + return this.websocket; + } + + // Close any stale websocket before creating a new one + this.closeWebSocket(); + const wsUrl = `ws://${HttpRpcEngine.getHostAndPort(this.port)}/websocket`; this.websocket = new WebSocket(wsUrl); this.websocket.onopen = () => this.onWebsocketConnected(); @@ -72,6 +88,22 @@ export class HttpRpcEngine extends EngineBase { return this.websocket; } + /** + * Closes the current websocket if one exists, clearing event handlers + * to prevent spurious callbacks. + */ + private closeWebSocket(): void { + if (this.websocket === undefined) return; + + // Clear handlers to prevent callbacks from a closing socket + this.websocket.onopen = null; + this.websocket.onmessage = null; + this.websocket.onclose = null; + this.websocket.onerror = null; + this.websocket.close(); + this.websocket = undefined; + } + private onWebsocketError(e: Event): void { if (this.disposed) return; const readyState = (e.target as WebSocket)?.readyState; @@ -89,7 +121,7 @@ export class HttpRpcEngine extends EngineBase { this.retryTimeoutId = setTimeout(() => { if (this.disposed) return; console.debug('Attempting WebSocket reconnection...'); - this.initWebSocket(); + this.getOrCreateWebSocket(); }, this.retryDelayMs); // Exponential backoff with cap @@ -189,8 +221,6 @@ export class HttpRpcEngine extends EngineBase { this.retryTimeoutId = undefined; } - const websocket = this.websocket; - this.websocket = undefined; - websocket?.close(); + this.closeWebSocket(); } }