Skip to content

Commit 78ca4ed

Browse files
pallyoungpallyoung
committed
fix: stabilize workspace runtime recovery
1 parent de832ef commit 78ca4ed

3 files changed

Lines changed: 81 additions & 30 deletions

File tree

‎apps/web/src/features/workspace/runtime-attach.ts‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,29 +3,37 @@ import type { WorkspaceRuntimeSnapshot } from "../../types/app.ts";
33

44
type WithServiceFallback = <T>(operation: () => Promise<T>, fallback: T) => Promise<T>;
55

6-
const ATTACH_RUNTIME_RETRY_DELAYS_MS = [0, 250, 750] as const;
6+
export const ATTACH_RUNTIME_RETRY_DELAYS_MS = [0, 250, 750, 1500, 3000, 5000] as const;
77

8-
export const attachWorkspaceRuntimeWithRetry = async (
9-
workspaceId: string,
10-
deviceId: string,
11-
clientId: string,
12-
withServiceFallback: WithServiceFallback,
13-
): Promise<WorkspaceRuntimeSnapshot | null> => {
14-
for (const delayMs of ATTACH_RUNTIME_RETRY_DELAYS_MS) {
8+
export const runAttachWithRetry = async <T>(
9+
attach: () => Promise<T | null>,
10+
retryDelaysMs: readonly number[] = ATTACH_RUNTIME_RETRY_DELAYS_MS,
11+
): Promise<T | null> => {
12+
for (const delayMs of retryDelaysMs) {
1513
if (delayMs > 0) {
1614
await new Promise<void>((resolve) => {
17-
window.setTimeout(resolve, delayMs);
15+
globalThis.setTimeout(resolve, delayMs);
1816
});
1917
}
2018

21-
const runtimeSnapshot = await withServiceFallback(
22-
() => attachWorkspaceRuntime(workspaceId, deviceId, clientId),
23-
null,
24-
);
19+
const runtimeSnapshot = await attach();
2520
if (runtimeSnapshot) {
2621
return runtimeSnapshot;
2722
}
2823
}
2924

3025
return null;
3126
};
27+
28+
export const attachWorkspaceRuntimeWithRetry = async (
29+
workspaceId: string,
30+
deviceId: string,
31+
clientId: string,
32+
withServiceFallback: WithServiceFallback,
33+
): Promise<WorkspaceRuntimeSnapshot | null> => {
34+
return runAttachWithRetry(() => withServiceFallback(
35+
() => attachWorkspaceRuntime(workspaceId, deviceId, clientId),
36+
null,
37+
),
38+
);
39+
};

‎tests/e2e/transport.spec.ts‎

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,7 @@ test.describe('workspace transport baseline', () => {
366366
);
367367

368368
await page.reload();
369-
await expect(page.getByTestId('workspace-topbar')).toBeVisible();
369+
await waitForWorkspaceTopbar(page);
370370
await waitForBackendSocket(page);
371371
await expect(page.getByTestId('workspace-read-only-banner')).toBeHidden();
372372

@@ -540,7 +540,7 @@ test.describe('workspace transport baseline', () => {
540540
});
541541

542542
await page.reload();
543-
await expect(page.getByTestId('workspace-topbar')).toBeVisible();
543+
await waitForWorkspaceTopbar(page);
544544
await waitForBackendSocket(page);
545545
const sessionCard = page.locator(`.agent-pane-card[data-session-id="${session.id}"]`).first();
546546
const controllerAfterReload = await currentWorkspaceController(page, workspace.workspaceId, ids);
@@ -559,7 +559,7 @@ test.describe('workspace transport baseline', () => {
559559
TRANSPORT_EVENT_TIMEOUT_MS,
560560
);
561561
await page.reload();
562-
await expect(page.getByTestId('workspace-topbar')).toBeVisible();
562+
await waitForWorkspaceTopbar(page);
563563
await waitForBackendSocket(page);
564564
const runtimeAfterReload = await invokeRpc<{
565565
lifecycle_events?: Array<{ session_id: string; kind: string }>;
@@ -583,7 +583,7 @@ test.describe('workspace transport baseline', () => {
583583
await page.goto('about:blank');
584584
await page.waitForTimeout(2200);
585585
await page.goto(`/workspace/${workspace.workspaceId}`);
586-
await expect(page.getByTestId('workspace-topbar')).toBeVisible();
586+
await waitForWorkspaceTopbar(page);
587587
await waitForBackendSocket(page);
588588
await expect.poll(async () =>
589589
page.locator(`.agent-pane-card[data-session-id="${session.id}"]`).first().getAttribute('data-session-status')
@@ -619,7 +619,7 @@ test.describe('workspace transport baseline', () => {
619619
await waitForBackendSocket(controller);
620620

621621
await observer.goto(`/workspace/${workspace.workspaceId}`);
622-
await expect(observer.getByTestId('workspace-topbar')).toBeVisible();
622+
await waitForWorkspaceTopbar(observer);
623623
await waitForBackendSocket(observer);
624624
await expect(observer.getByTestId('runtime-validation-overlay')).toHaveCount(0);
625625

@@ -676,7 +676,7 @@ test.describe('workspace transport baseline', () => {
676676
await waitForBackendSocket(controller);
677677

678678
await observer.goto(`/workspace/${workspace.workspaceId}`);
679-
await expect(observer.getByTestId('workspace-topbar')).toBeVisible();
679+
await waitForWorkspaceTopbar(observer);
680680
await waitForBackendSocket(observer);
681681
await expect(observer.getByTestId('workspace-read-only-banner')).toBeVisible();
682682
await observer.bringToFront();
@@ -738,7 +738,7 @@ test.describe('workspace transport baseline', () => {
738738
await waitForBackendSocket(controller);
739739

740740
await observer.goto(`/workspace/${workspace.workspaceId}`);
741-
await expect(observer.getByTestId('workspace-topbar')).toBeVisible();
741+
await waitForWorkspaceTopbar(observer);
742742
await waitForBackendSocket(observer);
743743
await expect(observer.getByTestId('workspace-read-only-banner')).toBeVisible();
744744

@@ -821,7 +821,7 @@ test.describe('workspace transport baseline', () => {
821821
await waitForBackendSocket(controller);
822822

823823
await reopened.goto(`/workspace/${workspace.workspaceId}`);
824-
await expect(reopened.getByTestId('workspace-topbar')).toBeVisible();
824+
await waitForWorkspaceTopbar(reopened);
825825
await waitForBackendSocket(reopened);
826826
await expect(reopened.getByTestId('runtime-validation-overlay')).toHaveCount(0);
827827
await expect(reopened.getByTestId('workspace-read-only-banner')).toBeVisible();
@@ -900,7 +900,7 @@ test.describe('workspace transport baseline', () => {
900900
},
901901
});
902902
await page.reload();
903-
await expect(page.getByTestId('workspace-topbar')).toBeVisible();
903+
await waitForWorkspaceTopbar(page);
904904
await waitForBackendSocket(page);
905905
const controllerAfterReload = await currentWorkspaceController(page, workspace.workspaceId, ids);
906906
const interruptedSessionCard = page.locator(`.agent-pane-card[data-session-id="${session.id}"]`).first();
@@ -946,7 +946,7 @@ test.describe('workspace transport baseline', () => {
946946
);
947947

948948
await page.reload();
949-
await expect(page.getByTestId('workspace-topbar')).toBeVisible();
949+
await waitForWorkspaceTopbar(page);
950950
await waitForBackendSocket(page);
951951
let resumedStatus: string | null = null;
952952
await expect.poll(async () => {
@@ -1357,13 +1357,14 @@ async function openWorkspace(
13571357
});
13581358

13591359
const workspaceId = launch.snapshot.workspace.workspace_id;
1360-
await Promise.all([
1361-
page.waitForResponse((response) =>
1362-
response.request().method() === 'POST' && response.url().includes('/api/rpc/workbench_bootstrap'),
1363-
),
1364-
page.goto(`/workspace/${workspaceId}`),
1365-
]);
1366-
await expect(page.getByTestId('workspace-topbar')).toBeVisible();
1360+
const bootstrapRequest = page.waitForResponse((response) =>
1361+
response.request().method() === 'POST' && response.url().includes('/api/rpc/workbench_bootstrap'),
1362+
{
1363+
timeout: 15000,
1364+
}).catch(() => null);
1365+
await page.goto(`/workspace/${workspaceId}`);
1366+
await bootstrapRequest;
1367+
await waitForWorkspaceTopbar(page);
13671368

13681369
return {
13691370
workspaceId,
@@ -1466,6 +1467,10 @@ async function invokeRpc<T>(page: Page, command: string, payload: Record<string,
14661467
return body.data as T;
14671468
}
14681469

1470+
async function waitForWorkspaceTopbar(page: Page, timeout = 15000) {
1471+
await expect(page.getByTestId('workspace-topbar')).toBeVisible({ timeout });
1472+
}
1473+
14691474
async function closeAllOpenWorkspaces(page: Page) {
14701475
const bootstrap = await invokeRpc<{
14711476
ui_state: {

‎tests/runtime-attach.test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import test from "node:test";
2+
import assert from "node:assert/strict";
3+
import {
4+
ATTACH_RUNTIME_RETRY_DELAYS_MS,
5+
runAttachWithRetry,
6+
} from "../apps/web/src/features/workspace/runtime-attach.ts";
7+
8+
test("runAttachWithRetry keeps retrying until attach succeeds", async () => {
9+
const snapshot = { workspace: "ws-1" };
10+
let attempts = 0;
11+
12+
const result = await runAttachWithRetry(async () => {
13+
attempts += 1;
14+
return attempts < 4 ? null : snapshot;
15+
}, [0, 0, 0, 0]);
16+
17+
assert.equal(result, snapshot);
18+
assert.equal(attempts, 4);
19+
});
20+
21+
test("runAttachWithRetry returns null after exhausting its retry budget", async () => {
22+
let attempts = 0;
23+
24+
const result = await runAttachWithRetry(async () => {
25+
attempts += 1;
26+
return null;
27+
}, [0, 0, 0]);
28+
29+
assert.equal(result, null);
30+
assert.equal(attempts, 3);
31+
});
32+
33+
test("runtime attach retry delays reserve a longer recovery window for slow environments", () => {
34+
assert.deepEqual(
35+
ATTACH_RUNTIME_RETRY_DELAYS_MS,
36+
[0, 250, 750, 1500, 3000, 5000],
37+
);
38+
});

0 commit comments

Comments
 (0)