Skip to content

Commit c03ace3

Browse files
thymikeethymikee
committed
Add boot diagnostics classification and deadline-based boot retries
1 parent 835b59f commit c03ace3

7 files changed

Lines changed: 369 additions & 25 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
"prepack": "pnpm build:node && pnpm build:axsnapshot",
2727
"typecheck": "tsc -p tsconfig.json",
2828
"test": "node --test",
29-
"test:unit": "node --test src/core/__tests__/*.test.ts src/daemon/__tests__/*.test.ts src/daemon/handlers/__tests__/*.test.ts src/platforms/**/__tests__/*.test.ts",
29+
"test:unit": "node --test src/core/__tests__/*.test.ts src/daemon/__tests__/*.test.ts src/daemon/handlers/__tests__/*.test.ts src/platforms/**/__tests__/*.test.ts src/utils/**/__tests__/*.test.ts",
3030
"test:smoke": "node --test test/integration/smoke-*.test.ts",
3131
"test:integration": "node --test test/integration/*.test.ts"
3232
},

‎src/platforms/__tests__/boot-diagnostics.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import test from 'node:test';
2+
import assert from 'node:assert/strict';
3+
import { classifyBootFailure } from '../boot-diagnostics.ts';
4+
import { AppError } from '../../utils/errors.ts';
5+
6+
test('classifyBootFailure maps timeout errors', () => {
7+
const reason = classifyBootFailure({ message: 'bootstatus timed out after 120s' });
8+
assert.equal(reason, 'BOOT_TIMEOUT');
9+
});
10+
11+
test('classifyBootFailure maps adb offline errors', () => {
12+
const reason = classifyBootFailure({ stderr: 'error: device offline' });
13+
assert.equal(reason, 'DEVICE_OFFLINE');
14+
});
15+
16+
test('classifyBootFailure maps tool missing from AppError code', () => {
17+
const reason = classifyBootFailure({
18+
error: new AppError('TOOL_MISSING', 'adb not found in PATH'),
19+
});
20+
assert.equal(reason, 'TOOL_MISSING');
21+
});
22+
23+
test('classifyBootFailure reads stderr from AppError details', () => {
24+
const reason = classifyBootFailure({
25+
error: new AppError('COMMAND_FAILED', 'adb failed', {
26+
stderr: 'error: device unauthorized',
27+
}),
28+
});
29+
assert.equal(reason, 'PERMISSION_DENIED');
30+
});

‎src/platforms/android/devices.ts‎

Lines changed: 84 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { runCmd, whichCmd } from '../../utils/exec.ts';
2-
import { AppError } from '../../utils/errors.ts';
2+
import { AppError, asAppError } from '../../utils/errors.ts';
33
import type { DeviceInfo } from '../../utils/device.ts';
4+
import { Deadline, retryWithPolicy } from '../../utils/retry.ts';
5+
import { classifyBootFailure } from '../boot-diagnostics.ts';
46

57
export async function listAndroidDevices(): Promise<DeviceInfo[]> {
68
const adbAvailable = await whichCmd('adb');
@@ -59,13 +61,86 @@ export async function isAndroidBooted(serial: string): Promise<boolean> {
5961
}
6062

6163
export async function waitForAndroidBoot(serial: string, timeoutMs = 60000): Promise<void> {
62-
const start = Date.now();
63-
while (Date.now() - start < timeoutMs) {
64-
if (await isAndroidBooted(serial)) return;
65-
await new Promise((resolve) => setTimeout(resolve, 1000));
64+
const deadline = Deadline.fromTimeoutMs(timeoutMs);
65+
const maxAttempts = Math.max(1, Math.ceil(timeoutMs / 1000));
66+
let lastBootResult: { stdout: string; stderr: string; exitCode: number } | null = null;
67+
let timedOut = false;
68+
try {
69+
await retryWithPolicy(
70+
async ({ deadline: attemptDeadline }) => {
71+
if (attemptDeadline?.isExpired()) {
72+
timedOut = true;
73+
throw new AppError('COMMAND_FAILED', 'Android boot deadline exceeded', {
74+
serial,
75+
timeoutMs,
76+
elapsedMs: deadline.elapsedMs(),
77+
message: 'timeout',
78+
});
79+
}
80+
const result = await runCmd(
81+
'adb',
82+
['-s', serial, 'shell', 'getprop', 'sys.boot_completed'],
83+
{ allowFailure: true },
84+
);
85+
lastBootResult = result;
86+
if ((result.stdout as string).trim() === '1') return;
87+
throw new AppError('COMMAND_FAILED', 'Android device is still booting', {
88+
serial,
89+
stdout: result.stdout,
90+
stderr: result.stderr,
91+
exitCode: result.exitCode,
92+
});
93+
},
94+
{
95+
maxAttempts,
96+
baseDelayMs: 1000,
97+
maxDelayMs: 1000,
98+
jitter: 0,
99+
shouldRetry: (error) => {
100+
const reason = classifyBootFailure({
101+
error,
102+
stdout: lastBootResult?.stdout,
103+
stderr: lastBootResult?.stderr,
104+
});
105+
return reason !== 'PERMISSION_DENIED' && reason !== 'TOOL_MISSING' && reason !== 'BOOT_TIMEOUT';
106+
},
107+
},
108+
{ deadline },
109+
);
110+
} catch (error) {
111+
const appErr = asAppError(error);
112+
const reason = classifyBootFailure({
113+
error,
114+
stdout: lastBootResult?.stdout,
115+
stderr: lastBootResult?.stderr,
116+
});
117+
const baseDetails = {
118+
serial,
119+
timeoutMs,
120+
elapsedMs: deadline.elapsedMs(),
121+
reason,
122+
stdout: lastBootResult?.stdout,
123+
stderr: lastBootResult?.stderr,
124+
exitCode: lastBootResult?.exitCode,
125+
};
126+
if (timedOut || reason === 'BOOT_TIMEOUT') {
127+
throw new AppError('COMMAND_FAILED', 'Android device did not finish booting in time', baseDetails);
128+
}
129+
if (appErr.code === 'TOOL_MISSING' || reason === 'TOOL_MISSING') {
130+
throw new AppError('TOOL_MISSING', appErr.message, {
131+
...baseDetails,
132+
...(appErr.details ?? {}),
133+
});
134+
}
135+
if (reason === 'PERMISSION_DENIED' || reason === 'DEVICE_UNAVAILABLE' || reason === 'DEVICE_OFFLINE') {
136+
throw new AppError('COMMAND_FAILED', appErr.message, {
137+
...baseDetails,
138+
...(appErr.details ?? {}),
139+
});
140+
}
141+
throw new AppError(appErr.code, appErr.message, {
142+
...baseDetails,
143+
...(appErr.details ?? {}),
144+
}, appErr.cause);
66145
}
67-
throw new AppError('COMMAND_FAILED', 'Android device did not finish booting in time', {
68-
serial,
69-
timeoutMs,
70-
});
71146
}

‎src/platforms/boot-diagnostics.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { asAppError } from '../utils/errors.ts';
2+
3+
export type BootFailureReason =
4+
| 'BOOT_TIMEOUT'
5+
| 'DEVICE_UNAVAILABLE'
6+
| 'DEVICE_OFFLINE'
7+
| 'PERMISSION_DENIED'
8+
| 'TOOL_MISSING'
9+
| 'BOOT_COMMAND_FAILED'
10+
| 'UNKNOWN';
11+
12+
export function classifyBootFailure(input: {
13+
error?: unknown;
14+
message?: string;
15+
stdout?: string;
16+
stderr?: string;
17+
}): BootFailureReason {
18+
const appErr = input.error ? asAppError(input.error) : null;
19+
if (appErr?.code === 'TOOL_MISSING') return 'TOOL_MISSING';
20+
const details = (appErr?.details ?? {}) as Record<string, unknown>;
21+
const detailMessage = typeof details.message === 'string' ? details.message : undefined;
22+
const detailStdout = typeof details.stdout === 'string' ? details.stdout : undefined;
23+
const detailStderr = typeof details.stderr === 'string' ? details.stderr : undefined;
24+
const nestedBoot = details.boot && typeof details.boot === 'object'
25+
? (details.boot as Record<string, unknown>)
26+
: null;
27+
const nestedBootstatus = details.bootstatus && typeof details.bootstatus === 'object'
28+
? (details.bootstatus as Record<string, unknown>)
29+
: null;
30+
31+
const haystack = [
32+
input.message,
33+
appErr?.message,
34+
input.stdout,
35+
input.stderr,
36+
detailMessage,
37+
detailStdout,
38+
detailStderr,
39+
typeof nestedBoot?.stdout === 'string' ? nestedBoot.stdout : undefined,
40+
typeof nestedBoot?.stderr === 'string' ? nestedBoot.stderr : undefined,
41+
typeof nestedBootstatus?.stdout === 'string' ? nestedBootstatus.stdout : undefined,
42+
typeof nestedBootstatus?.stderr === 'string' ? nestedBootstatus.stderr : undefined,
43+
]
44+
.filter(Boolean)
45+
.join('\n')
46+
.toLowerCase();
47+
48+
if (haystack.includes('timed out') || haystack.includes('timeout')) return 'BOOT_TIMEOUT';
49+
if (
50+
haystack.includes('device not found') ||
51+
haystack.includes('no devices') ||
52+
haystack.includes('unable to locate device') ||
53+
haystack.includes('invalid device')
54+
) {
55+
return 'DEVICE_UNAVAILABLE';
56+
}
57+
if (haystack.includes('offline')) return 'DEVICE_OFFLINE';
58+
if (
59+
haystack.includes('permission denied') ||
60+
haystack.includes('not authorized') ||
61+
haystack.includes('unauthorized')
62+
) {
63+
return 'PERMISSION_DENIED';
64+
}
65+
if (appErr?.code === 'COMMAND_FAILED' || haystack.length > 0) return 'BOOT_COMMAND_FAILED';
66+
return 'UNKNOWN';
67+
}

‎src/platforms/ios/index.ts‎

Lines changed: 87 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
import { runCmd } from '../../utils/exec.ts';
22
import { AppError } from '../../utils/errors.ts';
33
import type { DeviceInfo } from '../../utils/device.ts';
4+
import { Deadline, retryWithPolicy } from '../../utils/retry.ts';
5+
import { classifyBootFailure } from '../boot-diagnostics.ts';
46

57
const ALIASES: Record<string, string> = {
68
settings: 'com.apple.Preferences',
79
};
810

11+
const IOS_BOOT_TIMEOUT_MS = resolveTimeoutMs(process.env.AGENT_DEVICE_IOS_BOOT_TIMEOUT_MS, 120_000, 5_000);
12+
913
export async function resolveIosApp(device: DeviceInfo, app: string): Promise<string> {
1014
const trimmed = app.trim();
1115
if (trimmed.includes('.')) return trimmed;
@@ -207,8 +211,82 @@ export async function ensureBootedSimulator(device: DeviceInfo): Promise<void> {
207211
if (device.kind !== 'simulator') return;
208212
const state = await getSimulatorState(device.id);
209213
if (state === 'Booted') return;
210-
await runCmd('xcrun', ['simctl', 'boot', device.id], { allowFailure: true });
211-
await runCmd('xcrun', ['simctl', 'bootstatus', device.id, '-b'], { allowFailure: true });
214+
const deadline = Deadline.fromTimeoutMs(IOS_BOOT_TIMEOUT_MS);
215+
let bootResult: { stdout: string; stderr: string; exitCode: number } | null = null;
216+
let bootStatusResult: { stdout: string; stderr: string; exitCode: number } | null = null;
217+
try {
218+
await retryWithPolicy(
219+
async () => {
220+
const currentState = await getSimulatorState(device.id);
221+
if (currentState === 'Booted') return;
222+
bootResult = await runCmd('xcrun', ['simctl', 'boot', device.id], { allowFailure: true });
223+
const bootOutput = `${bootResult.stdout}\n${bootResult.stderr}`.toLowerCase();
224+
const bootAlreadyDone =
225+
bootOutput.includes('already booted') || bootOutput.includes('current state: booted');
226+
if (bootResult.exitCode !== 0 && !bootAlreadyDone) {
227+
throw new AppError('COMMAND_FAILED', 'simctl boot failed', {
228+
stdout: bootResult.stdout,
229+
stderr: bootResult.stderr,
230+
exitCode: bootResult.exitCode,
231+
});
232+
}
233+
bootStatusResult = await runCmd('xcrun', ['simctl', 'bootstatus', device.id, '-b'], {
234+
allowFailure: true,
235+
});
236+
if (bootStatusResult.exitCode !== 0) {
237+
throw new AppError('COMMAND_FAILED', 'simctl bootstatus failed', {
238+
stdout: bootStatusResult.stdout,
239+
stderr: bootStatusResult.stderr,
240+
exitCode: bootStatusResult.exitCode,
241+
});
242+
}
243+
const nextState = await getSimulatorState(device.id);
244+
if (nextState !== 'Booted') {
245+
throw new AppError('COMMAND_FAILED', 'Simulator is still booting', {
246+
state: nextState,
247+
});
248+
}
249+
},
250+
{
251+
maxAttempts: 3,
252+
baseDelayMs: 500,
253+
maxDelayMs: 2000,
254+
jitter: 0.2,
255+
shouldRetry: (error) => {
256+
const reason = classifyBootFailure({
257+
error,
258+
stdout: bootStatusResult?.stdout ?? bootResult?.stdout,
259+
stderr: bootStatusResult?.stderr ?? bootResult?.stderr,
260+
});
261+
return reason !== 'PERMISSION_DENIED' && reason !== 'TOOL_MISSING';
262+
},
263+
},
264+
{ deadline },
265+
);
266+
} catch (error) {
267+
const reason = classifyBootFailure({
268+
error,
269+
stdout: bootStatusResult?.stdout ?? bootResult?.stdout,
270+
stderr: bootStatusResult?.stderr ?? bootResult?.stderr,
271+
});
272+
throw new AppError('COMMAND_FAILED', 'iOS simulator failed to boot', {
273+
platform: 'ios',
274+
deviceId: device.id,
275+
timeoutMs: IOS_BOOT_TIMEOUT_MS,
276+
elapsedMs: deadline.elapsedMs(),
277+
reason,
278+
boot: bootResult
279+
? { exitCode: bootResult.exitCode, stdout: bootResult.stdout, stderr: bootResult.stderr }
280+
: undefined,
281+
bootstatus: bootStatusResult
282+
? {
283+
exitCode: bootStatusResult.exitCode,
284+
stdout: bootStatusResult.stdout,
285+
stderr: bootStatusResult.stderr,
286+
}
287+
: undefined,
288+
});
289+
}
212290
}
213291

214292
async function getSimulatorState(udid: string): Promise<string | null> {
@@ -229,3 +307,10 @@ async function getSimulatorState(udid: string): Promise<string | null> {
229307
}
230308
return null;
231309
}
310+
311+
function resolveTimeoutMs(raw: string | undefined, fallback: number, min: number): number {
312+
if (!raw) return fallback;
313+
const parsed = Number(raw);
314+
if (!Number.isFinite(parsed)) return fallback;
315+
return Math.max(min, Math.floor(parsed));
316+
}

‎src/utils/__tests__/retry.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import test from 'node:test';
2+
import assert from 'node:assert/strict';
3+
import { Deadline, retryWithPolicy } from '../retry.ts';
4+
5+
test('Deadline tracks remaining and expiration', async () => {
6+
const deadline = Deadline.fromTimeoutMs(25);
7+
assert.equal(deadline.isExpired(), false);
8+
await new Promise((resolve) => setTimeout(resolve, 30));
9+
assert.equal(deadline.isExpired(), true);
10+
assert.equal(deadline.remainingMs(), 0);
11+
});
12+
13+
test('retryWithPolicy retries until success', async () => {
14+
let attempts = 0;
15+
const result = await retryWithPolicy(
16+
async () => {
17+
attempts += 1;
18+
if (attempts < 3) {
19+
throw new Error('transient');
20+
}
21+
return 'ok';
22+
},
23+
{ maxAttempts: 3, baseDelayMs: 1, maxDelayMs: 1, jitter: 0 },
24+
);
25+
assert.equal(result, 'ok');
26+
assert.equal(attempts, 3);
27+
});

0 commit comments

Comments
 (0)