Skip to content

Commit b7c24b4

Browse files
thymikeethymikee
committed
fix: retry iOS runner prepare launch
1 parent 6babdfb commit b7c24b4

2 files changed

Lines changed: 165 additions & 48 deletions

File tree

‎src/platforms/ios/__tests__/runner-command-retry.test.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,56 @@ test('prepareIosRunner invalidates rebuilt sessions when bad-cache recovery heal
128128
]);
129129
});
130130

131+
test('prepareIosRunner retries a fresh launch session when the health check cannot connect', async () => {
132+
const stuckSession = makeRunnerSession({ port: 8100 });
133+
const relaunchedSession = makeRunnerSession({ port: 8101 });
134+
135+
mockEnsureRunnerSession.mockResolvedValueOnce(stuckSession).mockResolvedValueOnce(relaunchedSession);
136+
mockExecuteRunnerCommandWithSession
137+
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'Runner did not accept connection'))
138+
.mockResolvedValueOnce({ uptimeMs: 42 });
139+
140+
const result = await prepareIosRunner(IOS_SIMULATOR, {
141+
healthTimeoutMs: 90_000,
142+
buildTimeoutMs: 300_000,
143+
});
144+
145+
assert.deepEqual(result.runner, { uptimeMs: 42 });
146+
assert.equal(result.recoveryReason, 'Runner did not accept connection');
147+
assert.deepEqual(mockInvalidateRunnerSession.mock.calls, [
148+
[stuckSession, 'prepare_runner_health_retry'],
149+
]);
150+
assert.equal(mockEnsureRunnerSession.mock.calls.length, 2);
151+
assert.equal(mockEnsureRunnerSession.mock.calls[1]?.[1]?.cleanStaleBundles, true);
152+
assert.equal(mockEnsureRunnerSession.mock.calls[1]?.[1]?.forceRunnerXctestrunRebuild, undefined);
153+
assert.equal(mockExecuteRunnerCommandWithSession.mock.calls[1]?.[1], relaunchedSession);
154+
assert.ok(
155+
mockEmitDiagnostic.mock.calls.some(
156+
([event]) => event.phase === 'ios_runner_prepare_health_retry',
157+
),
158+
);
159+
});
160+
161+
test('prepareIosRunner invalidates the relaunched session when health still cannot connect', async () => {
162+
const stuckSession = makeRunnerSession({ port: 8100 });
163+
const relaunchedSession = makeRunnerSession({ port: 8101 });
164+
165+
mockEnsureRunnerSession.mockResolvedValueOnce(stuckSession).mockResolvedValueOnce(relaunchedSession);
166+
mockExecuteRunnerCommandWithSession
167+
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'Runner did not accept connection'))
168+
.mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'Runner did not accept connection'));
169+
170+
await assert.rejects(
171+
() => prepareIosRunner(IOS_SIMULATOR, { healthTimeoutMs: 90_000 }),
172+
/Runner did not accept connection/,
173+
);
174+
175+
assert.deepEqual(mockInvalidateRunnerSession.mock.calls, [
176+
[stuckSession, 'prepare_runner_health_retry'],
177+
[relaunchedSession, 'prepare_runner_health_failed'],
178+
]);
179+
});
180+
131181
test('mutating commands restart stale ready sessions when the preflight probe never reaches the runner', async () => {
132182
const staleSession = makeRunnerSession({ port: 8100, ready: true });
133183
const freshSession = makeRunnerSession({ port: 8101, ready: false });

‎src/platforms/ios/runner-lifecycle.ts‎

Lines changed: 115 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ import { handleRunnerTransportErrorAfterCommandSend } from './runner-command-rec
2828
export type PrepareIosRunnerOptions = AppleRunnerPrepareOptions;
2929
export type PrepareIosRunnerResult = AppleRunnerPrepareResult;
3030

31+
const PREPARE_RUNNER_HEALTH_MAX_SESSION_ATTEMPTS = 2;
32+
3133
export async function prepareLocalIosRunner(
3234
device: DeviceInfo,
3335
options: PrepareIosRunnerOptions,
@@ -36,66 +38,131 @@ export async function prepareLocalIosRunner(
3638
const signal = getRequestSignal(options.requestId);
3739
const command = withRunnerCommandId({ command: 'uptime' });
3840
let session: RunnerSession | undefined;
39-
try {
40-
const connectStartedAt = Date.now();
41-
session = await ensureRunnerSession(device, options);
42-
const connectMs = Date.now() - connectStartedAt;
43-
return recordPrepareResult(
44-
device,
45-
await runPrepareHealthCheck(device, session, command, options, signal, connectMs),
46-
);
47-
} catch (err) {
48-
const appErr = err instanceof AppError ? err : new AppError('COMMAND_FAILED', String(err));
49-
if (!session || !shouldRecoverBadCachedRunnerArtifact(appErr, session)) {
50-
throw err;
51-
}
52-
const reason = appErr.message || 'runner_health_failed';
53-
await invalidateRunnerSession(session, 'prepare_cached_runner_health_failed');
54-
await markRunnerXctestrunArtifactBadForRun(session.xctestrunArtifact, reason);
55-
const connectStartedAt = Date.now();
56-
const rebuiltSession = await ensureRunnerSession(device, {
57-
...options,
58-
cleanStaleBundles: true,
59-
forceRunnerXctestrunRebuild: true,
60-
});
61-
const connectMs = Date.now() - connectStartedAt;
41+
let recoveryReason: string | undefined;
42+
for (let attempt = 1; attempt <= PREPARE_RUNNER_HEALTH_MAX_SESSION_ATTEMPTS; attempt += 1) {
6243
try {
63-
const recovered = await runPrepareHealthCheck(
44+
const connectStartedAt = Date.now();
45+
session = await ensureRunnerSession(device, {
46+
...options,
47+
cleanStaleBundles: attempt > 1 ? true : options.cleanStaleBundles,
48+
});
49+
const connectMs = Date.now() - connectStartedAt;
50+
return recordPrepareResult(
6451
device,
65-
rebuiltSession,
66-
command,
67-
options,
68-
signal,
69-
connectMs,
70-
{ recoveryReason: reason },
52+
await runPrepareHealthCheck(device, session, command, options, signal, connectMs, {
53+
recoveryReason,
54+
}),
7155
);
56+
} catch (err) {
57+
const appErr = err instanceof AppError ? err : new AppError('COMMAND_FAILED', String(err));
58+
if (!session) {
59+
throw err;
60+
}
61+
if (shouldRecoverBadCachedRunnerArtifact(appErr, session)) {
62+
return await recoverBadCachedRunnerArtifact({
63+
device,
64+
session,
65+
command,
66+
options,
67+
signal,
68+
error: appErr,
69+
});
70+
}
71+
if (!shouldRetryPrepareRunnerHealthFailure(appErr)) {
72+
throw err;
73+
}
74+
const reason = appErr.message || 'runner_health_failed';
75+
if (attempt >= PREPARE_RUNNER_HEALTH_MAX_SESSION_ATTEMPTS) {
76+
await invalidateRunnerSession(session, 'prepare_runner_health_failed');
77+
throw err;
78+
}
79+
recoveryReason ??= reason;
80+
await invalidateRunnerSession(session, 'prepare_runner_health_retry');
7281
emitDiagnostic({
73-
level: 'info',
74-
phase: 'ios_runner_prepare_bad_cache_recovered',
82+
level: 'warn',
83+
phase: 'ios_runner_prepare_health_retry',
7584
data: {
7685
command: command.command,
7786
commandId: command.commandId,
78-
sessionId: rebuiltSession.sessionId,
79-
xctestrunPath: rebuiltSession.xctestrunArtifact?.xctestrunPath,
87+
sessionId: session.sessionId,
88+
attempt,
89+
maxAttempts: PREPARE_RUNNER_HEALTH_MAX_SESSION_ATTEMPTS,
8090
reason,
8191
},
8292
});
83-
return recordPrepareResult(device, recovered);
84-
} catch (retryErr) {
85-
await invalidateRunnerSession(rebuiltSession, 'prepare_rebuilt_runner_health_failed');
86-
const wrapped = wrapPrepareHealthFailure(retryErr, rebuiltSession, reason);
87-
emitPrepareDiagnostic(device, {
88-
cache: rebuiltSession.xctestrunArtifact?.cache,
89-
artifact: rebuiltSession.xctestrunArtifact?.artifact,
90-
buildMs: rebuiltSession.xctestrunArtifact?.buildMs,
91-
connectMs,
92-
healthCheckMs: 0,
93-
xctestrunPath: rebuiltSession.xctestrunArtifact?.xctestrunPath,
94-
failureReason: wrapped.message,
95-
});
96-
throw wrapped;
93+
} finally {
94+
session = undefined;
9795
}
9896
}
97+
98+
throw new AppError('COMMAND_FAILED', 'iOS runner prepare failed');
99+
}
100+
101+
async function recoverBadCachedRunnerArtifact(params: {
102+
device: DeviceInfo;
103+
session: RunnerSession & {
104+
xctestrunArtifact: NonNullable<RunnerSession['xctestrunArtifact']>;
105+
};
106+
command: RunnerCommand;
107+
options: PrepareIosRunnerOptions;
108+
signal: AbortSignal | undefined;
109+
error: AppError;
110+
}): Promise<PrepareIosRunnerResult> {
111+
const { device, session, command, options, signal, error } = params;
112+
const reason = error.message || 'runner_health_failed';
113+
await invalidateRunnerSession(session, 'prepare_cached_runner_health_failed');
114+
await markRunnerXctestrunArtifactBadForRun(session.xctestrunArtifact, reason);
115+
const connectStartedAt = Date.now();
116+
const rebuiltSession = await ensureRunnerSession(device, {
117+
...options,
118+
cleanStaleBundles: true,
119+
forceRunnerXctestrunRebuild: true,
120+
});
121+
const connectMs = Date.now() - connectStartedAt;
122+
try {
123+
const recovered = await runPrepareHealthCheck(
124+
device,
125+
rebuiltSession,
126+
command,
127+
options,
128+
signal,
129+
connectMs,
130+
{ recoveryReason: reason },
131+
);
132+
emitDiagnostic({
133+
level: 'info',
134+
phase: 'ios_runner_prepare_bad_cache_recovered',
135+
data: {
136+
command: command.command,
137+
commandId: command.commandId,
138+
sessionId: rebuiltSession.sessionId,
139+
xctestrunPath: rebuiltSession.xctestrunArtifact?.xctestrunPath,
140+
reason,
141+
},
142+
});
143+
return recordPrepareResult(device, recovered);
144+
} catch (retryErr) {
145+
await invalidateRunnerSession(rebuiltSession, 'prepare_rebuilt_runner_health_failed');
146+
const wrapped = wrapPrepareHealthFailure(retryErr, rebuiltSession, reason);
147+
emitPrepareDiagnostic(device, {
148+
cache: rebuiltSession.xctestrunArtifact?.cache,
149+
artifact: rebuiltSession.xctestrunArtifact?.artifact,
150+
buildMs: rebuiltSession.xctestrunArtifact?.buildMs,
151+
connectMs,
152+
healthCheckMs: 0,
153+
xctestrunPath: rebuiltSession.xctestrunArtifact?.xctestrunPath,
154+
failureReason: wrapped.message,
155+
});
156+
throw wrapped;
157+
}
158+
}
159+
160+
function shouldRetryPrepareRunnerHealthFailure(error: AppError): boolean {
161+
return (
162+
isRetryableRunnerError(error) ||
163+
shouldRetryRunnerConnectError(error) ||
164+
isPrepareHealthTimeout(error)
165+
);
99166
}
100167

101168
// fallow-ignore-next-line complexity

0 commit comments

Comments
 (0)