Skip to content

feat(ios): support external xctest runner artifact #806

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hannojg wants to merge 2 commits into
base: main
Choose a base branch
from
+197 −33
Open

feat(ios): support external xctest runner artifact #806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5459da4
feat(ios): support external xctest runner artifact
hannojg Jun 15, 2026
da1ddd8
fix(ios): pass derived data path to xctest runner
hannojg Jun 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/platforms/ios/__tests__/runner-session.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,9 @@ test('runner session starts xcodebuild through provider seams and reuses an aliv
assert.equal(session.xctestrunPath, '/tmp/session-runner.xctestrun');
assert.equal(mockRunCmdBackground.mock.calls.length, 1);
assert.equal(mockRunCmdBackground.mock.calls[0]?.[0], 'xcodebuild');
const xcodebuildArgs = mockRunCmdBackground.mock.calls[0]?.[1];
assert.ok(Array.isArray(xcodebuildArgs));
assert.equal(xcodebuildArgs[xcodebuildArgs.indexOf('-derivedDataPath') + 1], '/tmp/derived');
assert.deepEqual(mockPrepareXctestrunWithEnv.mock.calls[0]?.[1], {
AGENT_DEVICE_RUNNER_PORT: '8123',
});
Expand Down
107 changes: 107 additions & 0 deletions src/platforms/ios/__tests__/runner-xctestrun.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ import { withCommandExecutorOverride } from '../../../utils/exec.ts';
import {
__resetRunnerToolchainFingerprintCacheForTests,
acquireXcodebuildSimulatorSetRedirect,
ensureXctestrunArtifact,
findXctestrun,
markRunnerXctestrunArtifactBadForRun,
prepareXctestrunWithEnv,
resolveExpectedRunnerCacheMetadata,
resolveXcodebuildSimulatorDeviceSetPath,
Expand Down Expand Up @@ -285,6 +287,16 @@ function restoreEnvVar(name: string, value: string | undefined): void {
process.env[name] = value;
}

async function withEnvVar<T>(name: string, value: string, fn: () => Promise<T>): Promise<T> {
const previous = process.env[name];
process.env[name] = value;
try {
return await fn();
} finally {
restoreEnvVar(name, previous);
}
}

test('prepareXctestrunWithEnv avoids XCTest screen recordings for nested and legacy targets', async () => {
await withTempDir('runner-xctestrun-policy-', async (root) => {
const xctestrunPath = path.join(root, 'AgentDeviceRunner.xctestrun');
Expand Down Expand Up @@ -319,6 +331,54 @@ test('prepareXctestrunWithEnv avoids XCTest screen recordings for nested and leg
});
});

test('prepareXctestrunWithEnv writes env overlays into configured env dir', async () => {
await withTempDir('runner-xctestrun-env-dir-', async (root) => {
const xctestrunPath = path.join(root, 'readonly-artifacts', 'AgentDeviceRunner.xctestrun');
const envDir = path.join(root, 'writable-env');
fs.mkdirSync(path.dirname(xctestrunPath), { recursive: true });
fs.writeFileSync(
xctestrunPath,
JSON.stringify({
TestConfigurations: [{ TestTargets: [{ TestBundlePath: 'AgentDeviceRunnerUITests' }] }],
}),
);

await withEnvVar('AGENT_DEVICE_IOS_XCTEST_ENV_DIR', envDir, async () => {
const prepared = await withCommandExecutorOverride(
(cmd, args) => {
if (cmd !== 'plutil') return undefined;
if (args[0] === '-convert' && args[1] === 'json' && args[2] === '-o' && args[3] === '-') {
return Promise.resolve({
stdout: fs.readFileSync(String(args[4]), 'utf8'),
stderr: '',
exitCode: 0,
});
}
if (args[0] === '-convert' && args[1] === 'xml1' && args[2] === '-o') {
fs.copyFileSync(String(args[4]), String(args[3]));
return Promise.resolve({ stdout: '', stderr: '', exitCode: 0 });
}
return Promise.resolve({
stdout: '',
stderr: `unexpected plutil args: ${args.join(' ')}`,
exitCode: 1,
});
},
() => prepareXctestrunWithEnv(xctestrunPath, runnerPortEnv, 'aws session'),
);

assert.equal(path.dirname(prepared.xctestrunPath), envDir);
assert.equal(path.dirname(prepared.jsonPath), envDir);
assert.equal(
path.basename(prepared.xctestrunPath),
'AgentDeviceRunner.env.aws_session.xctestrun',
);
assert.equal(fs.existsSync(prepared.xctestrunPath), true);
assert.equal(fs.existsSync(prepared.jsonPath), true);
});
});
});

test('prepareXctestrunWithEnv leaves unrelated targets without capture policy', async () => {
await withTempDir('runner-xctestrun-policy-', async (root) => {
const xctestrunPath = path.join(root, 'AgentDeviceRunner.xctestrun');
Expand All @@ -337,6 +397,53 @@ test('prepareXctestrunWithEnv leaves unrelated targets without capture policy',
});
});

test('ensureXctestrunArtifact uses configured external xctestrun artifact', async () => {
await withTempDir('runner-xctestrun-external-', async (root) => {
const xctestrunPath = path.join(root, 'aws', 'AgentDeviceRunner.xctestrun');
const derivedPath = path.join(root, 'derived');
fs.mkdirSync(path.dirname(xctestrunPath), { recursive: true });
fs.writeFileSync(xctestrunPath, '{}');

await withEnvVar('AGENT_DEVICE_IOS_XCTESTRUN_FILE', xctestrunPath, async () => {
await withEnvVar('AGENT_DEVICE_IOS_XCTEST_DERIVED_DATA_PATH', derivedPath, async () => {
const artifact = await ensureXctestrunArtifact(iosDevice, {
forceRunnerXctestrunRebuild: true,
});

assert.equal(artifact.xctestrunPath, xctestrunPath);
assert.equal(artifact.derived, derivedPath);
assert.equal(artifact.cache, 'external');
assert.equal(artifact.artifact, 'valid');
assert.equal(artifact.buildMs, 0);
assert.equal(artifact.xctestrunPathSource, 'external');
});
});
});
});

test('markRunnerXctestrunArtifactBadForRun preserves configured external artifacts', async () => {
await withTempDir('runner-xctestrun-external-bad-', async (root) => {
const derivedPath = path.join(root, 'derived');
const xctestrunPath = path.join(root, 'aws', 'AgentDeviceRunner.xctestrun');
fs.mkdirSync(derivedPath, { recursive: true });
fs.mkdirSync(path.dirname(xctestrunPath), { recursive: true });
fs.writeFileSync(path.join(derivedPath, 'keep.txt'), 'derived');
fs.writeFileSync(xctestrunPath, 'xctestrun');

await markRunnerXctestrunArtifactBadForRun(
{
xctestrunPath,
derived: derivedPath,
cache: 'external',
},
'runner health failed',
);

assert.equal(fs.existsSync(path.join(derivedPath, 'keep.txt')), true);
assert.equal(fs.existsSync(xctestrunPath), true);
});
});

test('resolveXcodebuildSimulatorDeviceSetPath uses XCTestDevices under the user home', () => {
assert.equal(
resolveXcodebuildSimulatorDeviceSetPath('/tmp/agent-device-home'),
Expand Down
53 changes: 26 additions & 27 deletions src/platforms/ios/runner-session.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,38 +166,37 @@ async function startRunnerSessionWithLease(
);
let child: ExecBackgroundResult['child'] | undefined;
let testPromise: Promise<ExecResult>;
const xcodebuildArgs = [
'test-without-building',
'-only-testing',
'AgentDeviceRunnerUITests/RunnerTests/testCommand',
'-parallel-testing-enabled',
'NO',
'-test-timeouts-enabled',
'NO',
'-collect-test-diagnostics',
'never',
resolveRunnerMaxConcurrentDestinationsFlag(device),
'1',
'-destination-timeout',
String(RUNNER_DESTINATION_TIMEOUT_SECONDS),
'-xctestrun',
xctestrunPath,
'-derivedDataPath',
xctestrunArtifact.derived,
'-destination',
resolveRunnerDestination(device),
];
try {
({ child, wait: testPromise } = await measureRunnerStartupStep(
startupTimings,
'launch_xcodebuild',
() =>
runCmdBackground(
'xcodebuild',
[
'test-without-building',
'-only-testing',
'AgentDeviceRunnerUITests/RunnerTests/testCommand',
'-parallel-testing-enabled',
'NO',
'-test-timeouts-enabled',
'NO',
'-collect-test-diagnostics',
'never',
resolveRunnerMaxConcurrentDestinationsFlag(device),
'1',
'-destination-timeout',
String(RUNNER_DESTINATION_TIMEOUT_SECONDS),
'-xctestrun',
xctestrunPath,
'-destination',
resolveRunnerDestination(device),
],
{
allowFailure: true,
env: { ...process.env, AGENT_DEVICE_RUNNER_PORT: String(port) },
detached: true,
},
),
runCmdBackground('xcodebuild', xcodebuildArgs, {
allowFailure: true,
env: { ...process.env, AGENT_DEVICE_RUNNER_PORT: String(port) },
detached: true,
}),
));
} catch (error) {
await simulatorSetRedirect?.release();
Expand Down
67 changes: 61 additions & 6 deletions src/platforms/ios/runner-xctestrun.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ const RUNNER_XCTESTRUN_CAPTURE_OPTIONS = {
SystemAttachmentLifetime: 'keepNever',
UserAttachmentLifetime: 'keepNever',
} as const;
const EXTERNAL_XCTESTRUN_FILE_ENV = 'AGENT_DEVICE_IOS_XCTESTRUN_FILE';
const EXTERNAL_XCTEST_DERIVED_DATA_PATH_ENV = 'AGENT_DEVICE_IOS_XCTEST_DERIVED_DATA_PATH';
const EXTERNAL_XCTEST_ENV_DIR_ENV = 'AGENT_DEVICE_IOS_XCTEST_ENV_DIR';

const runnerXctestrunBuildLocks = new Map<string, Promise<unknown>>();
const badRunnerArtifactsForRun = new Set<string>();
Expand Down Expand Up @@ -110,7 +113,7 @@ export type RunnerXctestrunCacheMetadata = {
artifacts?: RunnerXctestrunCacheArtifacts;
};

export type RunnerXctestrunCacheKind = 'exact' | 'restore-key' | 'miss';
export type RunnerXctestrunCacheKind = 'exact' | 'restore-key' | 'miss' | 'external';
export type RunnerXctestrunArtifactState = 'valid' | 'rebuilt';

export type RunnerXctestrunArtifact = {
Expand All @@ -119,7 +122,7 @@ export type RunnerXctestrunArtifact = {
cache: RunnerXctestrunCacheKind;
artifact: RunnerXctestrunArtifactState;
buildMs: number;
xctestrunPathSource: 'manifest' | 'scan' | 'build';
xctestrunPathSource: 'manifest' | 'scan' | 'build' | 'external';
reason?: string;
};

Expand Down Expand Up @@ -436,6 +439,9 @@ export async function ensureXctestrunArtifact(
forceRunnerXctestrunRebuild?: boolean;
},
): Promise<RunnerXctestrunArtifact> {
const external = resolveExternalXctestrunArtifact();
if (external) return external;

const projectRoot = findProjectRoot();
const expectedCacheMetadata = resolveExpectedRunnerCacheMetadata(device, projectRoot);
const derived = resolveRunnerDerivedPath(device, expectedCacheMetadata);
Expand All @@ -456,6 +462,42 @@ export async function ensureXctestrunArtifact(
});
}

function resolveExternalXctestrunArtifact(
env: NodeJS.ProcessEnv = process.env,
): RunnerXctestrunArtifact | null {
const configuredXctestrunPath = env[EXTERNAL_XCTESTRUN_FILE_ENV]?.trim();
if (!configuredXctestrunPath) {
return null;
}

const xctestrunPath = path.resolve(configuredXctestrunPath);
if (!fs.existsSync(xctestrunPath)) {
throw new AppError('COMMAND_FAILED', 'Configured iOS XCTest runner .xctestrun file not found', {
env: EXTERNAL_XCTESTRUN_FILE_ENV,
xctestrunPath,
});
}

const configuredDerivedPath = env[EXTERNAL_XCTEST_DERIVED_DATA_PATH_ENV]?.trim();
const derived = configuredDerivedPath
? path.resolve(configuredDerivedPath)
: path.dirname(xctestrunPath);

emitRunnerXctestrunDecision('reuse', 'external_xctestrun', {
derived,
xctestrunPath,
});

return {
xctestrunPath,
derived,
cache: 'external',
artifact: 'valid',
buildMs: 0,
xctestrunPathSource: 'external',
};
}

async function ensureXctestrunUnderCacheLock(params: {
device: DeviceInfo;
options: { verbose?: boolean; logPath?: string; traceLogPath?: string; buildTimeoutMs?: number };
Expand Down Expand Up @@ -766,9 +808,18 @@ export function writeRunnerCacheMetadata(
}

export async function markRunnerXctestrunArtifactBadForRun(
artifact: Pick<RunnerXctestrunArtifact, 'derived' | 'xctestrunPath'>,
artifact: Pick<RunnerXctestrunArtifact, 'cache' | 'derived' | 'xctestrunPath'>,
reason: string,
): Promise<void> {
if (artifact.cache === 'external') {
emitRunnerXctestrunDecision('preserve', 'external_bad_artifact', {
derived: artifact.derived,
xctestrunPath: artifact.xctestrunPath,
reason,
});
return;
}

badRunnerArtifactsForRun.add(artifact.derived);
const releaseCacheLock = await acquireRunnerXctestrunCacheLock(artifact.derived);
try {
Expand Down Expand Up @@ -1150,7 +1201,9 @@ export async function prepareXctestrunWithEnv(
envVars: Record<string, string>,
suffix: string,
): Promise<{ xctestrunPath: string; jsonPath: string }> {
const dir = path.dirname(xctestrunPath);
const configuredEnvDir = process.env[EXTERNAL_XCTEST_ENV_DIR_ENV]?.trim();
const dir = configuredEnvDir ? path.resolve(configuredEnvDir) : path.dirname(xctestrunPath);
fs.mkdirSync(dir, { recursive: true });
const safeSuffix = suffix.replace(/[^a-zA-Z0-9._-]/g, '_');
const tmpJsonPath = path.join(dir, `AgentDeviceRunner.env.${safeSuffix}.json`);
const tmpXctestrunPath = path.join(dir, `AgentDeviceRunner.env.${safeSuffix}.xctestrun`);
Expand Down Expand Up @@ -1508,7 +1561,7 @@ async function evaluateExistingXctestrun(options: {
}

function emitRunnerXctestrunDecision(
action: 'clean' | 'reuse' | 'rebuild' | 'build',
action: 'clean' | 'reuse' | 'rebuild' | 'build' | 'preserve',
reason:
| 'forced_clean'
| 'missing_xctestrun'
Expand All @@ -1520,7 +1573,9 @@ function emitRunnerXctestrunDecision(
| 'reuse_ready'
| 'forced_rebuild'
| 'bad_artifact'
| 'built_new',
| 'built_new'
| 'external_xctestrun'
| 'external_bad_artifact',
data: Record<string, unknown>,
): void {
emitDiagnostic({
Expand Down
Loading