feat: cache ios runner artifacts during prepare

Author: thymikee

.github/actions/setup-apple-replay/action.yml

@@ -27,6 +27,10 @@ inputs:
     description: "Optional AGENT_DEVICE_IOS_CLEAN_DERIVED value"
     required: false
     default: ""
+  build-on-miss:
+    description: "Whether this setup action should build replay artifacts on cache miss"
+    required: false
+    default: "true"
 
 outputs:
   agent-home-dir:
@@ -68,7 +72,7 @@ runs:
       shell: bash
 
     - name: Build replay artifacts
-      if: steps.restore-prebuilt.outputs.cache-hit != 'true'
+      if: inputs.build-on-miss == 'true' && steps.restore-prebuilt.outputs.cache-hit != 'true'
       run: ${{ inputs.build-command }}
       shell: bash
       env:

.github/workflows/ios.yml

@@ -47,6 +47,7 @@ jobs:
           xcuitest-platform: ios
           xcuitest-destination: generic/platform=iOS Simulator
           clean-derived: "1"
+          build-on-miss: "false"
 
       - name: Boot iOS test simulator
         uses: ./.github/actions/boot-ios-test-simulator
@@ -57,7 +58,7 @@ jobs:
       - name: Prepare iOS runner
         run: |
           pnpm clean:daemon
-          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 300000 --json
 
       - name: Run iOS simulator smoke replay
         run: |

.github/workflows/replays-nightly.yml

@@ -70,6 +70,7 @@ jobs:
           xcuitest-platform: ios
           xcuitest-destination: generic/platform=iOS Simulator
           clean-derived: "1"
+          build-on-miss: "false"
 
       - name: Boot iOS test simulator
         uses: ./.github/actions/boot-ios-test-simulator
@@ -80,7 +81,7 @@ jobs:
       - name: Prepare iOS runner
         run: |
           pnpm clean:daemon
-          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 240000
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout 300000 --json
 
       - name: Run iOS simulator replay suite
         run: node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator --retries 2 --report-junit test/artifacts/replays-ios-simulator.junit.xml

scripts/write-xcuitest-cache-metadata.mjs

@@ -131,6 +131,57 @@ function resolveBuildDestinationFamily() {
   return `generic/platform=${platformName}`;
 }
 
+function resolveRunnerSdkName() {
+  const platformName = resolvePlatformName();
+  if (platformName === 'macOS') return 'macosx';
+  if (platformName === 'tvOS') {
+    return resolveDeviceKind() === 'simulator' ? 'appletvsimulator' : 'appletvos';
+  }
+  return resolveDeviceKind() === 'simulator' ? 'iphonesimulator' : 'iphoneos';
+}
+
+function runAppleToolFingerprintCommand(command, args) {
+  try {
+    return execFileSync(command, args, {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+      timeout: 5000,
+      maxBuffer: 128 * 1024,
+    }).trim() || 'unknown';
+  } catch {
+    return 'unknown';
+  }
+}
+
+function parseXcodeVersionOutput(output) {
+  return {
+    version: output.match(/^Xcode\s+(.+)$/m)?.[1]?.trim() || 'unknown',
+    buildVersion: output.match(/^Build version\s+(.+)$/m)?.[1]?.trim() || 'unknown',
+  };
+}
+
+function resolveRunnerToolchainFingerprint() {
+  const xcode = parseXcodeVersionOutput(
+    runAppleToolFingerprintCommand('xcodebuild', ['-version']),
+  );
+  const sdkName = resolveRunnerSdkName();
+  return {
+    xcodeVersion: xcode.version,
+    xcodeBuildVersion: xcode.buildVersion,
+    sdkName,
+    sdkVersion: runAppleToolFingerprintCommand('xcrun', [
+      '--sdk',
+      sdkName,
+      '--show-sdk-version',
+    ]),
+    sdkBuildVersion: runAppleToolFingerprintCommand('xcrun', [
+      '--sdk',
+      sdkName,
+      '--show-sdk-build-version',
+    ]),
+  };
+}
+
 function resolveSigningBuildSettings() {
   if (platform !== 'macos') {
     return [];
@@ -149,6 +200,7 @@ const metadata = {
   schemaVersion: 1,
   packageVersion: readPackageVersion(),
   runnerSourceFingerprint: computeRunnerSourceFingerprint(),
+  ...resolveRunnerToolchainFingerprint(),
   platformName: resolvePlatformName(),
   deviceKind: resolveDeviceKind(),
   target: resolveTarget(),
@@ -175,14 +227,16 @@ function resolveRunnerCacheArtifacts() {
   const productPaths = resolveExistingXctestrunProductPaths(xctestrunPath);
   if (!productPaths || productPaths.length === 0) return null;
   const xctestrunMtimeMs = readFileMtimeMs(xctestrunPath);
-  if (xctestrunMtimeMs === null) return null;
+  const xctestrunSize = readFileSize(xctestrunPath);
+  if (xctestrunMtimeMs === null || xctestrunSize === null) return null;
   const productArtifacts = [];
   for (const productPath of productPaths) {
     const mtimeMs = readFileMtimeMs(productPath);
-    if (mtimeMs === null) return null;
-    productArtifacts.push({ path: productPath, mtimeMs });
+    const size = readFileSize(productPath);
+    if (mtimeMs === null || size === null) return null;
+    productArtifacts.push({ path: productPath, mtimeMs, size });
   }
-  return { xctestrunPath, xctestrunMtimeMs, productPaths: productArtifacts };
+  return { xctestrunPath, xctestrunMtimeMs, xctestrunSize, productPaths: productArtifacts };
 }
 
 function findXctestrun(root) {
@@ -378,6 +432,14 @@ function readFileMtimeMs(filePath) {
   }
 }
 
+function readFileSize(filePath) {
+  try {
+    return fs.statSync(filePath).size;
+  } catch {
+    return null;
+  }
+}
+
 function isRecord(value) {
   return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
 }

src/daemon/handlers/__tests__/session.test.ts

@@ -2135,7 +2135,11 @@ test('prepare ios-runner starts the XCTest runner on an explicit iOS selector',
     expect.objectContaining({ platform: 'ios', id: 'sim-1' }),
     { command: 'uptime' },
     expect.objectContaining({
+      buildTimeoutMs: 240000,
+      commandTimeoutMs: 90000,
       logPath: expect.stringMatching(/daemon\.log$/),
+      prepareDiagnostics: true,
+      recoverBadCachedRunnerArtifact: true,
       requestId: 'prepare-request',
     }),
   );

src/daemon/handlers/session.ts

@@ -41,6 +41,8 @@ const INVENTORY_COMMANDS = DAEMON_COMMAND_GROUPS.inventory;
 const STATE_COMMANDS = DAEMON_COMMAND_GROUPS.state;
 const OBSERVABILITY_COMMANDS = DAEMON_COMMAND_GROUPS.observability;
 const REPLAY_COMMANDS = DAEMON_COMMAND_GROUPS.replay;
+const PREPARE_IOS_RUNNER_DEFAULT_BUILD_TIMEOUT_MS = 5 * 60_000;
+const PREPARE_IOS_RUNNER_HEALTH_TIMEOUT_MS = 90_000;
 
 export const SESSION_COMMAND_HANDLERS = {
   ...Object.fromEntries([...INVENTORY_COMMANDS].map((command) => [command, true] as const)),
@@ -106,28 +108,49 @@ function buildPrepareIosRunnerOptions(
   session: SessionState | undefined,
   logPath: string,
 ): Parameters<typeof runIosRunnerCommand>[2] {
+  const buildTimeoutMs = readPrepareIosRunnerBuildTimeoutMs(req);
+  const commandTimeoutMs = Math.min(buildTimeoutMs, PREPARE_IOS_RUNNER_HEALTH_TIMEOUT_MS);
   return {
     verbose: req.flags?.verbose,
     logPath,
     traceLogPath: session?.trace?.outPath,
     requestId: req.meta?.requestId,
+    buildTimeoutMs,
+    commandTimeoutMs,
+    recoverBadCachedRunnerArtifact: true,
+    prepareDiagnostics: true,
   };
 }
 
+function readPrepareIosRunnerBuildTimeoutMs(req: DaemonRequest): number {
+  const value = req.flags?.timeoutMs;
+  return typeof value === 'number' && Number.isFinite(value) && value > 0
+    ? value
+    : PREPARE_IOS_RUNNER_DEFAULT_BUILD_TIMEOUT_MS;
+}
+
 function prepareIosRunnerResponseData(
   action: string,
   device: DeviceInfo,
   durationMs: number,
   runner: Awaited<ReturnType<typeof runIosRunnerCommand>>,
 ): Record<string, unknown> {
+  const { prepareDiagnostics, ...runnerData } = runner;
+  const diagnostics =
+    prepareDiagnostics &&
+    typeof prepareDiagnostics === 'object' &&
+    !Array.isArray(prepareDiagnostics)
+      ? (prepareDiagnostics as Record<string, unknown>)
+      : {};
   return {
     action,
     platform: device.platform,
     deviceId: device.id,
     deviceName: device.name,
     kind: device.kind,
     durationMs,
-    runner,
+    ...diagnostics,
+    runner: runnerData,
     message: `Prepared iOS runner: ${device.name}`,
   };
 }

src/platforms/ios/__tests__/runner-client.test.ts

@@ -48,7 +48,9 @@ import {
 } from '../runner-client.ts';
 import {
   acquireRunnerXctestrunCacheLock,
+  ensureXctestrunArtifact,
   ensureXctestrun,
+  markRunnerXctestrunArtifactBadForRun,
   resolveExpectedRunnerCacheMetadata,
   resolveRunnerDerivedPath,
   resolveRunnerCacheMetadataPath,
@@ -1132,6 +1134,57 @@ test('ensureXctestrun rebuilds cached runner when metadata package version misma
   assert.equal(rebuiltMetadata.artifacts?.xctestrunPath, rebuiltXctestrunPath);
 });
 
+test('ensureXctestrunArtifact stress-recovers after a bad restored artifact', async () => {
+  const projectRoot = repoRoot;
+  const tmpDir = await makeProjectTmpDir();
+  const derivedPath = path.join(tmpDir, 'custom-derived');
+  const productPath = path.join(derivedPath, 'Runner.app');
+  const cachedXctestrunPath = path.join(derivedPath, 'cached.xctestrun');
+  await fs.promises.mkdir(productPath, { recursive: true });
+  writeXctestrunFixture(cachedXctestrunPath, {
+    projectRoot,
+    productRelativePaths: ['Runner.app'],
+  });
+  writeRunnerCacheMetadataWithArtifacts({
+    derivedPath,
+    device: macOsDevice,
+    xctestrunPath: cachedXctestrunPath,
+    productPaths: [productPath],
+  });
+  withRunnerDerivedPathEnv(derivedPath);
+
+  const hit = await ensureXctestrunArtifact(macOsDevice, {});
+
+  assert.equal(hit.xctestrunPath, cachedXctestrunPath);
+  assert.equal(hit.cache, 'exact');
+  assert.equal(hit.artifact, 'valid');
+  assert.equal(hit.buildMs, 0);
+  assert.equal(mockRunCmdStreaming.mock.calls.length, 0);
+
+  await markRunnerXctestrunArtifactBadForRun(hit, 'stress health failed');
+  assert.equal(fs.existsSync(cachedXctestrunPath), false);
+
+  const rebuiltXctestrunPath = path.join(derivedPath, 'rebuilt', 'rebuilt.xctestrun');
+  mockRunCmdStreaming.mockImplementationOnce(async () => {
+    await fs.promises.mkdir(path.join(derivedPath, 'rebuilt', 'Runner.app'), { recursive: true });
+    writeXctestrunFixture(rebuiltXctestrunPath, {
+      projectRoot,
+      productRelativePaths: ['Runner.app'],
+    });
+  });
+
+  const rebuilt = await ensureXctestrunArtifact(macOsDevice, {
+    buildTimeoutMs: 300_000,
+  });
+
+  assert.equal(rebuilt.xctestrunPath, rebuiltXctestrunPath);
+  assert.equal(rebuilt.cache, 'miss');
+  assert.equal(rebuilt.artifact, 'rebuilt');
+  assert.equal(rebuilt.reason, 'missing_xctestrun');
+  assert.equal(mockRunCmdStreaming.mock.calls.length, 1);
+  assert.equal(mockRunCmdStreaming.mock.calls[0]?.[2]?.timeoutMs, 300_000);
+});
+
 test('ensureXctestrun rethrows unexpected cached macOS runner repair errors', async () => {
   const { derivedPath, existingXctestrunPath } = await makeCachedRunnerXctestrun();
 

src/platforms/ios/__tests__/runner-session.test.ts

@@ -12,7 +12,7 @@ import type { RunnerSession } from '../runner-session-types.ts';
 const {
   mockAcquireXcodebuildSimulatorSetRedirect,
   mockCleanupTempFile,
-  mockEnsureXctestrun,
+  mockEnsureXctestrunArtifact,
   mockGetFreePort,
   mockIsProcessAlive,
   mockIsProcessGroupAlive,
@@ -26,7 +26,7 @@ const {
 } = vi.hoisted(() => ({
   mockAcquireXcodebuildSimulatorSetRedirect: vi.fn(),
   mockCleanupTempFile: vi.fn(),
-  mockEnsureXctestrun: vi.fn(),
+  mockEnsureXctestrunArtifact: vi.fn(),
   mockGetFreePort: vi.fn(),
   mockIsProcessAlive: vi.fn(),
   mockIsProcessGroupAlive: vi.fn(),
@@ -86,7 +86,7 @@ vi.mock('../runner-xctestrun.ts', async () => {
   return {
     ...actual,
     acquireXcodebuildSimulatorSetRedirect: mockAcquireXcodebuildSimulatorSetRedirect,
-    ensureXctestrun: mockEnsureXctestrun,
+    ensureXctestrunArtifact: mockEnsureXctestrunArtifact,
     prepareXctestrunWithEnv: mockPrepareXctestrunWithEnv,
   };
 });
@@ -103,7 +103,14 @@ import {
 beforeEach(() => {
   vi.resetAllMocks();
   mockRunXcrun.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' });
-  mockEnsureXctestrun.mockResolvedValue('/tmp/base-runner.xctestrun');
+  mockEnsureXctestrunArtifact.mockResolvedValue({
+    xctestrunPath: '/tmp/base-runner.xctestrun',
+    derived: '/tmp/derived',
+    cache: 'miss',
+    artifact: 'rebuilt',
+    buildMs: 12,
+    xctestrunPathSource: 'build',
+  });
   mockGetFreePort.mockResolvedValue(8123);
   mockPrepareXctestrunWithEnv.mockResolvedValue({
     xctestrunPath: '/tmp/session-runner.xctestrun',