ci(ios): simulator boot / availability fixes

Poll simctl bootstatus with migration logging before app install, raise pre-boot and job timeouts, start simulator artifacts after boot, and document iOS simulator reliability in OKF.

Author: mikehardy

.github/workflows/scripts/boot-simulator.sh

@@ -1,30 +1,158 @@
 #!/bin/bash
 
-# Any command here that exits non-zero is an error
-set -e
+# Boot the Detox iOS simulator, wait until it is fully ready for testing (including
+# first-boot data migration on fresh simulators), then install the test app.
+# Uses the device *name* from tests/.detoxrc.js — no pinned UDID in the workflow.
+set -euo pipefail
+
+BOOT_POLL_INTERVAL_SECONDS="${BOOT_POLL_INTERVAL_SECONDS:-20}"
+BOOT_PROBE_TIMEOUT_SECONDS="${BOOT_PROBE_TIMEOUT_SECONDS:-12}"
+BOOT_MAX_WAIT_SECONDS="${BOOT_MAX_WAIT_SECONDS:-660}"
+
+run_with_timeout() {
+  local max="$1"
+  shift
+  "$@" &
+  local cmd_pid=$!
+  local waited=0
+  while kill -0 "$cmd_pid" 2>/dev/null && (( waited < max )); do
+    sleep 1
+    waited=$((waited + 1))
+  done
+  if kill -0 "$cmd_pid" 2>/dev/null; then
+    kill "$cmd_pid" 2>/dev/null
+    wait "$cmd_pid" 2>/dev/null || true
+    return 124
+  fi
+  wait "$cmd_pid"
+}
+
+log_boot_status() {
+  echo "[boot-status] $*"
+}
+
+describe_booted_device() {
+  local device="$1"
+  xcrun simctl list devices booted 2>/dev/null \
+    | grep -i "${device} (" \
+    | grep -v 'Phone:' \
+    | grep -v 'unavailable' \
+    | grep -v CoreSimulator \
+    | head -1 \
+    || true
+}
+
+log_migration_status() {
+  local device="$1"
+  local migration_output probe_rc
+
+  log_boot_status "probing data migration (bootstatus -d, up to ${BOOT_PROBE_TIMEOUT_SECONDS}s)..."
+  set +e
+  migration_output="$(run_with_timeout "$BOOT_PROBE_TIMEOUT_SECONDS" xcrun simctl bootstatus "$device" -d 2>&1)"
+  probe_rc=$?
+  set -e
+
+  if [[ "$probe_rc" -eq 124 ]]; then
+    log_boot_status "  data migration / system bring-up still in progress"
+    return 1
+  fi
+
+  if [[ -n "$migration_output" ]]; then
+    while IFS= read -r line; do
+      [[ -z "$line" ]] && continue
+      log_boot_status "  ${line}"
+    done <<<"$migration_output"
+  else
+    log_boot_status "  no migration details reported"
+  fi
+  return 0
+}
+
+wait_for_simulator_ready() {
+  local device="$1"
+  local start=$SECONDS
+
+  while (( SECONDS - start < BOOT_MAX_WAIT_SECONDS )); do
+    local elapsed=$(( SECONDS - start ))
+    local booted_line ready_rc
+
+    log_boot_status "elapsed=${elapsed}s phase=wait_for_full_boot device=\"${device}\""
+
+    booted_line="$(describe_booted_device "$device")"
+    if [[ -z "$booted_line" ]]; then
+      log_boot_status "  simctl list: not in Booted state yet"
+    else
+      log_boot_status "  simctl list: ${booted_line}"
+      log_migration_status "$device" || true
+    fi
+
+    set +e
+    run_with_timeout "$BOOT_PROBE_TIMEOUT_SECONDS" xcrun simctl bootstatus "$device" >/dev/null 2>&1
+    ready_rc=$?
+    set -e
+
+    if [[ "$ready_rc" -eq 0 ]]; then
+      log_boot_status "bootstatus: simulator ready after ${elapsed}s"
+      log_migration_status "$device" || true
+      return 0
+    fi
+
+    if [[ "$ready_rc" -eq 124 ]]; then
+      log_boot_status "bootstatus: still booting (probe timed out after ${BOOT_PROBE_TIMEOUT_SECONDS}s)"
+    else
+      log_boot_status "bootstatus: probe exited with status ${ready_rc}"
+    fi
+
+    sleep "$BOOT_POLL_INTERVAL_SECONDS"
+  done
+
+  log_boot_status "ERROR: timed out after ${BOOT_MAX_WAIT_SECONDS}s waiting for simulator to become ready"
+  return 1
+}
 
 # Get our simulator name from our test Detox config
-pushd "$(dirname "$0")/../../../tests" || exit 1
-SIM="$(cat .detoxrc.js | grep iPhone | cut -d"'" -f2)"
-echo "Attempting to boot iOS Simulator $SIM..."
+pushd "$(dirname "$0")/../../../tests" >/dev/null || exit 1
+SIM="$(grep iPhone .detoxrc.js | head -1 | cut -d"'" -f2)"
+popd >/dev/null || exit 1
+
+log_boot_status "phase=resolve_device name=\"${SIM}\" (from tests/.detoxrc.js)"
 
 # Clear up any existing attempts in case we are re-trying
-echo "...killing any existing Simulator processes..."
-killall Simulator || true
+log_boot_status "phase=shutdown_existing killing Simulator.app if running..."
+killall Simulator 2>/dev/null || true
+xcrun simctl shutdown "$SIM" 2>/dev/null || true
+
+log_boot_status "phase=boot_command starting simctl boot..."
+set +e
+boot_output="$(xcrun simctl boot "$SIM" 2>&1)"
+boot_rc=$?
+set -e
+if [[ "$boot_rc" -ne 0 ]]; then
+  log_boot_status "simctl boot exited ${boot_rc}: ${boot_output}"
+else
+  log_boot_status "simctl boot command returned (device may still be migrating data)"
+fi
+
+log_boot_status "phase=foreground_simulator opening Simulator.app..."
+open -a Simulator.app
 
-# Boot the simulator if not booted, make sure it is in the foreground
-echo "...booting $SIM and foregrounding Simulator..."
-(xcrun simctl boot "$SIM" || true) && open -a Simulator.app
+if ! wait_for_simulator_ready "$SIM"; then
+  exit 1
+fi
 
-# Is it booted?
-echo "...waiting to make sure $SIM is booted..."
-xcrun simctl list |grep -i "$SIM ("|grep -v 'Phone:'|grep -v 'unavailable'|grep -v CoreSimulator|grep Booted
+pushd "$(dirname "$0")/../../../tests" >/dev/null || exit 1
+BUILDDIR="$(find ios/build/Build/Products -type d -name 'testing.app' 2>/dev/null | head -1)"
 
-# Are we a Debug or Release build?
-BUILDDIR="$( find ios/build/Build/Products -type d |grep 'testing.app$' | head -1)"
+if [[ -z "$BUILDDIR" || ! -d "$BUILDDIR" ]]; then
+  log_boot_status "ERROR: could not find tests/ios/build/.../testing.app"
+  popd >/dev/null || exit 1
+  exit 1
+fi
 
-# Install our app (glob so Release or Debug works)
-echo "...installing the Test app build on $SIM..."
+log_boot_status "phase=install_app bundle=\"${BUILDDIR}\""
+install_start=$SECONDS
 xcrun simctl install "$SIM" "$BUILDDIR"
+log_boot_status "install complete in $((SECONDS - install_start))s"
+popd >/dev/null || exit 1
 
-echo "Successfully booted $SIM and installed test app."
+log_boot_status "phase=complete device=\"${SIM}\" ready with test app installed"

.github/workflows/tests_e2e_ios.yml

@@ -90,7 +90,7 @@ jobs:
     runs-on: macos-26
     needs: matrix_prep
     # TODO matrix across APIs, at least 11 and 15 (lowest to highest)
-    timeout-minutes: 80
+    timeout-minutes: 87
     env:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       CCACHE_SLOPPINESS: clang_index_store,file_stat_matches,include_file_ctime,include_file_mtime,ivfsoverlay,pch_defines,modules,system_headers,time_macros
@@ -322,34 +322,33 @@ jobs:
           curl --output /dev/null --silent --head --fail "http://localhost:8081/index.bundle?platform=ios&dev=true&minify=false&inlineSourceMap=true"
           echo "...javascript bundle ready"
 
-      - name: Start Screen and Simulator Recordings and System Logging
-        # With a little delay so the detox test below has time to spawn it, missing the first part of boot is fine
+      - name: Start Screen and System Logging
         continue-on-error: true
         run: |
           nohup sh -c "sleep 314159265 | screencapture -v -C -k -T0 -g screenrecording.mov > screenrecording.log 2>&1 &"
           nohup sh -c "log stream --backtrace --color none --style syslog > syslog.log 2>&1 &"
-          nohup sh -c "sleep 110 && xcrun simctl io booted recordVideo --codec=h264 -f simulator.mp4 2>&1 &"
-
-      - name: Create Simulator Log
-        # With a little delay so the detox test below has time to spawn it, missing the first part of boot is fine
-        # If you boot the simulator separately from detox, some other race fails and detox testee never sends ready to proxy
-        continue-on-error: true
-        run: nohup sh -c "sleep 110 && xcrun simctl spawn booted log stream --level debug --style compact > simulator.log 2>&1 &"
 
       - name: Pre-Boot Simulator
-        # The goal here is to separate Simulator boot from Detox run,
-        # So that Simulator boot issues we seem to have may be handled separately
+        # Separate Simulator boot from Detox run. boot-simulator.sh polls bootstatus and logs
+        # migration progress so long first-boot waits are visible in the step log.
         # https://github.com/nick-fields/retry/releases
         uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
         with:
-          timeout_minutes: 5
+          timeout_minutes: 12
           retry_wait_seconds: 60
           max_attempts: 3
           command: ./.github/workflows/scripts/boot-simulator.sh
 
+      - name: Start Simulator Recordings and Log
+        # Start after Pre-Boot so booted exists and logging covers the Detox run (not a fixed delay).
+        continue-on-error: true
+        run: |
+          nohup sh -c "xcrun simctl io booted recordVideo --codec=h264 -f simulator.mp4 2>&1 &"
+          nohup sh -c "xcrun simctl spawn booted log stream --level debug --style compact > simulator.log 2>&1 &"
+
       - name: Detox Test Debug
         if: contains(matrix.buildmode, 'debug')
-        timeout-minutes: 55
+        timeout-minutes: 62
         run: yarn tests:ios:test-cover
 
       - name: Process iOS native coverage
@@ -359,7 +358,7 @@ jobs:
 
       - name: Detox Test Release
         if: contains(matrix.buildmode, 'release')
-        timeout-minutes: 55
+        timeout-minutes: 62
         run: yarn tests:ios:test:release
 
       - name: Stop Screen and App Video and System Logging

okf-bundle/ci-workflows/android.md

@@ -0,0 +1,3 @@
+# Android CI workflows
+
+TBD — Gradle/Detox emulator reliability, `coverage.ec` pull behavior, and artifact troubleshooting.

okf-bundle/ci-workflows/index.md

@@ -0,0 +1,13 @@
+# CI workflows
+
+Knowledge for GitHub Actions workflows in this repository: how jobs are structured, platform-specific reliability concerns, and how to debug failures from CI artifacts.
+
+## Platforms
+
+* [iOS](ios.md) — simulator boot reliability, logging, and troubleshooting
+* [Android](android.md) — TBD
+* [Other](other.md) — macOS Detox (non-iOS), Windows, and shared workflow concerns — TBD
+
+## Related
+
+* [Testing / coverage design](../testing/coverage-design.md) — e2e coverage collection that runs inside the iOS workflow

okf-bundle/ci-workflows/ios.md

@@ -0,0 +1,91 @@
+# iOS CI workflows
+
+This document covers the **Testing E2E iOS** workflow (`.github/workflows/tests_e2e_ios.yml`) and scripts it uses under `.github/workflows/scripts/`.
+
+## Simulator reliability
+
+### Problem
+
+On GitHub Actions macOS runners (currently `macos-26` with `XCODE_VERSION: latest-stable`), booting an iOS Simulator for Detox is not instantaneous. A simulator can report `Booted` in `simctl list` while it is still unusable:
+
+1. **First-boot data migration** — `com.apple.datamigrator` can run for several minutes on a fresh simulator (observed ~4+ minutes on iOS 26.5). SpringBoard and app install are not reliable until migration finishes.
+2. **Ambiguous device names** — runners often have multiple simulators with the same marketing name (e.g. several `iPhone 17` entries across iOS runtimes). We intentionally use the **device name** from `tests/.detoxrc.js`, not a pinned UDID, so we do not churn workflow YAML when runner images change.
+3. **`Booted` ≠ ready for testing** — installing or launching the app during migration can block or fail; Detox may time out while the simulator is still migrating.
+
+### What we do
+
+**Pre-boot step** (`.github/workflows/scripts/boot-simulator.sh`), run via `nick-fields/retry` before Detox:
+
+| Phase | What happens |
+|--------|----------------|
+| `resolve_device` | Read simulator name from `tests/.detoxrc.js` (e.g. `iPhone 17`) |
+| `shutdown_existing` | Kill `Simulator.app` and `simctl shutdown` the target |
+| `boot_command` | `xcrun simctl boot <name>` |
+| `wait_for_full_boot` | Poll every 20s (up to 11 min) until `simctl bootstatus` reports ready |
+| `install_app` | `simctl install` the built `testing.app` **only after** bootstatus succeeds |
+
+During `wait_for_full_boot`, the script logs to the **GitHub Actions step log** with the `[boot-status]` prefix:
+
+- Whether `simctl list` shows the device as `Booted`
+- **Data migration** snippets from `xcrun simctl bootstatus <name> -d` (probed with a short timeout so the step keeps printing progress instead of looking hung)
+- Elapsed time per poll
+
+We wait for **`simctl bootstatus`** (full boot completion), not merely the `Booted` line in `simctl list`.
+
+**Timeouts** (tuned for first-boot migration on latest iOS):
+
+| Setting | Value | Notes |
+|---------|-------|--------|
+| Pre-Boot retry step | 12 min × 3 attempts | was 5 min |
+| Job `timeout-minutes` | 87 | +7 min vs previous 80 |
+| Detox test step | 62 min | +7 min vs previous 55 |
+
+Simulator **caching** of device data is intentionally deferred — caching a bad migration state would require cache invalidation policy.
+
+### Simulator logging and video (troubleshooting)
+
+Artifacts are uploaded on every run (`if: always()`), even when tests fail.
+
+| Artifact | Source | Use when |
+|----------|--------|----------|
+| `simulator-<buildmode>-<iteration>_log` | `xcrun simctl spawn booted log stream` → `simulator.log` | In-simulator system/app logs during Detox |
+| `simulator-<buildmode>-<iteration>_video` | `xcrun simctl io booted recordVideo` → `simulator.mp4` | Visual confirmation of UI state |
+| `screenrecording-<buildmode>-<iteration>` | `screencapture` of the Mac desktop | Includes Simulator.app window |
+| `screenrecording-setup-<buildmode>-<iteration>.mov` | Guidepup setup recording | Very early environment setup |
+| `emulator-scripts-logs-<buildmode>-<iteration>` | `.github/workflows/scripts/*.log` | Script output if redirected |
+
+**When to use which log**
+
+- **Boot / migration / “simulator won’t start”** — read the **Pre-Boot Simulator** step log in GitHub Actions first. Look for `[boot-status]` lines and `bootstatus -d` migration output. That captures first-boot migration even though `simulator.log` starts only after pre-boot succeeds.
+- **Detox / app / test failures** — download `simulator-*_log` and search for `com.invertase.testing`, `SpringBoard`, `xctest`, or `Detox`.
+- **UI regressions** — `simulator-*_video` or `screenrecording-*`.
+
+**Downloading artifacts**
+
+From the workflow run page: **Artifacts** section at the bottom, or:
+
+```bash
+gh run download <run-id> -n simulator-debug-0_log
+```
+
+**Analyzing `simulator.log`**
+
+The file is unified logging from the booted simulator (compact style). Useful patterns:
+
+```bash
+rg -i "datamigrator|Telemetry: duration|systemShellWillBootstrap" simulator.log
+rg -i "com\.invertase\.testing|installcoordination" simulator.log
+rg -i "test daemon not ready|xctest" simulator.log
+```
+
+A long gap with only `com.apple.datamigrator` activity and no `com.invertase.testing` usually means the simulator was still in first-boot migration or pre-boot had not finished installing the app yet.
+
+### Detox configuration
+
+Device type is defined in `tests/.detoxrc.js` (`devices.simulator.device.type`). The boot script and Detox both use this name. CI does not hard-code a UDID.
+
+### Operational notes
+
+- **Release vs debug** — matrix runs both; each has separate artifacts (`debug` / `release` in the artifact name).
+- **Retry** — Pre-Boot retries up to 3 times with 60s between attempts (clean shutdown + boot each time).
+- **Do not boot the simulator only inside Detox** — historical races where the testee never sent “ready” to the Detox proxy; pre-boot remains mandatory.

okf-bundle/ci-workflows/other.md

@@ -0,0 +1,3 @@
+# Other CI workflows
+
+TBD — macOS Detox (`tests_e2e_other.yml`), Windows, documentation workflows, and shared actions (caches, Codecov, etc.).

okf-bundle/index.md

@@ -6,6 +6,10 @@ okf_version: "0.1"
 
 Knowledge documents for react-native-firebase development, testing, and maintenance.
 
+# CI workflows
+
+* [CI workflows](/ci-workflows/index.md) — GitHub Actions reliability, logging, and troubleshooting (iOS simulator boot documented; Android/other TBD)
+
 # Testing
 
 * [Coverage design](/testing/coverage-design.md) - unit and e2e coverage goals, pipelines, and Codecov uploads