feat(tests): standardize test suites on user-layer API

Replace internal script sourcing and direct xcrun/adb calls in test
suites with user-layer commands (ios.sh, android.sh). This ensures
tests exercise the same commands users run.

New commands added:
- ios.sh deploy, ios.sh app status/stop, ios.sh simulator ready
- android.sh deploy, android.sh app status/stop, android.sh emulator ready

Suite namespacing via SUITE_NAME env var enables concurrent test
execution with isolated runtime state files.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: abueide

.github/workflows/e2e-full.yml

@@ -1,4 +1,4 @@
-name: Full E2E Tests
+tname: Full E2E Tests
 
 on:
   workflow_dispatch:

examples/android/devbox.d/android/devices/devices.lock

@@ -14,5 +14,5 @@
     }
   ],
   "checksum": "8df4d3393b61fbbb08e45cf8762f95c521316938e514527916e4fce88a849d57",
-  "generated_at": "2026-02-19T00:51:12Z"
+  "generated_at": "2026-02-19T04:09:58Z"
 }

examples/android/tests/test-suite.yaml

@@ -2,6 +2,7 @@ version: "0.5"
 
 environment:
   - "EMU_HEADLESS=${EMU_HEADLESS:-}"
+  - "SUITE_NAME=android-e2e"
   - "TEST_TIMEOUT=300"
   - "BOOT_TIMEOUT=90"
   - "TEST_TUI=${TEST_TUI:-false}"
@@ -14,130 +15,62 @@ is_strict: true
 processes:
   # Phase 1: Build - runs first (no dependency)
   build-app:
-    command: |
-      echo '[BUILD-APP] Starting build process (PID=$$)...'
-      # Source Android environment to set ANDROID_SDK_ROOT
-      . ${ANDROID_RUNTIME_DIR}/scripts/init/setup.sh
-      echo '📦 Building Android app...'
-      gradle assembleDebug --info
-      echo '✓ Build complete'
+    command: "android.sh build"
     availability:
       restart: "no"
 
   # Phase 2a: Sync AVDs - ensure all AVDs match device definitions
   sync-avds:
-    command: "echo '[SYNC-AVDS] Starting (PID=$$)...' && echo '[SYNC-AVDS] About to source setup.sh...' && . ${ANDROID_RUNTIME_DIR}/scripts/init/setup.sh && echo '[SYNC-AVDS] setup.sh sourced' && echo '🔄 Syncing AVDs with device definitions...' && android.sh devices sync && echo '✓ Sync complete'"
+    command: "android.sh devices sync"
     availability:
       restart: "no"
 
   # Phase 2b: Start emulator (depends on sync completing)
-  # Reuses existing emulator if already running
-  # Max boot time: 5 minutes, 1 retry allowed, total timeout: 10 minutes
   android-emulator:
     depends_on:
       sync-avds:
         condition: process_completed_successfully
     environment:
-      - "ANDROID_EMULATOR_FOREGROUND=1"  # Run emulator in foreground for process-compose monitoring
+      - "ANDROID_EMULATOR_FOREGROUND=1"
     command: |
-      echo "[ANDROID-EMULATOR] Starting emulator in foreground mode..."
-      # Source Android environment
-      . ${ANDROID_RUNTIME_DIR}/scripts/init/setup.sh
-
-      # Determine which device to start
+      echo "[ANDROID-EMULATOR] Starting emulator..."
       device="${ANDROID_DEFAULT_DEVICE:-max}"
 
-      # In pure mode, always start fresh
-      if [ "${IN_NIX_SHELL:-}" = "pure" ]; then
-        echo "🚀 Starting fresh Android emulator (pure mode): $device"
-        export ANDROID_EMULATOR_PURE=1
-
-        # Write expected serial for readiness probe (emulator always starts on 5554 first)
-        echo "emulator-5554" > ${ANDROID_RUNTIME_DIR}/emulator-serial.txt
-
-        # Start emulator in foreground - this blocks and runs the emulator
-        # Process-compose will monitor it and use readiness probe to check when ready
+      if [ "${IN_NIX_SHELL:-}" = "pure" ] || [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
+        echo "Starting fresh Android emulator (pure mode): $device"
         android.sh emulator start "$device"
       else
-        echo "🚀 Verifying Android emulator is running: $device"
-
-        # Write expected serial for readiness probe
-        serial="emulator-5554"
-        echo "$serial" > ${ANDROID_RUNTIME_DIR}/emulator-serial.txt
-
-        # Check if emulator is already running
-        if adb devices | grep -q "$serial"; then
-          echo "✓ Emulator already running: $serial"
-        else
-          echo "Starting emulator in background: $device"
-          # Start emulator in background (no foreground mode for reuse)
-          unset ANDROID_EMULATOR_FOREGROUND
-          android.sh emulator start "$device" &
-          echo "✓ Emulator starting (PID: $!)"
-        fi
-
-        # Wait for emulator to be fully booted
-        echo "Waiting for emulator to boot..."
-        max_attempts=60
-        attempt=0
-        while [ $attempt -lt $max_attempts ]; do
-          if adb -s $serial shell getprop sys.boot_completed 2>/dev/null | grep -q "1"; then
-            echo "✓ Emulator ready for testing"
-            break
-          fi
-          attempt=$((attempt + 1))
-          sleep 2
-        done
-
-        if [ $attempt -ge $max_attempts ]; then
-          echo "ERROR: Emulator did not boot within timeout"
-          exit 1
-        fi
+        echo "Starting Android emulator: $device"
+        # In non-pure mode, don't run foreground - start in background and wait
+        unset ANDROID_EMULATOR_FOREGROUND
+        android.sh emulator start "$device"
 
-        # Let process complete - emulator stays running in background
-        echo "✓ Emulator will remain running after tests complete"
+        echo "Emulator will remain running after tests complete"
         exit 0
       fi
     availability:
       restart: "no"
     readiness_probe:
       exec:
-        command: |
-          serial=$(cat ${ANDROID_RUNTIME_DIR}/emulator-serial.txt 2>/dev/null || echo "")
-          [ -n "$serial" ] && adb -s $serial shell getprop sys.boot_completed 2>/dev/null | grep -q "1"
+        command: "android.sh emulator ready"
       initial_delay_seconds: 10
       period_seconds: 10
       timeout_seconds: 180
       success_threshold: 1
       failure_threshold: 12
 
-  # Phase 3: Run app - depends on both build and emulator being ready
-  run-app:
+  # Phase 3: Deploy app - depends on both build and emulator being ready
+  deploy-app:
     command: |
-      # Source Android environment
-      . ${ANDROID_RUNTIME_DIR}/scripts/init/setup.sh
-
-      serial=$(cat ${ANDROID_RUNTIME_DIR}/emulator-serial.txt 2>/dev/null || echo emulator-5554)
-      export ANDROID_EMULATOR_SERIAL="$serial"
-
-      echo "📲 Running app on emulator: $serial"
-      # Use already-built APK from build-app phase
-      android.sh run ${ANDROID_APP_APK:-app/build/outputs/apk/debug/app-debug.apk}
+      android.sh deploy
 
       echo ""
-      # Resolve app ID: env var > auto-detected from APK > fail
-      app_id="${ANDROID_APP_ID:-$(cat ${ANDROID_RUNTIME_DIR}/app-id.txt 2>/dev/null || true)}"
-      if [ -z "$app_id" ]; then
-        echo "ERROR: Cannot determine app ID. Set ANDROID_APP_ID or run deploy first." >&2
-        exit 1
-      fi
-
-      echo "Waiting for app to start ($app_id)..."
+      echo "Waiting for app to start..."
       max_attempts=10
       attempt=0
       while [ $attempt -lt $max_attempts ]; do
-        if adb -s $serial shell pidof "$app_id" >/dev/null 2>&1; then
-          echo "✓ App process running (PID: $(adb -s $serial shell pidof "$app_id"))"
+        if android.sh app status; then
+          echo "App is running"
           mkdir -p reports
           touch reports/.e2e-passed
           exit 0
@@ -148,10 +81,6 @@ processes:
       done
 
       echo "ERROR: App did not start within timeout" >&2
-      echo "Installed packages:" >&2
-      adb -s $serial shell pm list packages | grep -i devbox || true
-      echo "Running processes:" >&2
-      adb -s $serial shell ps | grep -i devbox || true
       exit 1
     depends_on:
       build-app:
@@ -164,24 +93,16 @@ processes:
   # Cleanup - stop everything in pure mode, keep running in dev mode
   cleanup-app:
     command: |
-      # Source Android environment
-      . ${ANDROID_RUNTIME_DIR}/scripts/init/setup.sh
-
-      serial=$(cat ${ANDROID_RUNTIME_DIR}/emulator-serial.txt 2>/dev/null || echo emulator-5554)
-
-      # In pure mode (CI), stop app and emulator for reproducibility
-      if [ "${IN_NIX_SHELL:-}" = "pure" ]; then
-        echo "🧹 Cleaning up (pure mode): stopping app and emulator..."
-        app_id="${ANDROID_APP_ID:-$(cat ${ANDROID_RUNTIME_DIR}/app-id.txt 2>/dev/null || true)}"
-        [ -n "$app_id" ] && adb -s $serial shell am force-stop "$app_id" 2>/dev/null || true
+      if [ "${IN_NIX_SHELL:-}" = "pure" ] || [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
+        echo "Cleaning up (pure mode)..."
+        android.sh app stop || true
         android.sh emulator stop || true
-        echo "✓ App stopped, emulator stopped"
+        echo "App stopped, emulator stopped"
       else
-        # In dev mode, keep app and emulator running for interaction
-        echo "✓ Test complete - app and emulator still running"
+        echo "Test complete - app and emulator still running"
       fi
     depends_on:
-      run-app:
+      deploy-app:
         condition: process_completed
     availability:
       restart: "no"

examples/ios/tests/test-suite.yaml

@@ -2,6 +2,7 @@ version: "0.5"
 
 environment:
   - "IOS_DEVICE=${IOS_DEFAULT_DEVICE:-max}"
+  - "SUITE_NAME=ios-e2e"
   - "TEST_TIMEOUT=300"
   - "BOOT_TIMEOUT=120"
 
@@ -25,61 +26,20 @@ processes:
   # Phase 3: Start simulator - depends on sync completing
   ios-simulator:
     command: |
-      # Ensure reports/logs directory exists
-      mkdir -p reports/logs
+      ios.sh simulator start ${IOS_DEVICE}
 
-      # In pure mode (IN_NIX_SHELL=pure), start fresh simulator with clean state
-      # Otherwise, reuse existing simulator if available
-      if [ "${IN_NIX_SHELL:-}" = "pure" ]; then
-        echo "🚀 Starting fresh iOS simulator with clean state (${IOS_DEVICE})..."
-        output=$(ios.sh simulator start --pure ${IOS_DEVICE} 2>&1)
-        echo "$output"
-
-        # Extract and save test simulator UDID for cleanup
-        test_udid=$(echo "$output" | grep -oE '[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}' | tail -1)
-        if [ -n "$test_udid" ]; then
-          echo "$test_udid" > reports/logs/ios-test-sim-udid.txt
-          echo "✓ Test simulator UDID saved: $test_udid"
-        fi
-      else
-        echo "🚀 Starting iOS simulator (${IOS_DEVICE})..."
-        ios.sh simulator start ${IOS_DEVICE}
-
-        # Wait for simulator to be fully booted
-        echo "Waiting for simulator to boot..."
-        DEVICE_UDID=$(xcrun simctl list devices | grep "Booted" | grep -oE '[0-9A-F-]{36}' | head -1 || echo "")
-        if [ -z "$DEVICE_UDID" ]; then
-          echo "ERROR: No booted simulator found"
-          exit 1
-        fi
-        xcrun simctl bootstatus "$DEVICE_UDID" >/dev/null 2>&1 || true
-
-        echo "✓ Simulator ready for testing"
-        echo "✓ Simulator will remain running after tests complete"
-
-        # In reuse mode, let process complete - simulator stays running in background
-        exit 0
+      # In pure mode, keep process alive so cleanup can shut down test simulator
+      if [ "${IN_NIX_SHELL:-}" = "pure" ] || [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
+        tail -f /dev/null
       fi
-
-      # Pure mode: Keep process alive so cleanup can shut down test simulator
-      tail -f /dev/null
     depends_on:
       sync-simulators:
         condition: process_completed_successfully
     availability:
       restart: "no"
     readiness_probe:
       exec:
-        command: |
-          # Find booted simulator (either test simulator or regular)
-          if [ "${IN_NIX_SHELL:-}" = "pure" ]; then
-            # In pure mode, look for test simulator
-            DEVICE_UDID=$(xcrun simctl list devices | grep "Test" | grep "Booted" | grep -oE '[0-9A-F-]{36}' | head -1 || echo "")
-          else
-            # In normal mode, look for regular simulator by device selection
-            DEVICE_UDID=$(xcrun simctl list devices | grep "Booted" | grep -oE '[0-9A-F-]{36}' | head -1 || echo "")
-          fi
-          [ -n "$DEVICE_UDID" ] && xcrun simctl bootstatus "$DEVICE_UDID" >/dev/null 2>&1
+        command: "ios.sh simulator ready"
       initial_delay_seconds: 10
       period_seconds: 5
       timeout_seconds: 120
@@ -89,31 +49,15 @@ processes:
   # Phase 4: Deploy app - depends on both build and simulator
   deploy-app:
     command: |
-      # Find the booted simulator (test or regular)
-      if [ "${IN_NIX_SHELL:-}" = "pure" ]; then
-        DEVICE_UDID=$(xcrun simctl list devices | grep "Test" | grep "Booted" | grep -oE '[0-9A-F-]{36}' | head -1)
-      else
-        DEVICE_UDID=$(xcrun simctl list devices | grep "Booted" | grep -oE '[0-9A-F-]{36}' | head -1)
-      fi
-
-      if [ -z "$DEVICE_UDID" ]; then
-        echo "ERROR: No booted simulator found"
-        exit 1
-      fi
-
-      echo "📲 Installing app on simulator: $DEVICE_UDID"
-      xcrun simctl install "$DEVICE_UDID" "${IOS_APP_ARTIFACT}"
-
-      echo "🚀 Launching app: ${IOS_APP_BUNDLE_ID}"
-      xcrun simctl launch "$DEVICE_UDID" "${IOS_APP_BUNDLE_ID}"
+      ios.sh deploy
 
       echo ""
       echo "Waiting for app to start..."
       max_attempts=10
       attempt=0
       while [ $attempt -lt $max_attempts ]; do
-        if xcrun simctl spawn "$DEVICE_UDID" launchctl list | grep -q "${IOS_APP_BUNDLE_ID}"; then
-          echo "✓ App is running"
+        if ios.sh app status; then
+          echo "App is running"
           exit 0
         fi
         attempt=$((attempt + 1))
@@ -122,8 +66,6 @@ processes:
       done
 
       echo "ERROR: App did not start within timeout" >&2
-      echo "Checking app status:" >&2
-      xcrun simctl get_app_container "$DEVICE_UDID" "${IOS_APP_BUNDLE_ID}" 2>&1 || true
       exit 1
     depends_on:
       build-app:
@@ -136,16 +78,8 @@ processes:
   # Phase 5: Verify app is running (liveness check)
   verify-app-running:
     command: |
-      # Find the booted simulator
-      if [ "${IN_NIX_SHELL:-}" = "pure" ]; then
-        DEVICE_UDID=$(xcrun simctl list devices | grep "Test" | grep "Booted" | grep -oE '[0-9A-F-]{36}' | head -1)
-      else
-        DEVICE_UDID=$(xcrun simctl list devices | grep "Booted" | grep -oE '[0-9A-F-]{36}' | head -1)
-      fi
-
-      echo "Verifying app is running on simulator: $DEVICE_UDID"
-      if xcrun simctl spawn "$DEVICE_UDID" launchctl list | grep -q "${IOS_APP_BUNDLE_ID}"; then
-        echo "✓ App is running: ${IOS_APP_BUNDLE_ID}"
+      if ios.sh app status; then
+        echo "App is running"
         mkdir -p reports
         touch reports/.e2e-passed
         exit 0
@@ -162,36 +96,13 @@ processes:
   # Cleanup - stop everything in pure mode, keep running in dev mode
   cleanup:
     command: |
-      # In pure mode (CI), stop app and delete test simulator for reproducibility
-      if [ "${IN_NIX_SHELL:-}" = "pure" ]; then
-        echo "🧹 Cleaning up (pure mode): stopping app and deleting test simulator..."
-
-        # Get test simulator UDID
-        test_sim_udid=$(cat reports/logs/ios-test-sim-udid.txt 2>/dev/null || echo "")
-
-        if [ -z "$test_sim_udid" ]; then
-          # Fallback: find test simulator by name
-          test_sim_udid=$(xcrun simctl list devices | grep "Test" | grep -oE '[0-9A-F-]{36}' | head -1 || true)
-        fi
-
-        if [ -n "$test_sim_udid" ]; then
-          # Stop the app first
-          xcrun simctl terminate "$test_sim_udid" "${IOS_APP_BUNDLE_ID}" 2>/dev/null || true
-
-          # Shutdown and delete the simulator
-          xcrun simctl shutdown "$test_sim_udid" >/dev/null 2>&1 || true
-          xcrun simctl delete "$test_sim_udid" >/dev/null 2>&1 || true
-
-          # Clean up temp file
-          rm -f reports/logs/ios-test-sim-udid.txt
-
-          echo "✓ App stopped, test simulator deleted"
-        else
-          echo "✓ No test simulator to clean up"
-        fi
+      if [ "${IN_NIX_SHELL:-}" = "pure" ] || [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
+        echo "Cleaning up (pure mode)..."
+        ios.sh app stop || true
+        ios.sh simulator stop
+        echo "App stopped, test simulator deleted"
       else
-        # In dev mode, keep app and simulator running for interaction
-        echo "✓ Test complete - app and simulator still running"
+        echo "Test complete - app and simulator still running"
       fi
     depends_on:
       verify-app-running: