fix(ci): Fix E2E test flakiness with stable checks instead of retries

Replace retry-based approach with deterministic fixes:
- Per-flow process isolation in cli.mjs (no retries, crash.yml runs last)
- Proper simulator boot wait using xcrun simctl bootstatus + Settings.app warm-up
- Search all envelopes for app start transaction (handles slow VM delivery)
- Sort envelopes by timestamp for consistent ordering
- Allow-list for TTID/TTFD ops (navigation, ui.load) instead of fragile deny-list
- Bump MAESTRO_DRIVER_STARTUP_TIMEOUT to 180s for Cirrus Labs Tart VMs
- Use wait_for_boot + erase_before_boot on simulator-action

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: antonis

.github/workflows/e2e-v2.yml

@@ -508,12 +508,27 @@ jobs:
         with:
           model: ${{ env.IOS_DEVICE }}
           os_version: ${{ env.IOS_VERSION }}
+          wait_for_boot: true
+          erase_before_boot: false
+
+      - name: Wait for iOS simulator to be fully ready
+        if: ${{ steps.platform-check.outputs.skip != 'true' && matrix.platform == 'ios' }}
+        run: |
+          # Wait for boot to complete at the system level
+          xcrun simctl bootstatus booted -b
+          # Launch and dismiss Settings.app to ensure SpringBoard and system services
+          # are fully initialized — this avoids Maestro connecting to a half-booted
+          # simulator on Cirrus Labs Tart VMs.
+          xcrun simctl launch booted com.apple.Preferences
+          sleep 5
+          xcrun simctl terminate booted com.apple.Preferences
 
       - name: Run tests on iOS
         if: ${{ steps.platform-check.outputs.skip != 'true' && matrix.platform == 'ios' }}
         env:
-          # Increase timeout for Maestro iOS driver startup (default is 60s, some CI runners need more time)
-          MAESTRO_DRIVER_STARTUP_TIMEOUT: 120000
+          # Increase timeout for Maestro iOS driver startup from default 60s to 180s.
+          # Cirrus Labs Tart VMs can be slow to respond after simulator boot.
+          MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000
         run: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test
 
       - name: Upload logs

.github/workflows/sample-application.yml

@@ -14,7 +14,7 @@ concurrency:
 env:
   SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
   MAESTRO_VERSION: '2.3.0'
-  MAESTRO_DRIVER_STARTUP_TIMEOUT: 90000 # Increase timeout from default 30s to 90s for CI stability
+  MAESTRO_DRIVER_STARTUP_TIMEOUT: 180000 # Increase timeout from default 30s to 180s for CI stability on Cirrus Labs Tart VMs
   RN_SENTRY_POD_NAME: RNSentry
   IOS_APP_ARCHIVE_PATH: sentry-react-native-sample.app.zip
   ANDROID_APP_ARCHIVE_PATH: sentry-react-native-sample.apk.zip
@@ -332,6 +332,16 @@ jobs:
         with:
           model: ${{ env.IOS_DEVICE }}
           os_version: ${{ env.IOS_VERSION }}
+          wait_for_boot: true
+          erase_before_boot: false
+
+      - name: Wait for iOS simulator to be fully ready
+        if: ${{ steps.platform-check.outputs.skip != 'true' && matrix.platform == 'ios' }}
+        run: |
+          xcrun simctl bootstatus booted -b
+          xcrun simctl launch booted com.apple.Preferences
+          sleep 5
+          xcrun simctl terminate booted com.apple.Preferences
 
       - name: Run iOS Tests
         if: ${{ steps.platform-check.outputs.skip != 'true' && matrix.platform == 'ios' }}

dev-packages/e2e-tests/cli.mjs

@@ -290,35 +290,73 @@ if (actions.includes('test')) {
   if (!sentryAuthToken) {
     console.log('Skipping maestro test due to unavailable or empty SENTRY_AUTH_TOKEN');
   } else {
-    try {
-      execSync(
-        `maestro test maestro \
-          --env=APP_ID="${appId}" \
-          --env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \
-          --debug-output maestro-logs \
-          --flatten-debug-output`,
-        {
-          stdio: 'inherit',
-          cwd: e2eDir,
-        },
-      );
-    } finally {
-      // Always redact sensitive data, even if the test fails
-      const redactScript = `
-        if [[ "$(uname)" == "Darwin" ]]; then
-          find ./maestro-logs -type f -exec sed -i '' "s/${sentryAuthToken}/[REDACTED]/g" {} +
-          echo 'Redacted sensitive data from logs on MacOS'
-        else
-          find ./maestro-logs -type f -exec sed -i "s/${sentryAuthToken}/[REDACTED]/g" {} +
-          echo 'Redacted sensitive data from logs on Ubuntu'
-        fi
-      `;
+    const maestroDir = path.join(e2eDir, 'maestro');
+    const flowFiles = fs.readdirSync(maestroDir)
+      .filter(f => f.endsWith('.yml') && !f.startsWith('utils'))
+      .sort((a, b) => {
+        // Run crash.yml last — it kills the app, and running it last avoids
+        // any potential simulator instability affecting subsequent flows.
+        if (a === 'crash.yml') return 1;
+        if (b === 'crash.yml') return -1;
+        return a.localeCompare(b);
+      });
 
+    console.log(`Discovered ${flowFiles.length} Maestro flows: ${flowFiles.join(', ')}`);
+
+    const failedFlows = [];
+
+    // Run each flow in its own process to prevent crash cascade —
+    // when crash.yml kills the app, a shared Maestro session would fail
+    // all subsequent flows.
+    for (const flow of flowFiles) {
+      const flowPath = path.join('maestro', flow);
+      console.log(`\n--- Running flow: ${flow} ---`);
       try {
-        execSync(redactScript, { stdio: 'inherit', cwd: e2eDir, shell: '/bin/bash' });
+        execFileSync('maestro', [
+          'test',
+          flowPath,
+          '--env', `APP_ID=${appId}`,
+          '--env', `SENTRY_AUTH_TOKEN=${sentryAuthToken}`,
+          '--debug-output', 'maestro-logs',
+          '--flatten-debug-output',
+        ], {
+          stdio: 'inherit',
+          cwd: e2eDir,
+        });
+        console.log(`--- Flow ${flow}: PASSED ---`);
       } catch (error) {
-        console.warn('Failed to redact sensitive data from logs:', error.message);
+        console.error(`--- Flow ${flow}: FAILED ---`);
+        failedFlows.push(flow);
       }
     }
+
+    // Always redact sensitive data, even if some tests failed
+    try {
+      const logDir = path.join(e2eDir, 'maestro-logs');
+      if (fs.existsSync(logDir)) {
+        const redactFiles = (dir) => {
+          for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+              redactFiles(fullPath);
+            } else {
+              const content = fs.readFileSync(fullPath, 'utf8');
+              if (content.includes(sentryAuthToken)) {
+                fs.writeFileSync(fullPath, content.replaceAll(sentryAuthToken, '[REDACTED]'));
+              }
+            }
+          }
+        };
+        redactFiles(logDir);
+        console.log('Redacted sensitive data from logs');
+      }
+    } catch (error) {
+      console.warn('Failed to redact sensitive data from logs:', error.message);
+    }
+
+    if (failedFlows.length > 0) {
+      console.error(`\nFailed flows: ${failedFlows.join(', ')}`);
+      process.exit(1);
+    }
   }
 }

dev-packages/e2e-tests/maestro/crash.yml

@@ -4,6 +4,6 @@ jsEngine: graaljs
 - runFlow: utils/launchTestAppClear.yml
 - tapOn: "Crash"
 
-- launchApp
-
-- runFlow: utils/assertTestReady.yml
+# Verify the app can recover from a native crash by fully resetting
+# and relaunching (killApp + launchApp clearState + assertTestReady).
+- runFlow: utils/launchTestAppClear.yml

samples/react-native/e2e/tests/captureErrorScreenTransaction/captureErrorsScreenTransaction.test.ts

@@ -31,15 +31,23 @@ describe('Capture Errors Screen Transaction', () => {
   });
 
   it('envelope contains transaction context', async () => {
-    const envelope = getErrorsEnvelope();
-
-    const items = envelope[1];
-    const transactions = items.filter(([header]) => header.type === 'transaction');
-    const appStartTransaction = transactions.find(([_header, payload]) => {
-      const event = payload as any;
-      return event.transaction === 'ErrorsScreen' &&
-            event.contexts?.trace?.origin === 'auto.app.start';
-    });
+    // The app start transaction may arrive in a separate envelope on slow CI VMs,
+    // so search all matching envelopes instead of just the first one.
+    const allEnvelopes = sentryServer.getAllEnvelopes(
+      containingTransactionWithName('ErrorsScreen'),
+    );
+
+    let appStartTransaction: EventItem | undefined;
+    for (const envelope of allEnvelopes) {
+      const items = envelope[1];
+      const transactions = items.filter(([header]) => header.type === 'transaction') as EventItem[];
+      appStartTransaction = transactions.find(([_header, payload]) => {
+        const event = payload as any;
+        return event.transaction === 'ErrorsScreen' &&
+              event.contexts?.trace?.origin === 'auto.app.start';
+      });
+      if (appStartTransaction) break;
+    }
 
     expect(appStartTransaction).toBeDefined();
 

samples/react-native/e2e/tests/captureSpaceflightNewsScreenTransaction/captureSpaceflightNewsScreenTransaction.test.ts

@@ -42,6 +42,13 @@ describe('Capture Spaceflight News Screen Transaction', () => {
     await waitForSpaceflightNewsTx;
 
     newsEnvelopes = sentryServer.getAllEnvelopes(containingNewsScreen);
+    // Sort by transaction timestamp — envelope delivery order may vary on slow CI VMs,
+    // but test assertions depend on chronological order.
+    newsEnvelopes.sort((a, b) => {
+      const aItem = getItemOfTypeFrom<EventItem>(a, 'transaction');
+      const bItem = getItemOfTypeFrom<EventItem>(b, 'transaction');
+      return (aItem?.[1]?.timestamp ?? 0) - (bItem?.[1]?.timestamp ?? 0);
+    });
     allTransactionEnvelopes = sentryServer.getAllEnvelopes(
       containingTransaction,
     );
@@ -64,9 +71,11 @@ describe('Capture Spaceflight News Screen Transaction', () => {
     allTransactionEnvelopes
       .filter(envelope => {
         const item = getItemOfTypeFrom<EventItem>(envelope, 'transaction');
-        // Only check navigation transactions, not user interaction transactions
-        // User interaction transactions (ui.action.touch) don't have time-to-display measurements
-        return item?.[1]?.contexts?.trace?.op !== 'ui.action.touch';
+        // Only navigation and app start transactions have time-to-display measurements.
+        // Filter with an allow-list — other ops like 'ui.action.touch' or
+        // 'navigation.processing' do not include TTID/TTFD.
+        const op = item?.[1]?.contexts?.trace?.op;
+        return op === 'navigation' || op === 'ui.load';
       })
       .forEach(envelope => {
         expectToContainTimeToDisplayMeasurements(