benchmarking.yml

name: Benchmarking
on:
  push:
    branches:
      - main

  pull_request:

# Concurrency configuration:
# - We use workflow-specific concurrency groups to prevent multiple benchmark runs on the same code,
#   as benchmarks are extremely resource-intensive and require dedicated device time on SauceLabs.
# - For pull requests, we cancel in-progress runs when new commits are pushed to avoid wasting
#   expensive external testing resources and provide timely performance feedback.
# - For main branch pushes, we never cancel benchmarks to ensure we have complete performance
#   baselines for every main branch commit, which are critical for performance regression detection.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  files-changed:
    name: Detect File Changes
    runs-on: ubuntu-latest
    outputs:
      run_benchmarking_for_prs: ${{ steps.changes.outputs.run_benchmarking_for_prs }}
    steps:
      - uses: actions/checkout@v5
      - name: Get changed files
        id: changes
        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        with:
          token: ${{ github.token }}
          filters: .github/file-filters.yml

  build-benchmark-test-target:
    name: Build App and Test Runner
    # Run the job only for PRs with related changes or non-PR events.
    if: github.event_name != 'pull_request' || needs.files-changed.outputs.run_benchmarking_for_prs == 'true'
    needs: files-changed
    runs-on: macos-14
    steps:
      - uses: actions/checkout@v5
      - run: ./scripts/ci-select-xcode.sh 15.4
      - uses: ruby/setup-ruby@v1
        with:
          bundler-cache: true
      - name: Install old xcodegen
        # Install xcodegen 2.43.0 for since newer versions don't work with Xcode older than Xcode 16
        # We need to set HOMEBREW_DEVELOPER=1 to allow installing from a local formula
        run: |
          HOMEBREW_DEVELOPER=1 brew install --formula ./scripts/xcodegen.rb
          brew pin xcodegen
      - run: make init-ci-build
      - run: make xcode-ci
      - name: Install SentryCli
        run: brew install getsentry/tools/sentry-cli
      - name: Cache iOS-Swift App and dSYM build products
        id: ios-swift-cache
        uses: actions/cache@v4
        with:
          path: |
            DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM
            DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app
          key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/**') }}-${{ hashFiles('Sources/Sentry/**') }}
      - name: Cache iOS-Swift UI Test Runner App build product
        id: ios-swift-benchmark-runner-cache
        uses: actions/cache@v4
        with:
          path: |
            DerivedData/Build/Products/Debug-iphoneos/iOS-Benchmarking-Runner.app
          key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Benchmarking/**') }}
      - run: bundle exec fastlane build_ios_swift_for_tests
        env:
          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
      - run: bundle exec fastlane build_ios_benchmark_test
        env:
          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
      - name: Upload dSYMs
        run: |
          sentry-cli --auth-token ${{ secrets.SENTRY_AUTH_TOKEN }} upload-dif --org sentry-sdks --project sentry-cocoa DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM
      - name: Archiving DerivedData
        uses: actions/upload-artifact@v4
        with:
          name: DerivedData-Xcode
          path: |
            **/Debug-iphoneos/iOS-Swift.app
            **/Debug-iphoneos/iOS-Benchmarking-Runner.app
      - name: Run CI Diagnostics
        if: failure()
        run: ./scripts/ci-diagnostics.sh

  run-ui-tests-with-sauce:
    name: Run Benchmarks ${{matrix.suite}}
    # Run the job only for PRs with related changes or non-PR events.
    if: github.event_name != 'pull_request' || needs.files-changed.outputs.run_benchmarking_for_prs == 'true'
    runs-on: ubuntu-latest
    needs:
      [
        files-changed,
        build-benchmark-test-target,
      ]
    strategy:
      fail-fast: false
      matrix:
        suite: ["High-end device", "Mid-range device", "Low-end device"]
    steps:
      - uses: actions/checkout@v5
      - uses: actions/download-artifact@v5
        with:
          name: DerivedData-Xcode
      - run: npm install -g saucectl@0.186.0
      - name: Run Benchmarks in SauceLab
        id: run-benchmarks-in-sauce-lab
        env:
          SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
          SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
        # Note: We are not setting continue-on-error here, because we want the step to be marked as failed.
        run: |
          set -o pipefail && saucectl run \
            --select-suite "${{matrix.suite}}" \
            --config .sauce/benchmarking-config.yml \
            --tags benchmark \
            --verbose \
            2>&1 | tee output.log

      - name: Recovery - Extract Test ID from output
        id: should-retry-test
        # Note: We need to use always() here, because the previous run step might be marked as failed.
        if: ${{ always() && steps.run-benchmarks-in-sauce-lab.outcome == 'failure' }}
        uses: actions/github-script@v8
        env:
          SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
          SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
        with:
          # Retry Logic: This step only runs when the benchmark test has failed.
          # We determine if the failure was due to SauceLabs infrastructure issues
          # (which can be fixed by retrying) or legitimate test failures (which cannot).
          # SauceLabs internal errors frequently cause CI failures that can be resolved
          # by re-running the test. See scripts/saucelabs-helpers.js for detailed logic.
          script: |
            const fs = require('fs');
            const { execSync } = require('child_process');

            console.log("Extracting test ID from output log");
            const outputLog = fs.readFileSync('output.log', 'utf8');

            // Lookup for the test ID in the output log
            // Note: The CLI output might change over time, so this might need to be updated.
            const match = outputLog.match(/https:\/\/app\.saucelabs\.com\/tests\/([^\s]+)/);
            const testId = match?.[1] ?? '';

            if (!testId) {
              core.warning("No SauceLabs test ID found in CLI output, it might have changed, retrying...");
              core.setOutput('RETRY_TEST', 'true');

              return;
            }

            try {
              console.log(`Checking if the test exists in SauceLabs: ${testId}`);
              execSync(`saucectl jobs get ${testId}`, {
                env: process.env,
                stdio: 'inherit'
              });

              console.log("Test exists but failed, not retrying.");
              core.setFailed('Test exists but failed');
            } catch (error) {
              console.log("Failed to get job, retrying...");
              core.setOutput('RETRY_TEST', 'true');
            }

      - name: Run Benchmarks in SauceLab - Retry 1
        id: run-benchmarks-in-sauce-lab-retry-1
        # Note: We need to use always() here, because the previous run step might be marked as failed.
        if: ${{ always() && steps.should-retry-test.outputs.RETRY_TEST == 'true' }}
        env:
          SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
          SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
        run: |
          echo "::warning SauceLabs benchmark tests need to be retried"
          saucectl run \
            --select-suite "${{matrix.suite}}" \
            --config .sauce/benchmarking-config.yml \
            --tags benchmark \
            --verbose

      - name: Run CI Diagnostics
        if: failure()
        run: ./scripts/ci-diagnostics.sh

  benchmarking-required-check:
    needs:
      [
        files-changed,
        build-benchmark-test-target,
        run-ui-tests-with-sauce,
      ]
    name: Benchmarking
    # This is necessary since a failed/skipped dependent job would cause this job to be skipped
    if: always()
    runs-on: ubuntu-latest
    steps:
      # If any jobs we depend on fails gets cancelled or times out, this job will fail.
      # Skipped jobs are not considered failures.
      - name: Check for failures
        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
        run: |
          echo "One of the benchmark jobs has failed." && exit 1