pr-tests.yml

name: PR Test Comparison

on:
  pull_request:
    branches: [main]

# Cancel in-progress runs for the same PR
concurrency:
  group: pr-tests-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  test-main:
    name: Test (main)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout main branch
        uses: actions/checkout@v4
        with:
          ref: main
          submodules: true

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "20"
          cache: "npm"

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install uv (Python package manager)
        uses: astral-sh/setup-uv@v5

      - name: Get latest Sentry SDK versions
        id: sentry-versions
        run: |
          echo "js=$(npm view @sentry/node version)" >> $GITHUB_OUTPUT
          echo "python=$(curl -s https://pypi.org/pypi/sentry-sdk/json | python3 -c 'import sys,json; print(json.load(sys.stdin)["info"]["version"])')" >> $GITHUB_OUTPUT
          echo "php=$(curl -s https://repo.packagist.org/p2/sentry/sentry-laravel.json | python3 -c 'import sys,json; print(json.load(sys.stdin)["packages"]["sentry/sentry-laravel"][0]["version"])')" >> $GITHUB_OUTPUT

      - name: Cache Node test environments
        uses: actions/cache@v4
        with:
          path: runs/node
          key: node-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/node/**/config.json') }}
          restore-keys: |
            node-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-

      - name: Cache Browser test environments
        uses: actions/cache@v4
        with:
          path: runs/browser
          key: browser-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/browser/**/config.json') }}
          restore-keys: |
            browser-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-

      - name: Cache Next.js test environments
        uses: actions/cache@v4
        with:
          path: runs/nextjs
          key: nextjs-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/nextjs/**/config.json') }}
          restore-keys: |
            nextjs-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-

      - name: Cache Python test environments
        uses: actions/cache@v4
        with:
          path: runs/python
          key: python-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.python }}-${{ hashFiles('src/runner/templates/**/python/**/config.json') }}
          restore-keys: |
            python-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.python }}-

      - name: Cache PHP test environments
        uses: actions/cache@v4
        with:
          path: runs/php
          key: php-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.php }}-${{ hashFiles('src/runner/templates/**/php/**/config.json') }}
          restore-keys: |
            php-envs-main-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.php }}-

      - name: Setup PHP
        uses: shivammathur/setup-php@v2
        with:
          php-version: "8.4"
          tools: composer

      - name: Install dependencies and build
        run: |
          npm ci
          npm run build

      - name: Install Playwright browsers (for browser platform only)
        run: npx playwright install --with-deps chromium

      - name: Run tests
        id: run-tests
        continue-on-error: true
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GOOGLE_GENAI_API_KEY: ${{ secrets.GOOGLE_GENAI_API_KEY }}
          CO_API_KEY: ${{ secrets.CO_API_KEY }}
        run: npm run test -- -j=12

      - name: Upload CTRF report
        uses: actions/upload-artifact@v4
        with:
          name: ctrf-main
          path: test-results/ctrf-report-*.json
          retention-days: 1

  test-pr:
    name: Test (PR)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout PR branch
        uses: actions/checkout@v4
        with:
          submodules: true

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "20"
          cache: "npm"

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install uv (Python package manager)
        uses: astral-sh/setup-uv@v5

      - name: Get latest Sentry SDK versions
        id: sentry-versions
        run: |
          echo "js=$(npm view @sentry/node version)" >> $GITHUB_OUTPUT
          echo "python=$(curl -s https://pypi.org/pypi/sentry-sdk/json | python3 -c 'import sys,json; print(json.load(sys.stdin)["info"]["version"])')" >> $GITHUB_OUTPUT
          echo "php=$(curl -s https://repo.packagist.org/p2/sentry/sentry-laravel.json | python3 -c 'import sys,json; print(json.load(sys.stdin)["packages"]["sentry/sentry-laravel"][0]["version"])')" >> $GITHUB_OUTPUT

      - name: Cache Node test environments
        uses: actions/cache@v4
        with:
          path: runs/node
          key: node-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/node/**/config.json') }}
          restore-keys: |
            node-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-

      - name: Cache Browser test environments
        uses: actions/cache@v4
        with:
          path: runs/browser
          key: browser-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/browser/**/config.json') }}
          restore-keys: |
            browser-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-

      - name: Cache Next.js test environments
        uses: actions/cache@v4
        with:
          path: runs/nextjs
          key: nextjs-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-${{ hashFiles('src/runner/templates/**/nextjs/**/config.json') }}
          restore-keys: |
            nextjs-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.js }}-

      - name: Cache Python test environments
        uses: actions/cache@v4
        with:
          path: runs/python
          key: python-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.python }}-${{ hashFiles('src/runner/templates/**/python/**/config.json') }}
          restore-keys: |
            python-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.python }}-

      - name: Cache PHP test environments
        uses: actions/cache@v4
        with:
          path: runs/php
          key: php-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.php }}-${{ hashFiles('src/runner/templates/**/php/**/config.json') }}
          restore-keys: |
            php-envs-pr-${{ runner.os }}-sentry-${{ steps.sentry-versions.outputs.php }}-

      - name: Setup PHP
        uses: shivammathur/setup-php@v2
        with:
          php-version: "8.4"
          tools: composer

      - name: Install dependencies and build
        run: |
          npm ci
          npm run build

      - name: Install Playwright browsers (for browser platform only)
        run: npx playwright install --with-deps chromium

      - name: Run tests
        id: run-tests
        continue-on-error: true
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GOOGLE_GENAI_API_KEY: ${{ secrets.GOOGLE_GENAI_API_KEY }}
          CO_API_KEY: ${{ secrets.CO_API_KEY }}
        run: npm run test -- -j=12

      - name: Upload CTRF report
        uses: actions/upload-artifact@v4
        with:
          name: ctrf-pr
          path: test-results/ctrf-report-*.json
          retention-days: 1

  compare-results:
    name: Compare Results
    needs: [test-main, test-pr]
    runs-on: ubuntu-latest
    if: always() && needs.test-main.result != 'cancelled' && needs.test-pr.result != 'cancelled'
    steps:
      - name: Checkout PR branch
        uses: actions/checkout@v4

      - name: Download main CTRF report
        uses: actions/download-artifact@v4
        with:
          name: ctrf-main
          path: ctrf-main

      - name: Download PR CTRF report
        uses: actions/download-artifact@v4
        with:
          name: ctrf-pr
          path: ctrf-pr

      - name: Run comparison
        id: compare
        run: |
          # Find the latest CTRF reports (they have timestamps in filename)
          MAIN_REPORT=$(ls -t ctrf-main/ctrf-report-*.json 2>/dev/null | head -1)
          PR_REPORT=$(ls -t ctrf-pr/ctrf-report-*.json 2>/dev/null | head -1)

          if [ -z "$MAIN_REPORT" ] || [ -z "$PR_REPORT" ]; then
            echo "Missing CTRF reports"
            echo "Main: $MAIN_REPORT"
            echo "PR: $PR_REPORT"
            exit 1
          fi

          node .github/scripts/compare-ctrf.cjs \
            "$MAIN_REPORT" \
            "$PR_REPORT" \
            comparison-output.md

          # Check if there are regressions
          if grep -q "^HAS_REGRESSIONS=true$" comparison-output.env 2>/dev/null; then
            echo "has_regressions=true" >> $GITHUB_OUTPUT
          else
            echo "has_regressions=false" >> $GITHUB_OUTPUT
          fi

      - name: Write job summary
        run: cat comparison-output.md >> $GITHUB_STEP_SUMMARY

      - name: Find existing comment
        uses: peter-evans/find-comment@v3
        id: find-comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: "github-actions[bot]"
          body-includes: "AI SDK Integration Test Results"

      - name: Create or update PR comment
        uses: peter-evans/create-or-update-comment@v4
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-id: ${{ steps.find-comment.outputs.comment-id }}
          edit-mode: replace
          body-path: comparison-output.md

      - name: Fail if regressions
        if: steps.compare.outputs.has_regressions == 'true'
        run: |
          echo "::error::Test regressions detected! Tests that were passing on main are now failing."
          exit 1