test.yml

name: Test Suite

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main, develop ]
  # schedule:
  #   # Run tests daily at 6 AM UTC
  #   - cron: '0 6 * * *'

jobs:
  test-unit:
    name: Unit Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    
    - name: Cache pip dependencies
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        # Install jsonschema for schema validation tests
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
    
    - name: Run unit tests
      run: |
        pytest tests/unit/ -v \
          --cov=stagehand \
          --cov-report=xml \
          --cov-report=term-missing \
          --junit-xml=junit-unit-${{ matrix.python-version }}.xml
    
    - name: Upload unit test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: unit-test-results-${{ matrix.python-version }}
        path: junit-unit-${{ matrix.python-version }}.xml
    
    - name: Upload coverage data
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: coverage-data-${{ matrix.python-version }}
        path: |
          .coverage
          coverage.xml
    
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      if: matrix.python-version == '3.11'
      with:
        file: ./coverage.xml
        flags: unit
        name: unit-tests

  test-integration:
    name: Integration Tests
    runs-on: ubuntu-latest
    needs: test-unit
    strategy:
      matrix:
        test-category: ["api", "browser", "end_to_end"]
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
        # Install Playwright browsers for integration tests
        playwright install chromium
    
    - name: Run integration tests - ${{ matrix.test-category }}
      run: |
        # Check if test directory exists and has test files before running pytest
        if [ -d "tests/integration/${{ matrix.test-category }}" ] && find "tests/integration/${{ matrix.test-category }}" -name "test_*.py" -o -name "*_test.py" | grep -q .; then
          pytest tests/integration/${{ matrix.test-category }}/ -v \
            --cov=stagehand \
            --cov-report=xml \
            --junit-xml=junit-integration-${{ matrix.test-category }}.xml
        else
          echo "No test files found in tests/integration/${{ matrix.test-category }}/, skipping..."
          # Create empty junit file to prevent workflow failure
          echo '<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0" timestamp="$(date -Iseconds)" hostname="$(hostname)"></testsuite></testsuites>' > junit-integration-${{ matrix.test-category }}.xml
        fi
      env:
        # Mock environment variables for testing
        BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY || 'mock-api-key' }}
        BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID || 'mock-project-id' }}
        MODEL_API_KEY: ${{ secrets.MODEL_API_KEY || 'mock-model-key' }}
        STAGEHAND_API_URL: "http://localhost:3000"
    
    - name: Upload integration test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: integration-test-results-${{ matrix.test-category }}
        path: junit-integration-${{ matrix.test-category }}.xml
    
    - name: Upload coverage data
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: coverage-data-integration-${{ matrix.test-category }}
        path: |
          .coverage
          coverage.xml

  test-browserbase:
    name: Browserbase Integration Tests
    runs-on: ubuntu-latest
    needs: test-unit
    if: github.event_name == 'schedule' || contains(github.event.head_commit.message, '[test-browserbase]')
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
    
    - name: Run Browserbase tests
      run: |
        pytest tests/ -v \
          --cov=stagehand \
          --cov-report=xml \
          --junit-xml=junit-browserbase.xml \
          -m "browserbase" \
          --tb=short
      env:
        BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
        BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }}
        MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
        STAGEHAND_API_URL: ${{ secrets.STAGEHAND_API_URL }}
    
    - name: Upload Browserbase test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: browserbase-test-results
        path: junit-browserbase.xml

  test-performance:
    name: Performance Tests
    runs-on: ubuntu-latest
    needs: test-unit
    if: github.event_name == 'schedule' || contains(github.event.head_commit.message, '[test-performance]')
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
        playwright install chromium
    
    - name: Run performance tests
      run: |
        pytest tests/performance/ -v \
          --junit-xml=junit-performance.xml \
          -m "performance" \
          --tb=short
      env:
        MODEL_API_KEY: ${{ secrets.MODEL_API_KEY || 'mock-model-key' }}
    
    - name: Upload performance test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: performance-test-results
        path: junit-performance.xml

  smoke-tests:
    name: Smoke Tests
    runs-on: ubuntu-latest
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
    
    - name: Run smoke tests
      run: |
        pytest tests/ -v \
          --junit-xml=junit-smoke.xml \
          -m "smoke" \
          --tb=line \
          --maxfail=5
    
    - name: Upload smoke test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: smoke-test-results
        path: junit-smoke.xml

  coverage-report:
    name: Coverage Report
    runs-on: ubuntu-latest
    needs: [test-unit, test-integration]
    if: always() && (needs.test-unit.result == 'success')
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install coverage[toml] codecov
    
    - name: Download coverage artifacts
      uses: actions/download-artifact@v4
      with:
        pattern: coverage-data-*
        path: coverage-reports/
    
    - name: Combine coverage reports
      run: |
        # List downloaded artifacts for debugging
        echo "Downloaded coverage artifacts:"
        find coverage-reports/ -name ".coverage*" -o -name "coverage.xml" | sort || echo "No coverage files found"
        
        # Find and combine coverage files
        COVERAGE_FILES=$(find coverage-reports/ -name ".coverage" -type f 2>/dev/null | head -10)
        if [ -n "$COVERAGE_FILES" ]; then
          echo "Found coverage files:"
          echo "$COVERAGE_FILES"
          
          # Copy coverage files to current directory for combining
          for file in $COVERAGE_FILES; do
            cp "$file" ".coverage.$(basename $(dirname $file))"
          done
          
          # Combine coverage files
          coverage combine .coverage.* || echo "Failed to combine coverage files"
          coverage report --show-missing || echo "No coverage data to report"
          coverage html || echo "No coverage data for HTML report" 
          coverage xml || echo "No coverage data for XML report"
        else
          echo "No .coverage files found to combine"
          # Create minimal coverage.xml to prevent downstream failures
          echo '<?xml version="1.0" encoding="UTF-8"?><coverage version="0" timestamp="0" lines-valid="0" lines-covered="0" line-rate="0"></coverage>' > coverage.xml
        fi
    
    - name: Upload combined coverage
      uses: codecov/codecov-action@v3
      with:
        file: ./coverage.xml
        name: combined-coverage
    
    - name: Upload coverage HTML report
      uses: actions/upload-artifact@v4
      with:
        name: coverage-html-report
        path: htmlcov/

  test-summary:
    name: Test Summary
    runs-on: ubuntu-latest
    needs: [test-unit, test-integration, smoke-tests]
    if: always()
    
    steps:
    - name: Download all test artifacts
      uses: actions/download-artifact@v4
      with:
        path: test-results/
    
    - name: Generate test summary
      run: |
        echo "## Test Results Summary" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        
        # Count test files
        UNIT_TESTS=$(find test-results/ -name "junit-unit-*.xml" | wc -l)
        INTEGRATION_TESTS=$(find test-results/ -name "junit-integration-*.xml" | wc -l)
        
        echo "- Unit test configurations: $UNIT_TESTS" >> $GITHUB_STEP_SUMMARY
        echo "- Integration test categories: $INTEGRATION_TESTS" >> $GITHUB_STEP_SUMMARY
        
        # Check for test failures
        if [ -f test-results/*/junit-*.xml ]; then
          echo "- Test artifacts generated successfully ✅" >> $GITHUB_STEP_SUMMARY
        else
          echo "- Test artifacts missing ❌" >> $GITHUB_STEP_SUMMARY
        fi
        
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "Detailed results are available in the artifacts section." >> $GITHUB_STEP_SUMMARY