test.yml

name: Test Suite

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main, develop ]
    types: [opened, synchronize, reopened, labeled, unlabeled]

jobs:
  test-unit:
    name: Unit Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    
    - name: Cache pip dependencies
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        # Install jsonschema for schema validation tests
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
    
    - name: Run unit tests
      run: |
        pytest tests/unit/ -v --junit-xml=junit-unit-${{ matrix.python-version }}.xml
    
    - name: Upload unit test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: unit-test-results-${{ matrix.python-version }}
        path: junit-unit-${{ matrix.python-version }}.xml

  test-integration-local:
    name: Local Integration Tests
    runs-on: ubuntu-latest
    needs: test-unit
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y xvfb
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
        # Install Playwright browsers for integration tests
        playwright install chromium
        playwright install-deps chromium
    
    - name: Run local integration tests
      run: |
        # Run integration tests marked as 'integration' and 'local'
        xvfb-run -a pytest tests/integration/ -v \
          --cov=stagehand \
          --cov-report=xml \
          --junit-xml=junit-integration-local.xml \
          -m "integration and local" \
          --tb=short \
          --maxfail=5
      env:
        MODEL_API_KEY: ${{ secrets.MODEL_API_KEY || 'mock-model-key' }}
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || 'mock-openai-key' }}
        DISPLAY: ":99"
    
    - name: Upload integration test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: integration-test-results-local
        path: junit-integration-local.xml
    
    - name: Upload coverage data
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: coverage-data-integration-local
        path: |
          .coverage
          coverage.xml

  test-integration-api:
    name: API Integration Tests
    runs-on: ubuntu-latest
    needs: test-unit
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
    
    - name: Run API integration tests
      run: |
        pytest tests/integration/ -v \
          --cov=stagehand \
          --cov-report=xml \
          --junit-xml=junit-integration-api.xml \
          -m "integration and api" \
          --tb=short
      env:
        BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
        BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }}
        MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
        STAGEHAND_API_URL: ${{ secrets.STAGEHAND_API_URL }}
    
    - name: Upload API integration test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: api-integration-test-results
        path: junit-integration-api.xml

  smoke-tests:
    name: Smoke Tests
    runs-on: ubuntu-latest
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        pip install jsonschema
        pip install temp/google_genai-1.14.0-py3-none-any.whl
    
    - name: Run smoke tests
      run: |
        pytest tests/ -v \
          --junit-xml=junit-smoke.xml \
          -m "smoke" \
          --tb=line \
          --maxfail=5
    
    - name: Upload smoke test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: smoke-test-results
        path: junit-smoke.xml

  test-regression:
    name: Regression Tests
    runs-on: ubuntu-latest
    needs: test-unit
    if: |
      contains(github.event.pull_request.labels.*.name, 'test-regression') ||
      contains(github.event.pull_request.labels.*.name, 'regression')
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y xvfb
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
        playwright install chromium
        playwright install-deps chromium
    
    - name: Run regression tests
      run: |
        xvfb-run -a pytest tests/ -v \
          --cov=stagehand \
          --cov-report=xml \
          --junit-xml=junit-regression.xml \
          -m "regression" \
          --tb=short \
          --maxfail=10
      env:
        BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY || 'mock-api-key' }}
        BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID || 'mock-project-id' }}
        MODEL_API_KEY: ${{ secrets.MODEL_API_KEY || 'mock-model-key' }}
        STAGEHAND_API_URL: ${{ secrets.STAGEHAND_API_URL || 'http://localhost:3000' }}
    
    - name: Upload regression test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: regression-test-results
        path: junit-regression.xml
    
    - name: Upload coverage data
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: coverage-data-regression
        path: |
          .coverage
          coverage.xml

  test-e2e:
    name: End-to-End Tests
    runs-on: ubuntu-latest
    needs: test-unit
    if: |
      contains(github.event.pull_request.labels.*.name, 'test-e2e') ||
      contains(github.event.pull_request.labels.*.name, 'e2e')
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y xvfb
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
        playwright install chromium
        playwright install-deps chromium
    
    - name: Run E2E tests
      run: |
        xvfb-run -a pytest tests/ -v \
          --cov=stagehand \
          --cov-report=xml \
          --junit-xml=junit-e2e.xml \
          -m "e2e" \
          --tb=short
      env:
        BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY || 'mock-api-key' }}
        BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID || 'mock-project-id' }}
        MODEL_API_KEY: ${{ secrets.MODEL_API_KEY || 'mock-model-key' }}
        STAGEHAND_API_URL: ${{ secrets.STAGEHAND_API_URL || 'http://localhost:3000' }}
        DISPLAY: ":99"
    
    - name: Upload E2E test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: e2e-test-results
        path: junit-e2e.xml

  test-all:
    name: Complete Test Suite
    runs-on: ubuntu-latest
    needs: test-unit
    if: |
      contains(github.event.pull_request.labels.*.name, 'test-all') ||
      contains(github.event.pull_request.labels.*.name, 'full-test')
    
    steps:
    - uses: actions/checkout@v4
    
    - name: Set up Python 3.11
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"
        pip install jsonschema
        # Install temporary Google GenAI wheel
        pip install temp/google_genai-1.14.0-py3-none-any.whl
        playwright install chromium
    
    - name: Run complete test suite
      run: |
        pytest tests/ -v \
          --cov=stagehand \
          --cov-report=xml \
          --cov-report=html \
          --junit-xml=junit-all.xml \
          --maxfail=10 \
          --tb=short
      env:
        BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
        BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }}
        MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        STAGEHAND_API_URL: ${{ secrets.STAGEHAND_API_URL }}
    
    - name: Upload complete test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: complete-test-results
        path: |
          junit-all.xml
          htmlcov/