e2e.yml

name: E2E Tests

on:
  workflow_dispatch:
    inputs:
      agent:
        description: 'Run a single agent (leave empty for all)'
        required: false
        type: choice
        options:
          - ''
          - claude-code
          - opencode
          - gemini-cli
          - factoryai-droid
          - cursor-cli
          - copilot-cli
          - roger-roger
          - codex
  push:
    branches:
      - main

# Concurrency: only one E2E job runs at a time
concurrency:
  group: e2e-tests
  cancel-in-progress: true

jobs:
  # Build the matrix dynamically so workflow_dispatch can select a single agent.
  matrix-setup:
    runs-on: ubuntu-latest
    outputs:
      agents: ${{ steps.set.outputs.agents }}
    steps:
      - id: set
        run: |
          input="${{ github.event.inputs.agent }}"
          if [ -n "$input" ]; then
            echo "agents=[\"$input\"]" >> "$GITHUB_OUTPUT"
          else
            echo 'agents=["claude-code","opencode","gemini-cli","factoryai-droid","cursor-cli","copilot-cli","roger-roger","codex"]' >> "$GITHUB_OUTPUT"
          fi

  e2e-tests:
    needs: matrix-setup
    runs-on: ubuntu-latest
    timeout-minutes: 40
    permissions:
      actions: read
      contents: read
      copilot-requests: write
    strategy:
      fail-fast: false
      matrix:
        agent: ${{ fromJson(needs.matrix-setup.outputs.agents) }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

      - name: Setup mise
        uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4

      - name: Install system dependencies
        run: sudo apt-get update && sudo apt-get install -y tmux

      - name: Install agent CLI
        run: |
          case "${{ matrix.agent }}" in
            claude-code) curl -fsSL https://claude.ai/install.sh | bash ;;
            opencode)    curl -fsSL https://opencode.ai/install | bash ;;
            gemini-cli)  npm install -g @google/gemini-cli ;;
            codex)       npm install -g @openai/codex ;;
            cursor-cli)  curl https://cursor.com/install -fsS | bash ;;
            factoryai-droid) curl -fsSL https://app.factory.ai/cli | sh ;;
            copilot-cli)  npm install -g @github/copilot  ;;
            roger-roger) ;; # installed by mise (see mise.toml)
          esac
          echo "$HOME/.local/bin" >> $GITHUB_PATH

      - name: Verify roger-roger agent
        if: matrix.agent == 'roger-roger'
        run: |
          set -euo pipefail
          echo "Verifying roger-roger binaries on PATH..."
          command -v roger-roger
          command -v trace-agent-roger-roger

      - name: Bootstrap agent
        if: matrix.agent != 'roger-roger'
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
          FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }}
          COPILOT_GITHUB_TOKEN: ${{ github.token }}
        run: go run ./e2e/bootstrap

      - name: Run E2E Tests
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          E2E_CODEX_MODEL: ${{ matrix.agent == 'codex' && 'gpt-5.4-mini' || '' }}
          CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
          FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }}
          COPILOT_GITHUB_TOKEN: ${{ github.token }}
          E2E_CONCURRENT_TEST_LIMIT: ${{ matrix.agent == 'gemini-cli' && '6' || matrix.agent == 'factoryai-droid' && '1' || '' }}
        run: mise run test:e2e --agent ${{ matrix.agent }} ${{ matrix.agent == 'roger-roger' && 'TestExternalAgent' || '' }}

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: e2e-artifacts-${{ matrix.agent }}
          path: e2e/artifacts/
          retention-days: 7

  e2e-windows:
    uses: ./.github/workflows/e2e-windows.yml
    secrets: inherit

  notify-slack:
    runs-on: ubuntu-latest
    needs: [e2e-tests, e2e-windows]
    if: ${{ always() && (needs.e2e-tests.result == 'failure' || needs.e2e-windows.result == 'failure') && github.event_name == 'push' }}
    steps:
      - name: Get failed agents
        id: failed
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          failed=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs \
            --jq '[.jobs[] | select(.conclusion == "failure") | .name | if test("\\(") then capture("\\((?<agent>[^)]+)\\)") | .agent else . end] | join(", ")')
          echo "agents=$failed" >> "$GITHUB_OUTPUT"

      - name: Notify Slack of E2E failure
        uses: slackapi/slack-github-action@03ea5433c137af7c0495bc0cad1af10403fc800c # v3.0.2
        with:
          webhook: ${{ secrets.E2E_SLACK_WEBHOOK_URL }}
          webhook-type: incoming-webhook
          payload: |
            {
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": ":red_circle: *E2E Tests Failed* on `main`\n\nFailed agents: *${{ steps.failed.outputs.agents }}*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run details>"
                  }
                },
                {
                  "type": "context",
                  "elements": [
                    {
                      "type": "mrkdwn",
                      "text": "Commit: <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> by ${{ github.actor }}"
                    }
                  ]
                }
              ]
            }