[CCOR-13193] Run Integration(e2e) tests on PR-to-main (phase 1: test with sdkdev v5 server) #30
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Agent E2E | |
| # Runs the agent e2e suites (e2e/) against the Conductor OSS server boot | |
| # JAR (Maven Central) — the server this SDK ships against, with the agent | |
| # runtime on by default from 3.32.0-rc.8 onward. The Agentspan server JAR | |
| # is no longer used here; the Agentspan CLI binary is still downloaded | |
| # (its own separate pin) since e2e/test_suite16_cli_skills.py drives it | |
| # against this same server. | |
| # tests/integration/ai stays manual-only (same as upstream Agentspan, | |
| # which never ran its tests/integration in CI): run it locally against | |
| # a live server with `pytest tests/integration/ai`. | |
| # | |
| # These tests call real LLMs via the OPENAI_API_KEY / ANTHROPIC_API_KEY | |
| # repo secrets. Fork PRs cannot see repo secrets, so for them the run | |
| # fails at the silently-empty guard rather than passing vacuously. | |
| on: [pull_request, workflow_dispatch] | |
| # least privilege: the token is only used to download the public agentspan | |
| # CLI release asset (gh release download) — read access suffices | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: agent-e2e-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CONDUCTOR_SERVER_VERSION: "3.32.0-rc.8" # pinned server release — bump deliberately | |
| AGENTSPAN_CLI_VERSION: "0.4.4" # pinned CLI release — independent of the server (revert to 0.4.3 if this breaks) | |
| jobs: | |
| agent-e2e: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| AGENTSPAN_SERVER_URL: http://localhost:8080/api | |
| AGENTSPAN_CLI_PATH: ${{ github.workspace }}/agentspan | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| cache: 'pip' | |
| - name: Set up Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: '21' | |
| - name: Cache server JAR | |
| id: jar_cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: conductor-server.jar | |
| key: conductor-oss-server-${{ env.CONDUCTOR_SERVER_VERSION }} | |
| - name: Download server JAR from Maven Central | |
| if: steps.jar_cache.outputs.cache-hit != 'true' | |
| run: | | |
| curl -fL --retry 3 -o conductor-server.jar \ | |
| "https://repo1.maven.org/maven2/org/conductoross/conductor-server/${CONDUCTOR_SERVER_VERSION}/conductor-server-${CONDUCTOR_SERVER_VERSION}-boot.jar" | |
| - name: Download CLI binary from release | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh release download "v${AGENTSPAN_CLI_VERSION}" --repo agentspan-ai/agentspan \ | |
| --pattern "agentspan_linux_amd64" --output agentspan | |
| chmod +x agentspan | |
| - name: Install SDK + test deps | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e '.[agents]' | |
| pip install pytest pytest-asyncio pytest-xdist pytest-rerunfailures mcp-testkit | |
| - name: Start mcp-testkit | |
| run: | | |
| mcp-testkit --transport http --port 3001 & | |
| sleep 2 | |
| - name: Start server | |
| run: | | |
| java -jar conductor-server.jar --server.port=8080 > server.log 2>&1 & | |
| - name: Wait for server health | |
| run: | | |
| for i in $(seq 1 45); do | |
| if curl -sf http://localhost:8080/health | grep -q '"healthy"[[:space:]]*:[[:space:]]*true'; then | |
| echo "server healthy after ~$((i*2))s"; exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo "::error::server failed to become healthy within 90s" | |
| tail -100 server.log | |
| exit 1 | |
| - name: Run e2e suites | |
| run: | | |
| mkdir -p results | |
| pytest e2e/ -v --tb=short -n 3 --dist=loadgroup \ | |
| --junitxml=results/junit-e2e.xml | |
| # e2e/conftest.py pytest.skip()s the whole session when the server is | |
| # unreachable — without this guard a boot failure after the health gate | |
| # (or a future gate regression) would yield a green job that ran nothing. | |
| - name: Guard against silently-empty runs | |
| if: always() | |
| run: | | |
| python - <<'EOF' | |
| import glob | |
| import sys | |
| import xml.etree.ElementTree as ET | |
| total = executed = 0 | |
| for path in glob.glob("results/junit-*.xml"): | |
| root = ET.parse(path).getroot() | |
| for suite in root.iter("testsuite"): | |
| t = int(suite.get("tests", 0)) | |
| sk = int(suite.get("skipped", 0)) | |
| total += t | |
| executed += t - sk | |
| print(f"executed {executed}/{total} tests") | |
| sys.exit(0 if executed > 0 else 1) | |
| EOF | |
| - name: Generate HTML report | |
| if: always() | |
| run: | | |
| python e2e/report_generator.py results/junit-e2e.xml results/report.html || true | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: agent-e2e-results | |
| path: | | |
| results/ | |
| server.log | |
| retention-days: 14 |