Drift Tests #75
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Drift Tests | |
| on: | |
| schedule: | |
| - cron: "0 6 * * *" # Daily 6am UTC | |
| pull_request: | |
| paths: | |
| - "src/agui-types.ts" | |
| - "src/__tests__/drift/agui-schema.drift.ts" | |
| workflow_dispatch: # Manual trigger | |
| jobs: | |
| agui-schema-drift: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - name: Clone ag-ui repo | |
| run: git clone --depth 1 https://github.com/ag-ui-protocol/ag-ui.git ../ag-ui | |
| - name: Run AG-UI schema drift test | |
| run: npx vitest run src/__tests__/drift/agui-schema.drift.ts --config vitest.config.drift.ts | |
| drift: | |
| if: github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - name: Run drift tests | |
| id: drift | |
| run: | | |
| set +e | |
| npx tsx scripts/drift-report-collector.ts | |
| EXIT_CODE=$? | |
| set -e | |
| echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT | |
| if [ "$EXIT_CODE" -eq 2 ]; then | |
| : # critical drift found, continue | |
| elif [ "$EXIT_CODE" -ne 0 ]; then | |
| echo "::error::Collector script crashed with exit code $EXIT_CODE" | |
| exit $EXIT_CODE | |
| fi | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| - name: Upload drift report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: drift-report | |
| path: drift-report.json | |
| if-no-files-found: warn | |
| retention-days: 30 | |
| - name: Fail if critical drift detected | |
| if: steps.drift.outputs.exit_code == '2' | |
| run: exit 1 | |
| notify: | |
| if: always() && github.event_name != 'pull_request' | |
| needs: [drift, agui-schema-drift] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Check previous run status | |
| id: prev | |
| run: | | |
| PREV=$(gh run list --workflow="Drift Tests" --repo="${{ github.repository }}" --branch=main --limit=2 --json conclusion --jq '.[1].conclusion // "unknown"') | |
| echo "conclusion=$PREV" >> $GITHUB_OUTPUT | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Notify Slack | |
| run: | | |
| if [ -z "$SLACK_WEBHOOK" ]; then echo "SLACK_WEBHOOK not set, skipping"; exit 0; fi | |
| PREV="${{ steps.prev.outputs.conclusion }}" | |
| DRIFT_RESULT="${{ needs.drift.result }}" | |
| AGUI_RESULT="${{ needs.agui-schema-drift.result }}" | |
| HTTP_DRIFT=false | |
| AGUI_DRIFT=false | |
| INFRA_ERROR=false | |
| # Determine what happened in each job | |
| if [ "$DRIFT_RESULT" = "failure" ]; then | |
| HTTP_DRIFT=true | |
| elif [ "$DRIFT_RESULT" != "success" ] && [ "$DRIFT_RESULT" != "skipped" ]; then | |
| INFRA_ERROR=true | |
| fi | |
| if [ "$AGUI_RESULT" = "failure" ]; then | |
| AGUI_DRIFT=true | |
| elif [ "$AGUI_RESULT" != "success" ] && [ "$AGUI_RESULT" != "skipped" ]; then | |
| INFRA_ERROR=true | |
| fi | |
| RUN_URL="<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" | |
| # Both types of drift | |
| if [ "$HTTP_DRIFT" = "true" ] && [ "$AGUI_DRIFT" = "true" ]; then | |
| EMOJI="🚨" | |
| MSG="*Drift detected* in aimock — HTTP API drift + AG-UI schema drift. ${RUN_URL}" | |
| # HTTP API drift only | |
| elif [ "$HTTP_DRIFT" = "true" ]; then | |
| EMOJI="🚨" | |
| MSG="*HTTP API drift detected* in aimock — providers changed response formats. ${RUN_URL}" | |
| # AG-UI schema drift only | |
| elif [ "$AGUI_DRIFT" = "true" ]; then | |
| EMOJI="🚨" | |
| MSG="*AG-UI schema drift detected* in aimock — canonical ag-ui types changed. ${RUN_URL}" | |
| # Infra failure — always notify | |
| elif [ "$INFRA_ERROR" = "true" ]; then | |
| EMOJI="❌" | |
| MSG="*Drift tests failed* (infra error). ${RUN_URL}" | |
| # Recovery: previous was bad, now good — notify once | |
| elif [ "$PREV" = "failure" ]; then | |
| EMOJI="✅" | |
| MSG="Drift tests passing again — all providers and AG-UI schema match." | |
| # Good → good — stay quiet | |
| else | |
| exit 0 | |
| fi | |
| PAYLOAD=$(jq -n --arg text "${EMOJI} ${MSG}" '{text: $text}') | |
| curl -sf -X POST "$SLACK_WEBHOOK" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD" | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} |