mcp-data-platform/.github/workflows/bench.yml at main · txn2/mcp-data-platform · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
# Benchmark harness module checks + on-demand smoke (issue #930; phase 1 #942,
# phase 4 #945).
#
# Two jobs:
#   harness (PR/push) — build, vet, and unit-test the bench/ module, the
#     deterministic regression guard for the harness code itself (the pipeline
#     integration test and the -baseline regression gate both run against
#     in-process fakes, no Docker or model API).
#   smoke (workflow_dispatch) — boot the real compose stack + the a0 arm and run
#     the scripted, no-API-key smoke end to end, plus a live self-check of the
#     -baseline regression gate. DataHub is intentionally NOT in CI (it needs
#     `datahub docker quickstart`), so the smoke runs on a0 (raw toolkit tools,
#     no semantic layer), mirroring e2e-nightly. A tiny model-backed run is
#     available behind an opt-in input + the ANTHROPIC_API_KEY secret; it is off
#     by default so a dispatch spends nothing. Full model-backed runs happen at
#     release time, like verify-release.
#
# Not part of `make verify`.
name: Bench Harness

on:
  pull_request:
    paths:
      - "bench/**"
      - ".github/workflows/bench.yml"
  push:
    branches: [main]
    paths:
      - "bench/**"
      - ".github/workflows/bench.yml"
  workflow_dispatch:
    inputs:
      model_smoke:
        description: "Also run a tiny model-backed s1 run (needs the ANTHROPIC_API_KEY secret; spends a few cents)"
        type: boolean
        default: false

permissions:
  contents: read

jobs:
  harness:
    name: Harness module checks
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Set up Go
        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
        with:
          go-version: "1.26.5"
          cache: true
          cache-dependency-path: bench/go.sum

      - name: Build, vet, and test the harness
        run: make bench-test

      - name: Lint the harness
        uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0
        with:
          version: v2.11.4
          working-directory: bench

  smoke:
    name: Scripted smoke + regression gate (a0)
    if: github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Set up Go
        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
        with:
          go-version: "1.26.5"
          cache: true

      - name: Boot the a0 arm (compose stack + platform)
        run: make bench-up BENCH_ARM=a0

      - name: Scripted task smoke (no API key)
        run: make bench-smoke

      - name: Regression gate self-check (candidate vs itself passes)
        # The scripted smoke is deterministic, so gating it against its own
        # results must PASS — this exercises the -baseline flag end to end
        # (load, compare, exit code) on a live run. The gate's FAIL and
        # incompatible-baseline paths are covered deterministically in the harness
        # job by TestGateOnBaselineFailsOnRegression / RejectsArmMismatch (the
        # gateOnBaseline exit wiring) and TestCheckRegressionFailsOnDegradedConfig
        # (the comparison logic).
        run: |
          cp build/bench-results/results-a0.json build/bench-results/baseline-a0.json
          make bench-smoke BASELINE=build/bench-results/baseline-a0.json

      - name: Tiny model-backed s1 run (opt-in)
        if: inputs.model_smoke
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: make bench-run BENCH_ARM=a0 LLM=anthropic SUITE=s1 K=1

      - name: Collect platform + compose logs
        if: always()
        env:
          DOCKER_DEFAULT_PLATFORM: ""
        run: |
          cp build/mcp-data-platform-bench.log bench-platform.log 2>/dev/null || true
          docker compose -f docker-compose.e2e.yml logs --no-color > bench-compose-logs.txt 2>&1 || true

      - name: Upload logs
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bench-smoke-logs
          path: |
            bench-platform.log
            bench-compose-logs.txt
          if-no-files-found: warn
          retention-days: 14

      - name: Tear down
        if: always()
        run: make bench-down