|
| 1 | +name: Merge Queue Triage |
| 2 | + |
| 3 | +# Why this exists (#4859): red queue builds were being blind-requeued. On |
| 4 | +# 2026-08-03 one PR failed the queue four times and landed unchanged on the |
| 5 | +# fifth attempt (09:19 → 10:22), and every failure evicted and rebuilt every |
| 6 | +# entry queued behind it — the queue's perceived slowness that morning was |
| 7 | +# mostly this amplification, not build duration. |
| 8 | +# |
| 9 | +# A queue failure is a different animal from a PR failure: the PR's own CI ran |
| 10 | +# affected-only, while the queue runs the FULL suite on the speculative merge |
| 11 | +# result. The failing test is therefore often in a package the PR never |
| 12 | +# touched — a flake, or a semantic conflict with another queued PR — and |
| 13 | +# neither of those is fixed by re-queueing; re-queueing just burns another |
| 14 | +# ~10-minute build for every entry behind it. |
| 15 | +# |
| 16 | +# So: every red merge_group CI run gets a triage comment on its PR — the |
| 17 | +# failed jobs/steps, the failing test lines pulled from the logs (best |
| 18 | +# effort), how many times THIS PR has already failed in the queue in the last |
| 19 | +# 24 h, and the queue-wide failure count. The checklist tells the author |
| 20 | +# (human or agent) to diagnose before re-queueing. The comment is the |
| 21 | +# machine-readable signal the PM dispatch loop can key on, and the cross-PR |
| 22 | +# flake evidence lives in these comments: the same test name appearing in two |
| 23 | +# unrelated PRs' triage comments is a confirmed flake. |
| 24 | +# |
| 25 | +# Fires on conclusion == failure ONLY. 'cancelled' is the queue evicting an |
| 26 | +# entry because something AHEAD of it failed (or a manual cancel) — it says |
| 27 | +# nothing about this PR, so it gets no comment (the same run-lifecycle |
| 28 | +# reasoning as dogfood-gate's cancelled handling in ci.yml, from the other |
| 29 | +# side). |
| 30 | +# |
| 31 | +# workflow_run executes in the DEFAULT branch's context: this file must be on |
| 32 | +# main before it fires, it never checks out or runs PR code, and it holds the |
| 33 | +# minimum permissions (actions: read for logs, pull-requests: write for the |
| 34 | +# comment). |
| 35 | + |
| 36 | +on: |
| 37 | + workflow_run: |
| 38 | + workflows: [CI] |
| 39 | + types: [completed] |
| 40 | + |
| 41 | +permissions: {} |
| 42 | + |
| 43 | +jobs: |
| 44 | + triage: |
| 45 | + name: Comment queue-failure triage on the PR |
| 46 | + if: >- |
| 47 | + github.event.workflow_run.event == 'merge_group' && |
| 48 | + github.event.workflow_run.conclusion == 'failure' |
| 49 | + runs-on: ubuntu-latest |
| 50 | + timeout-minutes: 5 |
| 51 | + permissions: |
| 52 | + actions: read |
| 53 | + pull-requests: write |
| 54 | + steps: |
| 55 | + - name: Post the triage comment |
| 56 | + uses: actions/github-script@v9 |
| 57 | + with: |
| 58 | + script: | |
| 59 | + const run = context.payload.workflow_run; |
| 60 | + const { owner, repo } = context.repo; |
| 61 | +
|
| 62 | + // Queue branches are named gh-readonly-queue/<base>/pr-<N>-<sha>. |
| 63 | + const m = /^gh-readonly-queue\/.+\/pr-(\d+)-[0-9a-f]{40}$/.exec(run.head_branch ?? ''); |
| 64 | + if (!m) { |
| 65 | + core.info(`head_branch '${run.head_branch}' is not a merge-queue branch — nothing to do.`); |
| 66 | + return; |
| 67 | + } |
| 68 | + const prNumber = Number(m[1]); |
| 69 | + const marker = `<!-- merge-queue-triage:${run.id} -->`; |
| 70 | +
|
| 71 | + // Idempotency: workflow_run deliveries can repeat; one comment per run. |
| 72 | + const existing = await github.rest.issues.listComments({ |
| 73 | + owner, repo, issue_number: prNumber, per_page: 100, |
| 74 | + }); |
| 75 | + if (existing.data.some((c) => (c.body ?? '').includes(marker))) { |
| 76 | + core.info('triage comment for this run already exists — skipping.'); |
| 77 | + return; |
| 78 | + } |
| 79 | +
|
| 80 | + // Failed jobs and their failed steps. |
| 81 | + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { |
| 82 | + owner, repo, run_id: run.id, per_page: 100, |
| 83 | + }); |
| 84 | + const failedJobs = jobs.filter((j) => j.conclusion === 'failure'); |
| 85 | +
|
| 86 | + // Best-effort log harvest: the lines a human would grep for first. |
| 87 | + // Aggregate gate jobs (Test Core / Dogfood Regression Gate) fail |
| 88 | + // with no information of their own, so prefer real jobs when both |
| 89 | + // are present. A 4xx on the logs endpoint degrades to names only. |
| 90 | + const FAIL_LINE = /(?:^|[\s|])(?:✗|×|✕|FAIL\s|AssertionError|STALL)/; |
| 91 | + const informative = failedJobs.filter((j) => (j.steps ?? []).some( |
| 92 | + (s) => s.conclusion === 'failure' && !/^Verify .* results$/.test(s.name))); |
| 93 | + const details = []; |
| 94 | + for (const job of (informative.length ? informative : failedJobs).slice(0, 4)) { |
| 95 | + const steps = (job.steps ?? []) |
| 96 | + .filter((s) => s.conclusion === 'failure') |
| 97 | + .map((s) => s.name); |
| 98 | + let lines = []; |
| 99 | + try { |
| 100 | + const res = await github.request( |
| 101 | + 'GET /repos/{owner}/{repo}/actions/jobs/{job_id}/logs', |
| 102 | + { owner, repo, job_id: job.id }); |
| 103 | + const text = typeof res.data === 'string' |
| 104 | + ? res.data |
| 105 | + : Buffer.from(res.data).toString('utf8'); |
| 106 | + lines = text.split('\n') |
| 107 | + .map((l) => l.replace(/^[^ ]*Z /, '')) |
| 108 | + .filter((l) => FAIL_LINE.test(l)) |
| 109 | + .map((l) => l.trim().slice(0, 200)) |
| 110 | + .slice(0, 12); |
| 111 | + } catch (e) { |
| 112 | + core.info(`logs unavailable for job ${job.id}: ${e.message}`); |
| 113 | + } |
| 114 | + details.push({ name: job.name, url: job.html_url, steps, lines }); |
| 115 | + } |
| 116 | +
|
| 117 | + // History: this PR's earlier queue failures + queue-wide count, 24 h. |
| 118 | + const since = new Date(Date.now() - 24 * 3600 * 1000).toISOString(); |
| 119 | + const recent = await github.paginate(github.rest.actions.listWorkflowRunsForRepo, { |
| 120 | + owner, repo, event: 'merge_group', created: `>=${since}`, per_page: 100, |
| 121 | + }); |
| 122 | + const ciRuns = recent.filter((r) => r.workflow_id === run.workflow_id && r.id !== run.id); |
| 123 | + const priorFailuresThisPr = ciRuns.filter((r) => |
| 124 | + r.conclusion === 'failure' && (r.head_branch ?? '').includes(`/pr-${prNumber}-`)).length; |
| 125 | + const queueFailures24h = ciRuns.filter((r) => r.conclusion === 'failure').length; |
| 126 | +
|
| 127 | + const jobSections = details.map((d) => { |
| 128 | + const head = `- **[${d.name}](${d.url})** — 失败步骤: ${d.steps.join('、') || '(无步骤级结论)'}`; |
| 129 | + return d.lines.length |
| 130 | + ? `${head}\n\n \`\`\`\n ${d.lines.join('\n ')}\n \`\`\`` |
| 131 | + : `${head}（日志不可读，点进 job 看）`; |
| 132 | + }).join('\n'); |
| 133 | +
|
| 134 | + const flakeHint = priorFailuresThisPr > 0 |
| 135 | + ? `⚠️ **本 PR 过去 24h 已在队列失败 ${priorFailuresThisPr} 次（不含本次）。** 内容未变而反复失败 ⇒ 高度怀疑 flaky 测试或与同组 PR 的语义冲突，重排不解决。` |
| 136 | + : '本 PR 过去 24h 无队列失败记录（首次）。'; |
| 137 | +
|
| 138 | + const body = [ |
| 139 | + '### ⛔ merge queue 构建失败 — 先分诊，再决定要不要重排', |
| 140 | + '', |
| 141 | + `队列构建 [${run.id}](${run.html_url}) 红了。队列跑的是**全量**套件（PR 侧 CI 只跑 affected 子集），`, |
| 142 | + '所以失败的测试可能在本 PR 没碰过的包里 —— 那不是重排能修的。每次盲目重排都会让排在后面的所有 PR 重建一轮。', |
| 143 | + '', |
| 144 | + '**失败的 job（日志抽取，best effort）：**', |
| 145 | + '', |
| 146 | + jobSections || '- （没拿到 job 级信息，点上面的 run 链接看）', |
| 147 | + '', |
| 148 | + '**历史信号：**', |
| 149 | + `- ${flakeHint}`, |
| 150 | + `- 过去 24h 队列共有 ${queueFailures24h} 个失败构建（不含本次）。`, |
| 151 | + '', |
| 152 | + '**分诊清单：**', |
| 153 | + '1. 失败测试在本 PR 改动的包里 → 真回归，修 PR。', |
| 154 | + '2. 失败测试与本 PR 无关 → 在其他 PR 的同类评论里搜同名测试；出现过 ⇒ flaky 实锤，开 issue 修/隔离那条测试。修好前重排只会再烧一轮全队列。', |
| 155 | + '3. 两者都不是 → 可能与同组 PR 语义冲突；等前面的 PR 落地或失败出队后再重排一次即可，不要连续重排。', |
| 156 | + '', |
| 157 | + marker, |
| 158 | + '---', |
| 159 | + '_Generated by [Claude Code](https://claude.ai/code) · merge-queue-triage workflow (#4859)_', |
| 160 | + ].join('\n'); |
| 161 | +
|
| 162 | + await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body }); |
| 163 | + core.info(`triage comment posted on #${prNumber}.`); |
0 commit comments