diff --git a/.changeset/ci-merge-queue-throughput.md b/.changeset/ci-merge-queue-throughput.md new file mode 100644 index 0000000000..ca42c03bfc --- /dev/null +++ b/.changeset/ci-merge-queue-throughput.md @@ -0,0 +1,4 @@ +--- +--- + +CI-only (#4859): Test Core / Dogfood 各 2→3 分片 + merge-queue 失败自动分诊评论 workflow。仅 `.github/workflows/**`，不发布任何包。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6c1b40532..63080805ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,11 +75,13 @@ jobs: - '.github/workflows/ci.yml' test: - # Sharded 2-way BY PACKAGE: a core-touching PR ran the affected suite - # ~11½ min on one 4-vCPU runner — the longest pole in the whole workflow. - # scripts/partition-test-shards.mjs splits the package list into two - # deterministic, test-file-count-balanced halves (573/572 at the time of - # writing) and each shard runs its half through turbo. NOT the dogfood + # Sharded 3-way BY PACKAGE: a core-touching PR ran the affected suite + # ~11½ min on one 4-vCPU runner — the longest pole in the whole workflow — + # and at 2 shards the slower shard still ran ~10 min, keeping merge-queue + # builds at ~11 min end-to-end (#4859). + # scripts/partition-test-shards.mjs splits the package list into three + # deterministic, test-file-count-balanced thirds and each shard runs its + # slice through turbo. NOT the dogfood # job's vitest --shard passthrough, deliberately: that works for dogfood # because it is ONE package with ~60 files, but applied workspace-wide, # vitest 4 hard-fails every package with fewer test files than the shard @@ -90,13 +92,13 @@ jobs: # Branch protection requires the bare "Test Core" context, which a matrix # can never publish again — the test-gate job below carries that name # (the #3622 lesson; see dogfood-gate). - name: Test Core (${{ matrix.shard }}/2) + name: Test Core (${{ matrix.shard }}/3) needs: filter if: needs.filter.outputs.core == 'true' runs-on: ubuntu-latest # Backstop only — the stall guard on the test steps is the primary # detector for a #4250-style hang and fires well before this. 30 min is - # ~4× a normal sharded run (~6-7 min), with margin for a cold Turbo cache; + # ~5× a normal sharded run (~4-6 min), with margin for a cold Turbo cache; # the old 45 left a hung job "running" for half an hour past any plausible # healthy finish. timeout-minutes: 30 @@ -105,7 +107,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3] steps: - name: Checkout repository @@ -190,7 +192,7 @@ jobs: pnpm exec turbo ls --output=json > "$RUNNER_TEMP/turbo-ls.json" fi node scripts/partition-test-shards.mjs "$RUNNER_TEMP/turbo-ls.json" \ - --shard ${{ matrix.shard }}/2 --exclude @objectstack/dogfood \ + --shard ${{ matrix.shard }}/3 --exclude @objectstack/dogfood \ > "$RUNNER_TEMP/shard-packages.txt" echo "Packages on this shard:" cat "$RUNNER_TEMP/shard-packages.txt" @@ -499,18 +501,19 @@ jobs: test dogfood: - # Sharded 2-way: the suite is ~60 independent test files, each booting its - # own in-process app, and a single 4-vCPU runner needed ~7½ minutes for the - # lot — the longest pole in the whole workflow. vitest partitions the file - # list deterministically across shards; both shards must pass. If branch - # protection lists "Dogfood Regression Gate" as a required check, it must be - # updated to the two sharded check names. - name: Dogfood Regression Gate (${{ matrix.shard }}/2) + # Sharded 3-way: the suite is ~60 independent test files, each booting its + # own in-process app; a single 4-vCPU runner needed ~7½ minutes for the + # lot, and at 2 shards each half still ran ~7 min — the longest pole left + # once Test Core went 3-way (#4859). vitest partitions the file list + # deterministically across shards; all shards must pass. Branch protection + # requires only the bare "Dogfood Regression Gate" context, carried by the + # dogfood-gate job below — the shard count can change without touching it. + name: Dogfood Regression Gate (${{ matrix.shard }}/3) needs: filter if: needs.filter.outputs.core == 'true' runs-on: ubuntu-latest # Backstop only — the stall guard on the test step is the primary detector - # for a #4250-style hang (see Test Core). 30 min is ~4× a shard (~7 min; + # for a #4250-style hang (see Test Core). 30 min is ~6× a shard (~5 min; # the verify-CLI pass that used to ride shard 1 is its own parallel job # now — dogfood-verify below). timeout-minutes: 30 @@ -519,7 +522,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3] steps: - name: Checkout repository @@ -552,7 +555,10 @@ jobs: # Shard-scoped key: the turbo test hash differs per shard (pass-through # args are part of the task hash). Restore-only on PRs — see the Restore # Turbo cache comment in the test job; the save step at the end of this - # job seeds from main only. + # job seeds from main only. The job-level catch-all (same as Test Core's) + # is what keeps a NEW shard number warm before main has ever saved it: + # turbo's cache is content-addressed per task, so another shard's entries + # replay the shared build closure even when the test slice differs. - name: Restore Turbo cache uses: actions/cache/restore@v6 with: @@ -561,6 +567,7 @@ jobs: restore-keys: | ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}- ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}- + ${{ runner.os }}-turbo-${{ github.job }}- - name: Install dependencies run: pnpm install --frozen-lockfile @@ -585,7 +592,7 @@ jobs: mkdir -p "$RUNNER_TEMP/stall-reports" node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/dogfood.log" --stall-minutes 10 \ --report-dir "$RUNNER_TEMP/stall-reports" -- \ - pnpm turbo run test --filter=@objectstack/dogfood -- --shard=${{ matrix.shard }}/2 + pnpm turbo run test --filter=@objectstack/dogfood -- --shard=${{ matrix.shard }}/3 # Dogfood boots real apps in-process, so a native/OOM abort is likelier # here than in the unit suites — and a shard that dies silently looks like diff --git a/.github/workflows/merge-queue-triage.yml b/.github/workflows/merge-queue-triage.yml new file mode 100644 index 0000000000..1981f46dae --- /dev/null +++ b/.github/workflows/merge-queue-triage.yml @@ -0,0 +1,163 @@ +name: Merge Queue Triage + +# Why this exists (#4859): red queue builds were being blind-requeued. On +# 2026-08-03 one PR failed the queue four times and landed unchanged on the +# fifth attempt (09:19 → 10:22), and every failure evicted and rebuilt every +# entry queued behind it — the queue's perceived slowness that morning was +# mostly this amplification, not build duration. +# +# A queue failure is a different animal from a PR failure: the PR's own CI ran +# affected-only, while the queue runs the FULL suite on the speculative merge +# result. The failing test is therefore often in a package the PR never +# touched — a flake, or a semantic conflict with another queued PR — and +# neither of those is fixed by re-queueing; re-queueing just burns another +# ~10-minute build for every entry behind it. +# +# So: every red merge_group CI run gets a triage comment on its PR — the +# failed jobs/steps, the failing test lines pulled from the logs (best +# effort), how many times THIS PR has already failed in the queue in the last +# 24 h, and the queue-wide failure count. The checklist tells the author +# (human or agent) to diagnose before re-queueing. The comment is the +# machine-readable signal the PM dispatch loop can key on, and the cross-PR +# flake evidence lives in these comments: the same test name appearing in two +# unrelated PRs' triage comments is a confirmed flake. +# +# Fires on conclusion == failure ONLY. 'cancelled' is the queue evicting an +# entry because something AHEAD of it failed (or a manual cancel) — it says +# nothing about this PR, so it gets no comment (the same run-lifecycle +# reasoning as dogfood-gate's cancelled handling in ci.yml, from the other +# side). +# +# workflow_run executes in the DEFAULT branch's context: this file must be on +# main before it fires, it never checks out or runs PR code, and it holds the +# minimum permissions (actions: read for logs, pull-requests: write for the +# comment). + +on: + workflow_run: + workflows: [CI] + types: [completed] + +permissions: {} + +jobs: + triage: + name: Comment queue-failure triage on the PR + if: >- + github.event.workflow_run.event == 'merge_group' && + github.event.workflow_run.conclusion == 'failure' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: read + pull-requests: write + steps: + - name: Post the triage comment + uses: actions/github-script@v9 + with: + script: | + const run = context.payload.workflow_run; + const { owner, repo } = context.repo; + + // Queue branches are named gh-readonly-queue//pr--. + const m = /^gh-readonly-queue\/.+\/pr-(\d+)-[0-9a-f]{40}$/.exec(run.head_branch ?? ''); + if (!m) { + core.info(`head_branch '${run.head_branch}' is not a merge-queue branch — nothing to do.`); + return; + } + const prNumber = Number(m[1]); + const marker = ``; + + // Idempotency: workflow_run deliveries can repeat; one comment per run. + const existing = await github.rest.issues.listComments({ + owner, repo, issue_number: prNumber, per_page: 100, + }); + if (existing.data.some((c) => (c.body ?? '').includes(marker))) { + core.info('triage comment for this run already exists — skipping.'); + return; + } + + // Failed jobs and their failed steps. + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner, repo, run_id: run.id, per_page: 100, + }); + const failedJobs = jobs.filter((j) => j.conclusion === 'failure'); + + // Best-effort log harvest: the lines a human would grep for first. + // Aggregate gate jobs (Test Core / Dogfood Regression Gate) fail + // with no information of their own, so prefer real jobs when both + // are present. A 4xx on the logs endpoint degrades to names only. + const FAIL_LINE = /(?:^|[\s|])(?:✗|×|✕|FAIL\s|AssertionError|STALL)/; + const informative = failedJobs.filter((j) => (j.steps ?? []).some( + (s) => s.conclusion === 'failure' && !/^Verify .* results$/.test(s.name))); + const details = []; + for (const job of (informative.length ? informative : failedJobs).slice(0, 4)) { + const steps = (job.steps ?? []) + .filter((s) => s.conclusion === 'failure') + .map((s) => s.name); + let lines = []; + try { + const res = await github.request( + 'GET /repos/{owner}/{repo}/actions/jobs/{job_id}/logs', + { owner, repo, job_id: job.id }); + const text = typeof res.data === 'string' + ? res.data + : Buffer.from(res.data).toString('utf8'); + lines = text.split('\n') + .map((l) => l.replace(/^[^ ]*Z /, '')) + .filter((l) => FAIL_LINE.test(l)) + .map((l) => l.trim().slice(0, 200)) + .slice(0, 12); + } catch (e) { + core.info(`logs unavailable for job ${job.id}: ${e.message}`); + } + details.push({ name: job.name, url: job.html_url, steps, lines }); + } + + // History: this PR's earlier queue failures + queue-wide count, 24 h. + const since = new Date(Date.now() - 24 * 3600 * 1000).toISOString(); + const recent = await github.paginate(github.rest.actions.listWorkflowRunsForRepo, { + owner, repo, event: 'merge_group', created: `>=${since}`, per_page: 100, + }); + const ciRuns = recent.filter((r) => r.workflow_id === run.workflow_id && r.id !== run.id); + const priorFailuresThisPr = ciRuns.filter((r) => + r.conclusion === 'failure' && (r.head_branch ?? '').includes(`/pr-${prNumber}-`)).length; + const queueFailures24h = ciRuns.filter((r) => r.conclusion === 'failure').length; + + const jobSections = details.map((d) => { + const head = `- **[${d.name}](${d.url})** — 失败步骤: ${d.steps.join('、') || '(无步骤级结论)'}`; + return d.lines.length + ? `${head}\n\n \`\`\`\n ${d.lines.join('\n ')}\n \`\`\`` + : `${head}（日志不可读，点进 job 看）`; + }).join('\n'); + + const flakeHint = priorFailuresThisPr > 0 + ? `⚠️ **本 PR 过去 24h 已在队列失败 ${priorFailuresThisPr} 次（不含本次）。** 内容未变而反复失败 ⇒ 高度怀疑 flaky 测试或与同组 PR 的语义冲突，重排不解决。` + : '本 PR 过去 24h 无队列失败记录（首次）。'; + + const body = [ + '### ⛔ merge queue 构建失败 — 先分诊，再决定要不要重排', + '', + `队列构建 [${run.id}](${run.html_url}) 红了。队列跑的是**全量**套件（PR 侧 CI 只跑 affected 子集），`, + '所以失败的测试可能在本 PR 没碰过的包里 —— 那不是重排能修的。每次盲目重排都会让排在后面的所有 PR 重建一轮。', + '', + '**失败的 job（日志抽取，best effort）：**', + '', + jobSections || '- （没拿到 job 级信息，点上面的 run 链接看）', + '', + '**历史信号：**', + `- ${flakeHint}`, + `- 过去 24h 队列共有 ${queueFailures24h} 个失败构建（不含本次）。`, + '', + '**分诊清单：**', + '1. 失败测试在本 PR 改动的包里 → 真回归，修 PR。', + '2. 失败测试与本 PR 无关 → 在其他 PR 的同类评论里搜同名测试；出现过 ⇒ flaky 实锤，开 issue 修/隔离那条测试。修好前重排只会再烧一轮全队列。', + '3. 两者都不是 → 可能与同组 PR 语义冲突；等前面的 PR 落地或失败出队后再重排一次即可，不要连续重排。', + '', + marker, + '---', + '_Generated by [Claude Code](https://claude.ai/code) · merge-queue-triage workflow (#4859)_', + ].join('\n'); + + await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body }); + core.info(`triage comment posted on #${prNumber}.`);