From 7358e6d36f5f31d5e23526502af6a0a7edbe3be0 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 30 Apr 2026 13:36:55 -0700 Subject: [PATCH 1/8] Don't promote web-flow to author for non-Copilot bot PRs The PR dashboard's delegator lookup was applied to any bot author. For Renovate/Dependabot PRs, the first commit's committer is web-flow (GitHub's web-UI signer), which slipped past the bot filters and was wrongly surfaced as the human author. Scope the lookup to Copilot SWE-agent authors only, where there is actually a human delegator to find. --- .github/scripts/pull-request-dashboard.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/scripts/pull-request-dashboard.py b/.github/scripts/pull-request-dashboard.py index 7b5c5a855ea7..012e469e7a39 100644 --- a/.github/scripts/pull-request-dashboard.py +++ b/.github/scripts/pull-request-dashboard.py @@ -250,6 +250,12 @@ def role_for(login: str, author: str, reviewers: set[str]) -> str: # fix-up) pass through and are eligible to be picked as the delegator. _BOT_COMMITTER_LOGINS = {"copilot"} +# PR author logins that delegate work to a human (the Copilot SWE-agent +# opens the PR but a human triggered it). Only for these authors do we look +# up a human delegator from the first commit's committer. For other bots +# (renovate, dependabot, etc.) we keep the bot as the author. +_DELEGATING_BOT_AUTHORS = {"app/copilot-swe-agent", "copilot"} + def _is_bot_login(login: str) -> bool: if not login: @@ -312,9 +318,10 @@ def fetch_pr_context( # For Copilot SWE-agent PRs the API author is the bot; surface the human # who delegated the task so reviews/comments by that person are classified - # as "author" activity instead of "approver". + # as "author" activity instead of "approver". Other bot authors + # (renovate, dependabot, ...) have no human delegator, so we keep the bot. delegator = "" - if _is_bot_login(author): + if author.lower() in _DELEGATING_BOT_AUTHORS: delegator = detect_human_delegator(commits) if delegator: author = delegator From 7ea90352435a2a42f5ff23cfec590fcca7ea9621 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 30 Apr 2026 13:50:13 -0700 Subject: [PATCH 2/8] Don't pass CI/conflict status to the dashboard LLM CI status and merge-conflict status are already shown deterministically in dedicated columns. Including them in the LLM context tempted the model to bucket fresh PRs with pending CI as 'external' (waiting on something out of band) instead of 'approver' (waiting for review). Strip CI checks summary, mergeable/mergeStateStatus, and the related pre-computed signals from the rendered context. Tighten the prompt to clarify that 'external' requires explicit conversational evidence of a cross-repo dependency. --- .github/scripts/pull-request-dashboard.py | 34 ++++++++--------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/.github/scripts/pull-request-dashboard.py b/.github/scripts/pull-request-dashboard.py index 012e469e7a39..0d4265f863fb 100644 --- a/.github/scripts/pull-request-dashboard.py +++ b/.github/scripts/pull-request-dashboard.py @@ -49,21 +49,27 @@ the pull request dashboard. Decide who needs to act next on this PR using ONLY the context below. \ -Merge-conflict and CI-failure status are shown in separate columns of the \ -dashboard, so do NOT use those alone to decide; focus on the conversation. +The context deliberately omits CI status and merge-conflict status — \ +those are shown in separate deterministic columns of the dashboard. Do \ +not infer them or factor them into your decision; focus on the \ +conversation (comments, reviews, commits). Guidelines: - If the latest substantive activity is from the AUTHOR and there is an \ outstanding approver question, an approver should respond next. - If an approver's most recent comment asks for a specific change and the \ author has not responded, the author should act next. + - Use "external" ONLY when the conversation explicitly indicates the PR \ +is blocked on something outside this repo (e.g., an upstream PR, a \ +spec change, a release in another project). A new PR with no reviews \ +yet is NOT external — it is waiting on an approver. Respond with a single JSON object and nothing else (no prose, no fences): {{"side": "approver" | "author" | "external"}} Where: - "approver" = an approver should act (review, approve, request changes, decide to close) - - "author" = the PR author should act (respond, rebase, fix CI) + - "author" = the PR author should act (respond, rebase) - "external" = waiting on something outside this repo (upstream PR, etc.) ---BEGIN CONTEXT--- @@ -488,8 +494,8 @@ def render_context(ctx: dict[str, Any]) -> str: f"@{author} is treated as the effective author for triage.)" ) lines.append( - f"State: open | draft={pr.get('isDraft')} | mergeable={pr.get('mergeable')} " - f"| mergeStateStatus={pr.get('mergeStateStatus')} | reviewDecision={pr.get('reviewDecision')}" + f"State: open | draft={pr.get('isDraft')} " + f"| reviewDecision={pr.get('reviewDecision')}" ) lines.append(f"Created: {pr.get('createdAt')} ({pr_age}d ago)") lines.append(f"Updated: {pr.get('updatedAt')} ({updated_age}d ago)") @@ -507,18 +513,6 @@ def render_context(ctx: dict[str, Any]) -> str: lines.append(truncate(pr.get("body") or "", 800)) lines.append("") - # Checks - lines.append("CI checks summary:") - lines.append( - f" successful={len(ctx['checks_successful'])} failing={len(ctx['checks_failing'])} " - f"pending={len(ctx['checks_pending'])} skipped={len(ctx['checks_skipped'])}" - ) - for c in ctx["checks_failing"][:10]: - lines.append(f" FAIL: {c.get('name')} ({c.get('workflow') or ''})") - for c in ctx["checks_pending"][:5]: - lines.append(f" PENDING: {c.get('name')}") - lines.append("") - # Commits lines.append(f"Last {len(ctx['commits'])} commits (oldest first):") for c in ctx["commits"]: @@ -568,12 +562,6 @@ def render_context(ctx: dict[str, Any]) -> str: signals: list[str] = [] if pr.get("isDraft"): signals.append("PR is a draft") - if pr.get("mergeable") == "CONFLICTING" or pr.get("mergeStateStatus") == "DIRTY": - signals.append("merge conflict with base") - if ctx["checks_failing"]: - signals.append(f"{len(ctx['checks_failing'])} CI checks failing") - if pr.get("reviewDecision") == "APPROVED" and not ctx["checks_failing"] and pr.get("mergeStateStatus") == "CLEAN": - signals.append("approved + clean + green = mergeable") if last_role == "author" and ctx["approvers"]: signals.append("latest substantive activity is from author after approvals") if last_role == "approver" and last_sub: From 39297bbf23f0f1493ebb2ba5a1542b7c4de10b04 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 30 Apr 2026 13:57:31 -0700 Subject: [PATCH 3/8] Pass CI status to dashboard LLM as a single boolean Knowing whether CI is currently failing helps the model route renovate-style PRs whose checks have broken (e.g. an upstream tool release) into the 'external' bucket. But surfacing pending vs failing vs passing as a tri-state was tempting the model to bucket fresh PRs with pending CI as 'external'. Compromise: emit a single 'CI failing: yes/no' line, with pending treated as not-failing. --- .github/scripts/pull-request-dashboard.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/scripts/pull-request-dashboard.py b/.github/scripts/pull-request-dashboard.py index 0d4265f863fb..174185cf5a4b 100644 --- a/.github/scripts/pull-request-dashboard.py +++ b/.github/scripts/pull-request-dashboard.py @@ -49,10 +49,10 @@ the pull request dashboard. Decide who needs to act next on this PR using ONLY the context below. \ -The context deliberately omits CI status and merge-conflict status — \ -those are shown in separate deterministic columns of the dashboard. Do \ -not infer them or factor them into your decision; focus on the \ -conversation (comments, reviews, commits). +Merge-conflict status is shown in a separate deterministic column of the \ +dashboard — do not infer it. CI is summarized as a single boolean \ +(failing yes/no); pending checks are treated as not-failing. Focus on \ +the conversation (comments, reviews, commits). Guidelines: - If the latest substantive activity is from the AUTHOR and there is an \ @@ -69,7 +69,7 @@ Where: - "approver" = an approver should act (review, approve, request changes, decide to close) - - "author" = the PR author should act (respond, rebase) + - "author" = the PR author should act (respond, rebase, fix CI) - "external" = waiting on something outside this repo (upstream PR, etc.) ---BEGIN CONTEXT--- @@ -497,6 +497,8 @@ def render_context(ctx: dict[str, Any]) -> str: f"State: open | draft={pr.get('isDraft')} " f"| reviewDecision={pr.get('reviewDecision')}" ) + ci_failing = bool(ctx["checks_failing"]) + lines.append(f"CI failing: {'yes' if ci_failing else 'no'}") lines.append(f"Created: {pr.get('createdAt')} ({pr_age}d ago)") lines.append(f"Updated: {pr.get('updatedAt')} ({updated_age}d ago)") lines.append(f"Size: +{pr.get('additions', 0)}/-{pr.get('deletions', 0)} across {pr.get('changedFiles', 0)} files") From c00be24b6307d33a09552578458a7495ed5b83c7 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 30 Apr 2026 14:01:54 -0700 Subject: [PATCH 4/8] Tell dashboard LLM that CI failure alone isn't author/external PRs can still be reviewed and approved while CI is failing, so CI failure on its own should not shift the bucket. Treat the boolean only as weak supporting evidence; the conversation drives the decision. --- .github/scripts/pull-request-dashboard.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/scripts/pull-request-dashboard.py b/.github/scripts/pull-request-dashboard.py index 174185cf5a4b..891e4989cf71 100644 --- a/.github/scripts/pull-request-dashboard.py +++ b/.github/scripts/pull-request-dashboard.py @@ -51,8 +51,11 @@ Decide who needs to act next on this PR using ONLY the context below. \ Merge-conflict status is shown in a separate deterministic column of the \ dashboard — do not infer it. CI is summarized as a single boolean \ -(failing yes/no); pending checks are treated as not-failing. Focus on \ -the conversation (comments, reviews, commits). +(failing yes/no); pending checks are treated as not-failing. CI failure \ +on its own is NOT a reason to assign the PR to the author or to external: \ +PRs can still be reviewed and approved while CI is failing. Treat CI \ +status only as weak supporting evidence and focus on the conversation \ +(comments, reviews, commits). Guidelines: - If the latest substantive activity is from the AUTHOR and there is an \ @@ -62,7 +65,8 @@ - Use "external" ONLY when the conversation explicitly indicates the PR \ is blocked on something outside this repo (e.g., an upstream PR, a \ spec change, a release in another project). A new PR with no reviews \ -yet is NOT external — it is waiting on an approver. +yet is NOT external — it is waiting on an approver. CI failing alone \ +is NOT external. Respond with a single JSON object and nothing else (no prose, no fences): {{"side": "approver" | "author" | "external"}} From c4ab11bf25f308aa75f7fb356ae91fc8cb4c05fc Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 30 Apr 2026 14:04:51 -0700 Subject: [PATCH 5/8] Allow CI failure as evidence for 'external' bucket Renovate-style PRs whose CI breaks because of an upstream tool release legitimately belong in 'external'. The previous wording forbade CI failure as a signal for both author and external; loosen to forbid only the author shift. --- .github/scripts/pull-request-dashboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/pull-request-dashboard.py b/.github/scripts/pull-request-dashboard.py index 891e4989cf71..c4613c2c77aa 100644 --- a/.github/scripts/pull-request-dashboard.py +++ b/.github/scripts/pull-request-dashboard.py @@ -52,7 +52,7 @@ Merge-conflict status is shown in a separate deterministic column of the \ dashboard — do not infer it. CI is summarized as a single boolean \ (failing yes/no); pending checks are treated as not-failing. CI failure \ -on its own is NOT a reason to assign the PR to the author or to external: \ +on its own is NOT a reason to assign the PR to the author: \ PRs can still be reviewed and approved while CI is failing. Treat CI \ status only as weak supporting evidence and focus on the conversation \ (comments, reviews, commits). From d9141bd1f32ad2a50ea4053af142b6f6e2ff1037 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 30 Apr 2026 14:18:09 -0700 Subject: [PATCH 6/8] Tighten dashboard prompt: clearer rules with explicit ordering The previous prompt's guidelines were under-specified, so the model ran on intuition: when an approver had left review questions and the author hadn't replied (#16390), it picked 'approver' instead of 'author'. Restate the rules as an ordered first-match decision tree (external > author-owes-response > approver-default), with explicit edge cases: an approver *commit* does not put the ball in the author's court, and an approver comment linking to an upstream issue should bucket as external. --- .github/scripts/pull-request-dashboard.py | 33 ++++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/scripts/pull-request-dashboard.py b/.github/scripts/pull-request-dashboard.py index c4613c2c77aa..47b24b3fd9c0 100644 --- a/.github/scripts/pull-request-dashboard.py +++ b/.github/scripts/pull-request-dashboard.py @@ -57,16 +57,29 @@ status only as weak supporting evidence and focus on the conversation \ (comments, reviews, commits). -Guidelines: - - If the latest substantive activity is from the AUTHOR and there is an \ -outstanding approver question, an approver should respond next. - - If an approver's most recent comment asks for a specific change and the \ -author has not responded, the author should act next. - - Use "external" ONLY when the conversation explicitly indicates the PR \ -is blocked on something outside this repo (e.g., an upstream PR, a \ -spec change, a release in another project). A new PR with no reviews \ -yet is NOT external — it is waiting on an approver. CI failing alone \ -is NOT external. +The single most important signal is the latest substantive event in the \ +timeline. Apply these rules in order (first match wins): + + 1. EXTERNAL — Use "external" when the conversation explicitly indicates \ +the PR is blocked on something outside this repo (e.g., links to an \ +upstream PR/issue, "reported at ", a spec change, or a \ +release in another project). Look especially at the latest comments. \ +A new PR with no reviews yet is NOT external. CI failing alone is \ +NOT external unless an upstream cause is named. + 2. AUTHOR — If the latest substantive event is an approver review or \ +review-comment with content (a question, suggestion, change \ +request, clarification ask, or [APPROVED/CHANGES_REQUESTED] state) \ +and the AUTHOR has not posted any comment, review, or commit AFTER \ +it, the AUTHOR should act next. This holds even when the comment \ +is just a question or a soft suggestion — the ball is in the \ +author's court until they respond. (Note: a *commit* by an approver \ +does not count here — that's an approver pushing a fix, not asking \ +the author for something.) + 3. APPROVER — Otherwise, an APPROVER should act next. This includes: \ +fresh PRs with no reviews yet; PRs where the author has posted the \ +latest substantive event (comment, review, or commit) addressing \ +prior approver feedback; and PRs where an approver pushed the \ +latest commit (waiting for someone to review/merge). Respond with a single JSON object and nothing else (no prose, no fences): {{"side": "approver" | "author" | "external"}} From fc991664becc46f446a13fd16029a25e7661a26f Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 30 Apr 2026 14:21:37 -0700 Subject: [PATCH 7/8] Don't treat merge-from-base commits as substantive activity Pulling main into a PR branch (e.g. trask clicking 'Update branch' on #18090) was being recorded as the latest substantive event, masking earlier unanswered approver questions. Filter such commits out of the substantive timeline so the author still owes a response. --- .github/scripts/pull-request-dashboard.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/scripts/pull-request-dashboard.py b/.github/scripts/pull-request-dashboard.py index 47b24b3fd9c0..8aee0b27fc58 100644 --- a/.github/scripts/pull-request-dashboard.py +++ b/.github/scripts/pull-request-dashboard.py @@ -412,11 +412,29 @@ def fetch_pr_context( events = [e for e in events if e["ts"]] events.sort(key=lambda e: e["ts"]) + base_ref = pr.get("baseRefName") or "main" + merge_subject_prefixes = ( + f"merge branch '{base_ref}'", + f"merge remote-tracking branch 'upstream/{base_ref}'", + f"merge remote-tracking branch 'origin/{base_ref}'", + f"merge {base_ref} into", + ) + + def _is_merge_from_base(e: dict[str, Any]) -> bool: + if e["kind"] != "commit": + return False + subject = ((e.get("body") or "").splitlines() or [""])[0].strip().lower() + return any(subject.startswith(p) for p in merge_subject_prefixes) + # Last substantive event = last event whose body is non-empty OR whose - # kind is not "review:COMMENTED" (state changes always count). + # kind is not "review:COMMENTED" (state changes always count). Merges + # of the base branch into the PR don't count as substantive — they + # don't move the conversation forward. def is_substantive(e: dict[str, Any]) -> bool: if e["kind"].startswith("review:") and e["kind"] != "review:COMMENTED": return True + if _is_merge_from_base(e): + return False return bool((e.get("body") or "").strip()) substantive = [e for e in events if is_substantive(e)] From 38e122e2ca634d3585793f3b9871748d0fef3170 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 30 Apr 2026 14:24:45 -0700 Subject: [PATCH 8/8] Detect merge commits structurally via parent count Replace the substring-based 'Merge branch base into ...' filter with a check on the commit's parents array (>=2 parents = merge). The list endpoint returns parents per commit detail, which we already fetch in parallel for diffs. This catches merges with non-default subject lines and squash-merges from feature branches. --- .github/scripts/pull-request-dashboard.py | 26 +++++++---------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/.github/scripts/pull-request-dashboard.py b/.github/scripts/pull-request-dashboard.py index 8aee0b27fc58..d5cc0f1b321b 100644 --- a/.github/scripts/pull-request-dashboard.py +++ b/.github/scripts/pull-request-dashboard.py @@ -352,6 +352,7 @@ def fetch_pr_context( # Fetch per-commit diffs for the most recent commits, in parallel. recent_commits = commits[-MAX_COMMITS:] patches: dict[str, str] = {} + merge_shas: set[str] = set() if recent_commits: with ThreadPoolExecutor(max_workers=4) as pool: futs = { @@ -365,6 +366,8 @@ def fetch_pr_context( except Exception: detail = {} patches[sha] = format_commit_patch(detail, MAX_COMMIT_DIFF_CHARS) + if len(detail.get("parents") or []) >= 2: + merge_shas.add(sha) # Build unified activity timeline. events: list[dict[str, Any]] = [] @@ -386,6 +389,7 @@ def fetch_pr_context( "login": login, "body": body, "sha": sha_full[:7], + "is_merge": sha_full in merge_shas, }) for c in issue_comments: events.append({ @@ -412,28 +416,14 @@ def fetch_pr_context( events = [e for e in events if e["ts"]] events.sort(key=lambda e: e["ts"]) - base_ref = pr.get("baseRefName") or "main" - merge_subject_prefixes = ( - f"merge branch '{base_ref}'", - f"merge remote-tracking branch 'upstream/{base_ref}'", - f"merge remote-tracking branch 'origin/{base_ref}'", - f"merge {base_ref} into", - ) - - def _is_merge_from_base(e: dict[str, Any]) -> bool: - if e["kind"] != "commit": - return False - subject = ((e.get("body") or "").splitlines() or [""])[0].strip().lower() - return any(subject.startswith(p) for p in merge_subject_prefixes) - # Last substantive event = last event whose body is non-empty OR whose - # kind is not "review:COMMENTED" (state changes always count). Merges - # of the base branch into the PR don't count as substantive — they - # don't move the conversation forward. + # kind is not "review:COMMENTED" (state changes always count). Merge + # commits (≥2 parents — e.g. "Update branch" merging base into the PR) + # don't count as substantive: they don't move the conversation forward. def is_substantive(e: dict[str, Any]) -> bool: if e["kind"].startswith("review:") and e["kind"] != "review:COMMENTED": return True - if _is_merge_from_base(e): + if e.get("is_merge"): return False return bool((e.get("body") or "").strip())