ci: harden release gate required check matching

Author: justin808

.claude/hooks/main-ci-status.sh

@@ -158,32 +158,49 @@ checks_jsonl=$(gh api \
 # that each define a `detect-changes` job). Mirrors the Ruby dedup in
 # `validate_main_ci_status!` (rakelib/release.rake). Keep the two in sync.
 checks_json=$(echo "${checks_jsonl}" | jq -s '
-  [.[] | {id, name, status, conclusion, html_url, suite_id: (.check_suite.id // .id)}]
-  | group_by([.suite_id, .name])
+  [.[] | {id, name, status, conclusion, html_url, suite_id: (.check_suite.id // .id), app_id: (.app.id // null)}]
+  | group_by([.suite_id, .name, .app_id])
   | map(max_by(.id))
 ' 2>/dev/null) || fail_open "jq slurp failed"
 
 required_json=$(gh api \
   "repos/${repo_slug}/branches/main/protection/required_status_checks" \
-  --jq '(.contexts // []) + (.checks // [] | map(.context)) | unique' \
+  --jq '{contexts: (.contexts // []), checks: (.checks // [] | map({context, app_id}))}' \
   2>/dev/null || echo "null")
 case "${required_json}" in
-  \[*\]) ;;
+  \{*\}) ;;
   *) required_json="null" ;;
 esac
 
 # Aggregate counts with jq. `success`, `skipped`, `neutral` are all "passing".
 # Anything completed with another conclusion is a failure. Anything not yet
 # completed is in_progress.
 summary=$(echo "${checks_json}" | jq -r --argjson required_names "${required_json}" '
+  def app_wildcard($app_id): $app_id == null or $app_id == -1;
+  def check_matches($required):
+    .name == $required.context and (app_wildcard($required.app_id) or .app_id == $required.app_id);
+  def check_label:
+    if app_wildcard(.app_id) then .context else "\(.context) (app_id: \(.app_id))" end;
+
   . as $all
-  | ($all | map(.name)) as $observed_names
   | {
       total: length,
       passed: [.[] | select(.status == "completed" and (.conclusion | IN("success", "skipped", "neutral")))] | length,
       failed: [.[] | select(.status == "completed" and (.conclusion | IN("success", "skipped", "neutral") | not))],
       in_progress: [.[] | select(.status != "completed")],
-      missing_required: (if $required_names == null then [] else ($required_names - $observed_names) end)
+      missing_required: (
+        if $required_names == null then []
+        else
+          # NOTE: commit statuses are not fetched here; this hook is a fail-open
+          # display tool. The Ruby release gate owns legacy status enforcement.
+          (($required_names.contexts // [])
+            | map(select(. as $context | ($all | any(.name == $context) | not))))
+          +
+          (($required_names.checks // [])
+            | map(select(. as $required | ($all | any(. as $run | ($run | check_matches($required))) | not)))
+            | map(check_label))
+        end
+      )
     }
   | "TOTAL=\(.total)",
     "PASSED=\(.passed)",

rakelib/release.rake

@@ -608,9 +608,7 @@ def fetch_main_ci_checks(monorepo_root:, allow_override: false, dry_run: false)
   end
 
   begin
-    check_runs = output.lines.reject { |line| line.strip.empty? }.map do |line|
-      JSON.parse(line)
-    end
+    check_runs = parse_gh_jsonl(output)
   rescue JSON::ParserError => e
     handle_main_ci_status_violation!(
       message: "❌ Failed to parse check_runs response from gh: #{e.message}\n\nOutput:\n#{output}",
@@ -624,14 +622,93 @@ def fetch_main_ci_checks(monorepo_root:, allow_override: false, dry_run: false)
 end
 # rubocop:enable Metrics/MethodLength
 
+def parse_gh_jsonl(output)
+  output.lines.reject { |line| line.strip.empty? }.map do |line|
+    JSON.parse(line)
+  end
+end
+
+def fetch_main_commit_statuses(repo_slug:, sha:, allow_override:, dry_run:)
+  api_path = "repos/#{repo_slug}/commits/#{sha}/statuses"
+
+  begin
+    output, status = Open3.capture2e(
+      "gh", "api", "--paginate", "--jq", ".[]", api_path
+    )
+  rescue Errno::ENOENT
+    handle_main_ci_status_violation!(
+      message: "❌ GitHub CLI (`gh`) is not installed. Install it from https://cli.github.com/ and retry.",
+      allow_override: allow_override,
+      dry_run: dry_run
+    )
+    return nil
+  end
+
+  unless status.success?
+    handle_main_ci_status_violation!(
+      message: "❌ Unable to query GitHub Statuses API for #{sha}.\n\n#{output}",
+      allow_override: allow_override,
+      dry_run: dry_run
+    )
+    return nil
+  end
+
+  begin
+    parse_gh_jsonl(output)
+  rescue JSON::ParserError => e
+    handle_main_ci_status_violation!(
+      message: "❌ Failed to parse statuses response from gh: #{e.message}\n\nOutput:\n#{output}",
+      allow_override: allow_override,
+      dry_run: dry_run
+    )
+    nil
+  end
+end
+
+def normalize_status_as_check_run(status)
+  state = status["state"]
+  {
+    "id" => status["id"],
+    "name" => status["context"],
+    "status" => state == "pending" ? "pending" : "completed",
+    "conclusion" => state == "success" ? "success" : (state || "error"),
+    "html_url" => status["target_url"] || status["url"]
+  }
+end
+
+def latest_commit_statuses(statuses)
+  statuses
+    .group_by { |status| status["context"] }
+    .map { |_context, context_statuses| context_statuses.max_by { |status| status["id"].to_i } }
+end
+
+def normalize_required_check_entries(checks)
+  Array(checks).filter_map do |check|
+    context = check["context"].to_s
+    next if context.empty?
+
+    { context: context, app_id: check["app_id"]&.to_i }
+  end.uniq
+end
+
+def normalize_required_checks_payload(parsed)
+  return nil unless parsed.is_a?(Hash)
+
+  contexts = Array(parsed["contexts"]).map(&:to_s).reject(&:empty?).uniq
+  checks = normalize_required_check_entries(parsed["checks"])
+
+  # No required names parseable is treated the same as "no branch protection
+  # visible" — fail-safe to evaluating every check run.
+  contexts.empty? && checks.empty? ? nil : { contexts: contexts, checks: checks }
+end
+
 def required_check_names_for_main(monorepo_root:, repo_slug: nil)
   repo_slug ||= github_repo_slug(monorepo_root)
   api_path = "repos/#{repo_slug}/branches/main/protection/required_status_checks"
-  # Combine the legacy `contexts` list (older protection rules) with the newer
-  # `checks[].context` list. Branch protection set up via the `checks` API
-  # leaves `contexts` as `[]`, so reading only `contexts` would yield an empty
-  # array and trip the `:no_required_checks` abort path even when CI is green.
-  jq_query = "(.contexts // []) + (.checks // [] | map(.context)) | unique"
+  # Keep legacy `contexts` separate from modern `checks` entries. Modern
+  # required checks can be pinned to a GitHub App via `app_id`; legacy contexts
+  # may be satisfied by either a Checks API run or a commit-status context.
+  jq_query = "{contexts: (.contexts // []), checks: (.checks // [] | map({context, app_id}))}"
   # Precondition: `fetch_main_ci_checks` already verified `gh` is installed
   # before `validate_main_ci_status!` calls this helper. The remaining failure
   # mode here is "branch protection unknown", which returns nil so the caller
@@ -644,16 +721,54 @@ def required_check_names_for_main(monorepo_root:, repo_slug: nil)
 
   begin
     parsed = JSON.parse(output)
-    return nil unless parsed.is_a?(Array)
-
-    # Empty array (no required names parseable) is treated the same as "no
-    # branch protection visible" — fail-safe to evaluating every check run.
-    parsed.empty? ? nil : parsed
+    normalize_required_checks_payload(parsed)
   rescue JSON::ParserError
     nil
   end
 end
 
+def check_run_app_id(run)
+  app_id = run.dig("app", "id")
+  app_id&.to_i
+end
+
+def required_check_app_wildcard?(app_id)
+  app_id.nil? || app_id.to_i == -1
+end
+
+def required_check_matches_run?(required_check, run)
+  required_check[:context] == run["name"] &&
+    (required_check_app_wildcard?(required_check[:app_id]) || required_check[:app_id].to_i == check_run_app_id(run))
+end
+
+def required_check_label(required_check)
+  return required_check[:context] if required_check_app_wildcard?(required_check[:app_id])
+
+  "#{required_check[:context]} (app_id: #{required_check[:app_id]})"
+end
+
+def required_check_labels(required_checks)
+  (required_checks[:contexts] + required_checks[:checks].map { |check| required_check_label(check) }).uniq
+end
+
+def missing_required_check_labels(required_checks:, check_runs:, legacy_status_runs:)
+  missing_modern = required_checks[:checks].reject do |required_check|
+    check_runs.any? { |run| required_check_matches_run?(required_check, run) }
+  end
+  missing_legacy = required_checks[:contexts].reject do |context|
+    check_runs.any? { |run| run["name"] == context } || legacy_status_runs.any? { |run| run["name"] == context }
+  end
+
+  (missing_legacy + missing_modern.map { |check| required_check_label(check) }).uniq
+end
+
+def legacy_status_runs_for_required_contexts(required_checks:, check_runs:, statuses:)
+  latest_commit_statuses(statuses)
+    .select { |status| required_checks[:contexts].include?(status["context"]) }
+    .reject { |status| check_runs.any? { |run| run["name"] == status["context"] } }
+    .map { |status| normalize_status_as_check_run(status) }
+end
+
 def format_main_ci_status_violation(kind:, short_sha:, runs:) # rubocop:disable Metrics/CyclomaticComplexity
   header = case kind
            when :in_progress
@@ -721,15 +836,6 @@ def validate_main_ci_status!(monorepo_root:, is_prerelease:, allow_override:, dr
   repo_slug = data[:repo_slug]
   check_runs = data[:check_runs]
 
-  if check_runs.empty?
-    handle_main_ci_status_violation!(
-      message: format_main_ci_status_violation(kind: :no_checks, short_sha: short_sha, runs: nil),
-      allow_override: allow_override,
-      dry_run: dry_run
-    )
-    return
-  end
-
   # Collapse multiple runs per (check_suite_id, name) to the most recent
   # attempt (highest check_run id). The key intentionally includes
   # check_suite_id so we only collapse *true* reruns (same workflow run,
@@ -745,7 +851,7 @@ def validate_main_ci_status!(monorepo_root:, is_prerelease:, allow_override:, dr
   # another. The GitHub Actions Checks API always populates `check_suite`,
   # so this only matters for external check integrations.
   check_runs = check_runs
-               .group_by { |run| [run.dig("check_suite", "id") || run["id"], run["name"]] }
+               .group_by { |run| [run.dig("check_suite", "id") || run["id"], run["name"], check_run_app_id(run)] }
                .map { |_key, runs| runs.max_by { |run| run["id"].to_i } }
 
   # Always query branch-protection required checks (when configured) so the
@@ -756,12 +862,57 @@ def validate_main_ci_status!(monorepo_root:, is_prerelease:, allow_override:, dr
   required_args = { monorepo_root: monorepo_root }
   required_args[:repo_slug] = repo_slug if repo_slug
   required_names = required_check_names_for_main(**required_args)
+  legacy_status_runs = []
+  if required_names && required_names[:contexts].any?
+    statuses = fetch_main_commit_statuses(
+      repo_slug: repo_slug,
+      sha: sha,
+      allow_override: allow_override,
+      dry_run: dry_run
+    )
+    return if statuses.nil?
+
+    legacy_status_runs = legacy_status_runs_for_required_contexts(
+      required_checks: required_names,
+      check_runs: check_runs,
+      statuses: statuses
+    )
+  end
+
+  if check_runs.empty? && legacy_status_runs.empty?
+    handle_main_ci_status_violation!(
+      message: format_main_ci_status_violation(kind: :no_checks, short_sha: short_sha, runs: nil),
+      allow_override: allow_override,
+      dry_run: dry_run
+    )
+    return
+  end
+
   evaluated = if is_prerelease && required_names
-                check_runs.select { |run| required_names.include?(run["name"]) }
+                check_runs.select do |run|
+                  required_names[:contexts].include?(run["name"]) ||
+                    required_names[:checks].any? { |required_check| required_check_matches_run?(required_check, run) }
+                end + legacy_status_runs
               else
-                check_runs
+                check_runs + legacy_status_runs
               end
 
+  # Report visible failures before missing/in-progress runs. If both are
+  # present, the operator needs to know about the failure right away; this also
+  # prevents same-label legacy/modern required checks from hiding a failed
+  # legacy status behind a "missing required" message.
+  failed = evaluated.select do |run|
+    run["status"] == "completed" && !CI_PASSING_CONCLUSIONS.include?(run["conclusion"])
+  end
+  if failed.any?
+    handle_main_ci_status_violation!(
+      message: format_main_ci_status_violation(kind: :failed, short_sha: short_sha, runs: failed),
+      allow_override: allow_override,
+      dry_run: dry_run
+    )
+    return
+  end
+
   # When branch protection lists required checks, treat any missing required
   # check as blocking — for stable AND prerelease. Branch protection would
   # refuse the merge in this state, so a release that ignored the gap would
@@ -771,43 +922,31 @@ def validate_main_ci_status!(monorepo_root:, is_prerelease:, allow_override:, dr
   # required workflows ran, others never registered — usually a renamed or
   # deleted workflow that branch protection still requires).
   unless required_names.nil?
-    observed_names = check_runs.map { |run| run["name"] }
-    missing_names = required_names - observed_names
-    if missing_names.length == required_names.length
+    required_labels = required_check_labels(required_names)
+    missing_names = missing_required_check_labels(
+      required_checks: required_names,
+      check_runs: check_runs,
+      legacy_status_runs: legacy_status_runs
+    )
+    if missing_names.length == required_labels.length
       handle_main_ci_status_violation!(
         message: format_main_ci_status_violation(kind: :no_required_checks, short_sha: short_sha, runs: nil) +
-                 "\nRequired: #{required_names.join(', ')}",
+                 "\nRequired: #{required_labels.join(', ')}",
         allow_override: allow_override,
         dry_run: dry_run
       )
       return
     elsif missing_names.any?
       handle_main_ci_status_violation!(
         message: format_main_ci_status_violation(kind: :missing_required_checks, short_sha: short_sha, runs: nil) +
-                 "\nRequired: #{required_names.join(', ')}\nMissing: #{missing_names.join(', ')}",
+                 "\nRequired: #{required_labels.join(', ')}\nMissing: #{missing_names.join(', ')}",
         allow_override: allow_override,
         dry_run: dry_run
       )
       return
     end
   end
 
-  # Report failures before in-progress runs. If both are present, the operator
-  # needs to know about the failure right away — telling them to "wait or
-  # override" would just make them wait and re-run before seeing the real
-  # blocker.
-  failed = evaluated.select do |run|
-    run["status"] == "completed" && !CI_PASSING_CONCLUSIONS.include?(run["conclusion"])
-  end
-  if failed.any?
-    handle_main_ci_status_violation!(
-      message: format_main_ci_status_violation(kind: :failed, short_sha: short_sha, runs: failed),
-      allow_override: allow_override,
-      dry_run: dry_run
-    )
-    return
-  end
-
   in_progress = evaluated.select { |run| CI_INCOMPLETE_STATUSES.include?(run["status"]) }
   if in_progress.any?
     handle_main_ci_status_violation!(