check_required_contexts.py

#!/usr/bin/env python3
"""Verify required-status-check drift between workflows and branch-protection.

Walks every workflow under .github/workflows/ (minus an EXEMPT list),
collects each job's display name, and compares to the ``contexts`` arrays
in .github/branch-protection/{main,develop}.json.

Fails CI when:
- A job exists in a workflow but is missing from the contexts list
  (the drift class that lets a new check run without being required).
- A context is listed but no workflow job has that display name
  (stale names that silently stop blocking merges).

Usage (from repo root):

    uv run --with pyyaml python .github/scripts/check_required_contexts.py
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

import yaml

WORKFLOWS_DIR = Path(".github/workflows")
PROTECTION_DIR = Path(".github/branch-protection")

# Workflows whose jobs are intentionally not required status checks.
# Keep the list short and explain each entry.
EXEMPT_WORKFLOWS: dict[str, str] = {
    "codeql.yml": (
        "Placeholder — gated to workflow_dispatch pending GHAS subscription."
    ),
    "branch-protection.yml": (
        "Runs only on main / schedule; never appears on PR check sets."
    ),
    "eval-nightly.yml": (
        "Scheduled / workflow_dispatch only; runs against the configured LLM"
        " provider and never appears on PR check sets."
    ),
    "artifact-cleanup.yml": (
        "Scheduled / workflow_dispatch only; never appears on PR check sets."
    ),
    "release-drafter.yml": (
        "Runs on push to main + PR label events; drafts release notes,"
        " never appears on PR check sets."
    ),
    "release.yml": (
        "Tag-triggered (v*.*.*); builds image, generates SBOM, publishes"
        " release. Never appears on PR check sets."
    ),
}


def job_display_names(workflow_path: Path) -> list[str]:
    """Return the display name for every job in a workflow file.

    GitHub's status-check context is the job's ``name:`` field; when unset
    it falls back to the job key. Matrix jobs expand into one context per
    matrix dimension, but the base name is what the contexts array stores.
    """
    data = yaml.safe_load(workflow_path.read_text())
    if not isinstance(data, dict) or "jobs" not in data:
        return []
    return [job.get("name") or key for key, job in data["jobs"].items()]


def collect_actual_contexts() -> set[str]:
    """Every job display name from every non-exempt workflow."""
    contexts: set[str] = set()
    for path in sorted(WORKFLOWS_DIR.glob("*.yml")):
        if path.name in EXEMPT_WORKFLOWS:
            continue
        contexts.update(job_display_names(path))
    return contexts


def check_protection_file(path: Path, actual: set[str]) -> bool:
    """Compare one branch-protection file's contexts array to actual jobs.

    Returns True when they match exactly; logs GitHub-actions-style errors
    and returns False on any drift.
    """
    data = json.loads(path.read_text())
    declared: set[str] = set(data["required_status_checks"]["contexts"])

    missing = actual - declared
    extra = declared - actual

    if not (missing or extra):
        return True

    print(f"::error file={path}::drift detected in required_status_checks.contexts")
    for name in sorted(missing):
        print(f"::error file={path}::  + MISSING (job exists, not listed): {name!r}")
    for name in sorted(extra):
        print(f"::error file={path}::  - STALE   (listed, no such job): {name!r}")
    return False


def main() -> int:
    actual = collect_actual_contexts()
    if not actual:
        print("::error::No workflow jobs discovered — check WORKFLOWS_DIR path.")
        return 1

    all_ok = True
    for json_path in sorted(PROTECTION_DIR.glob("*.json")):
        all_ok &= check_protection_file(json_path, actual)

    if all_ok:
        print("Branch-protection contexts are in sync with workflow jobs.")
        print(
            f"  {len(actual)} required job(s) across "
            f"{sum(1 for _ in PROTECTION_DIR.glob('*.json'))} branch(es)."
        )
    else:
        print(
            "\nFix: update .github/branch-protection/{main,develop}.json "
            "contexts arrays to match the current workflow jobs, or add the "
            "workflow to EXEMPT_WORKFLOWS in this script if intentionally "
            "non-required."
        )
    return 0 if all_ok else 1


if __name__ == "__main__":
    sys.exit(main())