Report nightly failures as issues only, dropping the Slack card

Author: aidangarske

.github/scripts/ci_triage.py

@@ -32,18 +32,12 @@
 AUTO_RETRY = os.environ.get("AUTO_RETRY", "false") == "true"
 DRY_RUN = os.environ.get("DRY_RUN", "true") == "true"
 RETRY_DEADLINE_S = int(os.environ.get("RETRY_DEADLINE_S", "3600"))
-SLACK_WEBHOOK = os.environ.get("SLACK_WEBHOOK_URL", "")
-# Separate from DRY_RUN: DRY_RUN also gates issue filing, so folding "no webhook"
-# into it would silently stop filing issues on any repo without the secret.
-SLACK_DRY = DRY_RUN or not SLACK_WEBHOOK
 
 API = "https://api.github.com"
 POLL_INTERVAL_S = 30
 JOBS_PER_PAGE = 50  # the jobs endpoint 502s at per_page=100 on large runs
 AI_MAX_CALLS = 15
 LOG_TAIL_CHARS = 4000
-RETRY_CODES = {429, 500, 502, 503, 504}
-SPARK = "▁▂▃▄▅▆▇█"
 
 
 def gh(path, method="GET", body=None, retry=True):
@@ -528,226 +522,6 @@ def run_url():
     return f"https://github.com/{REPO}/actions/runs/{RUN_ID}"
 
 
-# --- Slack: the only "did the nightly pass" signal ---
-# Ported from wolfProvider's osp-triage.py. Green is otherwise communicated by
-# absence, which is indistinguishable from the nightly never running.
-
-
-def backoff_delay(err, attempt):
-    ra = err.headers.get("Retry-After", "") if hasattr(err, "headers") else ""
-    return int(ra) if ra.isdigit() else 2 ** attempt
-
-
-def section(text):
-    return {"type": "section", "text": {"type": "mrkdwn", "text": text}}
-
-
-def chunk_lines(lines, limit=2900):
-    """Slack hard-caps a block at 3000 chars."""
-    buf, size = [], 0
-    for ln in lines:
-        if size + len(ln) + 1 > limit and buf:
-            yield "\n".join(buf)
-            buf, size = [], 0
-        buf.append(ln)
-        size += len(ln) + 1
-    if buf:
-        yield "\n".join(buf)
-
-
-def render_text(blocks):
-    out = []
-    for b in blocks:
-        t = b.get("type")
-        if t == "header":
-            out.append(b["text"]["text"])
-        elif t == "divider":
-            out.append("-" * 48)
-        elif t == "section" and "fields" in b:
-            out.append("  ".join(f["text"].replace("\n", " ") for f in b["fields"]))
-        elif t == "section":
-            out.append(b["text"]["text"])
-        elif t == "context":
-            out.append(" ".join(e["text"] for e in b["elements"]))
-    return "\n".join(out)
-
-
-def post_slack(color, fallback, blocks):
-    payload = {"attachments": [{"color": color, "fallback": fallback,
-                                "blocks": blocks}]}
-    if SLACK_DRY:
-        print("=== DRY RUN (no Slack post) ===\n")
-        print(render_text(blocks))
-        return
-    data = json.dumps(payload).encode()
-    for attempt in range(4):
-        req = urllib.request.Request(SLACK_WEBHOOK, data=data, method="POST")
-        req.add_header("Content-Type", "application/json")
-        try:
-            with urllib.request.urlopen(req, timeout=30) as r:
-                r.read()
-            return
-        except urllib.error.HTTPError as e:
-            if e.code not in RETRY_CODES or attempt == 3:
-                raise
-            delay = backoff_delay(e, attempt)
-        except urllib.error.URLError:
-            if attempt == 3:
-                raise
-            delay = 2 ** attempt
-        time.sleep(delay)
-
-
-def pass_rate(run_id):
-    js = all_jobs(run_id)
-    s = sum(1 for j in js if j.get("conclusion") == "success")
-    f = sum(1 for j in js if j.get("conclusion") == "failure")
-    return (s, s + f) if (s + f) else None
-
-
-def history(n=6):
-    """Pass rates of the last n nightlies (newest first). Best-effort: [] on error."""
-    try:
-        runs = gh(f"/repos/{REPO}/actions/workflows/nightly.yml/runs"
-                  f"?per_page=20&status=completed").get("workflow_runs", [])
-    except Exception:
-        return []
-    out = []
-    for r in runs:
-        if str(r["id"]) == str(RUN_ID) or r.get("conclusion") not in ("success", "failure"):
-            continue
-        try:
-            pr = pass_rate(r["id"])
-        except Exception:
-            continue
-        if pr and pr[1] >= 50:
-            out.append(pr[0] / pr[1])
-        if len(out) >= n:
-            break
-    return out
-
-
-def sparkline(rates):
-    if not rates:
-        return ""
-    # map a 90-100% band onto the 8 spark levels (clamped)
-    return "".join(SPARK[min(7, max(0, int((x - 0.90) / 0.10 * 7)))] for x in rates)
-
-
-SEV_RANK = {"Critical": 0, "High": 1, "Medium": 2, "Low": 3}
-
-
-def post_report(obs, fail_refs, verdicts, filed, results, attempt, retry_note):
-    """One card per nightly, pass or fail. Everything here is already computed.
-
-    `filed` maps example id -> issue number (or None), collected from apply().
-    """
-    jobs = all_jobs(RUN_ID)
-    n_success = sum(1 for j in jobs if j.get("conclusion") == "success")
-    n_cancelled = sum(1 for j in jobs if j.get("conclusion") == "cancelled")
-    total_pf = sum(1 for j in jobs if j.get("conclusion") in ("success", "failure"))
-    pct = round(100 * n_success / total_pf) if total_pf else 0
-
-    # Per EXAMPLE DIR, to match the reals/flakes lists below. Counting result
-    # rows here instead would put rows and dirs on one line: "355 passed, 1 real"
-    # reads as 356 of something, and there is no such number.
-    n_pass = sum(1 for o in obs.values() if o == store.PASS)
-    # Rows are still worth stating -- one dir runs several targets -- but as its
-    # own unit, and skip/xfail only exist at row level.
-    tally = defaultdict(int)
-    for r in results:
-        tally[r["status"]] += 1
-    checks = sum(tally.values())
-
-    reals, flakes = [], []
-    for eid, o in sorted(obs.items()):
-        if o == store.PASS:
-            continue
-        ai = verdicts.get(eid) or {}
-        refs = ", ".join(fail_refs.get(eid, [])) or "unknown"
-        if o == store.FLAKE_ONLY:
-            flakes.append(f"`{eid}` ({refs})")
-            continue
-        sev = ai.get("severity") if ai.get("severity") in SEV_RANK else "Medium"
-        num = filed.get(eid)
-        reals.append({
-            "sev": sev,
-            "id": eid,
-            "refs": refs,
-            "issue": num,
-            "unconfirmed": o == store.UNCONFIRMED,
-            "symptom": store.scrub(ai.get("symptom", "") or "failed twice"),
-            "cause": store.scrub(ai.get("cause", "")),
-            "next": store.scrub(ai.get("next", "")),
-            "upstream": ai.get("upstream"),
-        })
-    reals.sort(key=lambda r: SEV_RANK.get(r["sev"], 9))
-
-    if not reals:
-        color, status = "good", "healthy"
-    elif pct >= 90:
-        color, status = "warning", f"{len(reals)} real — action needed"
-    else:
-        color, status = "danger", f"{len(reals)} real — degraded"
-
-    trend = sparkline(history())
-    date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-    blocks = [
-        {"type": "header", "text": {"type": "plain_text",
-                                    "text": "wolfssl-examples Nightly", "emoji": True}},
-        {"type": "section", "fields": [
-            {"type": "mrkdwn", "text": f"*Date:*\n{date_str}"},
-            {"type": "mrkdwn", "text": f"*Status:*\n{status}"},
-            {"type": "mrkdwn", "text": f"*Pass rate:*\n{n_success} / {total_pf} jobs  ({pct}%)"},
-            {"type": "mrkdwn", "text": f"*Trend:*\n{trend or 'building history'}"},
-        ]},
-        section(f"*Examples:*  \U0001F7E2 {n_pass} passed"
-                f"     \U0001F7E1 {len(flakes)} flaked"
-                f"     \U0001F534 {len(reals)} real"
-                + (f"     ⚪ {n_cancelled} cancelled job(s)" if n_cancelled else "")),
-        section(f"*Checks:*  {checks} across those examples"
-                f"  ·  {tally['pass']} pass"
-                f"  ·  {tally['fail']} fail"
-                + (f"  ·  {tally['skip']} skip" if tally["skip"] else "")
-                + (f"  ·  {tally['xfail']} known-fail" if tally["xfail"] else "")),
-    ]
-    if retry_note:
-        blocks.append(section(f":warning: {retry_note}"))
-    if reals:
-        meter = " · ".join(
-            f"{sum(1 for r in reals if r['sev'] == s)} {s}"
-            for s in ("Critical", "High", "Medium", "Low")
-            if any(r["sev"] == s for r in reals)
-        )
-        blocks += [section(f"*Severity:*   {meter}"), {"type": "divider"},
-                   section(f"*Real failures — {len(reals)}, action needed*")]
-        for r in reals:
-            link = f"  <{run_url()}|logs>"
-            iss = (f"  <https://github.com/{REPO}/issues/{r['issue']}|#{r['issue']}>"
-                   if r["issue"] else "")
-            lines = [f"*{r['sev']} · `{r['id']}`* ({r['refs']}){link}{iss}",
-                     f"   • {r['symptom']}"]
-            if r["unconfirmed"]:
-                lines.append("   • _unconfirmed: the retry never completed_")
-            if r["cause"]:
-                lines.append(f"   • _cause:_ {r['cause']}")
-            if r["next"]:
-                lines.append(f"   • _next:_ {r['next']}")
-            if r["upstream"]:
-                lines.append("   • _fails on master but not stable — likely a wolfSSL "
-                             "regression, not an example bug_")
-            blocks.append(section("\n".join(lines)))
-    if flakes:
-        blocks.append({"type": "divider"})
-        for chunk in chunk_lines([f"*Flaked (cleared on retry):* {', '.join(flakes)}"]):
-            blocks.append(section(chunk))
-    blocks.append({"type": "context", "elements": [
-        {"type": "mrkdwn", "text": f"<{run_url()}|run {RUN_ID}> · attempt {attempt}"}]})
-
-    fallback = f"wolfssl-examples nightly: {status} ({n_success}/{total_pf})"
-    post_slack(color, fallback, blocks)
-
-
 def apply(eid, observation, issue, ai, state, rows, close_allowed, target):
     """Returns the issue number now tracking this dir, or None."""
     now = datetime.now(timezone.utc).strftime("%Y-%m-%d")
@@ -919,14 +693,6 @@ def report(attempt, retry_note):
         filed[eid] = apply(eid, observation, issue, verdicts.get(eid), state, rows,
                            close_allowed, targets.get(eid, "examples"))
 
-    # Last, so the card can link the issues opened above. Never let a reporting
-    # failure lose the triage that already succeeded.
-    try:
-        post_report(obs, fail_refs, verdicts, filed, results, attempt, retry_note)
-    except Exception:
-        traceback.print_exc()
-        print("slack report failed; triage itself succeeded", file=sys.stderr)
-
 
 def main():
     run = gh(f"/repos/{REPO}/actions/runs/{RUN_ID}")

.github/workflows/ci-triage.yml

@@ -59,9 +59,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           CLAUDE_MODEL: claude-opus-4-8
-          # Unset is fine: the reporter prints the card to this log instead of
-          # posting, so a fork (or this repo before the secret exists) still works.
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
           AUTO_RETRY: ${{ github.event_name == 'workflow_run' && 'true' || inputs.auto_retry && 'true' || 'false' }}
           # Defaults TRUE on workflow_dispatch: the first thing anyone does with a
           # new issue-filing bot is point it at an old run to see what it says.