Triage every nightly target, label it, and comment the history each night

Author: aidangarske

.github/scripts/ci_triage.py

@@ -12,6 +12,7 @@
 
 import json
 import os
+import re
 import sys
 import time
 import traceback
@@ -240,7 +241,62 @@ def classify(run_id, attempt, retry_note):
             obs[eid] = store.FLAKE_ONLY
         else:
             obs[eid] = store.PASS
-    return obs, {k: sorted(v) for k, v in fail_refs.items()}
+    # every host dir came from examples.yml, which nightly.yml calls as `host`
+    targets = {eid: "examples" for eid in obs}
+
+    jobs_obs, jobs_refs, jobs_targets = classify_jobs(run_id, attempt, retry_note)
+    obs.update(jobs_obs)
+    fail_refs.update(jobs_refs)
+    targets.update(jobs_targets)
+    return obs, {k: sorted(v) for k, v in fail_refs.items()}, targets
+
+
+# Jobs the nightly fans out to that emit no results-*.json. Without this they
+# ran every night, went red, and told nobody -- 12 of the 13 targets.
+HOST_JOBS = {"host", "refs", "gate"}
+REF_RE = re.compile(r"wolfSSL (\S+)|\((master|v[\d.]+-stable)\)")
+
+
+def job_unit(name):
+    """`esp32 / Build / ESP32 (...) wolfSSL master` -> ('esp32', 'master').
+
+    The prefix is nightly.yml's job name, which is deliberately the target name
+    and therefore the ci:<target> label.
+    """
+    if " / " not in name:
+        return None, None
+    target, rest = name.split(" / ", 1)
+    if target in HOST_JOBS:
+        return None, None
+    m = REF_RE.search(rest)
+    return target, (m.group(1) or m.group(2)) if m else "unknown"
+
+
+def classify_jobs(run_id, attempt, retry_note):
+    def by_unit(jobs):
+        out = defaultdict(list)
+        for j in jobs:
+            t, ref = job_unit(j.get("name", ""))
+            if t:
+                out[t].append((j.get("conclusion"), ref))
+        return out
+
+    a1 = by_unit(all_jobs(run_id, attempt=1))
+    final = by_unit(all_jobs(run_id)) if attempt >= 2 else a1
+
+    obs, refs, targets = {}, defaultdict(set), {}
+    for t in set(a1) | set(final):
+        failed_now = any(c == "failure" for c, _ in final.get(t, []))
+        failed_before = any(c == "failure" for c, _ in a1.get(t, []))
+        if failed_now:
+            obs[t] = store.UNCONFIRMED if retry_note else store.REAL
+            refs[t] = {r for c, r in final[t] if c == "failure"}
+        elif failed_before and attempt >= 2:
+            obs[t] = store.FLAKE_ONLY
+        else:
+            obs[t] = store.PASS
+        targets[t] = t
+    return obs, refs, targets
 
 
 # --- AI: advisory only -----------------------------------------------------
@@ -401,10 +457,21 @@ def open_issues():
         page += 1
 
 
+NIGHTLY_TARGETS = ("examples", "esp32", "android", "rt1060", "uefi", "puf",
+                   "rpi-pico", "ebpf", "fullstack", "java", "cmake", "csharp",
+                   "emulated")
+
+
 def ensure_labels():
+    for t in NIGHTLY_TARGETS:
+        try:
+            gh(f"/repos/{REPO}/labels", method="POST", retry=False,
+               body={"name": store.target_label(t), "color": "1d76db",
+                     "description": f"Nightly {t} target"})
+        except urllib.error.HTTPError:
+            pass  # 422 = already exists
     for name, color, desc in [
         (store.LABEL, "b60205", "Filed by the nightly examples triage"),
-        (store.LABEL_FLAKY, "fbca04", "Flaps between pass and fail"),
         (store.LABEL_MUTE, "ededed", "Triage will not touch this issue"),
         (store.LABEL_UNCONFIRMED, "d4c5f9", "Retry did not run; not double-confirmed"),
     ]:
@@ -658,7 +725,7 @@ def post_report(obs, fail_refs, verdicts, filed, results, attempt, retry_note):
     post_slack(color, fallback, blocks)
 
 
-def apply(eid, observation, issue, ai, state, rows, close_allowed):
+def apply(eid, observation, issue, ai, state, rows, close_allowed, target):
     """Returns the issue number now tracking this dir, or None."""
     now = datetime.now(timezone.utc).strftime("%Y-%m-%d")
     action, why = store.decide(observation, issue, close_allowed)
@@ -674,16 +741,21 @@ def apply(eid, observation, issue, ai, state, rows, close_allowed):
     body = store.render_body(eid, state, ai, rows, run_url(), now)
 
     if action == "open":
-        created = gh(
-            f"/repos/{REPO}/issues",
-            method="POST",
-            body={
+        body_kw = {
                 "title": f"CI: `{eid}` failing in the nightly",
                 "body": body,
-                "labels": [store.LABEL]
+                "labels": [store.LABEL, store.target_label(target)]
                 + ([store.LABEL_UNCONFIRMED] if observation == store.UNCONFIRMED else []),
-            },
-        )
+                # wolfHSM assigns its nightly issues to the bot; retried
+                # without it below if the account is not assignable here.
+                "assignees": ["wolfSSL-Bot"],
+        }
+        try:
+            created = gh(f"/repos/{REPO}/issues", method="POST", body=body_kw)
+        except urllib.error.HTTPError:
+            # not assignable in this repo; file it anyway rather than lose it
+            body_kw.pop("assignees", None)
+            created = gh(f"/repos/{REPO}/issues", method="POST", body=body_kw)
         num = created["number"]
         print(f"  {eid}: opened #{num} ({why})")
     elif action == "reopen":
@@ -699,19 +771,20 @@ def apply(eid, observation, issue, ai, state, rows, close_allowed):
         )
         print(f"  {eid}: reopened #{issue['number']} ({why})")
     elif action == "update":
-        # Silent: issues.update sends no notification. A comment every night on a
-        # long-standing break is how a bot gets muted.
         gh(f"/repos/{REPO}/issues/{issue['number']}", method="PATCH", body={"body": body})
+        # A comment every night, so the thread IS the history. wolfHSM #412 does
+        # this with a bare "Still failing as of <url>" x15; carry the verdict,
+        # the retry outcome and what changed instead, so a reader can skim the
+        # thread and see how the failure evolved.
         reason = store.material_change(store.parse_state(issue.get("body", "")), state)
-        if reason:
-            gh(
-                f"/repos/{REPO}/issues/{issue['number']}/comments",
-                method="POST",
-                body={"body": f"{reason}. [Run]({run_url()})"},
-            )
-            print(f"  {eid}: updated #{issue['number']} + comment ({reason})")
-        else:
-            print(f"  {eid}: updated #{issue['number']} silently ({why})")
+        gh(
+            f"/repos/{REPO}/issues/{issue['number']}/comments",
+            method="POST",
+            body={"body": store.render_comment(state, ai, observation, reason,
+                                               run_url(), now)},
+        )
+        print(f"  {eid}: updated #{issue['number']} + comment"
+              + (f" ({reason})" if reason else ""))
     elif action == "close":
         gh(
             f"/repos/{REPO}/issues/{issue['number']}/comments",
@@ -731,7 +804,7 @@ def apply(eid, observation, issue, ai, state, rows, close_allowed):
 
 def report(attempt, retry_note):
     results = merged_results(RUN_ID, attempt)
-    obs, fail_refs = classify(RUN_ID, attempt, retry_note)
+    obs, fail_refs, targets = classify(RUN_ID, attempt, retry_note)
 
     logs = defaultdict(str)
     for r in results:
@@ -804,7 +877,7 @@ def report(attempt, retry_note):
         # collected, not read back from `issues`: a dir filed for the first time
         # tonight is not in that map, and its card entry would have no link
         filed[eid] = apply(eid, observation, issue, verdicts.get(eid), state, rows,
-                           close_allowed)
+                           close_allowed, targets.get(eid, "examples"))
 
     # Last, so the card can link the issues opened above. Never let a reporting
     # failure lose the triage that already succeeded.

.github/scripts/issue_store.py

@@ -12,7 +12,11 @@
 import re
 
 LABEL = "ci:nightly"
-LABEL_FLAKY = "ci:flaky"
+# ci:<target> -- the nightly job that failed (examples, esp32, android, ...).
+# Added only when that target fails, so a green night adds no labels.
+def target_label(target):
+    return f"ci:{target}"
+
 LABEL_MUTE = "ci:mute"
 LABEL_UNCONFIRMED = "ci:unconfirmed"
 
@@ -200,6 +204,34 @@ def decide(observation, issue, close_allowed=True):
     return "update", "still failing"
 
 
+def render_comment(state, ai, observation, reason, run_url, now):
+    """One comment per night, so the thread carries the history.
+
+    wolfHSM's nightly posts a bare "Still failing as of <url>" -- 15 of them on
+    one issue, none of which say what happened. Carry the verdict and the retry
+    outcome, so the thread is readable rather than just long.
+    """
+    n = state.get("streak", 1)
+    verdict = ("**Real** — retried once, failed again." if observation == REAL
+               else "**Unconfirmed** — the retry did not complete, so this is "
+                    "not double-checked.")
+    lines = [
+        f"**Night {n}** · [run]({run_url}) · {now}",
+        "",
+        verdict,
+    ]
+    if reason:
+        lines += ["", f"Changed since last night: **{reason}**."]
+    if ai:
+        lines += ["", f"`{ai.get('severity', '?')}` · {scrub(ai.get('symptom', ''))}"]
+        if ai.get("next"):
+            lines.append(f"_next:_ {scrub(ai.get('next', ''))}")
+    refs = state.get("failing") or []
+    if refs:
+        lines += ["", f"Failing on: {', '.join(f'`{r}`' for r in refs)}"]
+    return "\n".join(lines)
+
+
 def material_change(old_state, new_state):
     """Comment only on a material change. Body edits are silent; comments notify,
     and a comment every night on a long-standing break gets the bot muted."""