fixup! fixup! fixup! fixup! Add CI that builds and runs every example against wolfSSL master and stable

Author: aidangarske

.github/scripts/ci_triage.py

@@ -224,6 +224,13 @@ def classify(run_id, attempt, retry_note):
     for r in merged_results(run_id, attempt):
         if r["status"] == "fail":
             fail_refs[r["id"]].add(r.get("ref") or "unknown")
+    # A flake has no failing ref in the final attempt by definition, so without
+    # this its issue would name no ref at all. The ref that failed first is the
+    # whole point of the report.
+    a1_refs = defaultdict(set)
+    for r in fetch_results(run_id, 1):
+        if r["status"] == "fail":
+            a1_refs[r["id"]].add(r.get("ref") or "unknown")
 
     obs = {}
     for eid in set(a1) | set(final):
@@ -233,6 +240,7 @@ def classify(run_id, attempt, retry_note):
             obs[eid] = store.UNCONFIRMED if retry_note else store.REAL
         elif failed_before and attempt >= 2:
             obs[eid] = store.FLAKE_ONLY
+            fail_refs[eid] |= a1_refs.get(eid, set())
         else:
             obs[eid] = store.PASS
     # every host dir came from examples.yml, which nightly.yml calls as `host`
@@ -310,6 +318,8 @@ def by_unit(jobs):
             refs[t] = {r for c, r in final[t] if c == "failure"}
         elif failed_before and attempt >= 2:
             obs[t] = store.FLAKE_ONLY
+            # nothing failed in the final attempt, so name the ref that failed first
+            refs[t] = {r for c, r in a1[t] if c == "failure"}
         else:
             obs[t] = store.PASS
         targets[t] = t

.github/scripts/issue_store.py

@@ -198,16 +198,26 @@ def decide(observation, issue, close_allowed=True):
         # Absence of evidence is not recovery. Only an observed pass closes.
         return "noop", "absent from this run"
 
-    if observation == PASS or observation == FLAKE_ONLY:
+    if observation == PASS:
         if issue and issue.get("state") == "open":
             if not close_allowed:
                 return "update", "circuit breaker: refusing to close"
             if parse_state(issue.get("body", "")).get("sticky"):
                 return "update", "sticky (flaky): human closes"
-            why = "recovered on retry (flake)" if observation == FLAKE_ONLY else "passing again"
-            return "close", why
+            return "close", "passing again"
         return "noop", "passing"
 
+    if observation == FLAKE_ONLY:
+        # A flake is a failure a retry hid, not a pass. It used to fall in with
+        # PASS above, so a dir that flaked every night closed its own issue every
+        # night and left no trace -- which is how the dtls resume flake stayed
+        # invisible until two runs were diffed by hand. Only a clean night closes.
+        if not issue:
+            return "open", "flaked and recovered on retry"
+        if issue.get("state") == "closed":
+            return "reopen", "flaked again"
+        return "update", "flaked and recovered on retry"
+
     # REAL or UNCONFIRMED
     if not issue:
         return "open", "first failure"