Dispatch repair after CI activation failures

Author: mrjf

.github/workflows/evergreen.lock.yml

@@ -66,6 +66,7 @@ jobs:
           EXHAUSTED_LABEL: evergreen-exhausted
           REQUIRED_CHECKS_JSON: '["Test & Lint","Playground E2E (Playwright)","Build","Validate Python Examples"]'
           CHECK_GATE_MODE: configured
+          CI_ACTIVATION_WAIT_SECONDS: "300"
           MANUAL_PR: ${{ github.event.inputs.pr || '' }}
           MANUAL_HEAD_SHA: ${{ github.event.inputs.head_sha || '' }}
           MANUAL_REASON: ${{ github.event.inputs.reason || '' }}
@@ -187,6 +188,30 @@ jobs:
             return 1
           }
 
+          any_configured_check_failing() {
+            local payload="$1"
+            local required="$2"
+            local count
+            local state
+
+            count="$(jq 'length' <<<"$required")"
+            if [ "$count" -eq 0 ]; then
+              has_failing_check "$payload"
+              return
+            fi
+
+            while IFS= read -r name; do
+              state="$(check_state_for "$payload" "$name")"
+              case "$state" in
+                FAILURE|failure|FAILED|failed|ERROR|error|TIMED_OUT|timed_out|CANCELLED|cancelled)
+                  return 0
+                  ;;
+              esac
+            done < <(jq -r '.[]' <<<"$required")
+
+            return 1
+          }
+
           evaluate_readiness() {
             local payload="$1"
             local required
@@ -210,6 +235,11 @@ jobs:
               return 0
             fi
 
+            if any_configured_check_failing "$payload" "$required"; then
+              echo "needs_repair"
+              return 0
+            fi
+
             if any_configured_check_missing "$payload" "$required"; then
               echo "needs_ci"
               return 0
@@ -304,7 +334,7 @@ jobs:
 
             if [ -z "$run_id" ]; then
               echo "No CI run found for PR #$pr ($head_sha); leaving for scheduled/PR CI to start."
-              return 0
+              return 1
             fi
 
             case "$status" in
@@ -317,14 +347,58 @@ jobs:
             case "$conclusion" in
               success)
                 echo "CI run $run_id for PR #$pr ($head_sha) already succeeded; not rerunning green checks."
-                return 0
+                return 1
                 ;;
             esac
 
             echo "Reactivating CI run $run_id for PR #$pr ($head_sha) (status=$status conclusion=$conclusion)."
-            ci_gh gh run rerun "$run_id" --repo "$REPO" --failed || \
-              ci_gh gh run rerun "$run_id" --repo "$REPO" || \
-              echo "Could not reactivate CI run $run_id; scheduled monitor will retry."
+            if ci_gh gh run rerun "$run_id" --repo "$REPO" --failed; then
+              return 0
+            fi
+
+            echo "Failed-jobs rerun was unavailable for CI run $run_id; trying full rerun."
+            if ci_gh gh run rerun "$run_id" --repo "$REPO"; then
+              return 0
+            fi
+
+            echo "Could not reactivate CI run $run_id; scheduled monitor will retry."
+            return 1
+          }
+
+          wait_for_ci_activation() {
+            local pr="$1"
+            local head_sha="$2"
+            local timeout="${CI_ACTIVATION_WAIT_SECONDS:-0}"
+            local deadline payload current_head state
+
+            if ! grep -Eq '^[0-9]+$' <<<"$timeout" || [ "$timeout" -le 0 ]; then
+              return 1
+            fi
+
+            deadline=$((SECONDS + timeout))
+            while [ "$SECONDS" -lt "$deadline" ]; do
+              sleep 10
+              payload="$(pr_json "$pr")"
+              current_head="$(jq -r '.headRefOid' <<<"$payload")"
+              if [ "$current_head" != "$head_sha" ]; then
+                echo "PR #$pr head changed from $head_sha to $current_head while waiting for CI."
+                return 0
+              fi
+
+              state="$(evaluate_readiness "$payload")"
+              case "$state" in
+                waiting|needs_ci)
+                  echo "CI for PR #$pr is still $state; continuing to wait."
+                  ;;
+                *)
+                  echo "CI for PR #$pr settled to $state."
+                  return 0
+                  ;;
+              esac
+            done
+
+            echo "CI for PR #$pr did not settle within ${timeout}s; a later Evergreen run will continue."
+            return 1
           }
 
           update_branch_if_needed() {
@@ -402,7 +476,25 @@ jobs:
                 return 1
                 ;;
               needs_ci)
-                trigger_ci_if_needed "$pr" "$head_sha"
+                if trigger_ci_if_needed "$pr" "$head_sha" &&
+                   wait_for_ci_activation "$pr" "$head_sha"; then
+                  payload="$(pr_json "$pr")"
+                  head_sha="$(jq -r '.headRefOid' <<<"$payload")"
+                  state="$(evaluate_readiness "$payload")"
+                  if ! reconcile_ready_label "$pr" "$state" "$payload"; then
+                    echo "Could not reconcile $READY_LABEL for PR #$pr after CI activation; refusing to dispatch the agent."
+                    set_result "false" "$pr" "$head_sha" "blocked" "$reason:ready_label_failed"
+                    return 1
+                  fi
+
+                  if [ "$state" = "needs_repair" ]; then
+                    if ! claim_active_label "$pr" "$head_sha" "$reason:ci_failed_after_activation"; then
+                      return 1
+                    fi
+                    set_result "true" "$pr" "$head_sha" "$state" "$reason:ci_failed_after_activation"
+                    return 0
+                  fi
+                fi
                 return 1
                 ;;
               needs_branch_update)

.github/workflows/evergreen.md

@@ -42,9 +42,14 @@ readiness controller and the agentic orchestrator must both respect this file.
   (new head SHA, pending, failing, missing check, conflict, out of scope).
 - Current-head SHA policy: readiness is evaluated only against the current PR
   head SHA. A new push invalidates prior readiness.
-- Pending check policy: `waiting` — do not repair pending or in-progress checks.
+- Failing check policy: `needs_repair` — if a configured gate is visibly
+  failing for the current head SHA, dispatch the repair agent even when other
+  configured checks are still missing or skipped.
+- Pending check policy: `waiting` — do not repair pending or in-progress checks
+  unless another configured gate has already failed.
 - Missing/stale check policy: `needs_ci` — reactivate the latest `CI` run for the
-  head SHA once (see CI/CD Activation). Never rerun green checks.
+  head SHA once, wait briefly for a result, then dispatch repair in the same run
+  if the gate fails (see CI/CD Activation). Never rerun green checks.
 - Branch freshness ready criterion: not required to be up to date with `main`;
   freshness is only enforced when the branch is conflicted (`DIRTY`/`UNKNOWN`).
 - Additional deterministic ready criteria: PR open, has `evergreen`, not
@@ -100,7 +105,9 @@ readiness controller and the agentic orchestrator must both respect this file.
 - Workflows/checks Evergreen may dispatch: none by name in v1; activation is
   rerun-based only.
 - Stale check policy: reactivate the latest `CI` run for the head SHA once per
-  head; never rerun green checks; never re-trigger an already in-progress run.
+  head; briefly wait for it to settle; dispatch repair immediately if a
+  configured gate fails; never rerun green checks; never re-trigger an already
+  in-progress run.
 - Missing check policy: if no `CI` run exists for the head SHA, wait for the
   normal `pull_request`/schedule CI to start rather than forcing activation.
 - Empty commit policy: empty trigger commits are a last resort only, requested