feat(loop): execution-grounded Max-Mode candidate selection (ADR 0064)

README.md

@@ -95,7 +95,10 @@ project_board:
   max_concurrent: 1          # >1 builds features in parallel (each its own worktree)
   merge_poll: true           # poll merged PRs as a fallback to the webhook Done edge
   goal_verify: false         # flip true: verify the coder's diff vs acceptance_criteria before opening a PR
-  max_mode_n: 1              # >1 = best-of-N "Max-Mode": N coders per feature, a judge keeps the best diff (#21, WIP)
+  max_mode_n: 1              # >1 = best-of-N "Max-Mode": N coders per feature, keep the best diff
+  # With local_gate_cmd set, Max-Mode is EXECUTION-GROUNDED (ADR 0064): the winner is
+  # picked from candidates whose gate actually PASSES; the LLM judge only breaks ties
+  # among the passing set (or decides when no gate is set / none pass).
   # webhook_secret: "..."    # set before exposing /webhook/pr publicly
 ```
 

loop.py

@@ -951,6 +951,59 @@ async def _judge_candidates(self, feature: dict, base: str, worktrees: list[str]
                 return idx
         return nonempty[0]  # judge unclear → first non-empty candidate
 
+    async def _candidate_diff_indices(self, base: str, worktrees: list[str]) -> list[int]:
+        """Indices of candidates that produced a non-empty staged diff vs ``origin/<base>``.
+        Cheap name-only check; best-effort (a candidate we can't stage/diff is skipped)."""
+        out: list[int] = []
+        for i, wt in enumerate(worktrees):
+            try:
+                await worktree.stage_all(wt)
+                _rc, names, _err = await worktree._git(wt, "diff", "--cached", "--name-only", f"origin/{base}")
+            except Exception:  # noqa: BLE001 — best-effort, like _judge_candidates
+                names = ""
+            if (names or "").strip():
+                out.append(i)
+        return out
+
+    async def _select_candidate(self, feature: dict, base: str, worktrees: list[str]) -> int | None:
+        """Pick the winning Max-Mode candidate — EXECUTION-GROUNDED (ADR 0064).
+
+        When a pre-PR gate (``local_gate_cmd``) is configured, PREFER candidates whose
+        gate actually PASSES: run the candidates, don't just judge their diffs. An LLM
+        judge of code rewards plausible-looking diffs and can't catch subtle wrongness —
+        only running the tests discriminates. The judge (``_judge_candidates``) then only
+        breaks ties among the PASSING set (quality among the correct), or decides when no
+        gate is configured / none pass. With no gate this is exactly the old behavior.
+
+        Returns the winning index, or ``None`` when no candidate produced a diff."""
+        # No oracle → judge exactly as before (it does its own emptiness handling and
+        # returns None when every candidate is empty). Avoids a redundant diff pass.
+        if not self.local_gate_cmd:
+            return await self._judge_candidates(feature, base, worktrees)
+
+        nonempty = await self._candidate_diff_indices(base, worktrees)
+        if not nonempty:
+            return None
+        if len(nonempty) == 1:
+            return nonempty[0]
+
+        fid = feature.get("id")
+        gates = await asyncio.gather(*(self._run_local_gate(worktrees[i]) for i in nonempty))
+        passing = [i for i, gap in zip(nonempty, gates) if gap is None]
+        if not passing:
+            log.info(
+                "[project_board] %s execution-select: 0/%d candidates pass the gate — judging diffs", fid, len(nonempty)
+            )
+            return await self._judge_candidates(feature, base, worktrees)
+        log.info(
+            "[project_board] %s execution-select: %d/%d candidates pass the gate", fid, len(passing), len(nonempty)
+        )
+        if len(passing) == 1:
+            return passing[0]
+        # Tie-break among the PASSING (correct) candidates by quality, via the judge.
+        j = await self._judge_candidates(feature, base, [worktrees[i] for i in passing])
+        return passing[j] if j is not None else passing[0]
+
     async def _dispatch_max_mode(
         self, feature: dict, coder, prompt: str, repo: str, base: str, fid: str
     ) -> tuple[str, str, str]:
@@ -960,9 +1013,10 @@ async def _dispatch_max_mode(
         ``feat-<id>.c<k>`` so none collides with the canonical name), dispatches the coder
         into ALL of them concurrently — each keeps its own ``coder_timeout`` watchdog +
         ``finally`` subprocess teardown (``dispatch_coder``), and ``return_exceptions``
-        means one candidate timing out / erroring leaves an empty tree the judge skips
-        rather than sinking the batch. ``_judge_candidates`` (the best-of-N judge that
-        landed in #22) picks the winning index; the winner is PROMOTED into the canonical
+        means one candidate timing out / erroring leaves an empty tree the selector skips
+        rather than sinking the batch. ``_select_candidate`` picks the winning index —
+        EXECUTION-GROUNDED when a pre-PR gate is configured (prefer candidates whose tests
+        pass; ADR 0064), else the best-of-N LLM judge; the winner is PROMOTED into the canonical
         ``feat-<id>`` worktree / ``feat/<id>`` branch (so the rest of the lifecycle is
         unchanged) and the losers are reaped. All-empty → ``NoChangesError``, which
         ``_drive`` escalates/blocks exactly like a single coder that produced nothing.
@@ -984,7 +1038,7 @@ async def _dispatch_max_mode(
         for i, r in enumerate(results):
             if isinstance(r, Exception):
                 log.info("[project_board] %s max-mode candidate %d failed (skipped): %s", fid, i, r)
-        idx = await self._judge_candidates(feature, base, [wt for wt, _b in cands])
+        idx = await self._select_candidate(feature, base, [wt for wt, _b in cands])
         if idx is None:
             for cid in cand_ids:
                 await worktree.reap_feature_worktree(repo, self.root, cid)

protoagent.plugin.yaml

@@ -1,6 +1,6 @@
 id: project_board
 name: Project Board (coding orchestration)
-version: 0.24.0
+version: 0.25.0
 description: >-
   A board-driven coding-orchestration plugin: a lean 6-state board (backlog → ready
   → in_progress → in_review → done, + a blocked flag) backed by **beads** (`br`), an

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "project-board"
-version = "0.24.0"
+version = "0.25.0"
 description = "Board-driven coding-orchestration plugin for protoAgent (beads board + ACP spawn loop + planning layer + console view)."
 requires-python = ">=3.11"
 

tests/test_loop.py

@@ -1349,3 +1349,96 @@ async def _err(prompt, *, system=None, model_name=None):
     monkeypatch.setattr("graph.sdk.complete", _err)
     # both candidates non-empty → first non-empty index wins when the judge dies
     assert await loop._judge_candidates(FEATURE, "main", ["/wt/a", "/wt/b"]) == 0
+
+
+# ── execution-grounded candidate selection (ADR 0064) ────────────────────────────
+
+
+def _git_nonempty_for(nonempty_wts):
+    """A worktree._git stub: name-only diff is non-empty only for the given worktrees."""
+
+    async def _git(wt, *args, timeout=60):
+        if args and args[0] == "diff":
+            return (0, ("solution.py" if wt in nonempty_wts else ""), "")
+        return (0, "", "")
+
+    return _git
+
+
+async def test_select_candidate_prefers_passing_gate(monkeypatch):
+    """With a gate, the candidate whose gate PASSES wins even if the judge would pick another."""
+    loop = BoardLoop({"local_gate_cmd": "pytest", "max_mode_n": 3})
+    wts = ["/c0", "/c1", "/c2"]
+    monkeypatch.setattr(worktree, "_git", _git_nonempty_for(set(wts)))  # all have a diff
+
+    async def gate(wt):
+        return None if wt == "/c2" else "boom"  # only c2 passes
+
+    async def judge(*a, **k):
+        return 0  # the judge would (wrongly) pick c0 — must be overridden
+
+    monkeypatch.setattr(loop, "_run_local_gate", gate)
+    monkeypatch.setattr(loop, "_judge_candidates", judge)
+    assert await loop._select_candidate(FEATURE, "main", wts) == 2
+
+
+async def test_select_candidate_judges_only_among_passing(monkeypatch):
+    """Multiple candidates pass → the judge breaks the tie among the PASSING set only."""
+    loop = BoardLoop({"local_gate_cmd": "pytest", "max_mode_n": 3})
+    wts = ["/c0", "/c1", "/c2"]
+    monkeypatch.setattr(worktree, "_git", _git_nonempty_for(set(wts)))
+
+    async def gate(wt):
+        return None if wt in ("/c0", "/c2") else "boom"  # c0 + c2 pass, c1 fails
+
+    async def judge(feature, base, sub):
+        assert sub == ["/c0", "/c2"]  # judge sees only the passing candidates
+        return 1  # picks the 2nd of the sublist → original index 2
+
+    monkeypatch.setattr(loop, "_run_local_gate", gate)
+    monkeypatch.setattr(loop, "_judge_candidates", judge)
+    assert await loop._select_candidate(FEATURE, "main", wts) == 2
+
+
+async def test_select_candidate_falls_back_to_judge_when_none_pass(monkeypatch):
+    loop = BoardLoop({"local_gate_cmd": "pytest", "max_mode_n": 2})
+    wts = ["/c0", "/c1"]
+    monkeypatch.setattr(worktree, "_git", _git_nonempty_for(set(wts)))
+
+    async def gate(wt):
+        return "boom"  # none pass
+
+    async def judge(feature, base, sub):
+        assert sub == wts  # judges over ALL candidates
+        return 1
+
+    monkeypatch.setattr(loop, "_run_local_gate", gate)
+    monkeypatch.setattr(loop, "_judge_candidates", judge)
+    assert await loop._select_candidate(FEATURE, "main", wts) == 1
+
+
+async def test_select_candidate_no_gate_uses_judge_and_never_runs_gate(monkeypatch):
+    loop = BoardLoop({"max_mode_n": 2})  # no local_gate_cmd
+    wts = ["/c0", "/c1"]
+    monkeypatch.setattr(worktree, "_git", _git_nonempty_for(set(wts)))
+
+    async def gate(wt):
+        raise AssertionError("the gate must not run when local_gate_cmd is unset")
+
+    async def judge(*a, **k):
+        return 0
+
+    monkeypatch.setattr(loop, "_run_local_gate", gate)
+    monkeypatch.setattr(loop, "_judge_candidates", judge)
+    assert await loop._select_candidate(FEATURE, "main", wts) == 0
+
+
+async def test_select_candidate_none_when_no_diff(monkeypatch):
+    loop = BoardLoop({"local_gate_cmd": "pytest", "max_mode_n": 2})
+    monkeypatch.setattr(worktree, "_git", _git_nonempty_for(set()))  # all empty
+
+    async def gate(wt):
+        raise AssertionError("no diffs → nothing to gate")
+
+    monkeypatch.setattr(loop, "_run_local_gate", gate)
+    assert await loop._select_candidate(FEATURE, "main", ["/c0", "/c1"]) is None