Harden orchestrator review verdict prompt

Author: LSantha

.github/AGENTS.md

@@ -86,7 +86,7 @@ State lives in the master issue body as a hidden HTML comment:
 ### Phases
 
 - **DEV**: Initial agent run. Agent creates a PR. Transition to `REVIEW`.
-- **REVIEW**: Agent reviews the PR. If approved and no `auto-merge` label, transition to `HUMAN_REVIEW`. If `auto-merge`, transition to `MERGE`. If changes requested, transition to `FEEDBACK`.
+- **REVIEW**: Agent reviews the PR. The orchestrator posts `/oc review` with explicit instructions requiring the final line to be exactly `Verdict: approve` or `Verdict: request-changes`. If approved and no `auto-merge` label, transition to `HUMAN_REVIEW`. If `auto-merge`, transition to `MERGE`. If changes requested, transition to `FEEDBACK`.
 - **FEEDBACK**: Agent addresses review comments. Transition to `REVIEW`.
 - **HUMAN_REVIEW**: Orchestrator waits for native GitHub PR review from a human maintainer. Approval → `MERGE`, Request changes → `FEEDBACK`.
 - **MERGE**: Orchestrator squashes the PR and deletes the branch inline.

.github/scripts/orchestrator.js

@@ -46,6 +46,10 @@ module.exports = async ({ github, context, core }) => {
     return null;
   }
 
+  function getReviewPrompt() {
+    return '/oc review\n\nYou are reviewing a pull request for the orchestrator. Your final line must be exactly one of:\nVerdict: approve\nVerdict: request-changes\n\nUse "Verdict: request-changes" if the PR needs code changes. Use "Verdict: approve" only if the PR is correct and ready for the next orchestrator phase.';
+  }
+
   async function getAgentReviewVerdict(prNumber) {
     const comments = await github.rest.issues.listComments({
       owner: context.repo.owner,
@@ -410,7 +414,7 @@ module.exports = async ({ github, context, core }) => {
         core.info(`Orchestrator already in progress. Retriggering current task #${getTaskIssueNumber(state.current_task)}`);
         const target = isMultiStepTask(state.current_task) && state.current_task.pr ? state.current_task.pr : getTaskIssueNumber(state.current_task);
         const msg = isMultiStepTask(state.current_task) && state.current_task.phase === 'FEEDBACK' ? "/oc fix Address review feedback." 
-                  : isMultiStepTask(state.current_task) && state.current_task.phase === 'REVIEW' ? "/oc review" 
+                  : isMultiStepTask(state.current_task) && state.current_task.phase === 'REVIEW' ? getReviewPrompt()
                   : "/oc Please proceed with this task.";
         await triggerTask(target, msg);
       }
@@ -491,7 +495,7 @@ module.exports = async ({ github, context, core }) => {
                   task.pr = foundPr;
                   task.phase = 'REVIEW';
                   task.retries = 0;
-                  await triggerTask(task.pr, "/oc review");
+                  await triggerTask(task.pr, getReviewPrompt());
                 } else {
                   // No PR found, treat as completed
                   state.completed.push(task.issue);
@@ -559,7 +563,7 @@ module.exports = async ({ github, context, core }) => {
               } else {
                 task.phase = 'REVIEW';
                 task.retries = 0;
-                await triggerTask(task.pr, "/oc review");
+                await triggerTask(task.pr, getReviewPrompt());
               }
             }
             break;
@@ -576,7 +580,7 @@ module.exports = async ({ github, context, core }) => {
             isDone = true;
           } else {
             const target = task.pr ? task.pr : task.issue;
-            const msg = task.phase === 'FEEDBACK' ? "/oc fix Address review feedback." : task.phase === 'REVIEW' ? "/oc review" : "/oc Please proceed with this task.";
+            const msg = task.phase === 'FEEDBACK' ? "/oc fix Address review feedback." : task.phase === 'REVIEW' ? getReviewPrompt() : "/oc Please proceed with this task.";
             await triggerTask(target, msg);
             await updateMasterIssue(masterIssueNumber, state);
             return;

.github/scripts/tests/orchestrator.test.js

@@ -11,6 +11,7 @@ function createMocks(eventName = 'issue_comment', commentBody = '/orchestrate')
   };
 
   const calls = { getIssue: [], updateIssue: [], createComment: [], listComments: [], addLabels: [], removeLabel: [] };
+  const updateIssueDetails = [];
   let masterIssueBody = `- [ ] #2\n- [ ] #3`;
   let currentTaskData = { labels: [], state: 'open' };
   let prData = { labels: [], state: 'open', head: { ref: 'opencode/issue2-fix' } };
@@ -26,6 +27,7 @@ function createMocks(eventName = 'issue_comment', commentBody = '/orchestrate')
         },
         update: async ({ issue_number, body, state }) => {
           calls.updateIssue.push(issue_number);
+          updateIssueDetails.push({ issue_number, body, state });
           if (issue_number === 1 && body) masterIssueBody = body;
         },
         createComment: async ({ issue_number, body }) => {
@@ -69,7 +71,7 @@ function createMocks(eventName = 'issue_comment', commentBody = '/orchestrate')
     };
   }
 
-  return { core, github, context, logs, calls, setMasterBody: (b) => masterIssueBody = b, setTaskData: (d) => currentTaskData = d };
+  return { core, github, context, logs, calls, updateIssueDetails, setMasterBody: (b) => masterIssueBody = b, setTaskData: (d) => currentTaskData = d };
 }
 
 test('orchestrator.js test suite', async (t) => {
@@ -105,6 +107,21 @@ test('orchestrator.js test suite', async (t) => {
     assert.strictEqual(calls.createComment.length, 1, 'Should trigger review phase');
     assert.strictEqual(calls.createComment[0].issue_number, 99, 'Triggers on PR #99');
     assert.ok(calls.createComment[0].body.includes('/oc review'));
+    assert.ok(calls.createComment[0].body.includes('Verdict: approve'));
+    assert.ok(calls.createComment[0].body.includes('Verdict: request-changes'));
+  });
+
+  await t.test('Review phase accepts explicit approve verdict', async () => {
+    const { core, github, context, calls, updateIssueDetails, setMasterBody, setTaskData } = createMocks('workflow_run');
+
+    setMasterBody(`<!-- ORCHESTRATOR_STATE:\n{ "status": "IN_PROGRESS", "current_task": { "issue": 2, "pr": 99, "phase": "REVIEW", "turn": 0, "max_turns": 3, "retries": 0 }, "queue": [3], "completed": [], "failed": [], "order": [2, 3] }\n-->`);
+    setTaskData({ labels: [], state: 'open' });
+    github.rest.issues.listComments = async () => ({ data: [{ body: 'Verdict: approve' }] });
+
+    await runOrchestrator({ github, context, core });
+
+    assert.ok(calls.createComment.some(c => c.issue_number === 99 && c.body.includes('Agent review passed')));
+    assert.ok(updateIssueDetails.some(u => u.issue_number === 1 && u.body.includes('HUMAN_REVIEW')));
   });
 
   await t.test('Workflow run handles task failure (retries)', async () => {