Refine supervisor evaluation prompt

Author: pallyoung

packages/core/src/domain/supervisor.ts

@@ -22,8 +22,7 @@ export type CycleTrigger = "turn_completed" | "manual" | "scheduled";
 export type SupervisorStopReason =
   | "objective_complete"
   | "max_supervision_count_reached"
-  | "supervisor_uncertain"
-  | "needs_user_input";
+  | "supervisor_uncertain";
 
 export type SupervisorPlanStepStatus = "pending" | "in_progress" | "done";
 
@@ -55,7 +54,7 @@ export interface SupervisorCycleTargetRecord {
   startedAt: number;
   completedAt: number;
   result: "continue" | "stop" | "error";
-  stopReason?: "objective_complete" | "supervisor_uncertain" | "needs_user_input";
+  stopReason?: "objective_complete" | "supervisor_uncertain";
   reason?: string;
   guidance?: string;
   progressSummary?: string;

packages/server/src/supervisor/evaluator.test.ts

@@ -293,6 +293,21 @@ describe("SupervisorEvaluator", () => {
     });
   });
 
+  it("rejects needs_user_input as a stopReason", async () => {
+    const evaluator = makeEvaluator(
+      JSON.stringify({
+        status: "stop",
+        stopReason: "needs_user_input",
+        reason: "Need user input",
+      }),
+      "claude"
+    );
+
+    await expect(evaluator.evaluate(makeSupervisor("claude"), makeContext())).rejects.toThrow(
+      "Supervisor stop result is missing a valid stopReason"
+    );
+  });
+
   it("falls back to provider.defaultConfig when evaluator config is missing", async () => {
     const evaluator = new SupervisorEvaluator({
       providerRegistry: [
@@ -434,8 +449,23 @@ describe("SupervisorEvaluator", () => {
     ).rejects.toThrow();
 
     const prompt = (logger.warn.mock.calls[0]?.[0] as { prompt?: string } | undefined)?.prompt;
-    expect(prompt).toContain("You are supervising a target-scoped software task.");
+    expect(prompt).toContain("You are an autonomous supervisor for a target-scoped software task.");
     expect(prompt).toContain("Return JSON only.");
+    expect(prompt).toContain('Prefer "continue" whenever there is a reasonable next action.');
+    expect(prompt).toContain(
+      "Do not ask the user to decide, clarify, or choose among implementation options."
+    );
+    expect(prompt).toContain("Use the target memory as the current supervision state.");
+    expect(prompt).toContain("Identify which plan step is currently active.");
+    expect(prompt).toContain("If the active step is done, advance to the next useful step.");
+    expect(prompt).toContain(
+      "If the agent appears stuck or repeated the same action, give a different concrete next action."
+    );
+    expect(prompt).toContain('Use "supervisor_uncertain" only as a last resort');
+    expect(prompt).toContain('Guidance requirements for "continue":');
+    expect(prompt).toContain(
+      "Be specific enough for the supervised agent to act without asking the user."
+    );
     expect(prompt).toContain("Current objective:");
     expect(prompt).toContain("Ship the fix");
     expect(prompt).toContain("Current target memory:");

packages/server/src/supervisor/evaluator.ts

@@ -30,10 +30,7 @@ const NOOP_LOGGER: FastifyBaseLogger = {
 
 export interface SupervisorEvaluationResult {
   status: "continue" | "stop";
-  stopReason?: Extract<
-    SupervisorStopReason,
-    "objective_complete" | "supervisor_uncertain" | "needs_user_input"
-  >;
+  stopReason?: Extract<SupervisorStopReason, "objective_complete" | "supervisor_uncertain">;
   reason: string;
   guidance?: string;
   plan?: SupervisorPlanStep[];
@@ -135,20 +132,71 @@ export class SupervisorEvaluator {
 
 function buildPrompt(context: SupervisorEvaluationContext): string {
   const lines: string[] = [
-    "You are supervising a target-scoped software task.",
+    "You are an autonomous supervisor for a target-scoped software task.",
+    "Your job is to keep the agent moving toward the objective until the objective is complete.",
+    "",
     "Return JSON only.",
     "",
+    "Decision policy:",
+    '- Prefer "continue" whenever there is a reasonable next action.',
+    "- Do not ask the user to decide, clarify, or choose among implementation options.",
+    "- When information is incomplete, choose a conservative next action based on the objective, target memory, latest user input, and terminal snapshot.",
+    "- Stop only when the objective is complete, or when continuing would likely push the agent in an unsafe or clearly unsupported direction.",
+    "",
+    "Stage decision policy:",
+    "- Use the target memory as the current supervision state.",
+    "- Base your decision on the objective, current plan, activeStepId, progressSummary, lastGuidance, stalledCount, latest user input, and terminal snapshot.",
+    "- Identify which plan step is currently active.",
+    "- Decide whether the active step is done, still in progress, blocked, or obsolete.",
+    "- If the active step is done, advance to the next useful step.",
+    "- If the active step is still in progress, give guidance that moves it forward.",
+    "- If the agent appears stuck or repeated the same action, give a different concrete next action.",
+    "- If the plan is obsolete, update only the affected steps unless a full replacement is necessary.",
+    "",
     "Allowed statuses:",
     '- "continue": more work is needed; include "reason" and "guidance".',
     '- "stop": supervision should stop; include "stopReason" and "reason".',
     "",
     "Allowed stop reasons:",
     '- "objective_complete"',
     '- "supervisor_uncertain"',
-    '- "needs_user_input"',
     "",
-    "If planGenerated is false, bootstrap a plan with 3 to 7 milestone-sized steps.",
-    "If planGenerated is true, update progress incrementally; do not rewrite the full plan unless absolutely necessary.",
+    'Use "objective_complete" only when the objective has been satisfied.',
+    'Use "supervisor_uncertain" only as a last resort when no useful next action can be inferred and additional guidance would likely be misleading.',
+    "",
+    'Guidance requirements for "continue":',
+    "- Give one concrete next action or a short ordered set of concrete actions.",
+    "- Focus on the highest-value step toward completing the objective.",
+    "- Be specific enough for the supervised agent to act without asking the user.",
+    "- Avoid generic reminders, encouragement, or restating the objective.",
+    "- If verification is needed, tell the agent exactly what to verify next.",
+    "- If implementation is needed, point to the likely area, behavior, or file/module based on available evidence.",
+    "",
+    "Planning policy:",
+    "- If planGenerated is false, include a plan with 3 to 7 milestone-sized steps.",
+    "- If planGenerated is true, update progress incrementally.",
+    "- Do not rewrite the full plan unless the existing plan is clearly wrong or obsolete.",
+    "- Use stepUpdates to mark completed or active steps when the terminal snapshot shows progress.",
+    "- Keep activeStepId aligned with the next useful step.",
+    "",
+    "Output schema:",
+    "For continue:",
+    "{",
+    '  "status": "continue",',
+    '  "reason": "brief explanation of why more work is needed",',
+    '  "guidance": "specific next action for the supervised agent",',
+    '  "plan": optional array of plan steps,',
+    '  "activeStepId": optional step id,',
+    '  "progressSummary": optional brief progress summary,',
+    '  "stepUpdates": optional array of { "id": string, "status": "pending" | "in_progress" | "done" }',
+    "}",
+    "",
+    "For stop:",
+    "{",
+    '  "status": "stop",',
+    '  "stopReason": "objective_complete" | "supervisor_uncertain",',
+    '  "reason": "brief explanation"',
+    "}",
     "",
     "Current objective:",
     context.objective,
@@ -563,11 +611,7 @@ function parseSupervisorEvaluationResult(
 
   if (status === "stop") {
     const stopReason = record.stopReason;
-    if (
-      stopReason !== "objective_complete" &&
-      stopReason !== "supervisor_uncertain" &&
-      stopReason !== "needs_user_input"
-    ) {
+    if (stopReason !== "objective_complete" && stopReason !== "supervisor_uncertain") {
       throw new Error("Supervisor stop result is missing a valid stopReason");
     }
 

packages/web/src/locales/en.json

@@ -676,8 +676,7 @@
     "stop_reason": {
       "objective_complete": "Objective complete. Supervisor stopped automatically.",
       "max_supervision_count_reached": "Reached the configured max supervision count.",
-      "supervisor_uncertain": "Supervisor could not determine the next step and stopped automatic supervision.",
-      "needs_user_input": "Supervisor needs more user input before automatic supervision can continue."
+      "supervisor_uncertain": "Supervisor could not determine the next step and stopped automatic supervision."
     },
     "target_memory": {
       "title": "Target memory",

packages/web/src/locales/zh.json

@@ -676,8 +676,7 @@
     "stop_reason": {
       "objective_complete": "目标已达成，Supervisor 已自动停止。",
       "max_supervision_count_reached": "已达到配置的最大监督次数。",
-      "supervisor_uncertain": "Supervisor 暂时无法判断下一步，已停止自动监督。",
-      "needs_user_input": "Supervisor 需要更多用户输入后才能继续自动监督。"
+      "supervisor_uncertain": "Supervisor 暂时无法判断下一步，已停止自动监督。"
     },
     "target_memory": {
       "title": "目标记忆",