fix: prevent heuristic/verifier drift and surface partial steps in goal verification

Three issues addressed:

1. Heuristic/verifier step drift: The agent's keyword-based
   _advance_plan_steps() heuristic and the DemoController's VLM verifier
   operated on independent state, allowing them to disagree on which step
   was current. Fix: add _external_step_control flag to the agent that the
   DemoController sets at init, making _advance_plan_steps() a no-op when
   the controller manages step progression via VLM verification.

2. partially_verified invisible to goal verification: When steps were
   marked partially_verified, the final goal verification pass had no
   visibility into which steps had partial completions. Fix: _verify_goal()
   now builds a step verification summary and augments the goal text with
   it when noteworthy statuses (partially_verified, failed) exist.

3. Missing integration tests: Added TestHeuristicVerifierSync (4 tests)
   and TestGoalVerificationContext (5 tests) that verify the heuristic is
   properly disabled under controller management, step advancement is
   driven by VLM verification, and partial/failed step context reaches
   goal verification. Also added 2 agent-level tests for
   _external_step_control behavior.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: abrichr

openadapt_evals/agents/claude_computer_use_agent.py

@@ -315,6 +315,12 @@ def __init__(
         self._goal: str = ""
         self._consecutive_done_overrides: int = 0
 
+        # When True, _advance_plan_steps() is a no-op.  The DemoController
+        # sets this flag so that step progression is driven exclusively by
+        # VLM verification, preventing drift between the agent's keyword
+        # heuristic and the controller's verifier.
+        self._external_step_control: bool = False
+
         if self._parsed_demo:
             self._goal = self._parsed_demo["goal"]
             self._trajectory = self._parsed_demo["trajectory"]
@@ -727,9 +733,16 @@ def _advance_plan_steps(self, action: BenchmarkAction) -> None:
         based on superficial text matches (e.g., typing "Year" matching
         both the header step and the data entry step).
 
+        When ``_external_step_control`` is True (set by :class:`DemoController`),
+        this method is a no-op because step progression is managed by VLM
+        verification in the controller.
+
         Args:
             action: The BenchmarkAction being returned to the runner.
         """
+        if self._external_step_control:
+            return
+
         if not self._plan_steps:
             return
 

openadapt_evals/demo_controller.py

@@ -155,6 +155,13 @@ def __init__(
         # Parse the demo into a structured plan
         self.plan_state = self._parse_demo(demo_text)
 
+        # Disable the agent's internal heuristic step advancement so that
+        # step progression is driven exclusively by VLM verification here
+        # in the controller.  This prevents drift between the agent's
+        # keyword-based heuristic and the controller's verifier.
+        if hasattr(agent, "_external_step_control"):
+            agent._external_step_control = True
+
         logger.info(
             "DemoController initialized: goal=%r, %d plan steps, %d trajectory steps",
             self.plan_state.goal[:80],
@@ -742,7 +749,9 @@ def _verify_step(
     def _verify_goal(self, observation: BenchmarkObservation) -> bool:
         """Verify whether the overall goal has been achieved.
 
-        Delegates to :func:`plan_verify.verify_goal_completion`.
+        Delegates to :func:`plan_verify.verify_goal_completion`, augmenting
+        the goal text with a summary of per-step verification outcomes so
+        that the VLM knows which steps were only ``partially_verified``.
 
         Args:
             observation: Current observation with screenshot.
@@ -755,14 +764,59 @@ def _verify_goal(self, observation: BenchmarkObservation) -> bool:
             logger.warning("No screenshot for goal verification; assuming not done")
             return False
 
+        # Build step verification summary so goal verifier is aware of
+        # partial completions and failures.
+        step_summary = self._build_step_verification_summary()
+        augmented_goal = self.plan_state.goal
+        if step_summary:
+            augmented_goal = (
+                f"{self.plan_state.goal}\n\n"
+                f"STEP VERIFICATION SUMMARY (for context):\n{step_summary}"
+            )
+
         result = verify_goal_completion(
             screenshot_bytes,
-            self.plan_state.goal,
+            augmented_goal,
             model=self.verify_model,
             provider=self.verify_provider,
         )
         return result.effectively_verified
 
+    def _build_step_verification_summary(self) -> str:
+        """Build a concise summary of per-step verification outcomes.
+
+        Returns:
+            A multi-line string summarising each step's verification status
+            and any partial-verification explanations, or an empty string
+            if there is nothing noteworthy to report.
+        """
+        lines: list[str] = []
+        has_noteworthy = False
+
+        for step in self.plan_state.steps:
+            vr = step.verification_result
+            if vr is None:
+                status_text = step.status
+            else:
+                status_text = vr.status
+                if vr.status == "partially_verified":
+                    has_noteworthy = True
+
+            line = f"  Step {step.step_num} ({step.action[:60]}): {status_text}"
+            if vr and vr.status == "partially_verified":
+                line += f" -- {vr.explanation[:120]}"
+            elif step.status == "failed":
+                has_noteworthy = True
+                if vr:
+                    line += f" -- {vr.explanation[:120]}"
+            lines.append(line)
+
+        if not has_noteworthy:
+            # All steps fully verified or done; no extra context needed
+            return ""
+
+        return "\n".join(lines)
+
     # ------------------------------------------------------------------
     # Helpers
     # ------------------------------------------------------------------

tests/test_claude_computer_use_agent.py

@@ -1084,3 +1084,37 @@ def test_drift_scenario_from_live_eval(self, agent_with_multilevel_demo):
         assert agent._plan_steps[2]["status"] == "pending"
         assert agent._plan_steps[3]["status"] == "pending"
         assert agent._plan_steps[4]["status"] == "pending"
+
+    def test_external_step_control_suppresses_heuristic(
+        self, agent_with_multilevel_demo
+    ):
+        """When _external_step_control is True, _advance_plan_steps is a no-op.
+
+        The DemoController sets this flag so that step progression is driven
+        by VLM verification, not the agent's keyword heuristic.
+        """
+        agent = agent_with_multilevel_demo
+        assert agent._plan_steps[0]["status"] == "in_progress"
+
+        # Enable external step control (as DemoController does)
+        agent._external_step_control = True
+
+        # This action would normally advance step 1 -> step 2
+        action = BenchmarkAction(
+            type="type", text="Year",
+            raw_action={"claude_action": {"action": "type", "text": "Year"}},
+        )
+        agent._advance_plan_steps(action)
+
+        # With external control enabled, nothing should have changed
+        assert agent._plan_steps[0]["status"] == "in_progress"
+        assert agent._plan_steps[1]["status"] == "pending"
+
+    def test_external_step_control_default_false(
+        self, agent_with_multilevel_demo
+    ):
+        """_external_step_control defaults to False so the agent works
+        standalone (without DemoController).
+        """
+        agent = agent_with_multilevel_demo
+        assert agent._external_step_control is False