@@ -202,14 +202,19 @@ def to_prompt_text(self) -> str:
202202 return ""
203203
204204 lines = [
205- "DEMONSTRATION GUIDANCE:" ,
205+ "DEMONSTRATION GUIDANCE (current step hint) :" ,
206206 f" Expected action: { self .action_type } ,
207207 ]
208208 if self .target_description :
209209 lines .append (f" Target: { self .target_description } )
210210 if self .action_value :
211211 lines .append (f" Value: { self .action_value } )
212- lines .append (f" Instruction: { self .instruction } )
212+ # Use description (natural language) rather than
213+ # coordinate-based instruction when available.
214+ instruction = self .instruction
215+ if self .metadata .get ("description" ):
216+ instruction = self .metadata ["description" ]
217+ lines .append (f" Instruction: { instruction } )
213218 lines .append (
214219 " NOTE: Adapt if the current UI state differs from the demo. "
215220 "This is guidance, not a rigid script. "
@@ -1533,6 +1538,11 @@ def align_step(
15331538 if alignment_trace is not None :
15341539 guidance_metadata ["alignment_trace" ] = asdict (alignment_trace )
15351540
1541+ # Pass the step's natural-language description so to_prompt_text()
1542+ # can prefer it over coordinate-based instruction text.
1543+ if step .description :
1544+ guidance_metadata ["description" ] = step .description
1545+
15361546 return DemoGuidance (
15371547 available = True ,
15381548 step_index = step .step_index ,
@@ -1549,6 +1559,46 @@ def align_step(
15491559 visual_distance = visual_distance ,
15501560 )
15511561
1562+ def get_plan_overview (self , task_id : str ) -> str :
1563+ """Return a high-level strategy overview of the full demo.
1564+
1565+ Instead of one step at a time, this gives the planner ALL demo
1566+ steps as a numbered plan. The planner can then use this as a
1567+ strategy reference while making its own decisions based on the
1568+ actual screenshot.
1569+
1570+ Coordinates are intentionally omitted -- the planner should
1571+ decide WHERE to click based on what it sees, not on hardcoded
1572+ positions from a demo recorded on a different screen.
1573+
1574+ Returns:
1575+ Multi-line string suitable for injection into the planner
1576+ prompt, or empty string if no demo exists.
1577+ """
1578+ demo = self .get_demo (task_id )
1579+ if demo is None or not demo .steps :
1580+ return ""
1581+
1582+ lines = [
1583+ "DEMONSTRATION STRATEGY (from a successful prior execution):" ,
1584+ f"This task was previously completed in { len (demo .steps )} ,
1585+ ]
1586+ for step in demo .steps :
1587+ # Prefer description (natural language) over action_description
1588+ label = step .description or step .action_description
1589+ action_hint = ""
1590+ if step .action_type == "key" and step .action_value :
1591+ action_hint = f" [{ step .action_value }
1592+ lines .append (f" { step .step_index + 1 } { label } { action_hint } )
1593+
1594+ lines .append ("" )
1595+ lines .append (
1596+ "Use this as a STRATEGY GUIDE. Adapt to the current screen state -- "
1597+ "skip steps that are already done, and decide click targets based on "
1598+ "what you actually see, not memorized coordinates."
1599+ )
1600+ return "\n " .join (lines )
1601+
15521602 def enrich_demo (
15531603 self ,
15541604 task_id : str ,
@@ -1707,13 +1757,17 @@ def _build_enriched_instruction(
17071757) -> str :
17081758 """Build a human-readable instruction from a DemoStep.
17091759
1710- When the step has a VLM-generated ``description``, the instruction
1711- reads like *"Click on three-dot menu button in Chrome toolbar at
1712- approximately (0.960, 0.066)"* instead of the raw
1713- ``CLICK(0.960, 0.066)`` notation.
1760+ When the step has a natural-language ``description``, returns it
1761+ directly — the description already contains the action verb and
1762+ target (e.g., *"Double-click Google Chrome icon"*).
1763+
1764+ Coordinates from manual demos (hardcoded placeholders) are omitted
1765+ because they don't correspond to real screen positions. For VLM-
1766+ enriched demos with real screenshots, coordinates are included as
1767+ approximate hints.
17141768
1715- For non-click actions or steps without descriptions, falls back to
1716- the original ``action_description``.
1769+ For steps without descriptions, falls back to the original
1770+ ``action_description``.
17171771
17181772 Args:
17191773 step: The demo step to build an instruction for.
@@ -1723,14 +1777,18 @@ def _build_enriched_instruction(
17231777 Returns:
17241778 Human-readable instruction string.
17251779 """
1726- if step .description and step .action_type == "click" :
1727- # Rich instruction with element description
1728- if normalized_x is not None and normalized_y is not None :
1729- return (
1730- f"Click on { step .description }
1731- f"({ normalized_x :.3f} { normalized_y :.3f}
1732- )
1733- return f"Click on { step .description }
1780+ if step .description :
1781+ # If the description already reads as a complete sentence/
1782+ # instruction (starts with a verb), use it directly.
1783+ # Only append coordinates when we have real screenshots
1784+ # (manual demos have empty screenshot_path and fake coords).
1785+ if step .action_type == "click" and step .screenshot_path :
1786+ if normalized_x is not None and normalized_y is not None :
1787+ return (
1788+ f"{ step .description }
1789+ f"({ normalized_x :.3f} { normalized_y :.3f}
1790+ )
1791+ return step .description
17341792
17351793 # Fallback: original action description
17361794 return step .action_description
0 commit comments