update

Author: erikchwang

src/agentlab/agents/vl_agent/config.py

@@ -33,6 +33,7 @@
         use_error=True,
         use_abstract_example=True,
         use_concrete_example=False,
+        extra_instruction=None,
     )
 }
 

src/agentlab/agents/vl_agent/vl_agent/ui_agent.py

@@ -35,12 +35,7 @@ def __init__(
     @cost_tracker_decorator
     def get_action(self, obs: dict) -> tuple[str, dict]:
         ui_prompt = self.ui_prompt_args.make_prompt(
-            obs=obs,
-            thoughts=self.thoughts,
-            actions=self.actions,
-            action_set=self.action_set,
-            extra_instructions=None,
-            preliminary_answer=None,
+            obs=obs, thoughts=self.thoughts, actions=self.actions, action_set=self.action_set
         )
         try:
             messages = ui_prompt.get_messages()
@@ -62,7 +57,6 @@ def get_action(self, obs: dict) -> tuple[str, dict]:
                 thoughts=self.thoughts,
                 actions=self.actions,
                 action_set=self.action_set,
-                extra_instructions=None,
                 preliminary_answer=preliminary_answer,
             )
             try:

src/agentlab/agents/vl_agent/vl_prompt/ui_prompt.py

@@ -264,19 +264,19 @@ class UIPromptArgs(VLPromptArgs):
     use_error: bool
     use_abstract_example: bool
     use_concrete_example: bool
+    extra_instruction: Optional[str]
 
     def make_prompt(
         self,
         obs: dict,
         thoughts: list[str],
         actions: list[str],
         action_set: HighLevelActionSet,
-        extra_instruction: Optional[str] = None,
         preliminary_answer: Optional[dict] = None,
     ) -> UIPrompt:
         system_prompt_part = SystemPromptPart()
         instruction_prompt_part = InstructionPromptPart(
-            goal_object=obs["goal_object"], extra_instruction=extra_instruction
+            goal_object=obs["goal_object"], extra_instruction=self.extra_instruction
         )
         if self.use_screenshot:
             screenshot_prompt_part = ScreenshotPromptPart(obs["screenshot"])