Merge branch 'main' into generic_agent_hinter

Author: patricebechard

pyproject.toml

@@ -109,3 +109,4 @@ hint = [
 [project.scripts]
 agentlab-assistant = "agentlab.ui_assistant:main"
 agentlab-xray = "agentlab.analyze.agent_xray:main"
+agentlab-mentor = "agentlab.agents.hitl_agent.launch_hint_ui:main"

src/agentlab/agents/agent_utils.py

@@ -1,6 +1,11 @@
+import copy
+
 from PIL import Image, ImageDraw
 from playwright.sync_api import Page
 
+from agentlab.analyze import overlay_utils
+from agentlab.llm.llm_utils import img_to_base_64
+
 
 def draw_mouse_pointer(image: Image.Image, x: int, y: int) -> Image.Image:
     """
@@ -128,3 +133,24 @@ def zoom_webpage(page: Page, zoom_factor: float = 1.5):
 
     page.evaluate(f"document.documentElement.style.zoom='{zoom_factor*100}%'")
     return page
+
+
+def overlay_action(obs, action):
+    """Overlays actions on screenshot in-place"""
+    act_img = copy.deepcopy(obs["screenshot"])
+    act_img = Image.fromarray(act_img)
+
+    new_obs_properties = copy.deepcopy(obs["extra_element_properties"])
+    import os
+
+    if os.getenv("AGENTLAB_USE_RETINA"):
+        # HACK: divide everything by 2 in the obs
+        # TODO: make this more robust by changing login in annotate_action directly (or maybe in the obs section?)
+        for key, value in new_obs_properties.items():
+            try:
+                new_obs_properties[key]["bbox"] = [elem / 2 for elem in value["bbox"]]
+            except:
+                pass
+
+    overlay_utils.annotate_action(act_img, action, properties=new_obs_properties)
+    return img_to_base_64(act_img)

src/agentlab/agents/generic_agent/__init__.py

@@ -22,6 +22,22 @@
     AGENT_4o_MINI,
     AGENT_4o_MINI_VISION,
     AGENT_4o_VISION,
+    AGENT_AZURE_4o_MINI,
+    AGENT_AZURE_4o,
+    AGENT_AZURE_4o_VISION,
+    AGENT_AZURE_4o_MINI_VISION,
+    AGENT_AZURE_41,
+    AGENT_AZURE_41_MINI,
+    AGENT_AZURE_41_NANO,
+    AGENT_AZURE_41_VISION,
+    AGENT_AZURE_41_MINI_VISION,
+    AGENT_AZURE_41_NANO_VISION,
+    AGENT_AZURE_5,
+    AGENT_AZURE_5_MINI,
+    AGENT_AZURE_5_NANO,
+    AGENT_AZURE_5_VISION,
+    AGENT_AZURE_5_MINI_VISION,
+    AGENT_AZURE_5_NANO_VISION,
     AGENT_o1_MINI,
     AGENT_o3_MINI,
     FLAGS_GPT_4o,
@@ -46,6 +62,22 @@
     "AGENT_37_SONNET",
     "AGENT_4o_VISION",
     "AGENT_4o_MINI_VISION",
+    "AGENT_AZURE_4o_MINI",
+    "AGENT_AZURE_4o",
+    "AGENT_AZURE_4o_VISION",
+    "AGENT_AZURE_4o_MINI_VISION",
+    "AGENT_AZURE_41",
+    "AGENT_AZURE_41_MINI",
+    "AGENT_AZURE_41_NANO",
+    "AGENT_AZURE_41_VISION",
+    "AGENT_AZURE_41_MINI_VISION",
+    "AGENT_AZURE_41_NANO_VISION",
+    "AGENT_AZURE_5",
+    "AGENT_AZURE_5_MINI",
+    "AGENT_AZURE_5_NANO",
+    "AGENT_AZURE_5_VISION",
+    "AGENT_AZURE_5_MINI_VISION",
+    "AGENT_AZURE_5_NANO_VISION",
     "AGENT_CLAUDE_SONNET_35_VISION",
     "AGENT_GPT5_MINI",
 ]

src/agentlab/agents/generic_agent/agent_configs.py

@@ -262,6 +262,43 @@
     chat_model_args=CHAT_MODEL_ARGS_DICT["openai/gpt-4o-mini-2024-07-18"],
     flags=FLAGS_GPT_4o,
 )
+
+AGENT_AZURE_4o_MINI = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4o-mini-2024-07-18"],
+    flags=FLAGS_GPT_4o,
+)
+AGENT_AZURE_4o = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4o-2024-08-06"],
+    flags=FLAGS_GPT_4o,
+)
+AGENT_AZURE_41 = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4.1-2025-04-14"],
+    flags=FLAGS_GPT_4o,
+)
+AGENT_AZURE_41_MINI = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4.1-mini-2025-04-14"],
+    flags=FLAGS_GPT_4o,
+)
+AGENT_AZURE_41_NANO = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4.1-nano-2025-04-14"],
+    flags=FLAGS_GPT_4o,
+)
+
+AGENT_AZURE_5 = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-5-2025-08-07"],
+    flags=FLAGS_GPT_4o,
+)
+
+AGENT_AZURE_5_MINI = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-5-mini-2025-08-07"],
+    flags=FLAGS_GPT_4o,
+)
+
+AGENT_AZURE_5_NANO = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-5-nano-2025-08-07"],
+    flags=FLAGS_GPT_4o,
+)
+
 AGENT_CLAUDE_SONNET_35 = GenericAgentArgs(
     chat_model_args=CHAT_MODEL_ARGS_DICT["openrouter/anthropic/claude-3.5-sonnet:beta"],
     flags=FLAGS_GPT_4o,
@@ -298,6 +335,45 @@
     flags=FLAGS_GPT_4o_VISION,
 )
 
+AGENT_AZURE_4o_VISION = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4o-2024-08-06"],
+    flags=FLAGS_GPT_4o_VISION,
+)
+
+AGENT_AZURE_4o_MINI_VISION = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4o-mini-2024-07-18"],
+    flags=FLAGS_GPT_4o_VISION,
+)
+
+AGENT_AZURE_41_VISION = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4.1-2025-04-14"],
+    flags=FLAGS_GPT_4o_VISION,
+)
+
+AGENT_AZURE_41_MINI_VISION = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4.1-mini-2025-04-14"],
+    flags=FLAGS_GPT_4o_VISION,
+)
+AGENT_AZURE_41_NANO_VISION = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-4.1-nano-2025-04-14"],
+    flags=FLAGS_GPT_4o_VISION,
+)
+
+AGENT_AZURE_5_VISION = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-5-2025-08-07"],
+    flags=FLAGS_GPT_4o_VISION,
+)
+
+AGENT_AZURE_5_MINI_VISION = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-5-mini-2025-08-07"],
+    flags=FLAGS_GPT_4o_VISION,
+)
+
+AGENT_AZURE_5_NANO_VISION = GenericAgentArgs(
+    chat_model_args=CHAT_MODEL_ARGS_DICT["azure/gpt-5-nano-2025-08-07"],
+    flags=FLAGS_GPT_4o_VISION,
+)
+
 AGENT_CLAUDE_SONNET_35_VISION = GenericAgentArgs(
     chat_model_args=CHAT_MODEL_ARGS_DICT["openrouter/anthropic/claude-3.5-sonnet:beta"],
     flags=FLAGS_GPT_4o_VISION,

src/agentlab/agents/hitl_agent/base_multi_candidate_agent.py

@@ -0,0 +1,50 @@
+from typing_extensions import Protocol
+
+from agentlab.agents.agent_args import AgentArgs
+
+
+class MultiCandidateAgent(Protocol):
+    """
+    Protocol for agents that generate multiple candidates for get_action.
+
+    This protocol defines the contract for agents that can generate
+    multiple candidate actions and allow selection of one of them for execution.
+    """
+
+    def get_candidate_generations(
+        self, obs: dict, hint: list[str] | None = None, n_candidates: int = 3
+    ) -> "list[dict]":
+        """
+        Generate multiple candidate actions for the given observation.
+
+        You can pass extra info in agent_info to update internal state of the
+        agent based on the selected candidate. Your internal state management
+        should be robust to multiple calls to the get_candidate_generations method
+        in a single step.
+
+        Args:
+            obs: The current observation dictionary containing environment state
+            hint: Optional list of hint strings to guide candidate generation
+            n_candidates: Number of candidate actions to generate
+        """
+        ...
+
+    def update_agent_state_from_selected_candidate(self, output: dict):
+        """
+        Update the agent's internal state based on the selected candidate.
+        This can include any memory or planning updates.
+
+        Args:
+            output: The selected candidate action dictionary
+        """
+        pass
+
+
+class MultiCandidateAgentArgs(AgentArgs):
+    def make_agent(self) -> MultiCandidateAgent: ...
+
+    def __post_init__(self):
+        """Prefix subagent name with 'MC-'."""
+        super().__post_init__()
+        if hasattr(self, "agent_name") and self.agent_name:
+            self.agent_name = "MC-" + self.agent_name