diff --git a/gui_agents/s1/mllm/MultimodalEngine.py b/gui_agents/s1/mllm/MultimodalEngine.py index 4cb4ea90..d5927f1e 100644 --- a/gui_agents/s1/mllm/MultimodalEngine.py +++ b/gui_agents/s1/mllm/MultimodalEngine.py @@ -11,6 +11,10 @@ import openai import requests from anthropic import Anthropic +from gui_agents.utils import ( + anthropic_supports_temperature, + extract_anthropic_text, +) from openai import APIConnectionError, APIError, AzureOpenAI, OpenAI, RateLimitError from PIL import Image @@ -115,18 +119,16 @@ def __init__(self, api_key=None, model=None, **kwargs): ) def generate(self, messages, temperature=0.0, max_new_tokens=None, **kwargs): """Generate the next message based on previous messages""" - return ( - self.llm_client.messages.create( - system=messages[0]["content"][0]["text"], - model=self.model, - messages=messages[1:], - max_tokens=max_new_tokens if max_new_tokens else 4096, - temperature=temperature, - **kwargs, - ) - .content[0] - .text - ) + request_kwargs = { + "system": messages[0]["content"][0]["text"], + "model": self.model, + "messages": messages[1:], + "max_tokens": max_new_tokens if max_new_tokens else 4096, + **kwargs, + } + if anthropic_supports_temperature(self.model): + request_kwargs["temperature"] = temperature + return extract_anthropic_text(self.llm_client.messages.create(**request_kwargs)) class OpenAIEmbeddingEngine(LMMEngine): diff --git a/gui_agents/s2/core/engine.py b/gui_agents/s2/core/engine.py index cef83fd8..ae805421 100644 --- a/gui_agents/s2/core/engine.py +++ b/gui_agents/s2/core/engine.py @@ -3,6 +3,11 @@ import backoff import numpy as np from anthropic import Anthropic +from gui_agents.utils import ( + anthropic_supports_temperature, + extract_anthropic_text, + extract_anthropic_thinking, +) from openai import ( AzureOpenAI, APIConnectionError, @@ -215,21 +220,19 @@ def generate(self, messages, temperature=0.0, max_new_tokens=None, **kwargs): thinking={"type": "enabled", "budget_tokens": 4096}, **kwargs, ) - thoughts = full_response.content[0].thinking - print("CLAUDE 3.7 THOUGHTS:", thoughts) - return full_response.content[1].text - return ( - self.llm_client.messages.create( - system=messages[0]["content"][0]["text"], - model=self.model, - messages=messages[1:], - max_tokens=max_new_tokens if max_new_tokens else 4096, - temperature=temperature, - **kwargs, - ) - .content[0] - .text - ) + thoughts = extract_anthropic_thinking(full_response) + print("CLAUDE THOUGHTS:", thoughts) + return extract_anthropic_text(full_response) + request_kwargs = { + "system": messages[0]["content"][0]["text"], + "model": self.model, + "messages": messages[1:], + "max_tokens": max_new_tokens if max_new_tokens else 4096, + **kwargs, + } + if anthropic_supports_temperature(self.model): + request_kwargs["temperature"] = temperature + return extract_anthropic_text(self.llm_client.messages.create(**request_kwargs)) class LMMEngineGemini(LMMEngine): diff --git a/gui_agents/s2_5/core/engine.py b/gui_agents/s2_5/core/engine.py index 3b17de60..86e49837 100644 --- a/gui_agents/s2_5/core/engine.py +++ b/gui_agents/s2_5/core/engine.py @@ -2,6 +2,11 @@ import backoff from anthropic import Anthropic +from gui_agents.utils import ( + anthropic_supports_temperature, + extract_anthropic_text, + extract_anthropic_thinking, +) from openai import ( AzureOpenAI, APIConnectionError, @@ -107,20 +112,17 @@ def generate(self, messages, temperature=0.0, max_new_tokens=None, **kwargs): thinking={"type": "enabled", "budget_tokens": 4096}, **kwargs, ) - thoughts = full_response.content[0].thinking - return full_response.content[1].text - return ( - self.llm_client.messages.create( - system=messages[0]["content"][0]["text"], - model=self.model, - messages=messages[1:], - max_tokens=max_new_tokens if max_new_tokens else 4096, - temperature=temp, - **kwargs, - ) - .content[0] - .text - ) + return extract_anthropic_text(full_response) + request_kwargs = { + "system": messages[0]["content"][0]["text"], + "model": self.model, + "messages": messages[1:], + "max_tokens": max_new_tokens if max_new_tokens else 4096, + **kwargs, + } + if anthropic_supports_temperature(self.model): + request_kwargs["temperature"] = temp + return extract_anthropic_text(self.llm_client.messages.create(**request_kwargs)) @backoff.on_exception( backoff.expo, (APIConnectionError, APIError, RateLimitError), max_time=60 @@ -140,8 +142,8 @@ def generate_with_thinking( **kwargs, ) - thoughts = full_response.content[0].thinking - answer = full_response.content[1].text + thoughts = extract_anthropic_thinking(full_response) + answer = extract_anthropic_text(full_response) full_response = ( f"\n{thoughts}\n\n\n\n{answer}\n\n" ) diff --git a/gui_agents/s3/agents/worker.py b/gui_agents/s3/agents/worker.py index 2b4aff06..7af14d76 100644 --- a/gui_agents/s3/agents/worker.py +++ b/gui_agents/s3/agents/worker.py @@ -46,7 +46,7 @@ def __init__( """ super().__init__(worker_engine_params, platform) - self.temperature = worker_engine_params.get("temperature", 0.0) + self.temperature = worker_engine_params.get("temperature") or 0.0 self.use_thinking = worker_engine_params.get("model", "") in [ "claude-opus-4-20250514", "claude-sonnet-4-20250514", diff --git a/gui_agents/s3/cli_app.py b/gui_agents/s3/cli_app.py index 55816be0..e4306dd9 100644 --- a/gui_agents/s3/cli_app.py +++ b/gui_agents/s3/cli_app.py @@ -335,8 +335,9 @@ def main(): "model": args.model, "base_url": args.model_url, "api_key": args.model_api_key, - "temperature": getattr(args, "model_temperature", None), } + if args.model_temperature is not None: + engine_params["temperature"] = args.model_temperature # Load the grounding engine from a custom endpoint engine_params_for_grounding = { diff --git a/gui_agents/s3/core/engine.py b/gui_agents/s3/core/engine.py index 7bf90f14..4fefd9e9 100644 --- a/gui_agents/s3/core/engine.py +++ b/gui_agents/s3/core/engine.py @@ -2,6 +2,11 @@ import backoff from anthropic import Anthropic +from gui_agents.utils import ( + anthropic_supports_temperature, + extract_anthropic_text, + extract_anthropic_thinking, +) from openai import ( AzureOpenAI, APIConnectionError, @@ -95,8 +100,13 @@ def generate(self, messages, temperature=0.0, max_new_tokens=None, **kwargs): "An API Key needs to be provided in either the api_key parameter or as an environment variable named ANTHROPIC_API_KEY" ) self.llm_client = Anthropic(api_key=api_key) - # Use the instance temperature if not specified in the call - temp = self.temperature if temperature is None else temperature + # Prefer instance temperature when set; otherwise use call arg or default to 0.0 + if self.temperature is not None: + temp = self.temperature + elif temperature is not None: + temp = temperature + else: + temp = 0.0 if self.thinking: full_response = self.llm_client.messages.create( system=messages[0]["content"][0]["text"], @@ -106,20 +116,17 @@ def generate(self, messages, temperature=0.0, max_new_tokens=None, **kwargs): thinking={"type": "enabled", "budget_tokens": 4096}, **kwargs, ) - thoughts = full_response.content[0].thinking - return full_response.content[1].text - return ( - self.llm_client.messages.create( - system=messages[0]["content"][0]["text"], - model=self.model, - messages=messages[1:], - max_tokens=max_new_tokens if max_new_tokens else 4096, - temperature=temp, - **kwargs, - ) - .content[0] - .text - ) + return extract_anthropic_text(full_response) + request_kwargs = { + "system": messages[0]["content"][0]["text"], + "model": self.model, + "messages": messages[1:], + "max_tokens": max_new_tokens if max_new_tokens else 4096, + **kwargs, + } + if anthropic_supports_temperature(self.model): + request_kwargs["temperature"] = temp + return extract_anthropic_text(self.llm_client.messages.create(**request_kwargs)) @backoff.on_exception( backoff.expo, (APIConnectionError, APIError, RateLimitError), max_time=60 @@ -144,8 +151,8 @@ def generate_with_thinking( **kwargs, ) - thoughts = full_response.content[0].thinking - answer = full_response.content[1].text + thoughts = extract_anthropic_thinking(full_response) + answer = extract_anthropic_text(full_response) full_response = ( f"\n{thoughts}\n\n\n\n{answer}\n\n" ) diff --git a/gui_agents/utils.py b/gui_agents/utils.py index 00223422..463fcaaa 100644 --- a/gui_agents/utils.py +++ b/gui_agents/utils.py @@ -1,11 +1,52 @@ """General utility.""" import platform +import re import requests import zipfile import io import os +# Anthropic models that reject non-default sampling params (temperature, top_p, top_k). +_ANTHROPIC_NO_TEMPERATURE_PATTERNS = ( + re.compile(r"sonnet-5"), + re.compile(r"opus-4-7"), + re.compile(r"opus-4-8"), +) + + +def anthropic_supports_temperature(model: str) -> bool: + """Return False for Anthropic models that reject the temperature parameter.""" + model_lower = model.lower() + return not any( + pattern.search(model_lower) for pattern in _ANTHROPIC_NO_TEMPERATURE_PATTERNS + ) + + +def extract_anthropic_text(response) -> str: + """Concatenate text from an Anthropic Messages response. + + Newer models (e.g. Sonnet 5) enable adaptive thinking by default, so the + response content can contain ThinkingBlocks that have no `text` attribute. + This skips non-text blocks and returns only the assistant's text output. + """ + texts = [ + block.text + for block in response.content + if getattr(block, "type", None) == "text" + ] + return "".join(texts) + + +def extract_anthropic_thinking(response) -> str: + """Concatenate thinking/reasoning text from an Anthropic Messages response.""" + thoughts = [ + getattr(block, "thinking", "") + for block in response.content + if getattr(block, "type", None) == "thinking" + ] + return "".join(thoughts) + def download_kb_data( version="s2", diff --git a/tests/test_providers.py b/tests/test_providers.py index 2e1b24d2..bfd0426d 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -2,7 +2,32 @@ import unittest from unittest.mock import patch, MagicMock from gui_agents.s3.core.mllm import LMMAgent -from gui_agents.s3.core.engine import LMMEngineOpenAI +from gui_agents.s3.core.engine import LMMEngineOpenAI, LMMEngineAnthropic +from gui_agents.utils import ( + anthropic_supports_temperature, + extract_anthropic_text, + extract_anthropic_thinking, +) + + +def _text_block(text): + block = MagicMock() + block.type = "text" + block.text = text + return block + + +def _thinking_block(thinking): + block = MagicMock() + block.type = "thinking" + block.thinking = thinking + return block + + +def _anthropic_response(*blocks): + response = MagicMock() + response.content = list(blocks) + return response class TestProviders(unittest.TestCase): @@ -67,6 +92,94 @@ def test_qwen_init(self): "https://dashscope.aliyuncs.com/compatible-mode/v1", ) + def test_anthropic_supports_temperature(self): + """Test temperature support detection for Anthropic models.""" + self.assertTrue( + anthropic_supports_temperature("claude-sonnet-4-20250514") + ) + self.assertTrue( + anthropic_supports_temperature("claude-sonnet-4-5-20250929") + ) + self.assertFalse(anthropic_supports_temperature("claude-sonnet-5")) + self.assertFalse( + anthropic_supports_temperature("claude-sonnet-5-20260301") + ) + self.assertFalse(anthropic_supports_temperature("claude-opus-4-7")) + self.assertFalse(anthropic_supports_temperature("claude-opus-4-8")) + + def test_extract_anthropic_text_skips_thinking_blocks(self): + """Adaptive-thinking responses (thinking block first) must return text only.""" + response = _anthropic_response( + _thinking_block("let me reason about this"), + _text_block("the actual answer"), + ) + self.assertEqual(extract_anthropic_text(response), "the actual answer") + self.assertEqual( + extract_anthropic_thinking(response), "let me reason about this" + ) + + def test_extract_anthropic_text_plain(self): + """Plain text-only responses still return the text.""" + response = _anthropic_response(_text_block("hello")) + self.assertEqual(extract_anthropic_text(response), "hello") + + @patch("gui_agents.s3.core.engine.Anthropic") + def test_anthropic_sonnet_5_omits_temperature(self, mock_anthropic): + """Sonnet 5 requests must not include temperature.""" + mock_client = MagicMock() + mock_anthropic.return_value = mock_client + mock_client.messages.create.return_value = _anthropic_response( + _text_block("ok") + ) + + engine = LMMEngineAnthropic(model="claude-sonnet-5", api_key="test") + messages = [ + {"content": [{"text": "system prompt"}]}, + {"role": "user", "content": [{"type": "text", "text": "hello"}]}, + ] + result = engine.generate(messages, temperature=0.0) + + _, kwargs = mock_client.messages.create.call_args + self.assertNotIn("temperature", kwargs) + self.assertEqual(result, "ok") + + @patch("gui_agents.s3.core.engine.Anthropic") + def test_anthropic_sonnet_5_adaptive_thinking_response(self, mock_anthropic): + """Sonnet 5 responses with a leading thinking block must not crash.""" + mock_client = MagicMock() + mock_anthropic.return_value = mock_client + mock_client.messages.create.return_value = _anthropic_response( + _thinking_block("reasoning"), + _text_block("final answer"), + ) + + engine = LMMEngineAnthropic(model="claude-sonnet-5", api_key="test") + messages = [ + {"content": [{"text": "system prompt"}]}, + {"role": "user", "content": [{"type": "text", "text": "hello"}]}, + ] + result = engine.generate(messages, temperature=0.0) + self.assertEqual(result, "final answer") + + @patch("gui_agents.s3.core.engine.Anthropic") + def test_anthropic_sonnet_4_includes_temperature(self, mock_anthropic): + """Sonnet 4 requests should still include temperature.""" + mock_client = MagicMock() + mock_anthropic.return_value = mock_client + mock_client.messages.create.return_value = _anthropic_response( + _text_block("ok") + ) + + engine = LMMEngineAnthropic(model="claude-sonnet-4-20250514", api_key="test") + messages = [ + {"content": [{"text": "system prompt"}]}, + {"role": "user", "content": [{"type": "text", "text": "hello"}]}, + ] + engine.generate(messages, temperature=0.0) + + _, kwargs = mock_client.messages.create.call_args + self.assertEqual(kwargs["temperature"], 0.0) + if __name__ == "__main__": unittest.main()