diff --git a/harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py b/harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py index 3750d8f0..18d489d1 100644 --- a/harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py +++ b/harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py @@ -11,6 +11,16 @@ from harbor.models.agent.context import AgentContext from openai import AsyncOpenAI +def _is_reasoning_model(model: str) -> bool: + """Whether `model` is an OpenAI reasoning model. + + Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects + reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served + open models match none of these prefixes, so they keep the legacy shape. + """ + name = model.lower() + return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name + MAX_TURNS = 32 MAX_TOOL_OUTPUT_CHARS = 40_000 @@ -158,16 +168,24 @@ def _usage_value(value: Any, name: str) -> int: def _completion_kwargs( self, messages: list[dict[str, Any]], *, tools: bool = True ) -> dict[str, Any]: + # Reasoning models replaced max_tokens with max_completion_tokens and + # reject the old name outright ("Unsupported parameter: 'max_tokens' + # is not supported with this model"). Same capability test as the + # reasoning_effort gate below, so the two stay consistent. + _token_limit_key = ( + "max_completion_tokens" + if _is_reasoning_model(self._api_model) + else "max_tokens" + ) kwargs: dict[str, Any] = { "model": self._api_model, "messages": messages, - "max_tokens": 12_000, } + kwargs[_token_limit_key] = 12_000 if tools: kwargs["tools"] = TOOLS - # OpenAI reasoning models accept reasoning_effort; other providers - # (e.g. Fireworks-served open models) reject it. - if "fireworks" not in self._api_model: + # Capability, not provider: see _is_reasoning_model. + if _is_reasoning_model(self._api_model): kwargs["reasoning_effort"] = "medium" return kwargs diff --git a/harness-engineering-bench/gaia/baseline/target/src/gaia_agent/agent.py b/harness-engineering-bench/gaia/baseline/target/src/gaia_agent/agent.py index 8396bb29..62002d08 100644 --- a/harness-engineering-bench/gaia/baseline/target/src/gaia_agent/agent.py +++ b/harness-engineering-bench/gaia/baseline/target/src/gaia_agent/agent.py @@ -188,10 +188,14 @@ async def run( "instructions": INSTRUCTIONS, "input": next_input, "tools": TOOLS, - "reasoning": {"effort": "medium"}, "max_output_tokens": 8000, "parallel_tool_calls": False, } + # gpt-4o and other non-reasoning models reject `reasoning.effort` + # with HTTP 400; only send it to reasoning-capable models. + _model = self._api_model.lower() + if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model: + request["reasoning"] = {"effort": "medium"} if previous_response_id is not None: request["previous_response_id"] = previous_response_id response = await self._client.responses.create(**request) diff --git a/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py b/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py index dd4d90f8..ce7a3af1 100644 --- a/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py +++ b/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py @@ -13,6 +13,16 @@ from harbor.models.agent.context import AgentContext from openai import AsyncOpenAI +def _is_reasoning_model(model: str) -> bool: + """Whether `model` is an OpenAI reasoning model. + + Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects + reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served + open models match none of these prefixes, so they keep the legacy shape. + """ + name = model.lower() + return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name + MAX_TURNS = 24 MAX_TOOL_OUTPUT_CHARS = 20_000 MAX_IMAGE_BYTES = 20 * 1024 * 1024 @@ -172,17 +182,27 @@ def _usage_value(value: Any, name: str) -> int: def _completion_kwargs( self, messages: list[dict[str, Any]], *, tools: bool = True ) -> dict[str, Any]: + # Reasoning models replaced max_tokens with max_completion_tokens and + # reject the old name outright ("Unsupported parameter: 'max_tokens' + # is not supported with this model"). Same capability test as the + # reasoning_effort gate below, so the two stay consistent. + _token_limit_key = ( + "max_completion_tokens" + if _is_reasoning_model(self._api_model) + else "max_tokens" + ) kwargs: dict[str, Any] = { "model": self._api_model, "messages": messages, - "max_tokens": 8000, } + kwargs[_token_limit_key] = 8000 if tools: kwargs["tools"] = TOOLS - # OpenAI reasoning models accept reasoning_effort and parallel_tool_calls; - # other providers (e.g. Fireworks-served open models) reject them. - if "fireworks" not in self._api_model: + if _is_reasoning_model(self._api_model): kwargs["reasoning_effort"] = "medium" + # parallel_tool_calls is a separate axis: Fireworks-served models reject + # it, but gpt-4o supports it, so this one stays a provider check. + if "fireworks" not in self._api_model: kwargs["parallel_tool_calls"] = False return kwargs diff --git a/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py b/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py index b10067c0..471d40db 100644 --- a/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py +++ b/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py @@ -11,6 +11,16 @@ from harbor.models.agent.context import AgentContext from openai import AsyncOpenAI +def _is_reasoning_model(model: str) -> bool: + """Whether `model` is an OpenAI reasoning model. + + Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects + reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served + open models match none of these prefixes, so they keep the legacy shape. + """ + name = model.lower() + return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name + MAX_TURNS = 40 MAX_TOOL_OUTPUT_CHARS = 30_000 @@ -152,16 +162,23 @@ def _usage_value(value: Any, name: str) -> int: def _completion_kwargs( self, messages: list[dict[str, Any]], *, tools: bool = True ) -> dict[str, Any]: + # Reasoning models replaced max_tokens with max_completion_tokens and + # reject the old name outright ("Unsupported parameter: 'max_tokens' + # is not supported with this model"). Same capability test as the + # reasoning_effort gate below, so the two stay consistent. + _token_limit_key = ( + "max_completion_tokens" + if _is_reasoning_model(self._api_model) + else "max_tokens" + ) kwargs: dict[str, Any] = { "model": self._api_model, "messages": messages, - "max_tokens": 12_000, } + kwargs[_token_limit_key] = 12_000 if tools: kwargs["tools"] = TOOLS - # OpenAI reasoning models accept reasoning_effort; other providers - # (e.g. Fireworks-served open models) reject it. - if "fireworks" not in self._api_model: + if _is_reasoning_model(self._api_model): kwargs["reasoning_effort"] = "high" return kwargs diff --git a/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py b/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py index 061df527..1cac3594 100644 --- a/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py +++ b/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py @@ -28,6 +28,16 @@ from harbor.models.agent.context import AgentContext from openai import AsyncOpenAI +def _is_reasoning_model(model: str) -> bool: + """Whether `model` is an OpenAI reasoning model. + + Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects + reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served + open models match none of these prefixes, so they keep the legacy shape. + """ + name = model.lower() + return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name + MAX_TURNS = 80 MAX_TOOL_OUTPUT_CHARS = 30_000 PROTOCOL_VERSION = "2025-06-18" @@ -315,15 +325,22 @@ async def run( for turn in range(1, MAX_TURNS + 1): turns = turn + # Reasoning models replaced max_tokens with max_completion_tokens and + # reject the old name outright ("Unsupported parameter: 'max_tokens' + # is not supported with this model"). Same capability test as the + # reasoning_effort gate below, so the two stay consistent. + _token_limit_key = ( + "max_completion_tokens" + if _is_reasoning_model(self._api_model) + else "max_tokens" + ) kwargs: dict[str, Any] = { "model": self._api_model, "messages": messages, "tools": openai_tools, - "max_tokens": 8_000, } - # OpenAI reasoning models accept reasoning_effort; other providers - # (e.g. Fireworks-served open models) reject it. - if "fireworks" not in self._api_model: + kwargs[_token_limit_key] = 8_000 + if _is_reasoning_model(self._api_model): kwargs["reasoning_effort"] = "medium" response = await client.chat.completions.create(**kwargs) usage = response.usage