fix: only send reasoning.effort to models that support it

harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py

@@ -11,6 +11,16 @@
 from harbor.models.agent.context import AgentContext
 from openai import AsyncOpenAI
 
+def _is_reasoning_model(model: str) -> bool:
+    """Whether `model` is an OpenAI reasoning model.
+
+    Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects
+    reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served
+    open models match none of these prefixes, so they keep the legacy shape.
+    """
+    name = model.lower()
+    return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name
+
 MAX_TURNS = 32
 MAX_TOOL_OUTPUT_CHARS = 40_000
 
@@ -158,16 +168,24 @@ def _usage_value(value: Any, name: str) -> int:
     def _completion_kwargs(
         self, messages: list[dict[str, Any]], *, tools: bool = True
     ) -> dict[str, Any]:
+        # Reasoning models replaced max_tokens with max_completion_tokens and
+        # reject the old name outright ("Unsupported parameter: 'max_tokens'
+        # is not supported with this model"). Same capability test as the
+        # reasoning_effort gate below, so the two stay consistent.
+        _token_limit_key = (
+            "max_completion_tokens"
+            if _is_reasoning_model(self._api_model)
+            else "max_tokens"
+        )
         kwargs: dict[str, Any] = {
             "model": self._api_model,
             "messages": messages,
-            "max_tokens": 12_000,
         }
+        kwargs[_token_limit_key] = 12_000
         if tools:
             kwargs["tools"] = TOOLS
-        # OpenAI reasoning models accept reasoning_effort; other providers
-        # (e.g. Fireworks-served open models) reject it.
-        if "fireworks" not in self._api_model:
+        # Capability, not provider: see _is_reasoning_model.
+        if _is_reasoning_model(self._api_model):
             kwargs["reasoning_effort"] = "medium"
         return kwargs
 

harness-engineering-bench/gaia/baseline/target/src/gaia_agent/agent.py

@@ -188,10 +188,14 @@ async def run(
                 "instructions": INSTRUCTIONS,
                 "input": next_input,
                 "tools": TOOLS,
-                "reasoning": {"effort": "medium"},
                 "max_output_tokens": 8000,
                 "parallel_tool_calls": False,
             }
+            # gpt-4o and other non-reasoning models reject `reasoning.effort`
+            # with HTTP 400; only send it to reasoning-capable models.
+            _model = self._api_model.lower()
+            if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model:
+                request["reasoning"] = {"effort": "medium"}
             if previous_response_id is not None:
                 request["previous_response_id"] = previous_response_id
             response = await self._client.responses.create(**request)

harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py

@@ -13,6 +13,16 @@
 from harbor.models.agent.context import AgentContext
 from openai import AsyncOpenAI
 
+def _is_reasoning_model(model: str) -> bool:
+    """Whether `model` is an OpenAI reasoning model.
+
+    Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects
+    reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served
+    open models match none of these prefixes, so they keep the legacy shape.
+    """
+    name = model.lower()
+    return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name
+
 MAX_TURNS = 24
 MAX_TOOL_OUTPUT_CHARS = 20_000
 MAX_IMAGE_BYTES = 20 * 1024 * 1024
@@ -172,17 +182,27 @@ def _usage_value(value: Any, name: str) -> int:
     def _completion_kwargs(
         self, messages: list[dict[str, Any]], *, tools: bool = True
     ) -> dict[str, Any]:
+        # Reasoning models replaced max_tokens with max_completion_tokens and
+        # reject the old name outright ("Unsupported parameter: 'max_tokens'
+        # is not supported with this model"). Same capability test as the
+        # reasoning_effort gate below, so the two stay consistent.
+        _token_limit_key = (
+            "max_completion_tokens"
+            if _is_reasoning_model(self._api_model)
+            else "max_tokens"
+        )
         kwargs: dict[str, Any] = {
             "model": self._api_model,
             "messages": messages,
-            "max_tokens": 8000,
         }
+        kwargs[_token_limit_key] = 8000
         if tools:
             kwargs["tools"] = TOOLS
-        # OpenAI reasoning models accept reasoning_effort and parallel_tool_calls;
-        # other providers (e.g. Fireworks-served open models) reject them.
-        if "fireworks" not in self._api_model:
+        if _is_reasoning_model(self._api_model):
             kwargs["reasoning_effort"] = "medium"
+        # parallel_tool_calls is a separate axis: Fireworks-served models reject
+        # it, but gpt-4o supports it, so this one stays a provider check.
+        if "fireworks" not in self._api_model:
             kwargs["parallel_tool_calls"] = False
         return kwargs
 

harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py

@@ -11,6 +11,16 @@
 from harbor.models.agent.context import AgentContext
 from openai import AsyncOpenAI
 
+def _is_reasoning_model(model: str) -> bool:
+    """Whether `model` is an OpenAI reasoning model.
+
+    Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects
+    reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served
+    open models match none of these prefixes, so they keep the legacy shape.
+    """
+    name = model.lower()
+    return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name
+
 MAX_TURNS = 40
 MAX_TOOL_OUTPUT_CHARS = 30_000
 
@@ -152,16 +162,23 @@ def _usage_value(value: Any, name: str) -> int:
     def _completion_kwargs(
         self, messages: list[dict[str, Any]], *, tools: bool = True
     ) -> dict[str, Any]:
+        # Reasoning models replaced max_tokens with max_completion_tokens and
+        # reject the old name outright ("Unsupported parameter: 'max_tokens'
+        # is not supported with this model"). Same capability test as the
+        # reasoning_effort gate below, so the two stay consistent.
+        _token_limit_key = (
+            "max_completion_tokens"
+            if _is_reasoning_model(self._api_model)
+            else "max_tokens"
+        )
         kwargs: dict[str, Any] = {
             "model": self._api_model,
             "messages": messages,
-            "max_tokens": 12_000,
         }
+        kwargs[_token_limit_key] = 12_000
         if tools:
             kwargs["tools"] = TOOLS
-        # OpenAI reasoning models accept reasoning_effort; other providers
-        # (e.g. Fireworks-served open models) reject it.
-        if "fireworks" not in self._api_model:
+        if _is_reasoning_model(self._api_model):
             kwargs["reasoning_effort"] = "high"
         return kwargs
 

harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py

@@ -28,6 +28,16 @@
 from harbor.models.agent.context import AgentContext
 from openai import AsyncOpenAI
 
+def _is_reasoning_model(model: str) -> bool:
+    """Whether `model` is an OpenAI reasoning model.
+
+    Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects
+    reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served
+    open models match none of these prefixes, so they keep the legacy shape.
+    """
+    name = model.lower()
+    return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name
+
 MAX_TURNS = 80
 MAX_TOOL_OUTPUT_CHARS = 30_000
 PROTOCOL_VERSION = "2025-06-18"
@@ -315,15 +325,22 @@ async def run(
 
         for turn in range(1, MAX_TURNS + 1):
             turns = turn
+            # Reasoning models replaced max_tokens with max_completion_tokens and
+            # reject the old name outright ("Unsupported parameter: 'max_tokens'
+            # is not supported with this model"). Same capability test as the
+            # reasoning_effort gate below, so the two stay consistent.
+            _token_limit_key = (
+                "max_completion_tokens"
+                if _is_reasoning_model(self._api_model)
+                else "max_tokens"
+            )
             kwargs: dict[str, Any] = {
                 "model": self._api_model,
                 "messages": messages,
                 "tools": openai_tools,
-                "max_tokens": 8_000,
             }
-            # OpenAI reasoning models accept reasoning_effort; other providers
-            # (e.g. Fireworks-served open models) reject it.
-            if "fireworks" not in self._api_model:
+            kwargs[_token_limit_key] = 8_000
+            if _is_reasoning_model(self._api_model):
                 kwargs["reasoning_effort"] = "medium"
             response = await client.chat.completions.create(**kwargs)
             usage = response.usage