fix: pick the token-limit parameter by model capability too

The four Chat Completions agents send `max_tokens` unconditionally. Every
gpt-5 model rejects it: "Unsupported parameter: 'max_tokens' is not supported
with this model. Use 'max_completion_tokens' instead." So even with the
reasoning_effort gate corrected, these agents could not target any modern
OpenAI reasoning model, only Fireworks-served models and gpt-4o.

Select the parameter name from the same capability test, and fold the two
copies of that test into one `_is_reasoning_model` helper per agent so they
cannot drift apart. browsecomp-plus's reasoning_effort check moves onto the
helper as well; it had the same provider-shaped gate as the other three.

Verified live against the configured Azure endpoint, sending exactly the
shapes the patched agents now build:

  gpt-4o                   ['max_tokens']                              200
  gpt-5.4-mini-2026-03-17  ['max_completion_tokens','reasoning_effort'] 200

and the predicate classifies the suite's own targets correctly:
`fireworks_ai/deepseek-v4-flash` and `fireworks_ai/gpt-oss-120b` are both
non-reasoning, so the default configuration keeps the legacy shape and is
unaffected.

gaia and swe-bench-pro are untouched here: they use the Responses API, whose
`max_output_tokens` has no such split.

Refs #51.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: shehabyasser-scale

harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py

@@ -11,6 +11,16 @@
 from harbor.models.agent.context import AgentContext
 from openai import AsyncOpenAI
 
+def _is_reasoning_model(model: str) -> bool:
+    """Whether `model` is an OpenAI reasoning model.
+
+    Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects
+    reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served
+    open models match none of these prefixes, so they keep the legacy shape.
+    """
+    name = model.lower()
+    return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name
+
 MAX_TURNS = 32
 MAX_TOOL_OUTPUT_CHARS = 40_000
 
@@ -158,16 +168,24 @@ def _usage_value(value: Any, name: str) -> int:
     def _completion_kwargs(
         self, messages: list[dict[str, Any]], *, tools: bool = True
     ) -> dict[str, Any]:
+        # Reasoning models replaced max_tokens with max_completion_tokens and
+        # reject the old name outright ("Unsupported parameter: 'max_tokens'
+        # is not supported with this model"). Same capability test as the
+        # reasoning_effort gate below, so the two stay consistent.
+        _token_limit_key = (
+            "max_completion_tokens"
+            if _is_reasoning_model(self._api_model)
+            else "max_tokens"
+        )
         kwargs: dict[str, Any] = {
             "model": self._api_model,
             "messages": messages,
-            "max_tokens": 12_000,
         }
+        kwargs[_token_limit_key] = 12_000
         if tools:
             kwargs["tools"] = TOOLS
-        # OpenAI reasoning models accept reasoning_effort; other providers
-        # (e.g. Fireworks-served open models) reject it.
-        if "fireworks" not in self._api_model:
+        # Capability, not provider: see _is_reasoning_model.
+        if _is_reasoning_model(self._api_model):
             kwargs["reasoning_effort"] = "medium"
         return kwargs
 

harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py

@@ -13,6 +13,16 @@
 from harbor.models.agent.context import AgentContext
 from openai import AsyncOpenAI
 
+def _is_reasoning_model(model: str) -> bool:
+    """Whether `model` is an OpenAI reasoning model.
+
+    Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects
+    reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served
+    open models match none of these prefixes, so they keep the legacy shape.
+    """
+    name = model.lower()
+    return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name
+
 MAX_TURNS = 24
 MAX_TOOL_OUTPUT_CHARS = 20_000
 MAX_IMAGE_BYTES = 20 * 1024 * 1024
@@ -172,19 +182,23 @@ def _usage_value(value: Any, name: str) -> int:
     def _completion_kwargs(
         self, messages: list[dict[str, Any]], *, tools: bool = True
     ) -> dict[str, Any]:
+        # Reasoning models replaced max_tokens with max_completion_tokens and
+        # reject the old name outright ("Unsupported parameter: 'max_tokens'
+        # is not supported with this model"). Same capability test as the
+        # reasoning_effort gate below, so the two stay consistent.
+        _token_limit_key = (
+            "max_completion_tokens"
+            if _is_reasoning_model(self._api_model)
+            else "max_tokens"
+        )
         kwargs: dict[str, Any] = {
             "model": self._api_model,
             "messages": messages,
-            "max_tokens": 8000,
         }
+        kwargs[_token_limit_key] = 8000
         if tools:
             kwargs["tools"] = TOOLS
-        # Only reasoning-capable models accept reasoning_effort. A provider
-        # check is not sufficient: Azure gpt-4o is not Fireworks and still
-        # rejects it with "Unrecognized request argument supplied:
-        # reasoning_effort".
-        _model = self._api_model.lower()
-        if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model:
+        if _is_reasoning_model(self._api_model):
             kwargs["reasoning_effort"] = "medium"
         # parallel_tool_calls is a separate axis: Fireworks-served models reject
         # it, but gpt-4o supports it, so this one stays a provider check.

harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py

@@ -11,6 +11,16 @@
 from harbor.models.agent.context import AgentContext
 from openai import AsyncOpenAI
 
+def _is_reasoning_model(model: str) -> bool:
+    """Whether `model` is an OpenAI reasoning model.
+
+    Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects
+    reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served
+    open models match none of these prefixes, so they keep the legacy shape.
+    """
+    name = model.lower()
+    return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name
+
 MAX_TURNS = 40
 MAX_TOOL_OUTPUT_CHARS = 30_000
 
@@ -152,19 +162,23 @@ def _usage_value(value: Any, name: str) -> int:
     def _completion_kwargs(
         self, messages: list[dict[str, Any]], *, tools: bool = True
     ) -> dict[str, Any]:
+        # Reasoning models replaced max_tokens with max_completion_tokens and
+        # reject the old name outright ("Unsupported parameter: 'max_tokens'
+        # is not supported with this model"). Same capability test as the
+        # reasoning_effort gate below, so the two stay consistent.
+        _token_limit_key = (
+            "max_completion_tokens"
+            if _is_reasoning_model(self._api_model)
+            else "max_tokens"
+        )
         kwargs: dict[str, Any] = {
             "model": self._api_model,
             "messages": messages,
-            "max_tokens": 12_000,
         }
+        kwargs[_token_limit_key] = 12_000
         if tools:
             kwargs["tools"] = TOOLS
-        # Only reasoning-capable models accept reasoning_effort. A provider
-        # check is not sufficient: Azure gpt-4o is not Fireworks and still
-        # rejects it with "Unrecognized request argument supplied:
-        # reasoning_effort".
-        _model = self._api_model.lower()
-        if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model:
+        if _is_reasoning_model(self._api_model):
             kwargs["reasoning_effort"] = "high"
         return kwargs
 

harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py

@@ -27,6 +27,16 @@
 from harbor.models.agent.context import AgentContext
 from openai import AsyncOpenAI
 
+def _is_reasoning_model(model: str) -> bool:
+    """Whether `model` is an OpenAI reasoning model.
+
+    Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects
+    reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served
+    open models match none of these prefixes, so they keep the legacy shape.
+    """
+    name = model.lower()
+    return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name
+
 MAX_TURNS = 80
 MAX_TOOL_OUTPUT_CHARS = 30_000
 PROTOCOL_VERSION = "2025-06-18"
@@ -309,18 +319,22 @@ async def run(
 
         for turn in range(1, MAX_TURNS + 1):
             turns = turn
+            # Reasoning models replaced max_tokens with max_completion_tokens and
+            # reject the old name outright ("Unsupported parameter: 'max_tokens'
+            # is not supported with this model"). Same capability test as the
+            # reasoning_effort gate below, so the two stay consistent.
+            _token_limit_key = (
+                "max_completion_tokens"
+                if _is_reasoning_model(self._api_model)
+                else "max_tokens"
+            )
             kwargs: dict[str, Any] = {
                 "model": self._api_model,
                 "messages": messages,
                 "tools": openai_tools,
-                "max_tokens": 8_000,
             }
-            # Only reasoning-capable models accept reasoning_effort. A provider
-            # check is not sufficient: Azure gpt-4o is not Fireworks and still
-            # rejects it with "Unrecognized request argument supplied:
-            # reasoning_effort".
-            _model = self._api_model.lower()
-            if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model:
+            kwargs[_token_limit_key] = 8_000
+            if _is_reasoning_model(self._api_model):
                 kwargs["reasoning_effort"] = "medium"
             response = await client.chat.completions.create(**kwargs)
             usage = response.usage