From 1aec11fdbe9712fd518c884a891b9127ec969a4d Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Sat, 25 Jul 2026 06:56:25 +0300 Subject: [PATCH 1/3] fix: only send reasoning.effort to models that support it gaia and swe-bench-pro send `reasoning: {effort: ...}` unconditionally. A non-reasoning model rejects it with HTTP 400, so every call fails and the case is scored as an honest-looking task failure. Measured on a live gaia run: 715 of 2376 calls returned 400 (514 evaluation, 201 finalization). Gate it on the model being reasoning-capable. No behavior change for reasoning models; gpt-4o stops 400ing. Scoped to the two agents that have no gate at all. officeqa, tau3 and swe-atlas-qna gained `if "fireworks" not in self._api_model` in b7a5b80, so their hunks are dropped here rather than rewritten. That gate is provider-shaped rather than capability-shaped and still sends reasoning_effort to any non-Fireworks non-reasoning model such as Azure gpt-4o, which is raised as review on the PR rather than changed unilaterally. Refs #51. Co-Authored-By: Claude Opus 5 (1M context) --- .../gaia/baseline/target/src/gaia_agent/agent.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/harness-engineering-bench/gaia/baseline/target/src/gaia_agent/agent.py b/harness-engineering-bench/gaia/baseline/target/src/gaia_agent/agent.py index 8396bb29..62002d08 100644 --- a/harness-engineering-bench/gaia/baseline/target/src/gaia_agent/agent.py +++ b/harness-engineering-bench/gaia/baseline/target/src/gaia_agent/agent.py @@ -188,10 +188,14 @@ async def run( "instructions": INSTRUCTIONS, "input": next_input, "tools": TOOLS, - "reasoning": {"effort": "medium"}, "max_output_tokens": 8000, "parallel_tool_calls": False, } + # gpt-4o and other non-reasoning models reject `reasoning.effort` + # with HTTP 400; only send it to reasoning-capable models. + _model = self._api_model.lower() + if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model: + request["reasoning"] = {"effort": "medium"} if previous_response_id is not None: request["previous_response_id"] = previous_response_id response = await self._client.responses.create(**request) From 5bb029a7289d0942133a999ed566f7b7dbf64241 Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Sun, 26 Jul 2026 11:45:47 +0300 Subject: [PATCH 2/3] fix: gate reasoning.effort on capability in the three ported agents Restores the hunks dropped when this PR was restacked. b7a5b80 added `if "fireworks" not in self._api_model` to officeqa, tau3 and swe-atlas-qna, which is provider-shaped rather than capability-shaped: it correctly spares Fireworks-served open models and still sends `reasoning_effort` to any non-Fireworks model that cannot accept it. That is not theoretical. Measured today against the configured Azure endpoint, running the swe-atlas-qna seed agent over its 50-case held-out split: gpt-5.3-codex, provider gate 50/50 BadRequestError 400 "The requested operation is unsupported" (that model has no chat/completions surface on this resource at all) gpt-4o, provider gate 50/50 BadRequestError 400 "Unrecognized request argument supplied: reasoning_effort" gpt-4o, capability gate 0 BadRequestError, agent inference succeeded So the provider gate makes these three benchmarks unrunnable against any non-Fireworks upstream, which is every endpoint we currently have. The capability form also excludes `fireworks_ai/*`, so it is a strict superset of the current behaviour and changes nothing for the default configuration. officeqa gates two arguments on that one condition. Only `reasoning_effort` moves: `parallel_tool_calls` is a separate axis that Fireworks rejects and gpt-4o supports, so it keeps the provider check. Refs #51. Co-Authored-By: Claude Opus 5 (1M context) --- .../baseline/target/src/officeqa_agent/agent.py | 12 +++++++++--- .../baseline/target/src/atlas_agent/agent.py | 9 ++++++--- .../tau3/baseline/target/src/tau3_agent/agent.py | 9 ++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py b/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py index dd4d90f8..a28efc49 100644 --- a/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py +++ b/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py @@ -179,10 +179,16 @@ def _completion_kwargs( } if tools: kwargs["tools"] = TOOLS - # OpenAI reasoning models accept reasoning_effort and parallel_tool_calls; - # other providers (e.g. Fireworks-served open models) reject them. - if "fireworks" not in self._api_model: + # Only reasoning-capable models accept reasoning_effort. A provider + # check is not sufficient: Azure gpt-4o is not Fireworks and still + # rejects it with "Unrecognized request argument supplied: + # reasoning_effort". + _model = self._api_model.lower() + if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model: kwargs["reasoning_effort"] = "medium" + # parallel_tool_calls is a separate axis: Fireworks-served models reject + # it, but gpt-4o supports it, so this one stays a provider check. + if "fireworks" not in self._api_model: kwargs["parallel_tool_calls"] = False return kwargs diff --git a/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py b/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py index b10067c0..42b03df5 100644 --- a/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py +++ b/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py @@ -159,9 +159,12 @@ def _completion_kwargs( } if tools: kwargs["tools"] = TOOLS - # OpenAI reasoning models accept reasoning_effort; other providers - # (e.g. Fireworks-served open models) reject it. - if "fireworks" not in self._api_model: + # Only reasoning-capable models accept reasoning_effort. A provider + # check is not sufficient: Azure gpt-4o is not Fireworks and still + # rejects it with "Unrecognized request argument supplied: + # reasoning_effort". + _model = self._api_model.lower() + if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model: kwargs["reasoning_effort"] = "high" return kwargs diff --git a/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py b/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py index 061df527..6121eeb5 100644 --- a/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py +++ b/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py @@ -321,9 +321,12 @@ async def run( "tools": openai_tools, "max_tokens": 8_000, } - # OpenAI reasoning models accept reasoning_effort; other providers - # (e.g. Fireworks-served open models) reject it. - if "fireworks" not in self._api_model: + # Only reasoning-capable models accept reasoning_effort. A provider + # check is not sufficient: Azure gpt-4o is not Fireworks and still + # rejects it with "Unrecognized request argument supplied: + # reasoning_effort". + _model = self._api_model.lower() + if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model: kwargs["reasoning_effort"] = "medium" response = await client.chat.completions.create(**kwargs) usage = response.usage From c156c5920630869c856b6ee27b686053a45a29b4 Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Sun, 26 Jul 2026 12:16:00 +0300 Subject: [PATCH 3/3] fix: pick the token-limit parameter by model capability too The four Chat Completions agents send `max_tokens` unconditionally. Every gpt-5 model rejects it: "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead." So even with the reasoning_effort gate corrected, these agents could not target any modern OpenAI reasoning model, only Fireworks-served models and gpt-4o. Select the parameter name from the same capability test, and fold the two copies of that test into one `_is_reasoning_model` helper per agent so they cannot drift apart. browsecomp-plus's reasoning_effort check moves onto the helper as well; it had the same provider-shaped gate as the other three. Verified live against the configured Azure endpoint, sending exactly the shapes the patched agents now build: gpt-4o ['max_tokens'] 200 gpt-5.4-mini-2026-03-17 ['max_completion_tokens','reasoning_effort'] 200 and the predicate classifies the suite's own targets correctly: `fireworks_ai/deepseek-v4-flash` and `fireworks_ai/gpt-oss-120b` are both non-reasoning, so the default configuration keeps the legacy shape and is unaffected. gaia and swe-bench-pro are untouched here: they use the Responses API, whose `max_output_tokens` has no such split. Refs #51. Co-Authored-By: Claude Opus 5 (1M context) --- .../target/src/browsecomp_plus_agent/agent.py | 26 ++++++++++++++--- .../target/src/officeqa_agent/agent.py | 28 ++++++++++++++----- .../baseline/target/src/atlas_agent/agent.py | 28 ++++++++++++++----- .../baseline/target/src/tau3_agent/agent.py | 28 ++++++++++++++----- 4 files changed, 85 insertions(+), 25 deletions(-) diff --git a/harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py b/harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py index 3750d8f0..18d489d1 100644 --- a/harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py +++ b/harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py @@ -11,6 +11,16 @@ from harbor.models.agent.context import AgentContext from openai import AsyncOpenAI +def _is_reasoning_model(model: str) -> bool: + """Whether `model` is an OpenAI reasoning model. + + Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects + reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served + open models match none of these prefixes, so they keep the legacy shape. + """ + name = model.lower() + return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name + MAX_TURNS = 32 MAX_TOOL_OUTPUT_CHARS = 40_000 @@ -158,16 +168,24 @@ def _usage_value(value: Any, name: str) -> int: def _completion_kwargs( self, messages: list[dict[str, Any]], *, tools: bool = True ) -> dict[str, Any]: + # Reasoning models replaced max_tokens with max_completion_tokens and + # reject the old name outright ("Unsupported parameter: 'max_tokens' + # is not supported with this model"). Same capability test as the + # reasoning_effort gate below, so the two stay consistent. + _token_limit_key = ( + "max_completion_tokens" + if _is_reasoning_model(self._api_model) + else "max_tokens" + ) kwargs: dict[str, Any] = { "model": self._api_model, "messages": messages, - "max_tokens": 12_000, } + kwargs[_token_limit_key] = 12_000 if tools: kwargs["tools"] = TOOLS - # OpenAI reasoning models accept reasoning_effort; other providers - # (e.g. Fireworks-served open models) reject it. - if "fireworks" not in self._api_model: + # Capability, not provider: see _is_reasoning_model. + if _is_reasoning_model(self._api_model): kwargs["reasoning_effort"] = "medium" return kwargs diff --git a/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py b/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py index a28efc49..ce7a3af1 100644 --- a/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py +++ b/harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py @@ -13,6 +13,16 @@ from harbor.models.agent.context import AgentContext from openai import AsyncOpenAI +def _is_reasoning_model(model: str) -> bool: + """Whether `model` is an OpenAI reasoning model. + + Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects + reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served + open models match none of these prefixes, so they keep the legacy shape. + """ + name = model.lower() + return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name + MAX_TURNS = 24 MAX_TOOL_OUTPUT_CHARS = 20_000 MAX_IMAGE_BYTES = 20 * 1024 * 1024 @@ -172,19 +182,23 @@ def _usage_value(value: Any, name: str) -> int: def _completion_kwargs( self, messages: list[dict[str, Any]], *, tools: bool = True ) -> dict[str, Any]: + # Reasoning models replaced max_tokens with max_completion_tokens and + # reject the old name outright ("Unsupported parameter: 'max_tokens' + # is not supported with this model"). Same capability test as the + # reasoning_effort gate below, so the two stay consistent. + _token_limit_key = ( + "max_completion_tokens" + if _is_reasoning_model(self._api_model) + else "max_tokens" + ) kwargs: dict[str, Any] = { "model": self._api_model, "messages": messages, - "max_tokens": 8000, } + kwargs[_token_limit_key] = 8000 if tools: kwargs["tools"] = TOOLS - # Only reasoning-capable models accept reasoning_effort. A provider - # check is not sufficient: Azure gpt-4o is not Fireworks and still - # rejects it with "Unrecognized request argument supplied: - # reasoning_effort". - _model = self._api_model.lower() - if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model: + if _is_reasoning_model(self._api_model): kwargs["reasoning_effort"] = "medium" # parallel_tool_calls is a separate axis: Fireworks-served models reject # it, but gpt-4o supports it, so this one stays a provider check. diff --git a/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py b/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py index 42b03df5..471d40db 100644 --- a/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py +++ b/harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py @@ -11,6 +11,16 @@ from harbor.models.agent.context import AgentContext from openai import AsyncOpenAI +def _is_reasoning_model(model: str) -> bool: + """Whether `model` is an OpenAI reasoning model. + + Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects + reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served + open models match none of these prefixes, so they keep the legacy shape. + """ + name = model.lower() + return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name + MAX_TURNS = 40 MAX_TOOL_OUTPUT_CHARS = 30_000 @@ -152,19 +162,23 @@ def _usage_value(value: Any, name: str) -> int: def _completion_kwargs( self, messages: list[dict[str, Any]], *, tools: bool = True ) -> dict[str, Any]: + # Reasoning models replaced max_tokens with max_completion_tokens and + # reject the old name outright ("Unsupported parameter: 'max_tokens' + # is not supported with this model"). Same capability test as the + # reasoning_effort gate below, so the two stay consistent. + _token_limit_key = ( + "max_completion_tokens" + if _is_reasoning_model(self._api_model) + else "max_tokens" + ) kwargs: dict[str, Any] = { "model": self._api_model, "messages": messages, - "max_tokens": 12_000, } + kwargs[_token_limit_key] = 12_000 if tools: kwargs["tools"] = TOOLS - # Only reasoning-capable models accept reasoning_effort. A provider - # check is not sufficient: Azure gpt-4o is not Fireworks and still - # rejects it with "Unrecognized request argument supplied: - # reasoning_effort". - _model = self._api_model.lower() - if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model: + if _is_reasoning_model(self._api_model): kwargs["reasoning_effort"] = "high" return kwargs diff --git a/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py b/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py index 6121eeb5..1cac3594 100644 --- a/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py +++ b/harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py @@ -28,6 +28,16 @@ from harbor.models.agent.context import AgentContext from openai import AsyncOpenAI +def _is_reasoning_model(model: str) -> bool: + """Whether `model` is an OpenAI reasoning model. + + Capability, not provider: Azure gpt-4o is not Fireworks yet still rejects + reasoning_effort, and every gpt-5 model rejects max_tokens. Fireworks-served + open models match none of these prefixes, so they keep the legacy shape. + """ + name = model.lower() + return name.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in name + MAX_TURNS = 80 MAX_TOOL_OUTPUT_CHARS = 30_000 PROTOCOL_VERSION = "2025-06-18" @@ -315,18 +325,22 @@ async def run( for turn in range(1, MAX_TURNS + 1): turns = turn + # Reasoning models replaced max_tokens with max_completion_tokens and + # reject the old name outright ("Unsupported parameter: 'max_tokens' + # is not supported with this model"). Same capability test as the + # reasoning_effort gate below, so the two stay consistent. + _token_limit_key = ( + "max_completion_tokens" + if _is_reasoning_model(self._api_model) + else "max_tokens" + ) kwargs: dict[str, Any] = { "model": self._api_model, "messages": messages, "tools": openai_tools, - "max_tokens": 8_000, } - # Only reasoning-capable models accept reasoning_effort. A provider - # check is not sufficient: Azure gpt-4o is not Fireworks and still - # rejects it with "Unrecognized request argument supplied: - # reasoning_effort". - _model = self._api_model.lower() - if _model.startswith(("gpt-5", "o1", "o3", "o4")) or "codex" in _model: + kwargs[_token_limit_key] = 8_000 + if _is_reasoning_model(self._api_model): kwargs["reasoning_effort"] = "medium" response = await client.chat.completions.create(**kwargs) usage = response.usage