Skip to content

Commit 741bfa6

Browse files
varunursekarclaude
andcommitted
Add in-place 429 retry (max_retries=8) to the four baseline agents
Fireworks' shared-org rate limit surfaces as a RateLimitError inside the agent's OpenAI client; harbor only retries infra/env failures, not LLM 429s, so a burst lost trials. The SDK's default 2 retries were exhausted under sustained load. Bump to 8 so transient 429s back off in-place instead of failing the trial (durable fix is gateway-level retry, tracked separately). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 24c5f07 commit 741bfa6

4 files changed

Lines changed: 5 additions & 4 deletions

File tree

  • harness-engineering-bench
    • browsecomp-plus/baseline/target/src/browsecomp_plus_agent
    • officeqa/baseline/target/src/officeqa_agent
    • swe-atlas-qna/baseline/target/src/atlas_agent
    • tau3/baseline/target/src/tau3_agent

harness-engineering-bench/browsecomp-plus/baseline/target/src/browsecomp_plus_agent/agent.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
9999
if self.model_name is None:
100100
raise ValueError("BrowseComp-Plus agent requires a Harbor model")
101101
self._api_model = self.model_name.removeprefix("openai/")
102-
self._client = AsyncOpenAI()
102+
self._client = AsyncOpenAI(max_retries=8)
103103

104104
@override
105105
async def setup(self, environment: BaseEnvironment) -> None:

harness-engineering-bench/officeqa/baseline/target/src/officeqa_agent/agent.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
105105
if self.model_name is None:
106106
raise ValueError("OfficeQA agent requires a Harbor model")
107107
self._api_model = self.model_name.removeprefix("openai/")
108-
self._client = AsyncOpenAI()
108+
self._client = AsyncOpenAI(max_retries=8)
109109

110110
@override
111111
async def setup(self, environment: BaseEnvironment) -> None:

harness-engineering-bench/swe-atlas-qna/baseline/target/src/atlas_agent/agent.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
8888
# The metered per-evaluation gateway arrives on dedicated variables
8989
# (OPENAI_* carries the upstream for the task's rubric judge instead).
9090
self._client = AsyncOpenAI(
91+
max_retries=8,
9192
api_key=os.environ.get("VERO_AGENT_INFERENCE_API_KEY")
9293
or os.environ.get("OPENAI_API_KEY"),
9394
base_url=os.environ.get("VERO_AGENT_INFERENCE_BASE_URL")

harness-engineering-bench/tau3/baseline/target/src/tau3_agent/agent.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -279,9 +279,9 @@ async def run(
279279
gateway_key = os.environ.get("VERO_AGENT_INFERENCE_API_KEY")
280280
gateway_url = os.environ.get("VERO_AGENT_INFERENCE_BASE_URL")
281281
if gateway_key and gateway_url:
282-
client = AsyncOpenAI(api_key=gateway_key, base_url=gateway_url)
282+
client = AsyncOpenAI(api_key=gateway_key, base_url=gateway_url, max_retries=8)
283283
else:
284-
client = AsyncOpenAI() # OPENAI_API_KEY / OPENAI_BASE_URL from the env
284+
client = AsyncOpenAI(max_retries=8) # OPENAI_API_KEY / OPENAI_BASE_URL from the env
285285
input_tokens = output_tokens = cached_tokens = turns = 0
286286

287287
session_id, tools = await self._open_session(environment)

0 commit comments

Comments
 (0)