Skip to content

Commit 2a64639

Browse files
abrichrclaude
andauthored
feat: register HttpAgent in CLI with --agent http --agent-endpoint (#119)
Adds `http` as a valid agent type across all three CLI subcommands (mock, run, live). Requires `--agent-endpoint <URL>` when used. Usage: openadapt-evals run --agent http --agent-endpoint http://gpu-box:8080 openadapt-evals live --agent http --agent-endpoint http://gpu-box:8080 Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 694fddc commit 2a64639

1 file changed

Lines changed: 36 additions & 6 deletions

File tree

  • openadapt_evals/benchmarks

openadapt_evals/benchmarks/cli.py

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -265,9 +265,17 @@ def cmd_mock(args: argparse.Namespace) -> int:
265265
except RuntimeError as e:
266266
print(f"ERROR: {e}")
267267
return 1
268+
elif agent_type == "http":
269+
agent_endpoint = getattr(args, "agent_endpoint", None)
270+
if not agent_endpoint:
271+
print("ERROR: --agent-endpoint required for http agent")
272+
return 1
273+
from openadapt_evals.agents import HttpAgent
274+
agent = HttpAgent(endpoint_url=agent_endpoint)
275+
print(f"Using HttpAgent (endpoint={agent_endpoint})")
268276
else:
269277
print(f"ERROR: Unknown agent type: {agent_type}")
270-
print("Available for mock: mock, api-claude, api-openai, api-claude-cu, qwen3vl, smol")
278+
print("Available for mock: mock, api-claude, api-openai, api-claude-cu, qwen3vl, smol, http")
271279
return 1
272280

273281
# Create config for trace collection
@@ -438,9 +446,17 @@ def cmd_run(args: argparse.Namespace) -> int:
438446
except RuntimeError as e:
439447
print(f"ERROR: {e}")
440448
return 1
449+
elif agent_type == "http":
450+
agent_endpoint = getattr(args, "agent_endpoint", None)
451+
if not agent_endpoint:
452+
print("ERROR: --agent-endpoint required for http agent")
453+
return 1
454+
from openadapt_evals.agents import HttpAgent
455+
agent = HttpAgent(endpoint_url=agent_endpoint)
456+
print(f"Using HttpAgent (endpoint={agent_endpoint})")
441457
else:
442458
print(f"ERROR: Unknown agent type: {agent_type}")
443-
print("Available: noop, mock, api-claude, api-openai, api-claude-cu, qwen3vl, smol")
459+
print("Available: noop, mock, api-claude, api-openai, api-claude-cu, qwen3vl, smol, http")
444460
return 1
445461

446462
# Create config for trace collection
@@ -675,10 +691,18 @@ def cmd_live(args: argparse.Namespace) -> int:
675691
except RuntimeError as e:
676692
print(f"ERROR: {e}")
677693
return 1
694+
elif agent_type == "http":
695+
agent_endpoint = getattr(args, "agent_endpoint", None)
696+
if not agent_endpoint:
697+
print("ERROR: --agent-endpoint required for http agent")
698+
return 1
699+
from openadapt_evals.agents import HttpAgent
700+
agent = HttpAgent(endpoint_url=agent_endpoint)
701+
print(f"Using HttpAgent (endpoint={agent_endpoint})")
678702
else:
679703
print(f"ERROR: Unknown agent type: {agent_type}")
680704
print(
681-
"Available: mock, noop, api-claude, api-openai, api-claude-cu, qwen3vl, smol, retrieval-claude, retrieval-openai"
705+
"Available: mock, noop, api-claude, api-openai, api-claude-cu, qwen3vl, smol, http, retrieval-claude, retrieval-openai"
682706
)
683707
return 1
684708

@@ -2381,7 +2405,9 @@ def main() -> int:
23812405
mock_parser.add_argument("--tasks", type=int, default=10, help="Number of tasks")
23822406
mock_parser.add_argument("--max-steps", type=int, default=15, help="Max steps per task")
23832407
mock_parser.add_argument("--agent", type=str, default="mock",
2384-
help="Agent type: mock, api-claude, api-openai, api-claude-cu, qwen3vl, smol")
2408+
help="Agent type: mock, api-claude, api-openai, api-claude-cu, qwen3vl, smol, http")
2409+
mock_parser.add_argument("--agent-endpoint", type=str,
2410+
help="HTTP endpoint URL for http agent (e.g., http://localhost:8080)")
23852411
mock_parser.add_argument("--demo", type=str, help="Demo trajectory file for ApiAgent")
23862412
mock_parser.add_argument("--model-path", type=str, help="Model path for Qwen3VL agent")
23872413
mock_parser.add_argument("--model-endpoint", type=str, help="Remote endpoint for Qwen3VL ('modal' or HTTP URL)")
@@ -2406,7 +2432,9 @@ def main() -> int:
24062432
run_parser.add_argument("--evaluate-url", type=str, default=None,
24072433
help="Evaluate server URL (default: same as --server)")
24082434
run_parser.add_argument("--agent", type=str, default="api-openai",
2409-
help="Agent type: noop, mock, api-claude, api-openai, api-claude-cu, qwen3vl, smol")
2435+
help="Agent type: noop, mock, api-claude, api-openai, api-claude-cu, qwen3vl, smol, http")
2436+
run_parser.add_argument("--agent-endpoint", type=str,
2437+
help="HTTP endpoint URL for http agent (e.g., http://localhost:8080)")
24102438
run_parser.add_argument("--task", type=str,
24112439
help="Single task ID (e.g., notepad_1)")
24122440
run_parser.add_argument("--tasks", type=str,
@@ -2458,7 +2486,9 @@ def main() -> int:
24582486
live_parser.add_argument("--evaluate-url", type=str, default=None,
24592487
help="Evaluate server URL (default: same as --server)")
24602488
live_parser.add_argument("--agent", type=str, default="mock",
2461-
help="Agent type: mock, noop, api-claude, api-openai, api-claude-cu, qwen3vl, smol, retrieval-claude, retrieval-openai")
2489+
help="Agent type: mock, noop, api-claude, api-openai, api-claude-cu, qwen3vl, smol, http, retrieval-claude, retrieval-openai")
2490+
live_parser.add_argument("--agent-endpoint", type=str,
2491+
help="HTTP endpoint URL for http agent (e.g., http://localhost:8080)")
24622492
live_parser.add_argument("--demo", type=str, help="Demo trajectory file for ApiAgent")
24632493
live_parser.add_argument("--model-path", type=str, help="Model path for Qwen3VL agent")
24642494
live_parser.add_argument("--model-endpoint", type=str, help="Remote endpoint for Qwen3VL ('modal' or HTTP URL)")

0 commit comments

Comments
 (0)