Skip to content

Commit 49f7881

Browse files
committed
benchmark: add --agent hermes support + first comparison results
AIEB v1.0 now supports --agent odek|hermes. First head-to-head (deepseek-v4-flash): odek: 199s, 90%, 208K tokens hermes: 273s, 92%, N/A Hermes is 2pp more accurate but 37% slower — richer system prompt + memory overhead per invocation. Odek wins on trivial tasks (5s vs 31s), Hermes wins on refactoring.
1 parent b9fe4d2 commit 49f7881

1 file changed

Lines changed: 31 additions & 5 deletions

File tree

benchmark/run_aieb.py

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ def verify_refactor(_=None):
326326
"3.3": verify_refactor,
327327
}
328328

329-
# ─── Runner ───────────────────────────────────────────────────────────
329+
# ─── Runners ───────────────────────────────────────────────────────────
330330

331331
def run_odek(task):
332332
cmd = [ODEK_BIN, "run", "--model", "deepseek-v4-flash",
@@ -355,10 +355,32 @@ def run_odek(task):
355355
return {"wall_time": round(wall,1), "tokens_in": tokens_in, "tokens_out": tokens_out,
356356
"iterations": iters, "score": score, "error": None}
357357

358+
def run_hermes(task):
359+
cmd = ["hermes", "-z", task["prompt"],
360+
"-m", "deepseek-v4-flash", "--provider", "deepseek",
361+
"-t", "terminal,file,search"]
362+
363+
start = time.time()
364+
try:
365+
r = subprocess.run(cmd, capture_output=True, text=True, timeout=300,
366+
cwd=str(BENCHMARK_DIR))
367+
except subprocess.TimeoutExpired:
368+
return {"error": "timeout", "wall_time": time.time()-start, "score": 0}
369+
except Exception as e:
370+
return {"error": str(e), "wall_time": time.time()-start, "score": 0}
371+
372+
wall = time.time() - start
373+
output = (r.stdout or "") + "\n" + (r.stderr or "")
374+
score = SCORERS[task["id"]](output)
375+
376+
return {"wall_time": round(wall,1), "tokens_in": 0, "tokens_out": 0,
377+
"iterations": 0, "score": score, "error": None}
378+
358379
# ─── Main ─────────────────────────────────────────────────────────────
359380

360-
def main():
361-
print("AIEB v1.0 — odek + deepseek-v4-flash\n")
381+
def main(agent="odek"):
382+
runner = run_odek if agent == "odek" else run_hermes
383+
print(f"AIEB v1.0 — {agent} + deepseek-v4-flash\n")
362384
setup()
363385

364386
total_time = 0
@@ -369,7 +391,7 @@ def main():
369391
for task in TASKS:
370392
tier = f"T{task['tier']}"
371393
print(f" [{tier}.{task['id']}] {task['name']:12s}...", end=" ", flush=True)
372-
r = run_odek(task)
394+
r = runner(task)
373395

374396
if r.get("error"):
375397
print(f"❌ {r['error']}")
@@ -408,4 +430,8 @@ def main():
408430
print(f"\n Results saved to benchmark/results.json")
409431

410432
if __name__ == "__main__":
411-
main()
433+
import argparse
434+
p = argparse.ArgumentParser()
435+
p.add_argument("--agent", default="odek", choices=["odek", "hermes"])
436+
args = p.parse_args()
437+
main(agent=args.agent)

0 commit comments

Comments
 (0)