Skip to content

Commit 991b967

Browse files
john-b-yangjohn-b-yang
committed
Consolidate common models in elo; Update tracker
1 parent 5b38196 commit 991b967

3 files changed

Lines changed: 47 additions & 43 deletions

File tree

‎codeclash/analysis/metrics/elo.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
from codeclash.analysis.viz.utils import MODEL_TO_DISPLAY_NAME
1111
from codeclash.constants import LOCAL_LOG_DIR, RESULT_TIE
12-
from codeclash.games import ARENAS, DummyGame
12+
from codeclash.games import ARENAS, BattleCodeGame, DummyGame
1313

1414

1515
@dataclass
@@ -226,7 +226,7 @@ def _analyze_tournament(self, tournament_log_folder: Path) -> None:
226226
player2profile = {}
227227
for player_config in players:
228228
player_name = player_config["name"]
229-
model = player_config["config"]["model"]["model_name"].strip("@")
229+
model = player_config["config"]["model"]["model_name"].strip("@").split("/")[-1]
230230
key = f"{arena}.{model}"
231231
if key not in self._player_profiles:
232232
self._player_profiles[key] = ModelEloProfile(model=model, arena=arena, rating=self._starting_elo)
@@ -289,7 +289,7 @@ def print_results(self) -> None:
289289
# Claude 4 Sonnet & ... & & & \\
290290
# ...
291291
print("\nLaTeX formatted table:")
292-
arenas = [x.name for x in ARENAS if x != DummyGame]
292+
arenas = [x.name for x in ARENAS if x not in [DummyGame, BattleCodeGame]]
293293
lines = []
294294
arenas_small = [f"\\scriptsize{{{arena}}}" for arena in arenas]
295295
line = "& " + " & ".join(arenas_small) + " & \\textbf{All}" + r" \\"

‎codeclash/utils/update_tracker.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,13 @@
3131
if arena == "RoboCode":
3232
with open(arena_log / "metadata.json") as f:
3333
metadata = json.load(f)
34-
pvp = ".".join(
35-
sorted([p["config"]["model"]["model_name"].split("/")[-1] for p in metadata["config"]["players"]])
36-
)
34+
try:
35+
pvp = ".".join(
36+
sorted([p["config"]["model"]["model_name"].split("/")[-1] for p in metadata["config"]["players"]])
37+
)
38+
except KeyError as e:
39+
print(f"[{arena_log}] Error: Missing key in metadata: {e}")
40+
continue
3741

3842
if arena in tracker and setting in tracker[arena] and pvp in tracker[arena][setting]:
3943
tracker[arena][setting][pvp][0] += 1

‎configs/tracker.json‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,42 @@
11
{
22
"Halite": {
33
"r15.s250.p2": {
4-
"claude-sonnet-4-20250514.claude-sonnet-4-5-20250929": "0 (0 rounds)",
5-
"claude-sonnet-4-20250514.grok-code-fast-1": "0 (0 rounds)",
6-
"claude-sonnet-4-20250514.gemini-2.5-pro": "0 (0 rounds)",
4+
"claude-sonnet-4-20250514.claude-sonnet-4-5-20250929": "10 (148 rounds)",
5+
"claude-sonnet-4-20250514.grok-code-fast-1": "10 (160 rounds)",
6+
"claude-sonnet-4-20250514.gemini-2.5-pro": "10 (149 rounds)",
77
"claude-sonnet-4-20250514.glm-4-5": "0 (0 rounds)",
8-
"claude-sonnet-4-20250514.gpt-5": "0 (0 rounds)",
9-
"claude-sonnet-4-20250514.gpt-5-mini": "0 (0 rounds)",
10-
"claude-sonnet-4-20250514.qwen3-coder-plus-2025-09-23": "0 (0 rounds)",
11-
"claude-sonnet-4-20250514.o3": "0 (0 rounds)",
12-
"claude-sonnet-4-5-20250929.grok-code-fast-1": "0 (0 rounds)",
13-
"claude-sonnet-4-5-20250929.gemini-2.5-pro": "0 (0 rounds)",
8+
"claude-sonnet-4-20250514.gpt-5": "10 (160 rounds)",
9+
"claude-sonnet-4-20250514.gpt-5-mini": "10 (160 rounds)",
10+
"claude-sonnet-4-20250514.qwen3-coder-plus-2025-09-23": "10 (160 rounds)",
11+
"claude-sonnet-4-20250514.o3": "10 (160 rounds)",
12+
"claude-sonnet-4-5-20250929.grok-code-fast-1": "10 (160 rounds)",
13+
"claude-sonnet-4-5-20250929.gemini-2.5-pro": "10 (160 rounds)",
1414
"claude-sonnet-4-5-20250929.glm-4-5": "0 (0 rounds)",
15-
"claude-sonnet-4-5-20250929.gpt-5": "0 (0 rounds)",
16-
"claude-sonnet-4-5-20250929.gpt-5-mini": "0 (0 rounds)",
17-
"claude-sonnet-4-5-20250929.qwen3-coder-plus-2025-09-23": "0 (0 rounds)",
18-
"claude-sonnet-4-5-20250929.o3": "1 (12 rounds)",
19-
"gemini-2.5-pro.grok-code-fast-1": "0 (0 rounds)",
15+
"claude-sonnet-4-5-20250929.gpt-5": "10 (160 rounds)",
16+
"claude-sonnet-4-5-20250929.gpt-5-mini": "10 (160 rounds)",
17+
"claude-sonnet-4-5-20250929.qwen3-coder-plus-2025-09-23": "10 (160 rounds)",
18+
"claude-sonnet-4-5-20250929.o3": "11 (176 rounds)",
19+
"gemini-2.5-pro.grok-code-fast-1": "10 (151 rounds)",
2020
"glm-4-5.grok-code-fast-1": "0 (0 rounds)",
21-
"gpt-5.grok-code-fast-1": "0 (0 rounds)",
22-
"gpt-5-mini.grok-code-fast-1": "0 (0 rounds)",
23-
"grok-code-fast-1.qwen3-coder-plus-2025-09-23": "0 (0 rounds)",
24-
"grok-code-fast-1.o3": "0 (0 rounds)",
21+
"gpt-5.grok-code-fast-1": "10 (160 rounds)",
22+
"gpt-5-mini.grok-code-fast-1": "10 (160 rounds)",
23+
"grok-code-fast-1.qwen3-coder-plus-2025-09-23": "10 (160 rounds)",
24+
"grok-code-fast-1.o3": "10 (160 rounds)",
2525
"gemini-2.5-pro.glm-4-5": "0 (0 rounds)",
26-
"gemini-2.5-pro.gpt-5": "0 (0 rounds)",
27-
"gemini-2.5-pro.gpt-5-mini": "1 (16 rounds)",
28-
"gemini-2.5-pro.qwen3-coder-plus-2025-09-23": "0 (0 rounds)",
29-
"gemini-2.5-pro.o3": "0 (0 rounds)",
26+
"gemini-2.5-pro.gpt-5": "10 (149 rounds)",
27+
"gemini-2.5-pro.gpt-5-mini": "11 (166 rounds)",
28+
"gemini-2.5-pro.qwen3-coder-plus-2025-09-23": "10 (160 rounds)",
29+
"gemini-2.5-pro.o3": "10 (160 rounds)",
3030
"glm-4-5.gpt-5": "0 (0 rounds)",
3131
"glm-4-5.gpt-5-mini": "0 (0 rounds)",
3232
"glm-4-5.qwen3-coder-plus-2025-09-23": "0 (0 rounds)",
3333
"glm-4-5.o3": "0 (0 rounds)",
34-
"gpt-5.gpt-5-mini": "0 (0 rounds)",
35-
"gpt-5.qwen3-coder-plus-2025-09-23": "0 (0 rounds)",
36-
"gpt-5.o3": "0 (0 rounds)",
37-
"gpt-5-mini.qwen3-coder-plus-2025-09-23": "0 (0 rounds)",
38-
"gpt-5-mini.o3": "0 (0 rounds)",
39-
"o3.qwen3-coder-plus-2025-09-23": "0 (0 rounds)"
34+
"gpt-5.gpt-5-mini": "11 (176 rounds)",
35+
"gpt-5.qwen3-coder-plus-2025-09-23": "10 (160 rounds)",
36+
"gpt-5.o3": "10 (160 rounds)",
37+
"gpt-5-mini.qwen3-coder-plus-2025-09-23": "10 (160 rounds)",
38+
"gpt-5-mini.o3": "10 (160 rounds)",
39+
"o3.qwen3-coder-plus-2025-09-23": "10 (160 rounds)"
4040
}
4141
},
4242
"HuskyBench": {
@@ -47,36 +47,36 @@
4747
"claude-sonnet-4-20250514.glm-4-5": "0 (0 rounds)",
4848
"claude-sonnet-4-20250514.gpt-5": "10 (160 rounds)",
4949
"claude-sonnet-4-20250514.gpt-5-mini": "10 (160 rounds)",
50-
"claude-sonnet-4-20250514.qwen3-coder-plus-2025-09-23": "1 (1 rounds)",
50+
"claude-sonnet-4-20250514.qwen3-coder-plus-2025-09-23": "9 (144 rounds)",
5151
"claude-sonnet-4-20250514.o3": "10 (160 rounds)",
5252
"claude-sonnet-4-5-20250929.grok-code-fast-1": "10 (160 rounds)",
5353
"claude-sonnet-4-5-20250929.gemini-2.5-pro": "10 (160 rounds)",
5454
"claude-sonnet-4-5-20250929.glm-4-5": "0 (0 rounds)",
5555
"claude-sonnet-4-5-20250929.gpt-5": "10 (160 rounds)",
5656
"claude-sonnet-4-5-20250929.gpt-5-mini": "10 (160 rounds)",
57-
"claude-sonnet-4-5-20250929.qwen3-coder-plus-2025-09-23": "1 (1 rounds)",
57+
"claude-sonnet-4-5-20250929.qwen3-coder-plus-2025-09-23": "9 (144 rounds)",
5858
"claude-sonnet-4-5-20250929.o3": "10 (160 rounds)",
5959
"gemini-2.5-pro.grok-code-fast-1": "10 (160 rounds)",
6060
"glm-4-5.grok-code-fast-1": "0 (0 rounds)",
6161
"gpt-5.grok-code-fast-1": "10 (160 rounds)",
6262
"gpt-5-mini.grok-code-fast-1": "10 (160 rounds)",
63-
"grok-code-fast-1.qwen3-coder-plus-2025-09-23": "1 (1 rounds)",
63+
"grok-code-fast-1.qwen3-coder-plus-2025-09-23": "9 (143 rounds)",
6464
"grok-code-fast-1.o3": "10 (160 rounds)",
6565
"gemini-2.5-pro.glm-4-5": "0 (0 rounds)",
6666
"gemini-2.5-pro.gpt-5": "10 (160 rounds)",
6767
"gemini-2.5-pro.gpt-5-mini": "10 (160 rounds)",
68-
"gemini-2.5-pro.qwen3-coder-plus-2025-09-23": "1 (1 rounds)",
68+
"gemini-2.5-pro.qwen3-coder-plus-2025-09-23": "9 (144 rounds)",
6969
"gemini-2.5-pro.o3": "8 (128 rounds)",
7070
"glm-4-5.gpt-5": "0 (0 rounds)",
7171
"glm-4-5.gpt-5-mini": "0 (0 rounds)",
7272
"glm-4-5.qwen3-coder-plus-2025-09-23": "0 (0 rounds)",
7373
"glm-4-5.o3": "0 (0 rounds)",
7474
"gpt-5.gpt-5-mini": "10 (160 rounds)",
75-
"gpt-5.qwen3-coder-plus-2025-09-23": "1 (1 rounds)",
75+
"gpt-5.qwen3-coder-plus-2025-09-23": "10 (160 rounds)",
7676
"gpt-5.o3": "10 (160 rounds)",
77-
"gpt-5-mini.qwen3-coder-plus-2025-09-23": "1 (1 rounds)",
77+
"gpt-5-mini.qwen3-coder-plus-2025-09-23": "9 (144 rounds)",
7878
"gpt-5-mini.o3": "10 (160 rounds)",
79-
"o3.qwen3-coder-plus-2025-09-23": "1 (1 rounds)"
79+
"o3.qwen3-coder-plus-2025-09-23": "9 (139 rounds)"
8080
}
8181
},
8282
"RobotRumble": {
@@ -85,7 +85,7 @@
8585
"claude-sonnet-4-20250514.grok-code-fast-1": "10 (147 rounds)",
8686
"claude-sonnet-4-20250514.gemini-2.5-pro": "10 (160 rounds)",
8787
"claude-sonnet-4-20250514.glm-4-5": "0 (0 rounds)",
88-
"claude-sonnet-4-20250514.gpt-5": "10 (142 rounds)",
88+
"claude-sonnet-4-20250514.gpt-5": "10 (157 rounds)",
8989
"claude-sonnet-4-20250514.gpt-5-mini": "10 (160 rounds)",
9090
"claude-sonnet-4-20250514.qwen3-coder-plus-2025-09-23": "10 (160 rounds)",
9191
"claude-sonnet-4-20250514.o3": "10 (148 rounds)",
@@ -245,7 +245,7 @@
245245
"claude-sonnet-4-20250514.grok-code-fast-1": "10 (160 rounds)",
246246
"claude-sonnet-4-20250514.gemini-2.5-pro": "10 (160 rounds)",
247247
"claude-sonnet-4-20250514.glm-4-5": "0 (0 rounds)",
248-
"claude-sonnet-4-20250514.gpt-5": "10 (152 rounds)",
248+
"claude-sonnet-4-20250514.gpt-5": "11 (168 rounds)",
249249
"claude-sonnet-4-20250514.gpt-5-mini": "10 (160 rounds)",
250250
"claude-sonnet-4-20250514.qwen3-coder-plus-2025-09-23": "10 (160 rounds)",
251251
"claude-sonnet-4-20250514.o3": "10 (160 rounds)",

0 commit comments

Comments
 (0)