Skip to content

Commit 8c1e009

Browse files
john-b-yangjohn-b-yang
committed
Add per-round elo
1 parent 3136081 commit 8c1e009

7 files changed

Lines changed: 239 additions & 72 deletions

File tree

codeclash/ratings/elo.py renamed to codeclash/analysis/rate_elo.py

Lines changed: 74 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,40 @@ def calculate_round_weight_exponential(round_num, total_rounds, alpha=2.0):
5656
return raw_weight * norm_factor
5757

5858

59+
def update_profiles(prof_and_score, round_weight, k_factor):
60+
"""Update ELO profiles for two players based on their scores and round weight
61+
62+
Args:
63+
prof_and_score: List of tuples [(ModelEloProfile, score), ...] for two players
64+
round_weight: Weight for the current round (affects K-factor)
65+
k_factor: Base K-factor for ELO calculation
66+
"""
67+
p1_prof, p1_raw_score = prof_and_score[0]
68+
p2_prof, p2_raw_score = prof_and_score[1]
69+
70+
# Normalize scores so they sum to 1.0 (required for proper ELO)
71+
total_score = p1_raw_score + p2_raw_score
72+
if total_score > 0:
73+
p1_score = p1_raw_score / total_score
74+
p2_score = p2_raw_score / total_score
75+
else:
76+
# If both players scored 0, treat as a tie
77+
p1_score = p2_score = 0.5
78+
79+
expected_p1 = expected_score(p1_prof.rating, p2_prof.rating)
80+
81+
# Apply round weighting to K-factor
82+
weighted_k_factor = k_factor * round_weight
83+
rating_change = weighted_k_factor * (p1_score - expected_p1)
84+
85+
expected_p2 = expected_score(p2_prof.rating, p1_prof.rating)
86+
check = weighted_k_factor * (p2_score - expected_p2)
87+
assert abs(check + rating_change) < 1e-6, "Weighted ELO rating changes do not sum to zero!"
88+
89+
p1_prof.rating += rating_change
90+
p2_prof.rating -= rating_change # Zero-sum property
91+
92+
5993
def main(log_dir: Path, k_factor: int, starting_elo: int, weighting_function: str, alpha: float):
6094
print(f"Calculating weighted ELO ratings from logs in {log_dir} ...")
6195
print(f"Using K_FACTOR={k_factor}, STARTING_ELO={starting_elo}")
@@ -84,66 +118,46 @@ def main(log_dir: Path, k_factor: int, starting_elo: int, weighting_function: st
84118
# Determine total rounds for weighting calculation
85119
total_rounds = len([k for k in metadata["round_stats"].keys() if k != "0"])
86120

87-
if len(p2m) == 2:
121+
if len(p2m) != 2:
88122
# Only process if there are exactly 2 players
89-
for idx, stats in metadata["round_stats"].items():
90-
if idx == "0":
91-
# Skip initial round
92-
continue
93-
94-
# Calculate round weight
95-
current_round = int(idx)
96-
if weighting_function == "linear":
97-
round_weight = calculate_round_weight_linear(current_round, total_rounds)
98-
elif weighting_function == "exponential":
99-
round_weight = calculate_round_weight_exponential(current_round, total_rounds, alpha)
100-
else: # none
101-
round_weight = 1.0
102-
103-
prof_and_score = []
104-
valid_submits = sum(
105-
[x["valid_submit"] for x in stats["player_stats"].values() if x.get("valid_submit") is not None]
106-
)
107-
108-
for k, v in stats["player_stats"].items():
109-
if k != RESULT_TIE:
110-
if v["score"] is None:
111-
# Not sure why this happens, but just skip it
112-
continue
113-
s = v["score"] * 1.0 / sims
114-
if valid_submits == 1 and v["valid_submit"]:
115-
# FOR BACKWARDS COMPATIBILITY: If only one player submitted, give them full point
116-
s = 1.0
117-
prof = player_profiles[f"{arena}.{p2m[k]}"]
118-
prof.rounds_played += 1
119-
prof_and_score.append((prof, s))
120-
121-
# Update ELO ratings - should only happen once per match
122-
if len(prof_and_score) == 2:
123-
p1_prof, p1_raw_score = prof_and_score[0]
124-
p2_prof, p2_raw_score = prof_and_score[1]
125-
126-
# Normalize scores so they sum to 1.0 (required for proper ELO)
127-
total_score = p1_raw_score + p2_raw_score
128-
if total_score > 0:
129-
p1_score = p1_raw_score / total_score
130-
p2_score = p2_raw_score / total_score
131-
else:
132-
# If both players scored 0, treat as a tie
133-
p1_score = p2_score = 0.5
134-
135-
expected_p1 = expected_score(p1_prof.rating, p2_prof.rating)
136-
137-
# Apply round weighting to K-factor
138-
weighted_k_factor = k_factor * round_weight
139-
rating_change = weighted_k_factor * (p1_score - expected_p1)
140-
141-
expected_p2 = expected_score(p2_prof.rating, p1_prof.rating)
142-
check = weighted_k_factor * (p2_score - expected_p2)
143-
assert abs(check + rating_change) < 1e-6, "Weighted ELO rating changes do not sum to zero!"
144-
145-
p1_prof.rating += rating_change
146-
p2_prof.rating -= rating_change # Zero-sum property
123+
continue
124+
125+
for idx, stats in metadata["round_stats"].items():
126+
if idx == "0":
127+
# Skip initial round
128+
continue
129+
130+
# Calculate round weight
131+
current_round = int(idx)
132+
if weighting_function == "linear":
133+
round_weight = calculate_round_weight_linear(current_round, total_rounds)
134+
elif weighting_function == "exponential":
135+
round_weight = calculate_round_weight_exponential(current_round, total_rounds, alpha)
136+
else: # none
137+
round_weight = 1.0
138+
139+
prof_and_score = []
140+
valid_submits = sum(
141+
[x["valid_submit"] for x in stats["player_stats"].values() if x.get("valid_submit") is not None]
142+
)
143+
144+
for k, v in stats["player_stats"].items():
145+
if k != RESULT_TIE:
146+
if v["score"] is None:
147+
# Not sure why this happens, but just skip it
148+
continue
149+
s = v["score"] * 1.0 / sims
150+
if valid_submits == 1 and v["valid_submit"]:
151+
# FOR BACKWARDS COMPATIBILITY: If only one player submitted, give them full point
152+
s = 1.0
153+
prof = player_profiles[f"{arena}.{p2m[k]}"]
154+
prof.rounds_played += 1
155+
prof_and_score.append((prof, s))
156+
157+
# Update ELO ratings - should only happen once per match
158+
if len(prof_and_score) != 2:
159+
continue
160+
update_profiles(prof_and_score, round_weight, k_factor)
147161

148162
print("=" * 50)
149163
print("Player ELO profiles:")
@@ -162,7 +176,7 @@ def main(log_dir: Path, k_factor: int, starting_elo: int, weighting_function: st
162176
total_games[mid] = total_games.get(mid, 0) + profile.rounds_played
163177

164178
print("\nWeighted average ELO per player (across all games):")
165-
calc_avg_elo = lambda total_elo, games: total_elo / games if games > 0 else 0.0
179+
calc_avg_elo = lambda total_elo, games: total_elo / games
166180
lines = [
167181
f" - {pid}: Weighted Avg ELO {calc_avg_elo(weighted_elo[pid], total_games[pid]):.1f} (Games: {total_games[pid]})"
168182
for pid in weighted_elo

‎codeclash/analysis/rate_per_round_elo.py‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
import argparse
2+
import json
3+
from dataclasses import dataclass
4+
from pathlib import Path
5+
6+
from matplotlib import pyplot as plt
7+
from tqdm import tqdm
8+
9+
from codeclash.analysis.rate_elo import (
10+
calculate_round_weight_exponential,
11+
calculate_round_weight_linear,
12+
update_profiles,
13+
)
14+
from codeclash.constants import LOCAL_LOG_DIR, RESULT_TIE
15+
16+
17+
@dataclass
18+
class ModelRoundEloProfile:
19+
model: str
20+
arena: str
21+
rating: float
22+
round_idx: int
23+
rounds_played: int = 0
24+
25+
26+
def main(log_dir: Path, k_factor: int, starting_elo: int, weighting_function: str, alpha: float):
27+
print(f"Calculating weighted ELO ratings from logs in {log_dir} ...")
28+
print(f"Using K_FACTOR={k_factor}, STARTING_ELO={starting_elo}")
29+
print(
30+
f"Weighting function: {weighting_function}"
31+
+ (f" (alpha={alpha})" if weighting_function == "exponential" else "")
32+
)
33+
player_round_profiles = {}
34+
for game_log_folder in tqdm([x.parent for x in log_dir.rglob("game.log")]):
35+
arena = game_log_folder.name.split(".")[1]
36+
metadata = json.load(open(game_log_folder / "metadata.json"))
37+
try:
38+
p2m = {x["name"]: x["config"]["model"]["model_name"].strip("@") for x in metadata["config"]["players"]}
39+
except KeyError:
40+
print(f"Skipping {game_log_folder} (malformed metadata.json)")
41+
continue
42+
43+
if len(p2m) != 2:
44+
# Only process if there are exactly 2 players:
45+
continue
46+
47+
sims = metadata["game"]["config"]["sims_per_round"]
48+
49+
# Determine total rounds for weighting calculation
50+
total_rounds = len([k for k in metadata["round_stats"].keys() if k != "0"])
51+
52+
for idx, stats in metadata["round_stats"].items():
53+
if idx == "0":
54+
# Skip initial round
55+
continue
56+
57+
# Initialize profiles
58+
for model in p2m.values():
59+
key = f"{arena}.{model}.{idx}"
60+
if key not in player_round_profiles:
61+
player_round_profiles[key] = ModelRoundEloProfile(
62+
model=model, arena=arena, rating=starting_elo, round_idx=int(idx)
63+
)
64+
65+
# Calculate round weight
66+
current_round = int(idx)
67+
if weighting_function == "linear":
68+
round_weight = calculate_round_weight_linear(current_round, total_rounds)
69+
elif weighting_function == "exponential":
70+
round_weight = calculate_round_weight_exponential(current_round, total_rounds, alpha)
71+
else: # none
72+
round_weight = 1.0
73+
74+
prof_and_score = []
75+
valid_submits = sum(
76+
[x["valid_submit"] for x in stats["player_stats"].values() if x.get("valid_submit") is not None]
77+
)
78+
79+
for k, v in stats["player_stats"].items():
80+
if k != RESULT_TIE:
81+
if v["score"] is None:
82+
# Not sure why this happens, but just skip it
83+
continue
84+
s = v["score"] * 1.0 / sims
85+
if valid_submits == 1 and v["valid_submit"]:
86+
# If only one player submitted, give them full score
87+
s = 1.0
88+
prof = player_round_profiles[f"{arena}.{p2m[k]}.{idx}"]
89+
prof.rounds_played += 1
90+
prof_and_score.append((prof, s))
91+
92+
if len(prof_and_score) != 2:
93+
# Should always be 2 players here
94+
continue
95+
update_profiles(prof_and_score, round_weight, k_factor)
96+
97+
lines = {
98+
pid: [[] for _ in range(15)]
99+
for pid in {x.rsplit(".", 1)[0].split(".", 1)[-1] for x in player_round_profiles.keys()}
100+
}
101+
for pid, profile in player_round_profiles.items():
102+
k = pid.rsplit(".", 1)[0].split(".", 1)[-1]
103+
if 1 <= profile.round_idx <= 15:
104+
lines[k][profile.round_idx - 1].append(profile)
105+
106+
print("=" * 50)
107+
print("Player ELO progression per round:")
108+
109+
def aggregate_elos_across_games(profiles):
110+
return sum([p.rating * p.rounds_played for p in profiles]) / sum([p.rounds_played for p in profiles])
111+
112+
for pid, elos in lines.items():
113+
lines[pid] = [aggregate_elos_across_games(r) for r in elos]
114+
print(f" - {pid}: " + ", ".join([f"{e:.1f}" for e in lines[pid]]))
115+
116+
# Create line chart of ELO progression per player
117+
plt.figure(figsize=(10, 6))
118+
for pid, elos in lines.items():
119+
plt.plot(range(1, 16), elos, marker="o", label=pid)
120+
plt.title("ELO Progression per Round")
121+
plt.xlabel("Round")
122+
plt.ylabel("ELO Rating")
123+
plt.xticks(range(1, 16))
124+
plt.legend()
125+
plt.grid(True)
126+
plt.tight_layout()
127+
plt.savefig("elo_progression_per_round.png")
128+
print("ELO progression chart saved to elo_progression_per_round.png")
129+
130+
131+
if __name__ == "__main__":
132+
parser = argparse.ArgumentParser(description="Calculate weighted ELO ratings with configurable weighting functions")
133+
parser.add_argument("-d", "--log_dir", type=Path, help="Path to game logs (Default: logs/)", default=LOCAL_LOG_DIR)
134+
parser.add_argument("-k", "--k_factor", type=int, help="K-Factor for ELO calculation (Default: 32)", default=32)
135+
parser.add_argument(
136+
"-s", "--starting_elo", type=int, help="Starting ELO for new players (Default: 1200)", default=1200
137+
)
138+
parser.add_argument(
139+
"-w",
140+
"--weighting_function",
141+
choices=["none", "linear", "exponential"],
142+
default="none",
143+
help="Weighting function for rounds: 'linear' for gradual increase, 'exponential' for accelerating importance (Default: none)",
144+
)
145+
parser.add_argument(
146+
"-a",
147+
"--alpha",
148+
type=float,
149+
default=2.0,
150+
help="Alpha parameter for exponential weighting (Default: 2.0, ignored for linear weighting)",
151+
)
152+
args = parser.parse_args()
153+
main(**vars(args))

‎codeclash/analysis/rate_per_round_wins.py‎

Whitespace-only changes.

codeclash/ratings/win_rate.py renamed to codeclash/analysis/rate_wins.py

File renamed without changes.

codeclash/ratings/significance.py renamed to codeclash/analysis/significance.py

File renamed without changes.

‎codeclash/viewer/app.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
from flask import Flask, jsonify, redirect, render_template, request, send_file, url_for
1616

17-
from codeclash.ratings.significance import calculate_p_value
17+
from codeclash.analysis.significance import calculate_p_value
1818
from codeclash.tournaments.utils.git_utils import filter_git_diff, split_git_diff_by_files
1919

2020
logger = logging.getLogger(__name__)

‎configs/scripts/main_tracker.json‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -41,24 +41,24 @@
4141
},
4242
"BattleSnake": {
4343
"r15.s1000.p2": {
44-
"claude-sonnet-4-20250514.claude-sonnet-4-5-20250929": "0 (0 rounds)",
45-
"claude-sonnet-4-20250514.grok-code-fast-1": "0 (0 rounds)",
46-
"claude-sonnet-4-20250514.gemini-2.5-pro": "0 (0 rounds)",
44+
"claude-sonnet-4-20250514.claude-sonnet-4-5-20250929": "7 (112 rounds)",
45+
"claude-sonnet-4-20250514.grok-code-fast-1": "10 (159 rounds)",
46+
"claude-sonnet-4-20250514.gemini-2.5-pro": "10 (160 rounds)",
4747
"claude-sonnet-4-20250514.glm-4-5": "0 (0 rounds)",
4848
"claude-sonnet-4-20250514.gpt-5": "0 (0 rounds)",
49-
"claude-sonnet-4-20250514.gpt-5-mini": "0 (0 rounds)",
49+
"claude-sonnet-4-20250514.gpt-5-mini": "10 (152 rounds)",
5050
"claude-sonnet-4-20250514.qwen3-coder-480b-a35b-instruct": "0 (0 rounds)",
51-
"claude-sonnet-4-20250514.o3": "0 (0 rounds)",
52-
"claude-sonnet-4-5-20250929.grok-code-fast-1": "0 (0 rounds)",
53-
"claude-sonnet-4-5-20250929.gemini-2.5-pro": "0 (0 rounds)",
51+
"claude-sonnet-4-20250514.o3": "10 (160 rounds)",
52+
"claude-sonnet-4-5-20250929.grok-code-fast-1": "10 (152 rounds)",
53+
"claude-sonnet-4-5-20250929.gemini-2.5-pro": "10 (154 rounds)",
5454
"claude-sonnet-4-5-20250929.glm-4-5": "0 (0 rounds)",
5555
"claude-sonnet-4-5-20250929.gpt-5": "0 (0 rounds)",
56-
"claude-sonnet-4-5-20250929.gpt-5-mini": "0 (0 rounds)",
56+
"claude-sonnet-4-5-20250929.gpt-5-mini": "10 (148 rounds)",
5757
"claude-sonnet-4-5-20250929.qwen3-coder-480b-a35b-instruct": "0 (0 rounds)",
5858
"claude-sonnet-4-5-20250929.o3": "0 (0 rounds)",
59-
"gemini-2.5-pro.grok-code-fast-1": "10 (145 rounds)",
59+
"gemini-2.5-pro.grok-code-fast-1": "12 (177 rounds)",
6060
"glm-4-5.grok-code-fast-1": "0 (0 rounds)",
61-
"gpt-5.grok-code-fast-1": "10 (155 rounds)",
61+
"gpt-5.grok-code-fast-1": "11 (167 rounds)",
6262
"gpt-5-mini.grok-code-fast-1": "10 (160 rounds)",
6363
"grok-code-fast-1.qwen3-coder-480b-a35b-instruct": "0 (0 rounds)",
6464
"grok-code-fast-1.o3": "10 (158 rounds)",
@@ -71,7 +71,7 @@
7171
"glm-4-5.gpt-5-mini": "0 (0 rounds)",
7272
"glm-4-5.qwen3-coder-480b-a35b-instruct": "0 (0 rounds)",
7373
"glm-4-5.o3": "0 (0 rounds)",
74-
"gpt-5.gpt-5-mini": "10 (141 rounds)",
74+
"gpt-5.gpt-5-mini": "9 (139 rounds)",
7575
"gpt-5.qwen3-coder-480b-a35b-instruct": "0 (0 rounds)",
7676
"gpt-5.o3": "10 (160 rounds)",
7777
"gpt-5-mini.qwen3-coder-480b-a35b-instruct": "0 (0 rounds)",

0 commit comments

Comments
 (0)