Skip to content

Commit 4e886a1

Browse files
john-b-yangjohn-b-yang
committed
Update rating calcs
1 parent c654543 commit 4e886a1

4 files changed

Lines changed: 141 additions & 118 deletions

File tree

‎codeclash/ratings/elo.py‎

Lines changed: 91 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -5,100 +5,122 @@
55

66
from tqdm import tqdm
77

8-
from codeclash.constants import FILE_RESULTS, LOCAL_LOG_DIR
9-
10-
K_FACTOR = 32 # ELO constant, changeable
8+
from codeclash.constants import LOCAL_LOG_DIR, RESULT_TIE
119

1210

1311
@dataclass
14-
class PlayerEloProfile:
15-
player_id: str
16-
game_id: str
17-
rating: float = 1200.0 # Default starting ELO
18-
games_played: int = 0
12+
class ModelEloProfile:
13+
model: str
14+
arena: str
15+
rating: float
16+
rounds_played: int = 0
1917

2018

2119
def expected_score(rating_a, rating_b):
2220
return 1 / (1 + 10 ** ((rating_b - rating_a) / 400))
2321

2422

25-
def main(log_dir: Path):
26-
# Assuming directory structure is:
27-
# logs/<user_id>/<game_id>
28-
# - players/
29-
# - rounds/
30-
# - game.log
31-
# - metadata.json
23+
def main(log_dir: Path, k_factor: int, starting_elo: int):
24+
print(f"Calculating ELO ratings from logs in {log_dir} ...")
25+
print(f"Using K_FACTOR={k_factor}, STARTING_ELO={starting_elo}")
3226
player_profiles = {}
33-
for user_folder in log_dir.iterdir():
34-
print(f"Processing games under user `{user_folder.name}`")
35-
for game_log_folder in tqdm(list(user_folder.iterdir())):
36-
if not game_log_folder.is_dir():
37-
continue
38-
game_id = game_log_folder.name.split(".")[1]
39-
player_ids = [x.name for x in (game_log_folder / "players").iterdir() if x.is_dir()]
40-
# Initialize profiles
41-
for player in player_ids:
42-
key = f"{game_id}.{player}"
43-
if key not in player_profiles:
44-
player_profiles[key] = PlayerEloProfile(player_id=player, game_id=game_id)
45-
46-
for round_folder in (game_log_folder / "rounds").iterdir():
47-
if round_folder.name == "0":
27+
for game_log_folder in tqdm([x.parent for x in log_dir.rglob("game.log")]):
28+
arena = game_log_folder.name.split(".")[1]
29+
metadata = json.load(open(game_log_folder / "metadata.json"))
30+
try:
31+
p2m = {x["name"]: x["config"]["model"]["model_name"].strip("@") for x in metadata["config"]["players"]}
32+
except KeyError:
33+
print(f"Skipping {game_log_folder} (malformed metadata.json)")
34+
continue
35+
36+
# Initialize profiles
37+
for model in p2m.values():
38+
key = f"{arena}.{model}"
39+
if key not in player_profiles:
40+
player_profiles[key] = ModelEloProfile(model=model, arena=arena, rating=starting_elo)
41+
42+
sims = metadata["game"]["config"]["sims_per_round"]
43+
if len(p2m) == 2:
44+
# Only process if there are exactly 2 players
45+
for idx, stats in metadata["round_stats"].items():
46+
if idx == "0":
4847
# Skip initial round
4948
continue
50-
if not (round_folder / FILE_RESULTS).exists():
51-
continue
52-
round_results = json.load(open(round_folder / FILE_RESULTS))
53-
winner = round_results.get("winner")
54-
players = round_results.get("players", player_ids)
55-
# Only process if there are exactly 2 players
56-
if len(players) == 2:
57-
p1_key = f"{game_id}.{players[0]}"
58-
p2_key = f"{game_id}.{players[1]}"
59-
p1 = player_profiles[p1_key]
60-
p2 = player_profiles[p2_key]
61-
p1.games_played += 1
62-
p2.games_played += 1
63-
# Determine scores
64-
if winner == players[0]:
65-
s1, s2 = 1, 0
66-
elif winner == players[1]:
67-
s1, s2 = 0, 1
49+
50+
prof_and_score = []
51+
valid_submits = sum(
52+
[x["valid_submit"] for x in stats["player_stats"].values() if x.get("valid_submit") is not None]
53+
)
54+
55+
for k, v in stats["player_stats"].items():
56+
if k != RESULT_TIE:
57+
if v["score"] is None:
58+
# Not sure why this happens, but just skip it
59+
continue
60+
s = v["score"] * 1.0 / sims
61+
if valid_submits == 1 and v["valid_submit"]:
62+
# FOR BACKWARDS COMPATIBILITY: If only one player submitted, give them full point
63+
s = 1.0
64+
prof = player_profiles[f"{arena}.{p2m[k]}"]
65+
prof.rounds_played += 1
66+
prof_and_score.append((prof, s))
67+
68+
# Update ELO ratings - should only happen once per match
69+
if len(prof_and_score) == 2:
70+
p1_prof, p1_raw_score = prof_and_score[0]
71+
p2_prof, p2_raw_score = prof_and_score[1]
72+
73+
# Normalize scores so they sum to 1.0 (required for proper ELO)
74+
total_score = p1_raw_score + p2_raw_score
75+
if total_score > 0:
76+
p1_score = p1_raw_score / total_score
77+
p2_score = p2_raw_score / total_score
6878
else:
69-
s1, s2 = 0.5, 0.5 # Tie
70-
# Calculate expected scores
71-
e1 = expected_score(p1.rating, p2.rating)
72-
e2 = expected_score(p2.rating, p1.rating)
73-
# Update ratings
74-
p1.rating += K_FACTOR * (s1 - e1)
75-
p2.rating += K_FACTOR * (s2 - e2)
79+
# If both players scored 0, treat as a tie
80+
p1_score = p2_score = 0.5
81+
82+
expected_p1 = expected_score(p1_prof.rating, p2_prof.rating)
83+
rating_change = k_factor * (p1_score - expected_p1)
7684

85+
expected_p2 = expected_score(p2_prof.rating, p1_prof.rating)
86+
check = k_factor * (p2_score - expected_p2)
87+
assert abs(check + rating_change) < 1e-6, "ELO rating changes do not sum to zero!"
88+
89+
p1_prof.rating += rating_change
90+
p2_prof.rating -= rating_change # Zero-sum property
91+
92+
print("=" * 50)
7793
print("Player ELO profiles:")
78-
for profile in player_profiles.values():
79-
print(
80-
f" - {profile.player_id} (Game: {profile.game_id}) - ELO: {profile.rating:.1f} (Games: {profile.games_played})"
81-
)
94+
lines = [
95+
f" - {profile.model} (Arena: {profile.arena}) - ELO: {profile.rating:.1f} (Games: {profile.rounds_played})"
96+
for profile in player_profiles.values()
97+
]
98+
print("\n".join(sorted(lines)))
8299

83100
# Weighted average ELO per player across all games
84101
weighted_elo = {}
85102
total_games = {}
86103
for profile in player_profiles.values():
87-
pid = profile.player_id
88-
weighted_elo[pid] = weighted_elo.get(pid, 0) + profile.rating * profile.games_played
89-
total_games[pid] = total_games.get(pid, 0) + profile.games_played
104+
mid = profile.model
105+
weighted_elo[mid] = weighted_elo.get(mid, 0) + profile.rating * profile.rounds_played
106+
total_games[mid] = total_games.get(mid, 0) + profile.rounds_played
90107

91108
print("\nWeighted average ELO per player (across all games):")
92-
for pid in weighted_elo:
93-
if total_games[pid] > 0:
94-
avg_elo = weighted_elo[pid] / total_games[pid]
95-
else:
96-
avg_elo = 0.0
97-
print(f" - {pid}: Weighted Avg ELO {avg_elo:.1f} (Total Games: {total_games[pid]})")
109+
calc_avg_elo = lambda total_elo, games: total_elo / games if games > 0 else 0.0
110+
lines = [
111+
f" - {pid}: Weighted Avg ELO {calc_avg_elo(weighted_elo[pid], total_games[pid]):.1f} (Games: {total_games[pid]})"
112+
for pid in weighted_elo
113+
if total_games[pid] > 0
114+
]
115+
print("\n".join(sorted(lines)))
98116

99117

100118
if __name__ == "__main__":
101119
parser = argparse.ArgumentParser()
102120
parser.add_argument("-d", "--log_dir", type=Path, help="Path to game logs (Default: logs/)", default=LOCAL_LOG_DIR)
121+
parser.add_argument("-k", "--k_factor", type=int, help="K-Factor for ELO calculation (Default: 32)", default=32)
122+
parser.add_argument(
123+
"-s", "--starting_elo", type=int, help="Starting ELO for new players (Default: 1200)", default=1200
124+
)
103125
args = parser.parse_args()
104-
main(args.log_dir)
126+
main(**vars(args))

‎codeclash/ratings/win_rate.py‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,11 @@ def main(log_dir: Path):
6262
model_profiles[f"{game_id}.{player_to_model[winner]}"].wins += 1
6363

6464
print("Player profiles:")
65-
for profile in model_profiles.values():
66-
print(
67-
f" - {profile.model_name} (Game: {profile.game_id}) - Win Rate: {profile.win_rate:.2%} ({profile.wins}/{profile.count})"
68-
)
65+
lines = [
66+
f" - {profile.model_name} (Game: {profile.game_id}) - Win Rate: {profile.win_rate:.2%} ({profile.wins}/{profile.count})"
67+
for profile in model_profiles.values()
68+
]
69+
print("\n".join(sorted(lines)))
6970

7071
# Player-specific (game-agnostic) win rates (micro average)
7172
total_wins = {}
@@ -78,12 +79,12 @@ def main(log_dir: Path):
7879
model_names[mid] = profile.model_name
7980

8081
print("\nPlayer-specific win rates (game-agnostic, micro average):")
81-
for mid in total_wins:
82-
if total_games[mid] > 0:
83-
win_rate = total_wins[mid] / total_games[mid]
84-
else:
85-
win_rate = 0.0
86-
print(f" - {model_names[mid]}: Win Rate {win_rate:.2%} ({total_wins[mid]}/{total_games[mid]})")
82+
calc_win_rate = lambda w, c: w / c if c > 0 else 0.0
83+
lines = [
84+
f" - {model_names[mid]}: Win Rate {calc_win_rate(total_wins[mid], total_games[mid]):.2%} ({total_wins[mid]}/{total_games[mid]})"
85+
for mid in total_wins
86+
]
87+
print("\n".join(sorted(lines)))
8788

8889

8990
if __name__ == "__main__":

‎pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ ignore = [
163163
"E501", # line too long
164164
"E402", # import not on top of file
165165
"E722", # bare except
166+
"E731", # lambdas
166167
"E741", # ambiguous symbol
167168
# pytest
168169
"PT011",

0 commit comments

Comments
 (0)