Merge branch 'main' of github.com:emagedoc/CodeClash

Author: john-b-yang

codeclash/ratings/win_rate.py

@@ -5,12 +5,13 @@
 
 from tqdm import tqdm
 
-from codeclash.constants import DIR_LOGS, FILE_RESULTS, RESULT_TIE
+from codeclash.constants import DIR_LOGS, RESULT_TIE
 
 
 @dataclass
 class PlayerGameProfile:
     player_id: str
+    model_name: str
     game_id: str
     wins: int = 0
     count: int = 0
@@ -24,59 +25,65 @@ def main(log_dir: Path):
     # Assuming directory structure is:
     # logs/<user_id>/<game_id>
     # - players/
-    # - rounds/
     # - game.log
     # - metadata.json
-    player_profiles = {}
+    model_profiles = {}
     for user_folder in log_dir.iterdir():
         print(f"Processing games under user `{user_folder.name}`")
         for game_log_folder in tqdm(list(user_folder.iterdir())):
             if not game_log_folder.is_dir():
                 continue
             game_id = game_log_folder.name.split(".")[1]
             player_ids = [x.name for x in (game_log_folder / "players").iterdir() if x.is_dir()]
-            num_rounds = len(list((game_log_folder / "rounds").iterdir()))
+            metadata = json.load(open(game_log_folder / "metadata.json"))
+            player_to_model = {x["name"]: x["config"]["model"]["model_name"] for x in metadata["config"]["players"]}
+            print(player_to_model)
+            num_rounds = len(metadata["round_stats"])
 
-            for player in player_ids:
-                if f"{game_id}.{player}" in player_profiles:
-                    player_profiles[f"{game_id}.{player}"].count += num_rounds
+            # Only count each unique model once per game
+            unique_models = {player_to_model[player] for player in player_ids}
+            for model_name in unique_models:
+                k = f"{game_id}.{model_name}"
+                if k in model_profiles:
+                    model_profiles[k].count += num_rounds
                 else:
-                    player_profiles[f"{game_id}.{player}"] = PlayerGameProfile(
-                        player_id=player, game_id=game_id, count=num_rounds
+                    # Use the first player_id that matches this model_name for display
+                    player_id = next(pid for pid in player_ids if player_to_model[pid] == model_name)
+                    model_profiles[k] = PlayerGameProfile(
+                        player_id=player_id, model_name=model_name, game_id=game_id, count=num_rounds
                     )
 
-            for round_folder in (game_log_folder / "rounds").iterdir():
-                if round_folder.name == "0":
+            for round, details in metadata["round_stats"].items():
+                if round == "0":
                     # Skip initial round
                     continue
-                if not (round_folder / FILE_RESULTS).exists():
-                    continue
-                round_results = json.load(open(round_folder / FILE_RESULTS))
-                winner = round_results.get("winner")
+                winner = details["winner"]
                 if winner != RESULT_TIE:
-                    player_profiles[f"{game_id}.{winner}"].wins += 1
+                    model_profiles[f"{game_id}.{player_to_model[winner]}"].wins += 1
 
     print("Player profiles:")
-    for profile in player_profiles.values():
+    for profile in model_profiles.values():
         print(
-            f" - {profile.player_id} (Game: {profile.game_id}) - Win Rate: {profile.win_rate:.2%} ({profile.wins}/{profile.count})"
+            f" - {profile.model_name} (Game: {profile.game_id}) - Win Rate: {profile.win_rate:.2%} ({profile.wins}/{profile.count})"
         )
 
     # Player-specific (game-agnostic) win rates (micro average)
     total_wins = {}
     total_games = {}
-    for profile in player_profiles.values():
-        pid = profile.player_id
-        total_wins[pid] = total_wins.get(pid, 0) + profile.wins
-        total_games[pid] = total_games.get(pid, 0) + profile.count
+    model_names = {}
+    for profile in model_profiles.values():
+        mid = profile.model_name
+        total_wins[mid] = total_wins.get(mid, 0) + profile.wins
+        total_games[mid] = total_games.get(mid, 0) + profile.count
+        model_names[mid] = profile.model_name
 
     print("\nPlayer-specific win rates (game-agnostic, micro average):")
-    for pid in total_wins:
-        if total_games[pid] > 0:
-            win_rate = total_wins[pid] / total_games[pid]
+    for mid in total_wins:
+        if total_games[mid] > 0:
+            win_rate = total_wins[mid] / total_games[mid]
         else:
             win_rate = 0.0
-        print(f" - {pid}: Win Rate {win_rate:.2%} ({total_wins[pid]}/{total_games[pid]})")
+        print(f" - {model_names[mid]}: Win Rate {win_rate:.2%} ({total_wins[mid]}/{total_games[mid]})")
 
 
 if __name__ == "__main__":