Make ELO weighted; Remove sims_per_round from BattleCode

Author: john-b-yang

codeclash/games/battlecode/battlecode.py

@@ -1,13 +1,14 @@
+import random
 import re
 from pathlib import Path
 from typing import Any
 
-from tqdm.auto import tqdm
-
 from codeclash.constants import DIR_WORK, RESULT_TIE
 from codeclash.games.game import CodeGame, RoundStats
 from codeclash.utils.environment import copy_from_container
 
+BATTLECODE_LOG = "sim.log"
+
 
 class BattleCodeGame(CodeGame):
     name: str = "BattleCode"
@@ -27,27 +28,26 @@ def copy_logs_from_env(self, round_num):
         super().copy_logs_from_env(round_num)
         copy_from_container(
             container=self.environment,
-            src_path="/testbed/logs",
-            dest_path=self.log_local / "rounds" / str(round_num),
+            src_path="/testbed/sim.log",
+            dest_path=self.log_local / "rounds" / f"sim_{round_num}.log",
         )
 
     def get_stats(self, agents: list[Any]) -> RoundStats:
         winners = []
-        for sim_file in [f"logs/sim_{idx}.log" for idx in range(self.game_config["sims_per_round"])]:
-            ro = self.environment.execute(f"cat {sim_file}")["output"]
-            lines = ro.strip().split("\n")
-            # Get the third-to-last line which contains the winner info
-            winner_line = lines[-3] if len(lines) >= 3 else ""
-            self.logger.debug(f"Winner line: {winner_line}")
-            match = re.search(r"\s\((.*)\)\swins\s\(", winner_line)
-            if match:
-                winner_key = match.group(1)
-                self.logger.debug(f"Winner key from match: {winner_key}")
-                # Map A/B to actual agent names (much closer to original code)
-                winner = {"A": agents[0].name, "B": agents[1].name}.get(winner_key, RESULT_TIE)
-                winners.append(winner)
-            else:
-                winners.append(RESULT_TIE)
+        ro = self.environment.execute(f"cat {BATTLECODE_LOG}")["output"]
+        lines = ro.strip().split("\n")
+        # Get the third-to-last line which contains the winner info
+        winner_line = lines[-3] if len(lines) >= 3 else ""
+        self.logger.debug(f"Winner line: {winner_line}")
+        match = re.search(r"\s\((.*)\)\swins\s\(", winner_line)
+        if match:
+            winner_key = match.group(1)
+            self.logger.debug(f"Winner key from match: {winner_key}")
+            # Map A/B to actual agent names (much closer to original code)
+            winner = {"A": agents[0].name, "B": agents[1].name}.get(winner_key, RESULT_TIE)
+            winners.append(winner)
+        else:
+            winners.append(RESULT_TIE)
         return RoundStats(
             winner=max(set(winners), key=winners.count),
             scores={agent.name: winners.count(agent.name) for agent in agents},
@@ -57,11 +57,10 @@ def execute_round(self, agents: list[Any]):
         for agent in agents:
             src, dest = f"/{agent.name}/src/mysubmission/", str(DIR_WORK / "src" / agent.name)
             self.environment.execute(f"cp -r {src} {dest}")
+        random.shuffle(agents)  # Start position matters in BattleCode! Shuffle to be fair.
         args = [f"--p{idx + 1}-dir src --p{idx + 1} {agent.name}" for idx, agent in enumerate(agents)]
         cmd = f"{self.run_cmd_round} {' '.join(args)}"
         self.logger.info(f"Running game: {cmd}")
 
-        self.environment.execute("rm -rf logs; mkdir logs")
-        for idx in tqdm(range(self.game_config["sims_per_round"])):
-            response = self.environment.execute(cmd + f" > logs/sim_{idx}.log")
-            assert response["returncode"] == 0, response
+        response = self.environment.execute(cmd + f" > {BATTLECODE_LOG}")
+        assert response["returncode"] == 0, response

codeclash/ratings/elo.py

@@ -64,20 +64,21 @@ def main(log_dir: Path):
             f" - {profile.player_id} (Game: {profile.game_id}) - ELO: {profile.rating:.1f} (Games: {profile.games_played})"
         )
 
-    # Average ELO per player across all games
-    aggregated_elo = {}
-    game_counts = {}
+    # Weighted average ELO per player across all games
+    weighted_elo = {}
     total_games = {}
     for profile in player_profiles.values():
         pid = profile.player_id
-        aggregated_elo[pid] = aggregated_elo.get(pid, 0) + profile.rating
-        game_counts[pid] = game_counts.get(pid, 0) + 1
+        weighted_elo[pid] = weighted_elo.get(pid, 0) + profile.rating * profile.games_played
         total_games[pid] = total_games.get(pid, 0) + profile.games_played
 
-    print("\nAverage ELO per player (across all games):")
-    for pid in aggregated_elo:
-        avg_elo = aggregated_elo[pid] / game_counts[pid]
-        print(f" - {pid}: Avg ELO {avg_elo:.1f} (Total Games: {total_games[pid]})")
+    print("\nWeighted average ELO per player (across all games):")
+    for pid in weighted_elo:
+        if total_games[pid] > 0:
+            avg_elo = weighted_elo[pid] / total_games[pid]
+        else:
+            avg_elo = 0.0
+        print(f" - {pid}: Weighted Avg ELO {avg_elo:.1f} (Total Games: {total_games[pid]})")
 
 
 if __name__ == "__main__":

configs/pvp/battlecode.yaml

@@ -2,7 +2,7 @@ tournament:
   rounds: 25
 game:
   name: BattleCode
-  sims_per_round: 5
+  sims_per_round: 1 # NOTE: Setting to > 1 does nothing for BattleCode, since each round is deterministic
   args:
     maps: quack
 players:

configs/test/dummy_battlecode.yaml

@@ -2,7 +2,7 @@ tournament:
   rounds: 25
 game:
   name: BattleCode
-  sims_per_round: 2
+  sims_per_round: 1 # NOTE: Setting to > 1 does nothing for BattleCode, since each round is deterministic
   args:
     maps: quack
 players: