WIP HuskyBench

Author: john-b-yang

codeclash/games/huskybench/huskybench.py

@@ -1,19 +1,23 @@
+import re
 from pathlib import Path
 
 from codeclash.agents.player import Player
 from codeclash.games.game import CodeGame, RoundStats
 from codeclash.utils.environment import copy_from_container
 
 HB_LOG_DIR = Path("/testbed/engine/logs/")
+HB_REGEX_SCORE = re.compile(r"Player\s(\d+)\sdelta\supdated\:[\d\s\-\+\=]+,\smoney\:\s\d+\s\-\>\s(\d+)")
 
 
 class HuskyBenchGame(CodeGame):
     name: str = "HuskyBench"
 
     def __init__(self, config, *, tournament_id: str, local_output_dir: Path):
         super().__init__(config, tournament_id=tournament_id, local_output_dir=local_output_dir)
+        self.num_players: int = len(config["players"])
         self.run_cmd_round: str = (
-            f"python engine/main.py --port 8000 --sim --sim-rounds {self.game_config['sims_per_round']}"
+            f"python engine/main.py --port 8000 --players {self.num_players} "
+            f"--sim --sim-rounds {self.game_config['sims_per_round']}"
         )
         for arg, val in self.game_config.get("args", {}).items():
             if isinstance(val, bool):
@@ -32,17 +36,34 @@ def copy_logs_from_env(self, round_num):
         )
 
     def get_stats(self, agents: list[Player], round_num: int) -> RoundStats:
-        return RoundStats(winner="N/A", scores={})
+        map_id_to_agent = {}
+        for agent in agents:
+            with open(self.log_round(round_num) / f"logs/{agent.name}.log") as f:
+                for line in f:
+                    if line.startswith("My id:"):
+                        agent_id = line.strip().split()[-1]
+                        map_id_to_agent[agent_id] = agent.name
+        self.logger.info("Agent IDs: " + str(map_id_to_agent))
+
+        with open(self.log_round(round_num) / "logs/engine.log") as f:
+            score_updates = [
+                (match.group(1), int(match.group(2))) for l in f.readlines() if (match := HB_REGEX_SCORE.search(l))
+            ]
+            map_id_to_score = {k: v for k, v in score_updates[-self.num_players :]}
+        self.logger.info("Final Scores: " + str(map_id_to_score))
+        agent_to_score = {map_id_to_agent[agent_id]: score for agent_id, score in map_id_to_score.items()}
+        return RoundStats(winner=max(agent_to_score, key=agent_to_score.get), scores=agent_to_score)
 
     def execute_round(self, agents: list[Player]):
         self.environment.execute(f"rm -rf {HB_LOG_DIR}; mkdir -p {HB_LOG_DIR}")
         try:
-            self.logger.debug("Starting game servers")
-            self.environment.execute(f"{self.run_cmd_round} > {HB_LOG_DIR / 'engine.log'} &")
+            cmd = f"{self.run_cmd_round} > {HB_LOG_DIR / 'engine.log'} &"
+            self.logger.debug(f"Starting game engine with command: {cmd}")
+            self.environment.execute(cmd)
             for agent in agents:
-                self.environment.execute(
-                    f"python client/main.py --port 8000 > {HB_LOG_DIR / f'{agent.name}.log'} &", cwd=f"/{agent.name}"
-                )
+                cmd = f"python client/main.py --port 8000 > {HB_LOG_DIR / f'{agent.name}.log'} &"
+                self.logger.debug(f"Adding agent with command: {cmd}")
+                self.environment.execute(cmd, cwd=f"/{agent.name}")
         finally:
             # Kill all python servers when done
             self.environment.execute("pkill -f 'python client/main.py' || true")

codeclash/ratings/elo.py

@@ -3,6 +3,8 @@
 from dataclasses import dataclass
 from pathlib import Path
 
+from tqdm import tqdm
+
 from codeclash.constants import DIR_LOGS, FILE_RESULTS
 
 K_FACTOR = 32  # ELO constant, changeable
@@ -29,10 +31,10 @@ def main(log_dir: Path):
     # - metadata.json
     player_profiles = {}
     for user_folder in log_dir.iterdir():
-        for game_log_folder in user_folder.iterdir():
-            if game_log_folder.is_dir():
-                print(f"Processing game log folder: {game_log_folder}")
-
+        print(f"Processing games under user `{user_folder.name}`")
+        for game_log_folder in tqdm(list(user_folder.iterdir())):
+            if not game_log_folder.is_dir():
+                continue
             game_id = game_log_folder.name.split(".")[1]
             player_ids = [x.name for x in (game_log_folder / "players").iterdir() if x.is_dir()]
             # Initialize profiles
@@ -42,6 +44,9 @@ def main(log_dir: Path):
                     player_profiles[key] = PlayerEloProfile(player_id=player, game_id=game_id)
 
             for round_folder in (game_log_folder / "rounds").iterdir():
+                if round_folder.name == "0":
+                    # Skip initial round
+                    continue
                 if not (round_folder / FILE_RESULTS).exists():
                     continue
                 round_results = json.load(open(round_folder / FILE_RESULTS))

codeclash/ratings/win_rate.py

@@ -3,6 +3,8 @@
 from dataclasses import dataclass
 from pathlib import Path
 
+from tqdm import tqdm
+
 from codeclash.constants import DIR_LOGS, FILE_RESULTS, RESULT_TIE
 
 
@@ -27,10 +29,10 @@ def main(log_dir: Path):
     # - metadata.json
     player_profiles = {}
     for user_folder in log_dir.iterdir():
-        for game_log_folder in user_folder.iterdir():
-            if game_log_folder.is_dir():
-                print(f"Processing game log folder: {game_log_folder}")
-
+        print(f"Processing games under user `{user_folder.name}`")
+        for game_log_folder in tqdm(list(user_folder.iterdir())):
+            if not game_log_folder.is_dir():
+                continue
             game_id = game_log_folder.name.split(".")[1]
             player_ids = [x.name for x in (game_log_folder / "players").iterdir() if x.is_dir()]
             num_rounds = len(list((game_log_folder / "rounds").iterdir()))
@@ -44,6 +46,9 @@ def main(log_dir: Path):
                     )
 
             for round_folder in (game_log_folder / "rounds").iterdir():
+                if round_folder.name == "0":
+                    # Skip initial round
+                    continue
                 if not (round_folder / FILE_RESULTS).exists():
                     continue
                 round_results = json.load(open(round_folder / FILE_RESULTS))

configs/test/dummy_huskybench.yaml

@@ -1,8 +1,8 @@
 tournament:
-  rounds: 3
+  rounds: 25
 game:
   name: HuskyBench
-  sims_per_round: 1
+  sims_per_round: 30
 players:
 - agent: dummy
   name: p1