Clean up logging, configs

Author: john-b-yang

README.md

@@ -17,10 +17,11 @@ Make sure you have `GITHUB_TOKEN` (w/ access permissions for this organization)
 
 To run `n` rounds of 2+ models competing against one another on a game, run the following:
 ```bash
-python main.py configs/battlesnake.yaml
-python main.py configs/robocode.yaml
-python main.py configs/robotrumble.yaml
-python main.py configs/corewar.yaml
+python main.py configs/pvp/battlecode.yaml
+python main.py configs/pvp/battlesnake.yaml
+python main.py configs/pvp/robocode.yaml
+python main.py configs/pvp/robotrumble.yaml
+python main.py configs/pvp/corewar.yaml
 ```
 
 For storing `logs/`, we're maintaining an AWS S3 bucket (`s3://codeclash`).

codeclash/agents/abstract.py

@@ -30,7 +30,7 @@ def __init__(
         self.game_context = game_context
         self.logger = get_logger(
             self.name,
-            log_path=self.game_context.log_local / f"{self.name}.log",
+            log_path=self.game_context.log_local / "players" / self.name / "player.log",
             emoji="👤",
         )
         self._metadata = {

codeclash/agents/dummy.py

@@ -6,4 +6,3 @@ class Dummy(Player):
 
     def run(self):
         pass
-        # self.commit()  # now called in post_round_hook

codeclash/agents/minisweagent.py

@@ -93,7 +93,12 @@ def run(self):
             result = exc_message
             print(exc_message)
         finally:
-            traj_path = self.game_context.log_local / f"{self.name}_r{self.game_context.round}.traj.json"
+            traj_path = (
+                self.game_context.log_local
+                / "players"
+                / self.name
+                / f"{self.name}_r{self.game_context.round}.traj.json"
+            )
             save_traj(
                 self.agent,  # type: ignore
                 traj_path,
@@ -104,6 +109,5 @@ def run(self):
             copy_to_container(
                 self.environment,
                 traj_path,
-                self.game_context.log_env / traj_path.name,
+                self.game_context.log_env / "past_edits" / traj_path.name,
             )
-            # self.commit()  # now called in post_round_hook

codeclash/games/abstract.py

@@ -60,7 +60,7 @@ def __init__(self, config: dict, *, tournament_id: str, local_output_dir: Path):
             "game_id": tournament_id,
             "created_timestamp": int(time.time()),
         }
-        self.log_env: Path = (DIR_WORK / DIR_LOGS / self.game_id).resolve()
+        self.log_env: Path = (DIR_WORK / DIR_LOGS).resolve()
         self.log_local: Path = local_output_dir
         self.logger = get_logger(self.name, log_path=self.log_local / "game.log", emoji="🏓")
         self.environment: DockerEnvironment = self.get_environment()

codeclash/ratings/win_rate.py

@@ -0,0 +1,56 @@
+import argparse
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from codeclash.constants import RESULT_TIE
+
+
+@dataclass
+class PlayerGameProfile:
+    player_id: str
+    game_id: str
+    wins: int = 0
+    count: int = 0
+
+    @property
+    def win_rate(self) -> float:
+        return self.wins / self.count if self.count > 0 else 0.0
+
+
+def main(log_dir: Path):
+    player_profiles = {}
+    for game_log_folder in log_dir.iterdir():
+        if game_log_folder.is_dir():
+            print(f"Processing game log folder: {game_log_folder}")
+
+        game_id = game_log_folder.name.split(".")[1]
+        player_ids = [x.name for x in (game_log_folder / "players").iterdir() if x.is_dir()]
+        num_rounds = len(list((game_log_folder / "rounds").iterdir()))
+
+        for player in player_ids:
+            if f"{game_id}.{player}" in player_profiles:
+                player_profiles[f"{game_id}.{player}"].count += num_rounds
+            else:
+                player_profiles[f"{game_id}.{player}"] = PlayerGameProfile(
+                    player_id=player, game_id=game_id, count=num_rounds
+                )
+
+        for round_folder in (game_log_folder / "rounds").iterdir():
+            round_results = json.load(open(round_folder / "results.json"))
+            winner = round_results.get("winner")
+            if winner != RESULT_TIE:
+                player_profiles[f"{game_id}.{winner}"].wins += 1
+
+    print("Player profiles:")
+    for profile in player_profiles.values():
+        print(
+            f" - {profile.player_id} (Game: {profile.game_id}) - Win Rate: {profile.win_rate:.2%} ({profile.wins}/{profile.count})"
+        )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("log_dir", type=Path, help="Path to `logs/<user>` folder containing game logs")
+    args = parser.parse_args()
+    main(args.log_dir)

codeclash/tournaments/pvp.py

@@ -83,20 +83,23 @@ def run_training_round(self, round_num: int) -> None:
         self.logger.info(f"Round {round_num}:\n{record.stats}")
 
         # Create directory for round logs
-        (self.game.log_local / f"round_{round_num}").mkdir(parents=True, exist_ok=True)
+        (self.game.log_local / "rounds" / str(round_num)).mkdir(parents=True, exist_ok=True)
 
-        # Write log to file
+        # Write logs to file
         for idx, lo in enumerate(record.data.logs):
-            round_log_path = self.game.log_local / f"round_{round_num}" / f"sim_{idx}.log"
+            round_log_path = self.game.log_local / "rounds" / str(round_num) / f"sim_{idx}.log"
             round_log_path.write_text(lo)
+        results_file = self.game.log_local / "rounds" / str(round_num) / "results.json"
+        with open(results_file, "w") as f:
+            json.dump(record.stats.model_dump(), fp=f, indent=2)
 
         # Copy log to agent environments
         for agent in self.agents:
             self.logger.info(f"Copying round {round_num} log(s) to {agent.name}'s container...")
             copy_to_container(
                 agent.environment,
-                self.game.log_local / f"round_{round_num}",
-                f"logs/round_{round_num}/",
+                self.game.log_local / "rounds" / str(round_num),
+                f"logs/rounds/{round_num}/",
             )
 
         for agent in self.agents:
@@ -112,7 +115,8 @@ def run_agent(self, agent: Player, round_num: int) -> None:
 
     def end(self) -> None:
         """Save output files, clean up game resources and push agents if requested."""
-        (self.local_output_dir / "metadata.json").write_text(json.dumps(self.game.get_metadata()))
+        with open(self.local_output_dir / "metadata.json", "w") as f:
+            json.dump(self.game.get_metadata(), fp=f, indent=2)
         self.game.end(self.cleanup_on_end)
         if self.push_agent:
             for agent in self.agents:

codeclash/tournaments/single_player.py

@@ -3,6 +3,7 @@
 """
 
 import copy
+import json
 
 from codeclash.agents import get_agent
 from codeclash.agents.abstract import Player
@@ -95,16 +96,19 @@ def run_training_round(self, round_num: int) -> None:
 
         # Write log to file
         for idx, lo in enumerate(record.data.logs):
-            round_log_path = self.game.log_local / f"round_{round_num}" / f"sim_{idx}.log"
+            round_log_path = self.game.log_local / "rounds" / str(round_num) / f"sim_{idx}.log"
             round_log_path.parent.mkdir(parents=True, exist_ok=True)
             round_log_path.write_text(lo)
+        results_file = self.game.log_local / "rounds" / str(round_num) / "results.json"
+        with open(results_file, "w") as f:
+            json.dump(record.stats.model_dump(), fp=f, indent=2)
 
         # Copy log to main agent environment only
         self.logger.info(f"Copying round {round_num} log(s) to {self.agent.name}'s container...")
         copy_to_container(
             self.agent.environment,
-            self.game.log_local / f"round_{round_num}",
-            f"logs/round_{round_num}/",
+            self.game.log_local / "rounds" / str(round_num),
+            f"logs/rounds/{round_num}/",
         )
 
         self.run_main_agent(round_num)