Make log copying more flexible (#52)

* Add Huskybench basic scaffolding

* Remove RoundRecord; Add explicit copy logs method

Author: john-b-yang

codeclash/agents/player.py

@@ -24,7 +24,7 @@ def __init__(
     ) -> None:
         self.config = config
         self.name = config["name"]
-        self._player_unique_id = uuid.uuid4()
+        self._player_unique_id = str(uuid.uuid4())
         """Unique ID that doesn't clash even across multiple games. Used for git tags."""
         self.environment = environment
         self.game_context = game_context

codeclash/games/__init__.py

@@ -5,6 +5,7 @@
 from codeclash.games.corewar.corewar import CoreWarGame
 from codeclash.games.dummy.dummy_game import DummyGame
 from codeclash.games.game import CodeGame
+from codeclash.games.huskybench.huskybench import HuskyBenchGame
 from codeclash.games.robocode.robocode import RoboCodeGame
 from codeclash.games.robotrumble.robotrumble import RobotRumbleGame
 
@@ -18,6 +19,7 @@ def get_game(config: dict, *, tournament_id: str, local_output_dir: Path) -> Cod
             BattleSnakeGame,
             CoreWarGame,
             DummyGame,
+            HuskyBenchGame,
             RoboCodeGame,
             RobotRumbleGame,
         ]

codeclash/games/battlecode/battlecode.py

@@ -5,7 +5,8 @@
 from tqdm.auto import tqdm
 
 from codeclash.constants import DIR_WORK, RESULT_TIE
-from codeclash.games.game import CodeGame, RoundData, RoundStats
+from codeclash.games.game import CodeGame, RoundStats
+from codeclash.utils.environment import copy_from_container
 
 
 class BattleCodeGame(CodeGame):
@@ -22,9 +23,18 @@ def __init__(self, config, *, tournament_id: str, local_output_dir: Path):
             else:
                 self.run_cmd_round += f" --{arg} {val}"
 
-    def get_stats(self, result_outputs: list[str], agents: list[Any]) -> RoundStats:
+    def copy_logs_from_env(self, round_num):
+        super().copy_logs_from_env(round_num)
+        copy_from_container(
+            container=self.environment,
+            src_path="/testbed/logs",
+            dest_path=self.log_local / "rounds" / str(round_num),
+        )
+
+    def get_stats(self, agents: list[Any]) -> RoundStats:
         winners = []
-        for ro in result_outputs:
+        for sim_file in [f"logs/sim_{idx}.log" for idx in range(self.game_config["sims_per_round"])]:
+            ro = self.environment.execute(f"cat {sim_file}")["output"]
             lines = ro.strip().split("\n")
             # Get the third-to-last line which contains the winner info
             winner_line = lines[-3] if len(lines) >= 3 else ""
@@ -43,17 +53,15 @@ def get_stats(self, result_outputs: list[str], agents: list[Any]) -> RoundStats:
             scores={agent.name: winners.count(agent.name) for agent in agents},
         )
 
-    def execute_round(self, agents: list[Any]) -> RoundData:
+    def execute_round(self, agents: list[Any]):
         for agent in agents:
             src, dest = f"/{agent.name}/src/mysubmission/", str(DIR_WORK / "src" / agent.name)
             self.environment.execute(f"cp -r {src} {dest}")
         args = [f"--p{idx + 1}-dir src --p{idx + 1} {agent.name}" for idx, agent in enumerate(agents)]
         cmd = f"{self.run_cmd_round} {' '.join(args)}"
         self.logger.info(f"Running game: {cmd}")
-        outputs = []
-        for _ in tqdm(range(self.game_config["sims_per_round"])):
-            response = self.environment.execute(cmd)
+
+        self.environment.execute("rm -rf logs; mkdir logs")
+        for idx in tqdm(range(self.game_config["sims_per_round"])):
+            response = self.environment.execute(cmd + f" > logs/sim_{idx}.log")
             assert response["returncode"] == 0, response
-            # For BattleCode, log_outputs and result_outputs are the same
-            outputs.append(response["output"])
-        return RoundData(logs=outputs, results=outputs)

codeclash/games/battlesnake/battlesnake.py

@@ -1,15 +1,14 @@
 import json
 import time
-import uuid
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 
 from tqdm.auto import tqdm
 
 from codeclash.agents.player import Player
 from codeclash.constants import RESULT_TIE
-from codeclash.games.game import CodeGame, RoundData, RoundStats
-from codeclash.utils.environment import assert_zero_exit_code
+from codeclash.games.game import CodeGame, RoundStats
+from codeclash.utils.environment import assert_zero_exit_code, copy_from_container
 
 
 class BattleSnakeGame(CodeGame):
@@ -39,9 +38,18 @@ def _wait_for_ports(self, ports: list[int], timeout: float = 3.0) -> None:
 
             time.sleep(0.1)
 
-    def get_stats(self, result_outputs: list[str], agents: list[Player]) -> RoundStats:
+    def copy_logs_from_env(self, round_num):
+        super().copy_logs_from_env(round_num)
+        copy_from_container(
+            container=self.environment,
+            src_path=f"{self.environment.config.cwd}/game/logs",
+            dest_path=self.log_local / "rounds" / str(round_num),
+        )
+
+    def get_stats(self, agents: list[Player]) -> RoundStats:
         scores = {}
-        for ro in result_outputs:
+        for idx in range(self.game_config["sims_per_round"]):
+            ro = self.environment.execute(f"cat game/logs/sim_out_{idx}.json")["output"]
             lines = ro.strip().split("\n")
             results = json.loads(lines[-1]) if lines else {}  # Get the last line which contains the game result
             winner = RESULT_TIE if results["isDraw"] else results["winnerName"]
@@ -51,7 +59,7 @@ def get_stats(self, result_outputs: list[str], agents: list[Player]) -> RoundSta
         winner = RESULT_TIE if list(scores.values()).count(scores[winner]) > 1 else winner
         return RoundStats(winner=winner, scores=scores)
 
-    def execute_round(self, agents: list[Player]) -> RoundData:
+    def execute_round(self, agents: list[Player]):
         self.logger.debug("Starting game servers")
         cmd = []
         ports = []
@@ -68,46 +76,30 @@ def execute_round(self, agents: list[Player]) -> RoundData:
         self.logger.debug("All ports are ready")
 
         try:
-            log_outputs, result_outputs = [], []
             cmd = self.run_cmd_round + " " + " ".join(cmd)
             self.logger.info(f"Running game: {cmd}")
+            self.environment.execute("rm -rf logs; mkdir logs", cwd=f"{self.environment.config.cwd}/game")
 
             # Use ThreadPoolExecutor for parallel execution
             with ThreadPoolExecutor(20) as executor:
                 # Submit all simulations to the thread pool
                 futures = [
-                    executor.submit(self._run_single_simulation, cmd) for _ in range(self.game_config["sims_per_round"])
+                    executor.submit(self._run_single_simulation, cmd, idx)
+                    for idx in range(self.game_config["sims_per_round"])
                 ]
 
                 # Collect results as they complete
                 for future in tqdm(as_completed(futures), total=len(futures)):
-                    log_output, result_output = future.result()
-                    log_outputs.append(log_output)
-                    result_outputs.append(result_output)
-
-            return RoundData(logs=log_outputs, results=result_outputs)
+                    future.result()
         finally:
             # Kill all python servers when done
             self.environment.execute("pkill -f 'python main.py' || true")
 
-    def _run_single_simulation(self, cmd: str) -> tuple[str, str]:
+    def _run_single_simulation(self, cmd: str, idx: int) -> tuple[str, str]:
         """Run a single battlesnake simulation and return log and result outputs."""
-        # Create temporary output file for results
-        output_file = f"battlesnake_output_{uuid.uuid4().hex}.json"
-
-        # Run game
-        response = assert_zero_exit_code(
+        assert_zero_exit_code(
             self.environment.execute(
-                cmd + f" -o {output_file}",
+                cmd + f" -o logs/sim_out_{idx}.json",
                 cwd=f"{self.environment.config.cwd}/game",
             )
         )
-
-        # Read the output file for result information
-        result_response = self.environment.execute(f"cat game/{output_file}")
-        result_output = result_response["output"]
-
-        # Clean up the output file
-        self.environment.execute(f"rm -f game/{output_file}")
-
-        return response["output"], result_output

codeclash/games/corewar/corewar.py

@@ -3,7 +3,8 @@
 from pathlib import Path
 
 from codeclash.agents.player import Player
-from codeclash.games.game import CodeGame, RoundData, RoundStats
+from codeclash.games.game import CodeGame, RoundStats
+from codeclash.utils.environment import copy_from_container
 
 
 class CoreWarGame(CodeGame):
@@ -19,8 +20,16 @@ def __init__(self, config, *, tournament_id: str, local_output_dir: Path):
             else:
                 self.run_cmd_round += f" -{arg} {val}"
 
-    def get_stats(self, result_outputs: list[str], agents: list[Player]) -> RoundStats:
-        result_output = result_outputs[0]  # Get the first (and only) element
+    def copy_logs_from_env(self, round_num: int) -> None:
+        super().copy_logs_from_env(round_num)
+        copy_from_container(
+            container=self.environment,
+            src_path="/testbed/output.log",
+            dest_path=self.log_local / "rounds" / str(round_num) / "output.log",
+        )
+
+    def get_stats(self, agents: list[Player]) -> RoundStats:
+        result_output = self.environment.execute("cat output.log")["output"]
         self.logger.debug(f"Determining winner from result output: {result_output}")
         scores = []
         n = len(agents) * 2
@@ -50,10 +59,9 @@ def get_stats(self, result_outputs: list[str], agents: list[Player]) -> RoundSta
             self.logger.debug("No scores found, returning unknown")
             return RoundStats(winner="unknown", scores={agent.name: 0 for agent in agents})
 
-    def execute_round(self, agents: list[Player]) -> RoundData:
+    def execute_round(self, agents: list[Player]):
         args = [f"/{agent.name}/warriors/warrior.red" for agent in agents]
-        cmd = f"{self.run_cmd_round} {shlex.join(args)} -r {self.game_config['sims_per_round']}"
+        cmd = f"{self.run_cmd_round} {shlex.join(args)} -r {self.game_config['sims_per_round']} > output.log;"
         self.logger.info(f"Running game: {cmd}")
         response = self.environment.execute(cmd)
         assert response["returncode"] == 0, response
-        return RoundData(logs=[response["output"]], results=[response["output"]])

codeclash/games/dummy/dummy_game.py

@@ -1,14 +1,23 @@
 import re
 
 from codeclash.agents.player import Player
-from codeclash.games.game import CodeGame, RoundData, RoundStats
+from codeclash.games.game import CodeGame, RoundStats
+from codeclash.utils.environment import assert_zero_exit_code, copy_from_container
 
 
 class DummyGame(CodeGame):
     name: str = "DummyGame"
 
-    def get_stats(self, result_outputs: list[str], agents: list[Player]) -> RoundStats:
-        result_output = result_outputs[0]  # Get the first (and only) element
+    def copy_logs_from_env(self, round_num):
+        super().copy_logs_from_env(round_num)
+        copy_from_container(
+            container=self.environment,
+            src_path="/testbed/result.log",
+            dest_path=self.log_local / "rounds" / str(round_num) / "result.log",
+        )
+
+    def get_stats(self, agents: list[Player]) -> RoundStats:
+        result_output = self.environment.execute("cat result.log")["output"]
         lines = result_output.split("FINAL_RESULTS")[-1].splitlines()
 
         scores = {}
@@ -25,10 +34,8 @@ def get_stats(self, result_outputs: list[str], agents: list[Player]) -> RoundSta
             details={"dummy": True},
         )
 
-    def execute_round(self, agents: list[Player]) -> RoundData:
+    def execute_round(self, agents: list[Player]) -> None:
         args = [f"/{agent.name}/main.py" for agent in agents]
-        cmd = f"python engine.py {' '.join(args)} -r {self.game_config['sims_per_round']}"
+        cmd = f"python engine.py {' '.join(args)} -r {self.game_config['sims_per_round']} > result.log;"
         self.logger.info(f"Running game: {cmd}")
-        response = self.environment.execute(cmd)
-        assert response["returncode"] == 0, response
-        return RoundData(logs=[response["output"]], results=[response["output"]])
+        assert_zero_exit_code(self.environment.execute(cmd))

codeclash/games/dummy/main.py

@@ -23,16 +23,6 @@ def __str__(self) -> str:
         return "\n".join([f"- Winner: {self.winner}", f"- Scores: {self.scores}"])
 
 
-class RoundData(BaseModel):
-    logs: list[str]
-    results: list[str]
-
-
-class RoundRecord(BaseModel):
-    data: RoundData
-    stats: RoundStats
-
-
 class CodeGame(ABC):
     name: str
 
@@ -97,7 +87,7 @@ def build_image(self):
         result = subprocess.run(
             (
                 "export $(cat .env | xargs);"
-                f"docker build --build-arg GITHUB_TOKEN=$GITHUB_TOKEN -t {self.image_name} -f docker/{self.name}.Dockerfile ."
+                f"docker build --no-cache --build-arg GITHUB_TOKEN=$GITHUB_TOKEN -t {self.image_name} -f docker/{self.name}.Dockerfile ."
             ),
             shell=True,
             capture_output=True,
@@ -168,12 +158,15 @@ def _pre_round_setup(self, agents: list[Player]):
             logger=self.logger,
         )
 
+    def copy_logs_from_env(self, round_num: int) -> None:
+        """Copy logs from the game's environment to the local machine."""
+        (self.log_local / "rounds" / str(round_num)).mkdir(parents=True, exist_ok=True)
+
     @abstractmethod
-    def get_stats(self, result_outputs: list[str], agents: list[Player]) -> RoundStats:
+    def get_stats(self, agents: list[Player]) -> RoundStats:
         """Determine the winner of the game based on the result output.
 
         Args:
-            result_outputs: The specific output(s) containing winning information
             agents: List of agents participating in the round
 
         Returns:
@@ -182,24 +175,22 @@ def get_stats(self, result_outputs: list[str], agents: list[Player]) -> RoundSta
         pass
 
     @abstractmethod
-    def execute_round(self, agents: list[Player]) -> RoundData:
+    def execute_round(self, agents: list[Player]):
         """Subclasses implement their game-specific logic here.
         This is the low level implementation, you probably want to use run_round instead, which
         includes the pre-round setup, post-round setup, and winner determination.
-
-        Returns:
-            RoundData object
         """
         pass
 
-    def run_round(self, agents: list[Player]) -> RoundRecord:
+    def run_round(self, agents: list[Player], round_num: int) -> RoundStats:
         """
         Run a single round of the game with the given agents.
 
         Returns the log output, result output, and winner name. All bookkeeping should be
         handled by the tournament class.
         """
         self._pre_round_setup(agents)
-        data = self.execute_round(agents)
-        stats = self.get_stats(data.results, agents)
-        return RoundRecord(data=data, stats=stats)
+        self.execute_round(agents)
+        stats = self.get_stats(agents)
+        self.copy_logs_from_env(round_num)
+        return stats

codeclash/games/game.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+
+from codeclash.agents.player import Player
+from codeclash.games.game import CodeGame, RoundStats
+
+
+class HuskyBenchGame(CodeGame):
+    name: str = "HuskyBench"
+
+    def __init__(self, config, *, tournament_id: str, local_output_dir: Path):
+        super().__init__(config, tournament_id=tournament_id, local_output_dir=local_output_dir)
+        self.run_cmd_round: str = (
+            f"python engine/main.py --port 8000 --sim --sim-rounds {self.game_config['sims_per_round']}"
+        )
+        for arg, val in self.game_config.get("args", {}).items():
+            if isinstance(val, bool):
+                if val:
+                    self.run_cmd_round += f" --{arg}"
+            else:
+                self.run_cmd_round += f" --{arg} {val}"
+
+    def get_stats(self, result_outputs: list[str], agents: list[Player]) -> RoundStats:
+        return RoundStats(winner="N/A", scores={})
+
+    def execute_round(self, agents: list[Player]):
+        try:
+            self.logger.debug("Starting game servers")
+            self.environment.execute(self.run_cmd_round + " > output.log &")
+            for agent in agents:
+                self.environment.execute("python client/main.py --port 8000 &", cwd=f"/{agent.name}")
+        finally:
+            # Kill all python servers when done
+            self.environment.execute("pkill -f 'python client/main.py' || true")
+            self.environment.execute("pkill -f 'python engine/main.py' || true")