Merge branch 'main' into john/sims_per_round

Author: john-b-yang

codeclash/agents/abstract.py

@@ -1,4 +1,5 @@
 import os
+import time
 import uuid
 from abc import ABC, abstractmethod
 
@@ -37,6 +38,8 @@ def __init__(
             "player_unique_id": self._player_unique_id,
             "diff": {0: ""},  # mapping round -> diff
             "incremental_diff": {0: ""},  # mapping round -> diff
+            "created_timestamp": int(time.time()),
+            "config": self.config,
         }
 
     # --- Main methods ---

codeclash/games/abstract.py

@@ -1,6 +1,6 @@
-import json
 import os
 import subprocess
+import time
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from pathlib import Path
@@ -56,19 +56,26 @@ def __init__(self, config: dict, *, tournament_id: str, local_output_dir: Path):
         self.url_gh: str = f"git@github.com:{GH_ORG}/{self.name}.git"
         self.artifacts: list[Path] = []
         """Artifact objects that we might want to clean up after the game."""
-        self.game_config: dict = config["game"]
         self.config: dict = config
-        self.game_id: str = tournament_id
+        self._metadata: dict = {
+            "name": self.name,
+            "config": self.config["game"],
+            "game_id": tournament_id,
+            "created_timestamp": int(time.time()),
+        }
         self.log_env: Path = (DIR_WORK / DIR_LOGS / self.game_id).resolve()
         self.log_local: Path = local_output_dir
         self.logger = get_logger(self.name, log_path=self.log_local / "game.log", emoji="🏓")
         self.environment: DockerEnvironment = self.get_environment()
         """The running docker environment for executing the game"""
-        self._metadata: dict = {
-            "name": self.name,
-            "config": self.config,
-            "game_id": self.game_id,
-        }
+
+    @property
+    def game_config(self) -> dict:
+        return self.config["game"]
+
+    @property
+    def game_id(self) -> str:
+        return self._metadata["game_id"]
 
     @property
     def image_name(self) -> str:
@@ -112,7 +119,6 @@ def get_metadata(self) -> dict:
         return self._metadata
 
     def end(self, cleanup: bool = False):
-        (self.log_local / "metadata.json").write_text(json.dumps(self.get_metadata()))
         if cleanup:
             for artifact in self.artifacts:
                 if artifact.exists():
@@ -127,6 +133,8 @@ def get_environment(self, branch_name: str | None = None) -> DockerEnvironment:
             cwd=str(DIR_WORK),
             env={"GITHUB_TOKEN": os.getenv("GITHUB_TOKEN", "")},
         )
+        # Logger setting will likely not take effect for initial container creation logs
+        environment.logger = get_logger("environment", emoji="🪴")
         # Ensure all future branches occur against branch
         branch_name = self.game_id if branch_name is None else branch_name
         for cmd in [

codeclash/tournaments/abstract.py

@@ -17,6 +17,8 @@ def __init__(self, config: dict, *, name: str, **kwargs):
         self._metadata: dict = {
             "name": self.name,
             "tournament_id": self.tournament_id,
+            "config": self.config,
+            "created_timestamp": int(time.time()),
         }
         self.logger = get_logger(self.name, log_path=self.local_output_dir / "tournament.log", emoji="🏆")
 

codeclash/tournaments/pvp.py

@@ -2,12 +2,14 @@
 PvP training mode where multiple agents compete against each other.
 """
 
+import json
+
 from codeclash.agents import get_agent
 from codeclash.agents.abstract import Player
 from codeclash.agents.utils import GameContext
 from codeclash.constants import DIR_WORK
 from codeclash.games import get_game
-from codeclash.games.abstract import CodeGame, RoundStats
+from codeclash.games.abstract import CodeGame
 from codeclash.tournaments.abstract import AbstractTournament
 from codeclash.utils.environment import copy_to_container
 from codeclash.utils.log import get_logger
@@ -27,12 +29,24 @@ def __init__(self, config: dict, *, cleanup: bool = False, push_agent: bool = Fa
         for agent_conf in self.config["players"]:
             self.agents.append(self.get_agent(agent_conf, self.config["prompts"]))
         self.logger = get_logger(self.game.name)
-        self.scoreboard: list[RoundStats] = []
+
+    @property
+    def scoreboard(self) -> list[tuple[int, str]]:
+        return self._metadata.setdefault("scoreboard", [])
 
     @property
     def rounds(self) -> int:
         return self.config["tournament"]["rounds"]
 
+    def get_metadata(self) -> dict:
+        # will be saved in end()
+        return {
+            **super().get_metadata(),
+            "scoreboard": self.scoreboard,
+            "game": self.game.get_metadata(),
+            "agents": [agent.get_metadata() for agent in self.agents],
+        }
+
     def get_agent(self, agent_config: dict, prompts: dict) -> Player:
         """Create an agent with environment and game context."""
         environment = self.game.get_environment(f"{self.game.game_id}.{agent_config['name']}")
@@ -57,7 +71,7 @@ def run(self) -> None:
             for round_num in range(1, self.rounds + 1):
                 self.run_training_round(round_num)
         finally:
-            self.cleanup()
+            self.end()
 
     def run_training_round(self, round_num: int) -> None:
         """Execute a single training round."""
@@ -96,8 +110,9 @@ def run_agent(self, agent: Player, round_num: int) -> None:
         agent.run()
         agent.post_run_hook(round=round_num)
 
-    def cleanup(self) -> None:
-        """Clean up game resources and push agents if requested."""
+    def end(self) -> None:
+        """Save output files, clean up game resources and push agents if requested."""
+        (self.local_output_dir / "metadata.json").write_text(json.dumps(self.game.get_metadata()))
         self.game.end(self.cleanup_on_end)
         if self.push_agent:
             for agent in self.agents:

codeclash/tournaments/single_player.py

@@ -10,14 +10,14 @@
 from codeclash.agents.utils import GameContext
 from codeclash.constants import DIR_WORK
 from codeclash.games import get_game
-from codeclash.games.abstract import CodeGame, RoundStats
+from codeclash.games.abstract import CodeGame
 from codeclash.tournaments.abstract import AbstractTournament
 from codeclash.tournaments.utils.git_utils import filter_git_diff
 from codeclash.utils.environment import copy_to_container
 
 
 class SinglePlayerTraining(AbstractTournament):
-    def __init__(self, config: dict, cleanup: bool = False):
+    def __init__(self, config: dict, *, cleanup: bool = False):
         super().__init__(config, name="SinglePlayerTraining")
         self.cleanup_on_end = cleanup
         self.game: CodeGame = get_game(
@@ -29,12 +29,23 @@ def __init__(self, config: dict, cleanup: bool = False):
         mirror_agent_config = copy.deepcopy(self.config["player"])
         mirror_agent_config["name"] = "mirror"
         self.mirror_agent: Player = self.get_agent(mirror_agent_config, round=0)
-        self.scoreboard: list[RoundStats] = []
+
+    @property
+    def scoreboard(self) -> list[tuple[int, str]]:
+        return self._metadata.setdefault("scoreboard", [])
 
     @property
     def rounds(self) -> int:
         return self.config["tournament"]["rounds"]
 
+    def get_metadata(self) -> dict:
+        return {
+            **super().get_metadata(),
+            "scoreboard": self.scoreboard,
+            "game": self.game.get_metadata(),
+            "agents": [self.agent.get_metadata(), self.mirror_agent.get_metadata()],
+        }
+
     def get_game_context(self, agent_config: dict, *, round: int) -> GameContext:
         """Create a game context for an agent."""
         return GameContext(
@@ -71,7 +82,7 @@ def run(self):
             if self.config["tournament"]["evaluate_matrix"]:
                 self.evaluate()
         finally:
-            self.cleanup()
+            self.end()
 
     def run_training_round(self, round_num: int) -> None:
         """Execute a single training round, i.e., run the game, then run the agent."""
@@ -113,11 +124,11 @@ def set_mirror_state_to_round(self, round_num: int):
         full_diff = filter_git_diff(full_diff)
         self.mirror_agent.reset_and_apply_patch(full_diff)
 
-    def cleanup(self):
+    def end(self):
         """Clean up game resources."""
         self.game.end(self.cleanup_on_end)
 
-    def evaluate(self, n_repetitions: int = 3):
+    def evaluate(self, n_repetitions: int = 3) -> None:
         """Evaluate the agent's performance by
         calculating the matrix of every round against each other.
         """
@@ -144,4 +155,4 @@ def evaluate(self, n_repetitions: int = 3):
                     self.logger.info(f"Round {p1_round} vs {p2_round} repetition {i_repetition} winner: {winner}")
                     matrix[p1_round][p2_round].append(winner)
         self.logger.info(f"Evaluation matrix: {matrix}")
-        return matrix
+        self._metadata.setdefault("evaluation", {})["matrix"] = matrix