Enh: Merge all metadata in tournament and write out

Author: klieret

codeclash/games/abstract.py

@@ -92,7 +92,6 @@ def get_metadata(self) -> dict:
         return self._metadata
 
     def end(self, cleanup: bool = False):
-        (self.log_local / "metadata.json").write_text(json.dumps(self.get_metadata()))
         if cleanup:
             for artifact in self.artifacts:
                 if artifact.exists():

codeclash/tournaments/abstract.py

@@ -3,10 +3,7 @@
 import traceback
 from pathlib import Path
 
-from codeclash.agents import get_agent
-from codeclash.agents.abstract import Player
-from codeclash.agents.utils import GameContext
-from codeclash.constants import DIR_LOGS, DIR_WORK
+from codeclash.constants import DIR_LOGS
 from codeclash.utils.environment import create_file_on_container
 from codeclash.utils.log import get_logger
 

codeclash/tournaments/pvp_training.py

@@ -2,6 +2,8 @@
 PvP training mode where multiple agents compete against each other.
 """
 
+import json
+
 from codeclash.agents import get_agent
 from codeclash.agents.abstract import Player
 from codeclash.agents.utils import GameContext
@@ -34,6 +36,15 @@ def __init__(
     def rounds(self) -> int:
         return self.config["tournament"]["rounds"]
 
+    def get_metadata(self) -> dict:
+        # will be saved in end()
+        return {
+            **super().get_metadata(),
+            "scoreboard": self.scoreboard,
+            "game": self.game.get_metadata(),
+            "agents": [agent.get_metadata() for agent in self.agents],
+        }
+
     def get_agent(self, agent_config: dict, prompts: dict) -> Player:
         """Create an agent with environment and game context."""
         environment = self.game.get_environment(
@@ -60,7 +71,7 @@ def run(self) -> None:
             for round_num in range(1, self.rounds + 1):
                 self.run_training_round(round_num)
         finally:
-            self.cleanup()
+            self.end()
 
     def run_training_round(self, round_num: int) -> None:
         """Execute a single training round."""
@@ -93,8 +104,11 @@ def run_agent(self, agent: Player, round_num: int) -> None:
         agent.run()
         agent.post_run_hook(round=round_num)
 
-    def cleanup(self) -> None:
-        """Clean up game resources and push agents if requested."""
+    def end(self) -> None:
+        """Save output files, clean up game resources and push agents if requested."""
+        (self.local_output_dir / "metadata.json").write_text(
+            json.dumps(self.game.get_metadata())
+        )
         self.game.end(self.cleanup_on_end)
         if self.push_agent:
             for agent in self.agents:

codeclash/tournaments/single_player_training.py

@@ -34,6 +34,14 @@ def __init__(self, config: dict, cleanup: bool = False):
     def rounds(self) -> int:
         return self.config["tournament"]["rounds"]
 
+    def get_metadata(self) -> dict:
+        return {
+            **super().get_metadata(),
+            "scoreboard": self.scoreboard,
+            "game": self.game.get_metadata(),
+            "agents": [self.agent.get_metadata(), self.mirror_agent.get_metadata()],
+        }
+
     def get_game_context(self, agent_config: dict, *, round: int) -> GameContext:
         """Create a game context for an agent."""
         return GameContext(
@@ -72,7 +80,7 @@ def run(self):
             if self.config["tournament"]["evaluate_matrix"]:
                 self.evaluate()
         finally:
-            self.cleanup()
+            self.end()
 
     def run_training_round(self, round_num: int) -> None:
         """Execute a single training round, i.e., run the game, then run the agent."""
@@ -110,11 +118,11 @@ def set_mirror_state_to_round(self, round_num: int):
         full_diff = filter_git_diff(full_diff)
         self.mirror_agent.reset_and_apply_patch(full_diff)
 
-    def cleanup(self):
+    def end(self):
         """Clean up game resources."""
         self.game.end(self.cleanup_on_end)
 
-    def evaluate(self, n_repetitions: int = 3):
+    def evaluate(self, n_repetitions: int = 3) -> None:
         """Evaluate the agent's performance by
         calculating the matrix of every round against each other.
         """
@@ -150,4 +158,4 @@ def evaluate(self, n_repetitions: int = 3):
                     )
                     matrix[p1_round][p2_round].append(winner)
         self.logger.info(f"Evaluation matrix: {matrix}")
-        return matrix
+        self._metadata.setdefault("evaluation", {})["matrix"] = matrix