Merge branch 'main' of github.com:emagedoc/CodeClash

Author: klieret

assets/texgyrepagella-bold.otf

@@ -188,6 +188,15 @@ def _extract_modified_files_from_diff(self, diff: str) -> dict[str, str]:
 
         file_contents = {}
         for file_path in file_paths:
+            # Check whether the file exists in the container before attempting to cat it.
+            # We avoid try/except by inspecting the returncode returned by the execute call.
+            ls_result = self.environment.execute(f"ls -la '{file_path}'")
+            if ls_result.get("returncode", 0) != 0:
+                # File was removed or is not present in this tree. Per request, record empty string.
+                self.logger.warning(f"File '{file_path}' not found; recording empty content.")
+                file_contents[file_path] = ""
+                continue
+
             out = assert_zero_exit_code(
                 self.environment.execute(f"cat '{file_path}'"),
                 logger=self.logger,

assets/texgyrepagella-regular.otf

@@ -6,7 +6,9 @@
 
 from tqdm import tqdm
 
+from codeclash.analysis.viz.utils import MODEL_TO_DISPLAY_NAME
 from codeclash.constants import LOCAL_LOG_DIR, RESULT_TIE
+from codeclash.games import ARENAS
 
 
 @dataclass
@@ -216,7 +218,7 @@ def print_results(self) -> None:
         print("=" * 50)
         print("Player ELO profiles:")
         lines = [
-            f" - {profile.model} (Arena: {profile.arena}) - ELO: {profile.rating:.1f} (Games: {profile.rounds_played})"
+            f" - {profile.model} ({profile.arena}) - ELO: {profile.rating:.1f} (Rounds: {profile.rounds_played})"
             for profile in self._player_profiles.values()
         ]
         print("\n".join(sorted(lines)))
@@ -231,18 +233,71 @@ def print_results(self) -> None:
 
         print("\nELO per player (across all games):")
         calc_avg_elo = lambda total_elo, games: total_elo / games
+        model_to_avg_elo = {mid: calc_avg_elo(weighted_elo[mid], total_games[mid]) for mid in weighted_elo}
         lines = sorted(
-            [
-                f"{pid}: {calc_avg_elo(weighted_elo[pid], total_games[pid]):.1f} (Games: {total_games[pid]})"
-                for pid in weighted_elo
-                if total_games[pid] > 0
-            ],
+            [f"{pid}: {model_to_avg_elo[pid]:.1f} (Rounds: {total_games[pid]})" for pid in model_to_avg_elo.keys()],
             key=lambda x: float(x.split(":")[1].split("(")[0]),
             reverse=True,
         )
         for i, line in enumerate(lines, 1):
             print(f"{i}. {line}")
 
+        # Print latex formatted table for results, formatted as
+        # Model & ELO & & & \\
+        # & BattleCode & BattleSnake & ... \\
+        # \midrule
+        # Claude 4 Sonnet & ... & & & \\
+        # ...
+        print("\nLaTeX formatted table:")
+        arenas = [x.name for x in ARENAS]
+        lines = []
+        arenas_small = [f"\\scriptsize{{{arena}}}" for arena in arenas]
+        line = "& " + " & ".join(arenas_small) + " & All" + r" \\"
+        line = line.replace("HuskyBench", "Poker")
+        lines.append(line)
+        lines.append(r"\midrule")
+        per_model_lines = {}
+        arena_rankings = {}
+
+        # Create a mapping of model -> arena -> elo and also arena -> list of (model, elo) for ranking
+        for profile in self._player_profiles.values():
+            if profile.model not in per_model_lines:
+                per_model_lines[profile.model] = {}
+            if profile.arena not in arena_rankings:
+                arena_rankings[profile.arena] = []
+            per_model_lines[profile.model][profile.arena] = round(profile.rating, 1)
+            arena_rankings[profile.arena].append((profile.model, profile.rating))
+
+        # Sort each arena ranking by ELO descending
+        for arena in arena_rankings:
+            arena_rankings[arena].sort(key=lambda x: x[1], reverse=True)
+            for rank, (model, _) in enumerate(arena_rankings[arena], 1):
+                per_model_lines[model][arena] = (
+                    f"{per_model_lines[model][arena]}" + f"\\textsuperscript{{\\textcolor{{gray}}{{{rank}}}}}"
+                )
+                if rank == 1:
+                    # Make it bold
+                    per_model_lines[model][arena] = f"\\textbf{{{per_model_lines[model][arena]}}}"
+
+        # Now print the table
+        overall_ranks = sorted(model_to_avg_elo.items(), key=lambda x: x[1], reverse=True)
+        model_rank = {model: rank + 1 for rank, (model, _) in enumerate(overall_ranks)}
+        for model in sorted(per_model_lines.keys()):
+            m = model.split("/", 1)[-1]
+            rank = f"\\textsuperscript{{\\textcolor{{gray}}{{{model_rank[model]}}}}}"
+            overall_elo = f"{model_to_avg_elo[model]:.1f}{rank}"
+            if model_rank[model] == 1:
+                overall_elo = f"\\textbf{{{overall_elo}}}"
+            line = (
+                f"\\small{{{MODEL_TO_DISPLAY_NAME[m]}}} & "
+                + " & ".join(str(per_model_lines[model].get(arena, "-")) for arena in arenas)
+                + " & "
+                + overall_elo
+                + " \\\\"
+            )
+            lines.append(line)
+        print("\n".join(lines))
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Calculate weighted ELO ratings with configurable weighting functions")

codeclash/agents/player.py

@@ -41,9 +41,11 @@
 from matplotlib import pyplot as plt
 from tqdm.auto import tqdm
 
+from codeclash.analysis.viz.utils import ASSETS_DIR, MODEL_TO_COLOR, MODEL_TO_DISPLAY_NAME
 from codeclash.constants import LOCAL_LOG_DIR
 
-OUTPUT_FILE = "cdf_command_diversity.png"
+OUTPUT_FILE = ASSETS_DIR / "cdf_command_diversity.png"
+DATA_CACHE = ASSETS_DIR / "cdf_command_diversity.json"
 
 
 def shannon_entropy(command_list):
@@ -123,42 +125,49 @@ def main():
     """
     model_to_diversity = {}
 
-    # Find all tournament directories by looking for metadata.json files
-    tournaments = [x.parent for x in LOCAL_LOG_DIR.rglob("metadata.json")]
-    for game_log_folder in tqdm(tournaments):
-        # Load tournament metadata to get player-to-model mapping
-        with open(game_log_folder / "metadata.json") as f:
-            metadata = json.load(f)
-        try:
-            # Extract mapping from player name to model name
-            p2m = {x["name"]: x["config"]["model"]["model_name"].strip("@") for x in metadata["config"]["players"]}
-            # Initialize diversity list for each model we encounter
-            for model in p2m.values():
-                if model not in model_to_diversity:
-                    model_to_diversity[model] = []
-        except KeyError:
-            # Skip tournaments with malformed metadata
-            continue
-
-        # Process each player's trajectory files
-        for name in p2m.keys():
-            traj_files = (game_log_folder / "players" / name).rglob("*.traj.json")
-            for traj_file in traj_files:
-                try:
-                    with open(traj_file) as f:
-                        traj = json.load(f)
-
-                    # Extract commands and calculate diversity for this session
-                    commands = extract_commands_from_trajectory(traj)
-                    if commands:  # Only calculate entropy if there are commands
-                        diversity = shannon_entropy(commands)
-                        model_to_diversity[p2m[name]].append(diversity)
-                except (json.JSONDecodeError, KeyError):
-                    # Skip malformed trajectory files
-                    continue
-
-    # Remove models with no valid data
-    model_to_diversity = {k: v for k, v in model_to_diversity.items() if v}
+    if not DATA_CACHE.exists():
+        # Find all tournament directories by looking for metadata.json files
+        tournaments = [x.parent for x in LOCAL_LOG_DIR.rglob("metadata.json")]
+        for game_log_folder in tqdm(tournaments):
+            # Load tournament metadata to get player-to-model mapping
+            with open(game_log_folder / "metadata.json") as f:
+                metadata = json.load(f)
+            try:
+                # Extract mapping from player name to model name
+                p2m = {x["name"]: x["config"]["model"]["model_name"].strip("@") for x in metadata["config"]["players"]}
+                # Initialize diversity list for each model we encounter
+                for model in p2m.values():
+                    if model not in model_to_diversity:
+                        model_to_diversity[model] = []
+            except KeyError:
+                # Skip tournaments with malformed metadata
+                continue
+
+            # Process each player's trajectory files
+            for name in p2m.keys():
+                traj_files = (game_log_folder / "players" / name).rglob("*.traj.json")
+                for traj_file in traj_files:
+                    try:
+                        with open(traj_file) as f:
+                            traj = json.load(f)
+
+                        # Extract commands and calculate diversity for this session
+                        commands = extract_commands_from_trajectory(traj)
+                        if commands:  # Only calculate entropy if there are commands
+                            diversity = shannon_entropy(commands)
+                            model_to_diversity[p2m[name]].append(diversity)
+                    except (json.JSONDecodeError, KeyError):
+                        # Skip malformed trajectory files
+                        continue
+
+        # Remove models with no valid data
+        model_to_diversity = {k: v for k, v in model_to_diversity.items() if v}
+
+        with open(DATA_CACHE, "w") as f:
+            json.dump(model_to_diversity, f, indent=2)
+
+    with open(DATA_CACHE) as f:
+        model_to_diversity = json.load(f)
 
     # Print summary statistics for each model
     print("Command Diversity Summary:")
@@ -170,18 +179,19 @@ def main():
         )
 
     # Generate CDF plot comparing all models
-    plt.figure(figsize=(12, 8))
-    colors = plt.cm.tab10(range(len(model_to_diversity)))
+    plt.figure(figsize=(8, 8))
 
     for i, (model, diversities) in enumerate(model_to_diversity.items()):
         # Sort diversity values and create cumulative probability values
         sorted_diversities = sorted(diversities)
         yvals = [i / len(sorted_diversities) for i in range(len(sorted_diversities))]
         # Plot as step function (standard for CDFs)
-        plt.step(sorted_diversities, yvals, label=model, where="post", color=colors[i])
+        plt.step(
+            sorted_diversities, yvals, label=MODEL_TO_DISPLAY_NAME[model], where="post", color=MODEL_TO_COLOR[model]
+        )
 
     plt.xlabel("Command Diversity (Shannon Entropy)")
-    plt.ylabel("Cumulative Probability")
+    # plt.ylabel("Cumulative Probability")
     plt.title("CDF of Command Diversity by Model\n(Higher entropy = more diverse command usage)")
     plt.legend(bbox_to_anchor=(1.05, 1), loc="upper left")  # Legend outside plot area
     plt.grid(True, alpha=0.3)