Merge branch 'main' of github.com:emagedoc/CodeClash

Author: john-b-yang

codeclash/analysis/metrics/elo2.py

@@ -871,7 +871,7 @@ def _create_elo_violin_plot(
         self._save_plot(output_dir, f"{self.game}_elo_violin_{self.bootstrap_type}")
         plt.close()
 
-    def run(self) -> None:
+    def run(self) -> dict:
         game = self.game
         assert game in self.builder.win_matrix, f"Game '{game}' not found in win matrix"
 
@@ -988,6 +988,15 @@ def run(self) -> None:
             self._create_rank_matrix_plot(players, rank_samples, bootstrap_dir)
             self._create_elo_violin_plot(players, elo_samples, baseline_elos, bootstrap_dir)
 
+        return {
+            "kendall_tau": mean_tau,
+            "spearman_rho": mean_rho,
+            "footrule": mean_foot,
+            "top1_consistency": top1_consistency,
+            "pairwise_agreement": pairwise_agreement,
+            "topk_overlap": {k: float(np.mean(topk_overlap[k])) for k in topks},
+        }
+
 
 class EloVsMaxRounds:
     def __init__(
@@ -1375,6 +1384,123 @@ def write_website_results(results: dict[str, dict], output_dir: Path) -> None:
     logger.info(f"Saved leaderboard JSON: {output_file}")
 
 
+def write_bootstrap_metrics_table(bootstrap_results: dict[str, dict], output_dir: Path, game: str = "ALL") -> None:
+    """Write LaTeX table comparing bootstrap metrics between methods.
+
+    Args:
+        bootstrap_results: Dictionary mapping bootstrap type to metrics dict
+        output_dir: Directory to save the LaTeX file
+        game: Game name for the table caption
+    """
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_file = output_dir / "bootstrap_metrics.tex"
+
+    lines = []
+    lines.append(r"\begin{tabular}{lcc}")
+    lines.append(r"\toprule")
+    lines.append(r"Metric & Nonparametric & Parametric \\")
+    lines.append(r"\midrule")
+
+    if "nonparametric" in bootstrap_results and "parametric" in bootstrap_results:
+        nonparam = bootstrap_results["nonparametric"]
+        param = bootstrap_results["parametric"]
+
+        metrics = [
+            ("Kendall's $\\tau$", "kendall_tau"),
+            ("Spearman's $\\rho$", "spearman_rho"),
+            ("Footrule (normalized)", "footrule"),
+            ("Top-1 consistency", "top1_consistency"),
+            ("Pairwise order agreement", "pairwise_agreement"),
+        ]
+
+        for display_name, key in metrics:
+            nonparam_val = nonparam.get(key, float("nan"))
+            param_val = param.get(key, float("nan"))
+            lines.append(rf"{display_name} & {nonparam_val:.3f} & {param_val:.3f} \\")
+
+    lines.append(r"\bottomrule")
+    lines.append(r"\end{tabular}")
+
+    output_file.write_text("\n".join(lines) + "\n")
+    logger.info(f"Saved bootstrap metrics table: {output_file}")
+
+
+def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
+    """Write LaTeX table with plain ELO scores and uncertainties (no bar charts).
+
+    Args:
+        results: Dictionary mapping game name to fit results
+        output_dir: Directory to save the LaTeX file
+    """
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_file = output_dir / "elo_table_plain.tex"
+
+    single_arena_games = ["BattleSnake", "CoreWar", "Halite", "HuskyBench", "RoboCode", "RobotRumble"]
+    games_in_table = [g for g in single_arena_games if g in results]
+
+    if "ALL" not in results:
+        logger.warning("No 'ALL' game found in results, skipping LaTeX table generation")
+        return
+
+    all_result = results["ALL"]
+    all_players = all_result["players"]
+    all_strengths = all_result["strengths"]
+    all_elos = {p: BradleyTerryFitter.bt_to_elo(s) for p, s in zip(all_players, all_strengths)}
+    sorted_players = sorted(all_elos.items(), key=lambda x: x[1], reverse=True)
+
+    has_uncertainties = "elo_std" in all_result
+
+    lines = []
+    lines.append(r"\begin{table}[t]")
+    lines.append(r"\centering")
+    lines.append(r"\caption{ELO ratings" + (" with uncertainties" if has_uncertainties else "") + r"}")
+    lines.append(r"\label{tab:elo_ratings}")
+    lines.append(r"\begin{tabular}{l" + "r" * len(games_in_table) + "r}")
+    lines.append(r"\toprule")
+
+    display_names = [g.replace("HuskyBench", "Poker") for g in games_in_table]
+    header_parts = ["Model"] + display_names + ["All"]
+    lines.append(" & ".join(header_parts) + r" \\")
+    lines.append(r"\midrule")
+
+    for player, all_elo in sorted_players:
+        display_name = MODEL_TO_DISPLAY_NAME.get(player, player)
+        row_parts = [display_name.replace("_", r"\_")]
+
+        for game_name in games_in_table:
+            if game_name in results:
+                game_result = results[game_name]
+                if player in game_result["players"]:
+                    idx = game_result["players"].index(player)
+                    strength = game_result["strengths"][idx]
+                    elo = BradleyTerryFitter.bt_to_elo(strength)
+                    if has_uncertainties and "elo_std" in game_result:
+                        sigma = game_result["elo_std"][idx]
+                        row_parts.append(rf"${int(elo)} \pm {int(sigma)}$")
+                    else:
+                        row_parts.append(str(int(elo)))
+                else:
+                    row_parts.append("--")
+            else:
+                row_parts.append("--")
+
+        if has_uncertainties:
+            all_idx = all_result["players"].index(player)
+            all_sigma = all_result["elo_std"][all_idx]
+            row_parts.append(rf"$\mathbf{{{int(all_elo)} \pm {int(all_sigma)}}}$")
+        else:
+            row_parts.append(rf"$\mathbf{{{int(all_elo)}}}$")
+
+        lines.append(" & ".join(row_parts) + r" \\")
+
+    lines.append(r"\bottomrule")
+    lines.append(r"\end{tabular}")
+    lines.append(r"\end{table}")
+
+    output_file.write_text("\n".join(lines) + "\n")
+    logger.info(f"Saved plain LaTeX table: {output_file}")
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Build win matrix and fit Bradley-Terry model")
     parser.add_argument("-d", "--log_dir", type=Path, default=LOCAL_LOG_DIR)
@@ -1448,10 +1574,12 @@ def write_website_results(results: dict[str, dict], output_dir: Path) -> None:
     plotter.create_elo_plots(args.output_dir)
     write_latex_table(results, args.output_dir)
     write_website_results(results, args.output_dir)
+    write_latex_table_plain(results, args.output_dir)
 
     if uncertainties_supported:
+        bootstrap_results = {}
         for bootstrap_type in ["nonparametric", "parametric"]:
-            BootStrapRankStability(
+            bootstrap_results[bootstrap_type] = BootStrapRankStability(
                 builder,
                 n_bootstrap=1000,
                 game="ALL",
@@ -1460,6 +1588,7 @@ def write_website_results(results: dict[str, dict], output_dir: Path) -> None:
                 bootstrap_type=bootstrap_type,
                 output_dir=args.output_dir,
             ).run()
+        write_bootstrap_metrics_table(bootstrap_results, args.output_dir, game="ALL")
 
     logger.info("Running EloVsMaxRounds analysis")
     EloVsMaxRounds(

codeclash/analysis/viz/round_score_distribution.py

@@ -5,9 +5,10 @@
 
 import matplotlib.pyplot as plt
 import numpy as np
+from matplotlib.ticker import AutoMinorLocator
 from tqdm import tqdm
 
-from codeclash.analysis.viz.utils import MODEL_TO_DISPLAY_NAME
+from codeclash.analysis.viz.utils import ASSETS_DIR, FONT_BOLD, MODEL_TO_DISPLAY_NAME
 from codeclash.constants import LOCAL_LOG_DIR
 from codeclash.games import BattleCodeGame, DummyGame
 from codeclash.utils.log import get_logger
@@ -88,79 +89,79 @@ def plot_stratified(
     data_by_category: dict[str, list[float]], output_path: Path, *, title: str, by_model: bool = False
 ) -> None:
     """Plot normalized scores stratified by category (game or model)."""
-    all_scores = [s for scores in data_by_category.values() for s in scores]
-
     # Determine category order
     if by_model:
-        category_names = sorted(data_by_category.keys(), key=lambda m: MODEL_TO_DISPLAY_NAME.get(m, m))
+        # Sort by full model name (including prefix), then strip prefix for display
+        category_names = sorted(data_by_category.keys())
     else:
         category_names = sorted(data_by_category.keys())
 
-    # Create subplots: 1 for all + 1 per category
-    n_plots = 1 + len(category_names)
-    n_cols = 2
+    # Create subplots: 3 columns, no "All" plot
+    n_plots = len(category_names)
+    n_cols = 3
     n_rows = (n_plots + n_cols - 1) // n_cols
 
-    fig, axes = plt.subplots(n_rows, n_cols, figsize=(14, 5 * n_rows))
+    # Use 4x4 for all plots
+    fig, axes = plt.subplots(n_rows, n_cols, figsize=(4 * n_cols, 4 * n_rows))
     axes = axes.flatten() if n_plots > 1 else [axes]
 
     bins = np.linspace(0, 1, 51)
 
-    # Plot all combined
-    ax = axes[0]
-    ax.hist(all_scores, bins=bins, edgecolor="black", alpha=0.7)
-    ax.set_xlabel("Normalized Score", fontsize=10, fontweight="bold")
-    ax.set_ylabel("Frequency", fontsize=10, fontweight="bold")
-    ax.set_title("All Models" if by_model else "All Games", fontsize=12, fontweight="bold")
-    ax.set_xlim(0, 1)
-    ax.grid(True, alpha=0.3, axis="y")
-
-    mean_score = np.mean(all_scores)
-    median_score = np.median(all_scores)
-    stats_text = f"Mean: {mean_score:.3f}\nMedian: {median_score:.3f}\nN: {len(all_scores)}"
-    ax.text(
-        0.98,
-        0.98,
-        stats_text,
-        transform=ax.transAxes,
-        verticalalignment="top",
-        horizontalalignment="right",
-        fontsize=9,
-        bbox=dict(boxstyle="round", facecolor="wheat", alpha=0.8),
-    )
-
     # Plot per category
-    for idx, category_name in enumerate(category_names, start=1):
+    for idx, category_name in enumerate(category_names):
         ax = axes[idx]
         scores = data_by_category[category_name]
 
-        ax.hist(scores, bins=bins, edgecolor="black", alpha=0.7)
-        ax.set_xlabel("Normalized Score", fontsize=10, fontweight="bold")
-        ax.set_ylabel("Frequency", fontsize=10, fontweight="bold")
-        display_name = MODEL_TO_DISPLAY_NAME.get(category_name, category_name) if by_model else category_name
-        ax.set_title(display_name, fontsize=12, fontweight="bold")
+        ax.hist(scores, bins=bins, edgecolor="black", alpha=0.7, density=True)
+        ax.set_xlabel("Normalized Score", fontproperties=FONT_BOLD, fontsize=12)
+        ax.set_ylabel("Density", fontproperties=FONT_BOLD, fontsize=12)
+
+        # Set tick labels to also use bold font
+        for label in ax.get_xticklabels() + ax.get_yticklabels():
+            label.set_fontproperties(FONT_BOLD)
+
+        if by_model:
+            display_name = MODEL_TO_DISPLAY_NAME.get(category_name, category_name)
+        else:
+            display_name = category_name.replace("Halite", "Poker")
+        ax.set_title(display_name, fontproperties=FONT_BOLD, fontsize=14)
         ax.set_xlim(0, 1)
-        ax.grid(True, alpha=0.3, axis="y")
-
-        mean_score = np.mean(scores)
-        median_score = np.median(scores)
-        stats_text = f"Mean: {mean_score:.3f}\nMedian: {median_score:.3f}\nN: {len(scores)}"
-        ax.text(
-            0.98,
-            0.98,
-            stats_text,
-            transform=ax.transAxes,
-            verticalalignment="top",
-            horizontalalignment="right",
-            fontsize=9,
-            bbox=dict(boxstyle="round", facecolor="wheat", alpha=0.8),
-        )
+
+        # Add minor ticks on both axes
+        ax.xaxis.set_minor_locator(AutoMinorLocator())
+        ax.yaxis.set_minor_locator(AutoMinorLocator())
+
+        # Show ticks on all sides
+        ax.tick_params(top=True, right=True, which="both")
+
+        # Add gridlines for both x and y
+        ax.grid(True, alpha=0.3, axis="both", which="major")
+
+        # Add dashed black line at 50%
+        ax.axvline(0.5, color="black", linestyle="--", linewidth=1.5, alpha=0.7)
+
+        # Add stats text only for by_model plots
+        if by_model:
+            mean_score = np.mean(scores)
+            median_score = np.median(scores)
+            stats_text = f"Mean: {mean_score:.3f}    Median: {median_score:.3f}"
+            ax.text(
+                0.5,
+                0.93,
+                stats_text,
+                transform=ax.transAxes,
+                verticalalignment="top",
+                horizontalalignment="center",
+                fontproperties=FONT_BOLD,
+                fontsize=10,
+                bbox=dict(boxstyle="square", facecolor="white", edgecolor="black", linewidth=1),
+            )
 
     # Hide unused subplots
     for idx in range(n_plots, len(axes)):
         axes[idx].set_visible(False)
 
-    plt.suptitle(title, fontsize=14, fontweight="bold")
+    plt.suptitle(title, fontproperties=FONT_BOLD, fontsize=16)
     plt.tight_layout(rect=[0, 0, 1, 0.97])
 
     plt.savefig(output_path, dpi=300, bbox_inches="tight")
@@ -169,7 +170,7 @@ def plot_stratified(
     plt.close()
 
 
-def main(log_dir: Path, output_by_game: Path, output_by_model: Path) -> None:
+def main(log_dir: Path) -> None:
     """Calculate normalized scores and plot histograms stratified by game and by model."""
     logger.info(f"Processing tournaments from {log_dir}")
 
@@ -205,6 +206,7 @@ def main(log_dir: Path, output_by_game: Path, output_by_model: Path) -> None:
     )
 
     # Plot by game
+    output_by_game = ASSETS_DIR / "round_score_distribution_by_game.pdf"
     plot_stratified(
         scores_by_game,
         output_by_game,
@@ -213,6 +215,7 @@ def main(log_dir: Path, output_by_game: Path, output_by_model: Path) -> None:
     )
 
     # Plot by model
+    output_by_model = ASSETS_DIR / "round_score_distribution_by_model.pdf"
     plot_stratified(
         scores_by_model,
         output_by_model,
@@ -224,18 +227,6 @@ def main(log_dir: Path, output_by_game: Path, output_by_model: Path) -> None:
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Plot distribution of normalized player scores (valid rounds only)")
     parser.add_argument("-d", "--log_dir", type=Path, default=LOCAL_LOG_DIR, help="Path to log directory")
-    parser.add_argument(
-        "--output_by_game",
-        type=Path,
-        default=Path("assets/round_score_distribution_by_game.pdf"),
-        help="Output path for by-game plot",
-    )
-    parser.add_argument(
-        "--output_by_model",
-        type=Path,
-        default=Path("assets/round_score_distribution_by_model.pdf"),
-        help="Output path for by-model plot",
-    )
     args = parser.parse_args()
 
-    main(args.log_dir, args.output_by_game, args.output_by_model)
+    main(args.log_dir)