Merge branch 'main' of github.com:emagedoc/CodeClash

Author: yolo h8cker 93

codeclash/ratings/win_rate.py

@@ -5,12 +5,13 @@
 
 from tqdm import tqdm
 
-from codeclash.constants import DIR_LOGS, FILE_RESULTS, RESULT_TIE
+from codeclash.constants import DIR_LOGS, RESULT_TIE
 
 
 @dataclass
 class PlayerGameProfile:
     player_id: str
+    model_name: str
     game_id: str
     wins: int = 0
     count: int = 0
@@ -24,59 +25,65 @@ def main(log_dir: Path):
     # Assuming directory structure is:
     # logs/<user_id>/<game_id>
     # - players/
-    # - rounds/
     # - game.log
     # - metadata.json
-    player_profiles = {}
+    model_profiles = {}
     for user_folder in log_dir.iterdir():
         print(f"Processing games under user `{user_folder.name}`")
         for game_log_folder in tqdm(list(user_folder.iterdir())):
             if not game_log_folder.is_dir():
                 continue
             game_id = game_log_folder.name.split(".")[1]
             player_ids = [x.name for x in (game_log_folder / "players").iterdir() if x.is_dir()]
-            num_rounds = len(list((game_log_folder / "rounds").iterdir()))
+            metadata = json.load(open(game_log_folder / "metadata.json"))
+            player_to_model = {x["name"]: x["config"]["model"]["model_name"] for x in metadata["config"]["players"]}
+            print(player_to_model)
+            num_rounds = len(metadata["round_stats"])
 
-            for player in player_ids:
-                if f"{game_id}.{player}" in player_profiles:
-                    player_profiles[f"{game_id}.{player}"].count += num_rounds
+            # Only count each unique model once per game
+            unique_models = {player_to_model[player] for player in player_ids}
+            for model_name in unique_models:
+                k = f"{game_id}.{model_name}"
+                if k in model_profiles:
+                    model_profiles[k].count += num_rounds
                 else:
-                    player_profiles[f"{game_id}.{player}"] = PlayerGameProfile(
-                        player_id=player, game_id=game_id, count=num_rounds
+                    # Use the first player_id that matches this model_name for display
+                    player_id = next(pid for pid in player_ids if player_to_model[pid] == model_name)
+                    model_profiles[k] = PlayerGameProfile(
+                        player_id=player_id, model_name=model_name, game_id=game_id, count=num_rounds
                     )
 
-            for round_folder in (game_log_folder / "rounds").iterdir():
-                if round_folder.name == "0":
+            for round, details in metadata["round_stats"].items():
+                if round == "0":
                     # Skip initial round
                     continue
-                if not (round_folder / FILE_RESULTS).exists():
-                    continue
-                round_results = json.load(open(round_folder / FILE_RESULTS))
-                winner = round_results.get("winner")
+                winner = details["winner"]
                 if winner != RESULT_TIE:
-                    player_profiles[f"{game_id}.{winner}"].wins += 1
+                    model_profiles[f"{game_id}.{player_to_model[winner]}"].wins += 1
 
     print("Player profiles:")
-    for profile in player_profiles.values():
+    for profile in model_profiles.values():
         print(
-            f" - {profile.player_id} (Game: {profile.game_id}) - Win Rate: {profile.win_rate:.2%} ({profile.wins}/{profile.count})"
+            f" - {profile.model_name} (Game: {profile.game_id}) - Win Rate: {profile.win_rate:.2%} ({profile.wins}/{profile.count})"
         )
 
     # Player-specific (game-agnostic) win rates (micro average)
     total_wins = {}
     total_games = {}
-    for profile in player_profiles.values():
-        pid = profile.player_id
-        total_wins[pid] = total_wins.get(pid, 0) + profile.wins
-        total_games[pid] = total_games.get(pid, 0) + profile.count
+    model_names = {}
+    for profile in model_profiles.values():
+        mid = profile.model_name
+        total_wins[mid] = total_wins.get(mid, 0) + profile.wins
+        total_games[mid] = total_games.get(mid, 0) + profile.count
+        model_names[mid] = profile.model_name
 
     print("\nPlayer-specific win rates (game-agnostic, micro average):")
-    for pid in total_wins:
-        if total_games[pid] > 0:
-            win_rate = total_wins[pid] / total_games[pid]
+    for mid in total_wins:
+        if total_games[mid] > 0:
+            win_rate = total_wins[mid] / total_games[mid]
         else:
             win_rate = 0.0
-        print(f" - {pid}: Win Rate {win_rate:.2%} ({total_wins[pid]}/{total_games[pid]})")
+        print(f" - {model_names[mid]}: Win Rate {win_rate:.2%} ({total_wins[mid]}/{total_games[mid]})")
 
 
 if __name__ == "__main__":

codeclash/viewer/app.py

@@ -612,6 +612,41 @@ def analyze_line_counts(self) -> dict[str, Any]:
 
         return {"all_files": all_files_list, "line_counts_by_round": line_counts_by_round}
 
+    def analyze_sim_wins_per_round(self) -> dict[str, Any]:
+        """Analyze number of simulations won per round for each competitor from round_stats in metadata.json"""
+        metadata_file = self.log_dir / "metadata.json"
+        if not metadata_file.exists():
+            return {"players": [], "rounds": [], "wins_by_player": {}}
+
+        try:
+            metadata = json.loads(metadata_file.read_text())
+            round_stats = metadata.get("round_stats", {})
+            # Collect all player names from all rounds
+            player_names = set()
+            for round_data in round_stats.values():
+                scores = round_data.get("scores", {})
+                player_names.update([k for k in scores.keys() if k != "Tie"])
+            player_names = sorted(player_names)
+
+            # Collect all round numbers (sorted)
+            round_nums = sorted([int(k) for k in round_stats.keys()])
+
+            # Build wins_by_player: {player: [wins_per_round]}
+            wins_by_player = {p: [] for p in player_names}
+            for round_num in round_nums:
+                round_data = round_stats.get(str(round_num), {})
+                scores = round_data.get("scores", {})
+                for p in player_names:
+                    wins_by_player[p].append(scores.get(p, 0))
+
+            return {
+                "players": player_names,
+                "rounds": round_nums,
+                "wins_by_player": wins_by_player,
+            }
+        except Exception:
+            return {"players": [], "rounds": [], "wins_by_player": {}}
+
     def load_matrix_analysis(self) -> dict[str, Any] | None:
         """Load and process matrix.json if it exists"""
         matrix_file = self.log_dir / "matrix.json"
@@ -790,6 +825,7 @@ def index():
 
     # Get analysis data
     analysis_data = parser.analyze_line_counts()
+    sim_wins_data = parser.analyze_sim_wins_per_round()
 
     # Get matrix analysis data
     matrix_data = parser.load_matrix_analysis()
@@ -804,6 +840,7 @@ def index():
         metadata=metadata,
         trajectories_by_round=trajectories_by_round,
         analysis_data=analysis_data,
+        sim_wins_data=sim_wins_data,
         matrix_data=matrix_data,
         is_static=STATIC_MODE,
     )

codeclash/viewer/static/js/analysis.js

@@ -184,6 +184,97 @@ function prepareChartData(fileName) {
 }
 
 // Initialize when DOM is loaded
+// --- Sim Wins Per Round Chart ---
+let simWinsChart = null;
+let simWinsData = null;
+
+function initializeSimWinsChart() {
+  const simWinsDataElement = document.getElementById("sim-wins-data");
+  if (!simWinsDataElement) return;
+  try {
+    simWinsData = JSON.parse(simWinsDataElement.textContent);
+    if (
+      simWinsData &&
+      simWinsData.players &&
+      simWinsData.players.length > 0 &&
+      simWinsData.rounds &&
+      simWinsData.rounds.length > 0
+    ) {
+      createSimWinsChart();
+    }
+  } catch (error) {
+    console.error("Error parsing sim wins data:", error);
+  }
+}
+
+function createSimWinsChart() {
+  const canvas = document.getElementById("sim-wins-chart");
+  if (!canvas || !simWinsData) return;
+  const ctx = canvas.getContext("2d");
+  if (simWinsChart) simWinsChart.destroy();
+
+  // Prepare datasets
+  const colors = [
+    "#FF6384",
+    "#36A2EB",
+    "#FFCE56",
+    "#4BC0C0",
+    "#9966FF",
+    "#FF9F40",
+    "#C9CBCF",
+    "#4BC0C0",
+    "#FF6384",
+  ];
+  let colorIndex = 0;
+  const datasets = simWinsData.players.map((player) => {
+    const data = simWinsData.rounds.map((round, i) => ({ x: round, y: simWinsData.wins_by_player[player][i] }));
+    const color = colors[colorIndex % colors.length];
+    colorIndex++;
+    return {
+      label: player,
+      data: data,
+      borderColor: color,
+      backgroundColor: color + "20",
+      borderWidth: 2,
+      fill: false,
+      tension: 0.1,
+    };
+  });
+
+  simWinsChart = new Chart(ctx, {
+    type: "line",
+    data: { datasets },
+    options: {
+      responsive: true,
+      maintainAspectRatio: false,
+      plugins: {
+        title: {
+          display: true,
+          text: `Simulations Won Per Round`,
+        },
+        legend: {
+          display: true,
+          position: "top",
+        },
+      },
+      scales: {
+        x: {
+          title: { display: true, text: "Round" },
+          type: "linear",
+          position: "bottom",
+        },
+        y: {
+          title: { display: true, text: "Simulations Won" },
+          beginAtZero: true,
+        },
+      },
+      interaction: { intersect: false, mode: "index" },
+    },
+  });
+}
+
+// Initialize both analyses when DOM is loaded
 document.addEventListener("DOMContentLoaded", function () {
   initializeAnalysis();
+  initializeSimWinsChart();
 });

codeclash/viewer/templates/includes/analysis.html

@@ -2,6 +2,23 @@
 <section class="analysis-section">
     <h2><i class="bi bi-graph-up"></i> Analysis</h2>
 
+    <!-- Sim Wins Per Round Analysis -->
+    <details class="foldout">
+        <summary><i class="bi bi-bar-chart-line"></i> Sim Wins Per Round</summary>
+        <div class="log-content-container">
+            {% if sim_wins_data and sim_wins_data.players and sim_wins_data.rounds %}
+                <div class="sim-wins-analysis-container">
+                    <div class="chart-container">
+                        <canvas id="sim-wins-chart" width="800" height="400"></canvas>
+                    </div>
+                    <script type="application/json" id="sim-wins-data">{{ sim_wins_data | tojson }}</script>
+                </div>
+            {% else %}
+                <p>No simulation win data available. This analysis requires games with round stats.</p>
+            {% endif %}
+        </div>
+    </details>
+
     <!-- Line Counting Analysis -->
     <details class="foldout">
         <summary><i class="bi bi-graph-up-arrow"></i> Line Counting Analysis</summary>

configs/README.md

@@ -0,0 +1,78 @@
+# List of model configurations
+Scratchpad for recording how different models should be configured for PvP.
+
+### OpenAI Family
+
+GPT 5 Mini
+```yaml
+  - agent: mini
+    name: openai/gpt-5-mini
+    config:
+      agent: !include mini/default.yaml
+      model:
+        model_class: "portkey"
+        model_name: openai/gpt-5-mini
+```
+
+GPT 5 High Thinking
+```yaml
+  - agent: mini
+    name: openai/gpt-5
+    config:
+      agent: !include mini/default.yaml
+      model:
+        model_class: "portkey"
+        model_name: openai/gpt-5
+        model_kwargs:
+          drop_params: true
+          reasoning_effort: "high"
+          verbosity: "medium"
+```
+
+o3
+```yaml
+  - agent: mini
+    name: openai/o3
+    config:
+      agent: !include mini/default.yaml
+      model:
+        model_class: "portkey"
+        model_name: openai/o3
+```
+
+### Anthropic Family
+
+Claude Sonnet 4
+```yaml
+  - agent: mini
+    name: anthropic/claude-sonnet-4-20250514
+    config:
+      agent: !include mini/default.yaml
+      model:
+        model_class: "portkey"
+        model_name: anthropic/claude-sonnet-4-20250514
+```
+
+Claude Sonnet 3.7
+```yaml
+  - agent: mini
+    name: anthropic/claude-sonnet-3-7
+    config:
+      agent: !include mini/default.yaml
+      model:
+        model_class: "portkey"
+        model_name: anthropic/claude-sonnet-3-7
+```
+
+### Qwen
+
+Qwen 3 Coder
+```yaml
+  - agent: mini
+    name: qwen/qwen-3-coder
+    config:
+      agent: !include mini/default.yaml
+      model:
+        model_class: "portkey"
+        model_name: qwen/qwen-3-coder
+```

configs/pvp/huskybench.yaml

@@ -0,0 +1,31 @@
+tournament:
+  rounds: 25
+game:
+  name: HuskyBench
+  sims_per_round: 1000
+players:
+  - agent: mini
+    name: p1
+    config:
+      agent: !include mini/default.yaml
+      model:
+        model_name: openai/gpt-5-mini
+  - agent: mini
+    name: p2
+    config:
+      agent: !include mini/default.yaml
+      model:
+        model_name: "anthropic/claude-sonnet-4-20250514"
+        model_kwargs:
+          temperature: 0.0
+prompts:
+  game_description: |
+    You are a software developer ({{player_id}}) competing in a coding game for Poker.
+    In this game, you will write code to control a poker-playing bot, aiming to outsmart your opponents and win chips.
+    Victory comes from crafting clever strategies—bluffing, reading opponents, and managing your chip stack effectively.
+
+    The game is played in {{rounds}} rounds. For every round, you (and your competitor) edit program code that controls your bot. This is round {{round}}.
+    After you and your competitor finish editing your codebases, the game is run automatically.
+
+    Your task: improve the bot in `warriors/warrior.red`, located in {{working_dir}}.
+    {{working_dir}} is your codebase, which contains both your bot and supporting assets.