Add p-value calculation

Author: klieret

codeclash/ratings/significance.py

@@ -0,0 +1,39 @@
+from scipy.stats import binomtest
+
+from codeclash.constants import RESULT_TIE
+
+
+def calculate_p_value(scores: dict[str, int]) -> float:
+    """Calculate the p-value for the statistical significance of the winner.
+
+    Input: scores as a dictionary of player names and number of games won.
+    Special case: 'Tie' (constants.RESULT_TIE) is a tie between all players.
+
+    Ties (RESULT_TIE) are excluded by conditioning on decisive games.
+
+    Uses an exact one-sided binomial test for the top player's share vs 1/K,
+    Bonferroni-corrected for choosing the winner post hoc among K players.
+    Returns 1.0 if there is no unique winner or no decisive games.
+    """
+    player_wins = {p: c for p, c in scores.items() if p != RESULT_TIE}
+    decisive_games = sum(player_wins.values())
+    n_players = len(player_wins)
+    assert n_players > 1, "At least two players are required to calculate significance"
+    if not player_wins or not decisive_games:
+        # No winner
+        return 1.0
+    top_player_wins = max(player_wins.values())
+    if sum(c == top_player_wins for c in player_wins.values()) != 1:
+        # Multiple players have the same number of wins
+        # Definitely not significant
+        return 1.0
+    # Null-hypothesis: The top player wins 1/n_players of the games
+    p0 = 1.0 / n_players
+    p_one = binomtest(top_player_wins, decisive_games, p=p0, alternative="greater").pvalue
+    # Bonferonni correction: Imagine a game with 100 players. And let's assume that there's one
+    # player that wins 10 games, and everyone else 0 or 1. The p-value would probably look pretty convincing
+    # However, since we have so many players, it's actually not unlikely that _some_ player will win 10 games
+    # by chance. So essentially by selecting the winner post-hoc in a multiple-comparison setting, we've inflated
+    # the significance. Bonferonni correction is a simple way to account for this.
+    p_bonferonni = p_one * n_players
+    return min(1.0, p_bonferonni)

codeclash/viewer/app.py

@@ -12,6 +12,7 @@
 
 from flask import Flask, jsonify, redirect, render_template, request, url_for
 
+from codeclash.ratings.significance import calculate_p_value
 from codeclash.tournaments.utils.git_utils import filter_git_diff, split_git_diff_by_files
 
 # Global variable to store the directory to search for logs
@@ -206,7 +207,7 @@ def process_round_results(round_results: dict[str, Any] | None) -> dict[str, Any
     processed["scores"] = scores
     processed["sorted_scores"] = list(scores.items())
 
-    # Calculate winner percentage
+    # Calculate winner percentage and p-value
     winner = round_results.get("winner")
     if winner and scores:
         total_games = sum(scores.values())
@@ -218,10 +219,16 @@ def process_round_results(round_results: dict[str, Any] | None) -> dict[str, Any
                 processed["winner_percentage"] = win_percentage
             else:
                 processed["winner_percentage"] = None  # No percentage for ties
+
+            # Calculate p-value for statistical significance
+            p_value = calculate_p_value(scores)
+            processed["p_value"] = round(p_value, 2)
         else:
             processed["winner_percentage"] = None
+            processed["p_value"] = None
     else:
         processed["winner_percentage"] = None
+        processed["p_value"] = None
 
     return processed
 

codeclash/viewer/static/css/style.css

@@ -299,6 +299,14 @@ body {
   font-weight: 500;
 }
 
+.p-value {
+  display: inline-block;
+  margin-left: 0.5rem;
+  font-size: 0.8rem;
+  color: var(--text-secondary);
+  font-weight: 500;
+}
+
 .score-breakdown {
   display: flex;
   flex-wrap: wrap;

codeclash/viewer/templates/index.html

@@ -129,6 +129,9 @@ <h2>📊 Overview</h2>
                                         {% if round_data.results.winner_percentage %}
                                             <span class="winner-percentage">({{ round_data.results.winner_percentage }}%)</span>
                                         {% endif %}
+                                        {% if round_data.results.p_value is not none %}
+                                            <span class="p-value">(p={{ "%.2f"|format(round_data.results.p_value) }})</span>
+                                        {% endif %}
                                     {% else %}
                                         <span class="no-result">-</span>
                                     {% endif %}
@@ -462,13 +465,15 @@ <h3>
                                 {%- set winner_wins = round_data.results.scores.get(winner, 0) -%}
                                 {%- set ties = round_data.results.scores.get('Tie', 0) -%}
                                 {%- set win_percentage = ((winner_wins + 0.5 * ties) / total_games * 100) | round(1) -%}
-                                - Winner: {{ winner }} with {{ win_percentage }}% (
+                                - Winner: {{ winner }} with {{ win_percentage }}%
+                                {%- if round_data.results.p_value is not none %} (p={{ "%.2f"|format(round_data.results.p_value) }}){% endif %} (
                                 {%- for player, score in round_data.results.scores.items() -%}
                                     {{- score -}}
                                     {%- if not loop.last -%}-{%- endif -%}
                                 {%- endfor -%})
                             {%- else -%}
-                                - Winner: Tie (
+                                - Winner: Tie
+                                {%- if round_data.results.p_value is not none %} (p={{ "%.2f"|format(round_data.results.p_value) }}){% endif %} (
                                 {%- for player, score in round_data.results.scores.items() -%}
                                     {{- score -}}
                                     {%- if not loop.last -%}-{%- endif -%}