Skip to content

Commit 64bd5df

Browse files
john-b-yangjohn-b-yang
committed
Add win streak vizs
1 parent ea973ae commit 64bd5df

2 files changed

Lines changed: 266 additions & 0 deletions

File tree

‎codeclash/analysis/viz/box_plots_win_streaks.py‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import argparse
2+
import json
3+
from collections import defaultdict
4+
from pathlib import Path
5+
6+
import matplotlib.pyplot as plt
7+
import numpy as np
8+
from tqdm import tqdm
9+
10+
from codeclash.constants import LOCAL_LOG_DIR
11+
12+
13+
def main(log_dir: Path):
14+
win_streaks = defaultdict(list)
15+
16+
for metadata_file in tqdm(list(log_dir.rglob("metadata.json")), desc="Processing tournaments"):
17+
try:
18+
metadata = json.load(open(metadata_file))
19+
p2m = {x["name"]: x["config"]["model"]["model_name"].strip("@") for x in metadata["config"]["players"]}
20+
21+
if len(p2m) != 2:
22+
continue
23+
24+
# Skip tournaments where both players use the same model
25+
models_in_tournament = list(p2m.values())
26+
if len(set(models_in_tournament)) < 2:
27+
continue
28+
29+
round_stats = metadata.get("round_stats", {})
30+
31+
# Track consecutive wins for each model
32+
current_streaks = defaultdict(int)
33+
models = list(p2m.values())
34+
35+
# Process rounds in order (skip round 0)
36+
for round_id in sorted(round_stats.keys(), key=int):
37+
if round_id == "0":
38+
continue
39+
40+
round_data = round_stats[round_id]
41+
winner = round_data.get("winner")
42+
43+
if winner in p2m:
44+
winner_model = p2m[winner]
45+
46+
# Update streaks
47+
for model in models:
48+
if model == winner_model:
49+
current_streaks[model] += 1
50+
else:
51+
if current_streaks[model] > 0:
52+
win_streaks[model].append(current_streaks[model])
53+
current_streaks[model] = 0
54+
55+
# Record any remaining streaks at tournament end
56+
for model in models:
57+
if current_streaks[model] > 0:
58+
win_streaks[model].append(current_streaks[model])
59+
60+
except:
61+
continue
62+
63+
# Create box plot visualization
64+
models = sorted(win_streaks.keys())
65+
clean_names = [m.split("/")[-1] for m in models]
66+
67+
# Prepare data for box plot - filter out models with no streaks
68+
box_data = []
69+
valid_models = []
70+
valid_clean_names = []
71+
72+
for i, model in enumerate(models):
73+
streaks = win_streaks[model]
74+
if streaks: # Only include models that have streaks
75+
box_data.append(streaks)
76+
valid_models.append(model)
77+
valid_clean_names.append(clean_names[i])
78+
79+
if not box_data:
80+
print("No streak data found!")
81+
return
82+
83+
plt.figure(figsize=(12, 8))
84+
85+
# Create box plot
86+
bp = plt.boxplot(box_data, tick_labels=valid_clean_names, patch_artist=True, showmeans=False)
87+
88+
# Customize box plot appearance
89+
for patch in bp["boxes"]:
90+
patch.set_facecolor("#3498db")
91+
patch.set_alpha(0.7)
92+
93+
for whisker in bp["whiskers"]:
94+
whisker.set_color("#2c3e50")
95+
whisker.set_linewidth(2)
96+
97+
for cap in bp["caps"]:
98+
cap.set_color("#2c3e50")
99+
cap.set_linewidth(2)
100+
101+
# Hide the default median lines
102+
for median in bp["medians"]:
103+
median.set_visible(False)
104+
105+
# Add mean and median markers
106+
means = [np.mean(streaks) for streaks in box_data]
107+
medians = [np.median(streaks) for streaks in box_data]
108+
109+
plt.scatter(range(1, len(means) + 1), means, color="#f39c12", s=50, zorder=3, marker="D", label="Mean")
110+
plt.scatter(range(1, len(medians) + 1), medians, color="#e74c3c", s=50, zorder=3, marker="s", label="Median")
111+
112+
plt.xlabel("Model")
113+
plt.ylabel("Win Streak Length")
114+
plt.title("Win Streak Length Distribution by Model", fontweight="bold")
115+
plt.xticks(rotation=45, ha="right")
116+
plt.grid(True, alpha=0.3, axis="y")
117+
118+
# Create custom legend
119+
from matplotlib.lines import Line2D
120+
121+
legend_elements = [
122+
Line2D([0], [0], marker="s", color="w", markerfacecolor="#e74c3c", markersize=8, label="Median"),
123+
Line2D([0], [0], marker="D", color="w", markerfacecolor="#f39c12", markersize=8, label="Mean"),
124+
]
125+
plt.legend(handles=legend_elements)
126+
plt.tight_layout()
127+
plt.savefig("box_plots_win_streaks.png", dpi=300, bbox_inches="tight")
128+
print("Win streak box plots saved to box_plots_win_streaks.png")
129+
130+
131+
if __name__ == "__main__":
132+
parser = argparse.ArgumentParser(description="Analyze model win streak distributions with box plots")
133+
parser.add_argument("-d", "--log_dir", type=Path, default=LOCAL_LOG_DIR, help="Path to logs")
134+
args = parser.parse_args()
135+
main(args.log_dir)

‎codeclash/analysis/viz/heatmap_win_streak_distribution.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import argparse
2+
import json
3+
from collections import defaultdict
4+
from pathlib import Path
5+
6+
import matplotlib.pyplot as plt
7+
import numpy as np
8+
from tqdm import tqdm
9+
10+
from codeclash.constants import LOCAL_LOG_DIR
11+
12+
13+
def main(log_dir: Path):
14+
win_streaks = defaultdict(list)
15+
16+
for metadata_file in tqdm(list(log_dir.rglob("metadata.json")), desc="Processing tournaments"):
17+
try:
18+
metadata = json.load(open(metadata_file))
19+
p2m = {x["name"]: x["config"]["model"]["model_name"].strip("@") for x in metadata["config"]["players"]}
20+
21+
if len(p2m) != 2:
22+
continue
23+
24+
# Skip tournaments where both players use the same model
25+
models_in_tournament = list(p2m.values())
26+
if len(set(models_in_tournament)) < 2:
27+
continue
28+
29+
round_stats = metadata.get("round_stats", {})
30+
31+
# Track consecutive wins for each model
32+
current_streaks = defaultdict(int)
33+
models = list(p2m.values())
34+
35+
# Process rounds in order (skip round 0)
36+
for round_id in sorted(round_stats.keys(), key=int):
37+
if round_id == "0":
38+
continue
39+
40+
round_data = round_stats[round_id]
41+
winner = round_data.get("winner")
42+
43+
if winner in p2m:
44+
winner_model = p2m[winner]
45+
46+
# Update streaks
47+
for model in models:
48+
if model == winner_model:
49+
current_streaks[model] += 1
50+
else:
51+
if current_streaks[model] > 0:
52+
win_streaks[model].append(current_streaks[model])
53+
current_streaks[model] = 0
54+
55+
# Record any remaining streaks at tournament end
56+
for model in models:
57+
if current_streaks[model] > 0:
58+
win_streaks[model].append(current_streaks[model])
59+
except:
60+
continue
61+
62+
# Create heatmap visualization
63+
models = sorted(win_streaks.keys())
64+
clean_names = [m.split("/")[-1] for m in models]
65+
66+
max_streaks = []
67+
for model in models:
68+
streaks = win_streaks[model]
69+
max_streaks.append(max(streaks) if streaks else 0)
70+
71+
max_streak_overall = max(max_streaks) if max_streaks else 1
72+
# Limit display to reasonable maximum (tournament length)
73+
max_streak_overall = min(max_streak_overall, 15)
74+
streak_matrix = np.zeros((len(models), max_streak_overall))
75+
76+
for i, model in enumerate(models):
77+
streaks = win_streaks[model]
78+
for streak_len in streaks:
79+
if streak_len <= max_streak_overall:
80+
streak_matrix[i, streak_len - 1] += 1
81+
82+
# Normalize by total streaks for each model
83+
for i in range(len(models)):
84+
total = np.sum(streak_matrix[i, :])
85+
if total > 0:
86+
streak_matrix[i, :] = streak_matrix[i, :] / total * 100
87+
88+
plt.figure(figsize=(15, 8))
89+
im = plt.imshow(streak_matrix, cmap="Reds", aspect="auto")
90+
91+
# Keep track of absolute counts for labels
92+
absolute_counts = np.zeros((len(models), max_streak_overall))
93+
for i, model in enumerate(models):
94+
streaks = win_streaks[model]
95+
for streak_len in streaks:
96+
if streak_len <= max_streak_overall:
97+
absolute_counts[i, streak_len - 1] += 1
98+
99+
# Add percentage and absolute count labels to ALL cells
100+
for i in range(len(models)):
101+
for j in range(max_streak_overall):
102+
percentage = streak_matrix[i, j]
103+
count = int(absolute_counts[i, j])
104+
text = f"{percentage:.1f}%\n({count})"
105+
plt.text(
106+
j,
107+
i,
108+
text,
109+
ha="center",
110+
va="center",
111+
color="white" if percentage > 40 else "black",
112+
fontweight="bold",
113+
fontsize=7,
114+
)
115+
116+
plt.xlabel("Win Streak Length")
117+
plt.ylabel("Model")
118+
plt.title("Win Streak Distribution (%)", fontweight="bold")
119+
plt.xticks(range(max_streak_overall), range(1, max_streak_overall + 1))
120+
plt.yticks(range(len(models)), clean_names)
121+
plt.colorbar(im, label="Percentage of Streaks")
122+
plt.tight_layout()
123+
plt.savefig("heatmap_win_streak_distribution.png", dpi=300, bbox_inches="tight")
124+
print("Win streak distribution heatmap saved to heatmap_win_streak_distribution.png")
125+
126+
127+
if __name__ == "__main__":
128+
parser = argparse.ArgumentParser(description="Analyze model win streak distributions")
129+
parser.add_argument("-d", "--log_dir", type=Path, default=LOCAL_LOG_DIR, help="Path to logs")
130+
args = parser.parse_args()
131+
main(args.log_dir)

0 commit comments

Comments
 (0)