Skip to content

Commit 5772eab

Browse files
john-b-yangjohn-b-yang
committed
Move config gen to codeclash/utils/ 
1 parent 248f8b7 commit 5772eab

8 files changed

Lines changed: 208 additions & 106 deletions

File tree

configs/scripts/generate_confs.py renamed to codeclash/utils/generate_confs.py

Lines changed: 64 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,29 @@
1+
"""
2+
Generate configuration files for tournaments between pairs of models in specified arenas.
3+
4+
Each configuration file specifies a tournament between two models in a given arena,
5+
including the number of rounds and simulations per round. The configurations are saved
6+
as YAML files in the specified output directory (default: configs/main/).
7+
8+
Also generates a tracking JSON file at configs/tracker.json to keep track of
9+
the number of tournaments and rounds played for each pair of models in each arena.
10+
11+
Usage:
12+
13+
python codeclash/utils/generate_confs.py -m configs/models.yaml -r 15 -s 1000
14+
"""
15+
116
import argparse
217
import json
318
from pathlib import Path
419

520
import yaml
621

722
from codeclash.games import (
8-
BattleCodeGame,
9-
BattleSnakeGame,
23+
ARENAS,
1024
CodeGame,
11-
CoreWarGame,
12-
HuskyBenchGame,
25+
DummyGame,
1326
RoboCodeGame,
14-
RobotRumbleGame,
1527
)
1628

1729

@@ -37,19 +49,12 @@ def literal_representer(dumper, data):
3749
yaml.SafeDumper.add_representer(IncludeTag, include_representer)
3850
yaml.SafeDumper.add_representer(LiteralString, literal_representer)
3951

40-
ARENAS: list[CodeGame] = [
41-
BattleCodeGame,
42-
BattleSnakeGame,
43-
CoreWarGame,
44-
HuskyBenchGame,
45-
RoboCodeGame,
46-
RobotRumbleGame,
47-
]
4852

4953
NUM_TOURNAMENTS = 10
54+
TRACKING_PATH = "configs/tracker.json"
5055

5156

52-
def prompt_game_desc(arena, rounds):
57+
def prompt_game_desc(arena, rounds, players):
5358
# Return as a LiteralString to get proper YAML literal block scalar formatting
5459
content = f"""You are a software developer ({{{{player_id}}}}) competing in a coding game called {arena.name}.
5560
{arena.description}
@@ -63,6 +68,41 @@ def prompt_game_desc(arena, rounds):
6368
return LiteralString(content)
6469

6570

71+
def get_config(rounds: int, simulations: int, arena: CodeGame, players: list[dict], prompt_func=prompt_game_desc):
72+
return {
73+
"tournament": {
74+
"rounds": rounds,
75+
},
76+
"game": {
77+
"name": arena.name,
78+
"sims_per_round": simulations,
79+
"args": arena.default_args,
80+
},
81+
"players": [
82+
{
83+
"agent": "mini",
84+
"name": get_name(p),
85+
"config": {"agent": IncludeTag("mini/default.yaml"), "model": p},
86+
}
87+
for p in players
88+
],
89+
"prompts": {"game_description": prompt_func(arena, rounds, players)},
90+
}
91+
92+
93+
def clean_config(config_path: str):
94+
# Post-process to remove quotes around include paths
95+
with open(config_path) as f:
96+
content = f.read()
97+
98+
# Remove quotes around include paths
99+
content = content.replace("!include 'mini/", "!include mini/")
100+
content = content.replace(".yaml'", ".yaml")
101+
102+
with open(config_path, "w") as f:
103+
f.write(content)
104+
105+
66106
def get_name(p):
67107
return p["model_name"].split("/")[-1]
68108

@@ -84,32 +124,16 @@ def main(models, arenas, rounds: int, simulations: int, record_ratio: float, out
84124

85125
tracking_dict = {}
86126
arenas_list = ARENAS if arenas == "all" else [a for a in ARENAS if a.name in arenas.split(",")]
127+
if DummyGame in arenas_list:
128+
arenas_list.remove(DummyGame) # Skip DummyGame for config generation
87129
if not arenas_list:
88130
print(f"No valid arenas found from {arenas}. Choose from {[a.name for a in ARENAS]}.")
89131
return # Stop execution if no valid arenas are found
90132
for arena in arenas_list:
91133
print(f"Generating {len(pairs)} configs for arena: {arena.name}")
92134
tracking_dict[arena.name] = {}
93135
for pair in pairs:
94-
config = {
95-
"tournament": {
96-
"rounds": rounds,
97-
},
98-
"game": {
99-
"name": arena.name,
100-
"sims_per_round": simulations,
101-
"args": arena.default_args,
102-
},
103-
"players": [
104-
{
105-
"agent": "mini",
106-
"name": get_name(p),
107-
"config": {"agent": IncludeTag("mini/default.yaml"), "model": p},
108-
}
109-
for p in pair
110-
],
111-
"prompts": {"game_description": prompt_game_desc(arena, rounds)},
112-
}
136+
config = get_config(rounds, simulations, arena, pair)
113137

114138
if arena == RoboCodeGame:
115139
robocode_adjustments(config, record_ratio)
@@ -127,34 +151,24 @@ def main(models, arenas, rounds: int, simulations: int, record_ratio: float, out
127151
Dumper=yaml.SafeDumper,
128152
)
129153

130-
# Post-process to remove quotes around include paths
131-
with open(output / config_name) as f:
132-
content = f.read()
133-
134-
# Remove quotes around include paths
135-
content = content.replace("!include 'mini/", "!include mini/")
136-
content = content.replace(".yaml'", ".yaml")
137-
138-
with open(output / config_name, "w") as f:
139-
f.write(content)
154+
clean_config(output / config_name)
140155

141156
pvp = ".".join(sorted([get_name(pair[0]), get_name(pair[1])]))
142157
tracking_key = f"r{rounds}.s{simulations}.p2"
143158
if tracking_key not in tracking_dict[arena.name]:
144159
tracking_dict[arena.name][tracking_key] = {}
145160
tracking_dict[arena.name][tracking_key][pvp] = 0
146161

147-
tracking_path = "configs/scripts/main_tracker.json"
148-
if Path(tracking_path).exists():
149-
with open(tracking_path) as f:
162+
if Path(TRACKING_PATH).exists():
163+
with open(TRACKING_PATH) as f:
150164
tracking_dict_current = json.load(f)
151165
tracking_dict.update(tracking_dict_current)
152166

153-
with open(tracking_path, "w") as f:
167+
with open(TRACKING_PATH, "w") as f:
154168
json.dump(tracking_dict, f, indent=2)
155-
print(f"Wrote tracking file to '{tracking_path}'.")
169+
print(f"Wrote tracking file to '{TRACKING_PATH}'.")
156170

157-
print(f"Generated {len(pairs) * len(arenas)} configuration files in '{output}'.")
171+
print(f"Generated {len(pairs) * len(arenas_list)} configuration files in '{output}'.")
158172
print(f"- # Models: {len(models)}")
159173
print(f"- # Arenas: {len(ARENAS)}")
160174
print(f"- r (rounds) {rounds}")
@@ -173,7 +187,7 @@ def main(models, arenas, rounds: int, simulations: int, record_ratio: float, out
173187
"-m",
174188
"--models",
175189
type=str,
176-
default="configs/scripts/models.yaml",
190+
default="configs/models.yaml",
177191
help="Path to model configurations.",
178192
)
179193
parser.add_argument(

‎codeclash/utils/generate_confs_multi.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import argparse
2+
from pathlib import Path
3+
4+
import yaml
5+
6+
from codeclash.games import (
7+
ARENAS,
8+
DummyGame,
9+
)
10+
from codeclash.utils.generate_confs import clean_config, get_config
11+
12+
13+
def main(models: str, arenas: str, rounds: int, simulations: int, record_ratio: float, output: Path):
14+
# Get all models
15+
models = yaml.safe_load(open(models))
16+
output.mkdir(parents=True, exist_ok=True)
17+
18+
# Get arenas
19+
arenas_list = ARENAS if arenas == "all" else [a for a in ARENAS if a.name in arenas.split(",")]
20+
if DummyGame in arenas_list:
21+
arenas_list.remove(DummyGame) # Skip DummyGame for config generation
22+
if not arenas_list:
23+
print(f"No valid arenas found from {arenas}. Choose from {[a.name for a in ARENAS]}.")
24+
return # Stop execution if no valid arenas are found
25+
26+
for arena in arenas_list:
27+
print(f"Generating config for arena: {arena.name}")
28+
config = get_config(rounds, simulations, arena, models)
29+
config_name = f"{arena.name}__p{len(models)}__r{rounds}__s{simulations}.yaml"
30+
with open(output / config_name, "w") as f:
31+
yaml.dump(
32+
config,
33+
f,
34+
default_style=None,
35+
sort_keys=False,
36+
allow_unicode=True,
37+
default_flow_style=False,
38+
Dumper=yaml.SafeDumper,
39+
)
40+
41+
clean_config(output / config_name)
42+
43+
print(f"Generated {len(arenas_list)} configuration files in '{output}'.")
44+
print(f"- # Models: {len(models)}")
45+
print(f"- # Arenas: {len(arenas_list)}")
46+
print(f"- r (rounds) {rounds}")
47+
print(f"- s (sims_per_round) {simulations}")
48+
49+
50+
if __name__ == "__main__":
51+
parser = argparse.ArgumentParser(description="Generate configuration files.")
52+
parser.add_argument(
53+
"-m",
54+
"--models",
55+
type=str,
56+
default="configs/models_multi.yaml",
57+
help="Path to model configurations.",
58+
)
59+
parser.add_argument(
60+
"-a",
61+
"--arenas",
62+
type=str,
63+
default="all",
64+
help="Comma separated list of arenas to generate configs for (default: all).",
65+
)
66+
parser.add_argument(
67+
"-r",
68+
"--rounds",
69+
type=int,
70+
default=15,
71+
help="Number of rounds per tournament for the configuration (default: 15).",
72+
)
73+
parser.add_argument(
74+
"-s",
75+
"--simulations",
76+
type=int,
77+
default=1000,
78+
help="Number of simulations to run per round (default: 1000).",
79+
)
80+
parser.add_argument(
81+
"--record_ratio",
82+
type=float,
83+
default=1,
84+
help="Fraction of simulations to record (default: 1 = all).",
85+
)
86+
parser.add_argument(
87+
"-o",
88+
"--output",
89+
type=Path,
90+
default=Path("configs/multi/"),
91+
help="Output directory for configuration files (default: multi/).",
92+
)
93+
args = parser.parse_args()
94+
main(**vars(args))

configs/scripts/update_tracker.py renamed to codeclash/utils/update_tracker.py

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,16 @@
1+
"""
2+
Script for updating `configs/tracker.json` (generated by running `generate_confs.py` file in this directory)
3+
4+
Usage:
5+
6+
python codeclash/utils/update_tracker.py
7+
"""
8+
19
import json
210
from pathlib import Path
311

4-
tracker = json.load(open("configs/scripts/main_tracker.json"))
12+
tracker_path = Path("configs/tracker.json")
13+
tracker = json.load(open(tracker_path))
514
arena_logs = [p.parent for p in Path("logs").rglob("metadata.json")]
615

716
# Set all tracker values to 0
@@ -38,6 +47,6 @@
3847
print(f" - {arena}.{setting}.{pvp}: {v_str}")
3948
tracker[arena][setting][pvp] = v_str
4049

41-
print("Updated tracking file to 'configs/scripts/main_tracker.json'.")
42-
with open("configs/scripts/main_tracker.json", "w") as f:
50+
print(f"Updated tracking file to '{tracker_path}'.")
51+
with open(tracker_path, "w") as f:
4352
json.dump(tracker, f, indent=2)

configs/scripts/models.yaml renamed to configs/models.yaml

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,25 @@
11
- model_name: "@anthropic/claude-sonnet-4-20250514"
22
model_class: portkey
33
model_kwargs:
4-
temperature: 0.0
4+
temperature: 0.2
55
max_tokens: 4096
66
- model_name: "@anthropic/claude-sonnet-4-5-20250929"
77
model_class: portkey
88
model_kwargs:
9-
temperature: 0.0
9+
temperature: 0.2
1010
max_tokens: 4096
1111
# - model_name: "anthropic/claude-opus-4-1-20250805"
1212
# model_kwargs:
1313
# temperature: 0.0
1414
- model_name: "@x-ai/grok-code-fast-1"
1515
model_class: portkey
1616
litellm_model_name_override: "xai/grok-code-fast-1"
17+
model_kwargs:
18+
temperature: 0.2
1719
- model_name: "@google/gemini-2.5-pro"
1820
model_class: portkey
21+
model_kwargs:
22+
temperature: 0.2
1923
- model_name: "@fireworks/accounts/fireworks/models/glm-4-5"
2024
model_class: portkey
2125
litellm_model_name_override: "deepinfra/zai-org/GLM-4.5"
@@ -34,5 +38,7 @@
3438
- model_name: "qwen3-coder-plus-2025-09-23"
3539
model_class: dashscope
3640
litellm_model_name_override: "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct"
41+
model_kwargs:
42+
temperature: 0.2
3743
- model_name: "@openai/o3"
3844
model_class: portkey

‎configs/models_multi.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# - model_name: "@anthropic/claude-sonnet-4-20250514"
2+
# model_class: portkey
3+
# model_kwargs:
4+
# temperature: 0.2
5+
# max_tokens: 4096
6+
- model_name: "@anthropic/claude-sonnet-4-5-20250929"
7+
model_class: portkey
8+
model_kwargs:
9+
temperature: 0.2
10+
max_tokens: 4096
11+
- model_name: "@x-ai/grok-code-fast-1"
12+
model_class: portkey
13+
litellm_model_name_override: "xai/grok-code-fast-1"
14+
model_kwargs:
15+
temperature: 0.2
16+
- model_name: "@google/gemini-2.5-pro"
17+
model_class: portkey
18+
model_kwargs:
19+
temperature: 0.2
20+
# - model_name: "@openai/gpt-5"
21+
# model_class: portkey
22+
# - model_name: "@openai/gpt-5-mini"
23+
# model_class: portkey
24+
- model_name: "qwen3-coder-plus-2025-09-23"
25+
model_class: dashscope
26+
litellm_model_name_override: "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct"
27+
model_kwargs:
28+
temperature: 0.2
29+
- model_name: "@openai/o3"
30+
model_class: portkey

0 commit comments

Comments
 (0)