Skip to content

Commit 0904dd1

Browse files
author
Player
committed
Merge branch 'main' into feat/add-bridge
2 parents fe5f9bd + 94ccfaa commit 0904dd1

18 files changed

Lines changed: 265 additions & 22 deletions

‎codeclash/agents/minisweagent.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@
1616
from codeclash.utils.environment import copy_to_container
1717

1818
os.environ["MSWEA_MODEL_RETRY_STOP_AFTER_ATTEMPT"] = "90"
19-
os.environ["LITELLM_MODEL_REGISTRY_PATH"] = str((REPO_DIR / "configs" / "litellm_custom_model_config.yaml").resolve())
19+
os.environ["LITELLM_MODEL_REGISTRY_PATH"] = str(
20+
(REPO_DIR / "configs" / "mini" / "litellm_custom_model_config.yaml").resolve()
21+
)
2022

2123

2224
class ClashAgent(DefaultAgent):

‎codeclash/arenas/robotrumble/robotrumble.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from codeclash.arenas.arena import CodeArena, RoundStats
99
from codeclash.constants import RESULT_TIE
1010

11+
DEFAULT_SIMS = 100
1112
MAP_EXT_TO_HEADER = {
1213
"js": "function robot(state, unit) {",
1314
"py": "def robot(state, unit):",
@@ -66,15 +67,16 @@ def execute_round(self, agents: list[Player]):
6667
# Submit all simulations to the thread pool
6768
futures = [
6869
executor.submit(self._run_single_simulation, agents, idx, cmd)
69-
for idx in range(self.game_config.get("sims_per_round", 100))
70+
for idx in range(self.game_config.get("sims_per_round", DEFAULT_SIMS))
7071
]
7172

7273
# Collect results as they complete
7374
i_completed = 0
7475
for future in as_completed(futures):
7576
future.result()
7677
i_completed += 1
77-
self.logger.info(f"Completed {i_completed} of {len(futures)} simulations")
78+
if i_completed % 10 == 0:
79+
self.logger.info(f"Completed {i_completed} of {len(futures)} simulations")
7880

7981
def _get_winner_txt(self, output_file: str, agents: list[Player]) -> str:
8082
try:
@@ -114,7 +116,7 @@ def _get_winner_json(self, output_file: str, agents: list[Player]) -> str:
114116

115117
def get_results(self, agents: list[Player], round_num: int, stats: RoundStats):
116118
winners = []
117-
for idx in range(self.game_config.get("sims_per_round", 100)):
119+
for idx in range(self.game_config.get("sims_per_round", DEFAULT_SIMS)):
118120
output_file = self.log_round(round_num) / f"sim_{idx}.{self.sim_ext}"
119121
if not output_file.exists():
120122
self.logger.warning(f"Simulation {idx} not found, skipping")

‎configs/ablations/ladder/README.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# CC:Ladder
2+
3+
For a more static and hill-climb-able version of CodeClash, we introduce CC:Ladder - for each arena, we curate a collection of human-written solutions, determine their relative rankings, and then see how "high up" the ladder models can climb.
4+
5+
For instance, for RobotRumble, we created a ladder by doing the following steps:
6+
1. From the online [leaderboard](https://robotrumble.org/boards/2), we manually crawled all open source, published bots and pushed them as branches to the [CC:RobotRumble](https://github.com/CodeClash-ai/RobotRumble) repository.
7+
2. We then created the `robotrumble.yaml` file in this folder.
8+
3. Next, from the repository root, we run `uv run python scripts/run_ladder.py configs/ablations/ladder/robotrumble.yaml`, which runs PvP Tournaments against all pairs of branches.
9+
4. From these logs, we then calculate win rate to rank all models.
10+
11+
You can follow these steps to create your own "CC:<arena>" ladder.
12+
The tricky part is typically finding a large collection of human solutions for a particular arena.
13+
We've typically found that googling for online leaderboards or awesome-<arena> repositories (e.g. [BattleSnake](https://github.com/BattlesnakeOfficial/awesome-battlesnake)) is a good strategy.

‎configs/ablations/ladder/robotrumble.yaml‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
tournament:
2+
rounds: 0
3+
game:
4+
name: RobotRumble
5+
sims_per_round: 250
6+
args:
7+
raw: false
8+
players:
9+
- agent: dummy
10+
branch_init: human/aaa/jippty5
11+
- agent: dummy
12+
branch_init: human/aaoutkine/dark-knight
13+
- agent: dummy
14+
branch_init: human/aaoutkine/school-bot
15+
- agent: dummy
16+
branch_init: human/aaoutkine/silo34
17+
- agent: dummy
18+
branch_init: human/aayyad/testbot
19+
- agent: dummy
20+
branch_init: human/anton/anton3000
21+
- agent: dummy
22+
branch_init: human/anton/anton4000
23+
- agent: dummy
24+
branch_init: human/anton/om-om
25+
- agent: dummy
26+
branch_init: human/anton/wallifier
27+
- agent: dummy
28+
branch_init: human/atl15/centerrr
29+
- agent: dummy
30+
branch_init: human/clay/diag-lattice
31+
- agent: dummy
32+
branch_init: human/devchris/black_magic
33+
- agent: dummy
34+
branch_init: human/devchris/first_test
35+
- agent: dummy
36+
branch_init: human/edward/flail
37+
- agent: dummy
38+
branch_init: human/entropicdrifter/gigachad
39+
- agent: dummy
40+
branch_init: human/entropicdrifter/glommer
41+
- agent: dummy
42+
branch_init: human/entropicdrifter/glommerv2
43+
- agent: dummy
44+
branch_init: human/entropicdrifter/seven-of-nine
45+
- agent: dummy
46+
branch_init: human/entropicdrifter/we-are-borg
47+
- agent: dummy
48+
branch_init: human/essickmango/fruity-test
49+
- agent: dummy
50+
branch_init: human/essickmango/pickle-up
51+
- agent: dummy
52+
branch_init: human/gerenuk/gere-ape
53+
- agent: dummy
54+
branch_init: human/happysquid/test
55+
- agent: dummy
56+
branch_init: human/jammyliu/sixty-nine-line
57+
- agent: dummy
58+
branch_init: human/jay0jayjay/naivestarter
59+
- agent: dummy
60+
branch_init: human/jiricodes/jiricodes-bot
61+
- agent: dummy
62+
branch_init: human/kalkin/artemis
63+
- agent: dummy
64+
branch_init: human/kalkin/artemis2
65+
- agent: dummy
66+
branch_init: human/kalkin/maxad
67+
- agent: dummy
68+
branch_init: human/ketza/arthur
69+
- agent: dummy
70+
branch_init: human/ketza/bob
71+
- agent: dummy
72+
branch_init: human/lanity/sivuy
73+
- agent: dummy
74+
branch_init: human/ldang/nemo
75+
- agent: dummy
76+
branch_init: human/ldang/nessy
77+
- agent: dummy
78+
branch_init: human/luisa/baselinegere
79+
- agent: dummy
80+
branch_init: human/luisa/luisasrobot
81+
- agent: dummy
82+
branch_init: human/mario31313/alpha_13
83+
- agent: dummy
84+
branch_init: human/mee42/follow-bot
85+
- agent: dummy
86+
branch_init: human/mitch84/crw_preempt
87+
- agent: dummy
88+
branch_init: human/mitch84/retreat_walk2
89+
- agent: dummy
90+
branch_init: human/mitch84/walk_retreat
91+
- agent: dummy
92+
branch_init: human/mjburgess/rule99
93+
- agent: dummy
94+
branch_init: human/mkap/test
95+
- agent: dummy
96+
branch_init: human/mountain/neuralbot1-1h
97+
- agent: dummy
98+
branch_init: human/mountain/neuralbot2-6h
99+
- agent: dummy
100+
branch_init: human/mountain/neuralbot4-3h
101+
- agent: dummy
102+
branch_init: human/mousetail/coward-bot
103+
- agent: dummy
104+
branch_init: human/mousetail/genetic-robot
105+
- agent: dummy
106+
branch_init: human/navster8/bash-brothers
107+
- agent: dummy
108+
branch_init: human/navster8/maginot-line
109+
- agent: dummy
110+
branch_init: human/sbasu3/meek-bot
111+
- agent: dummy
112+
branch_init: human/sivecano/clouded-mind
113+
- agent: dummy
114+
branch_init: human/suddenlyseals/control-center
115+
- agent: dummy
116+
branch_init: human/tabaxi3k/black-magic-1
117+
- agent: dummy
118+
branch_init: human/tabaxi3k/charles
119+
- agent: dummy
120+
branch_init: human/thesmilingturtl/naivefaa
121+
- agent: dummy
122+
branch_init: human/underscore/bot1
123+
- agent: dummy
124+
branch_init: human/wolfsleuth/simple
125+
prompts:
126+
game_description: |-
127+
RobotRumble ladder

‎configs/ablations/multi/README.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Multi-player competitions
2+
3+
The default CodeClash tournament setting pits players head to head.
4+
5+
In these configurations, we explore how competitive dynamics change when 3+ players are competing. Specifically, we run Core War tournaments lasting 15 rounds with 6 players. Our findings can be found in the original paper in Section 4.1, specifically *Multi-agent competitions (3+ players) reflect similar rankings*.
6+
7+
To enable multi-player competitions, simply add more players under the `players` field in your configuration, such as:
8+
9+
```yaml
10+
players:
11+
- agent: mini
12+
...
13+
- agent: mini
14+
...
15+
- agent: mini
16+
...
17+
```

‎configs/ablations/transparent/README.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Transparent Codebases
2+
3+
These set of configurations correspond to Section 4.1 of the original paper, specifically *Models have limited capacity for opponent analysis even with transparent codebases.*.
4+
5+
Under normal CodeClash circumstances, models' codebases are not made available to one another. One of CodeClash's challenges is to see whether models are capable of discerning opponent behavior via logs.
6+
7+
In this ablation, we explore lifting this restriction. Each round, in addition to the competition logs, opponents' codebases are also made available to each player. All that's required to enable this feature is to set:
8+
9+
```yaml
10+
tournament:
11+
...
12+
transparent: true
13+
``

‎configs/ablations/vs_human/README.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# vs. Human
2+
3+
These set of configurations correspond to Section 4.1 of the original paper, specifically the subsection *On RobotRumble, models trail substantially behind expert human programmers*.
4+
5+
Each configuration pits a model against an open source codebase written by a human expert for a particular arena. Across a tournament spanning 15 rounds, the model is allowed the evolve the codebase as it sees fit to beat the human expert's solution. The human's solution is *not* changing for the duration of the tournament.
6+
7+
To make models compete against static human solutions, do the following two steps.
8+
9+
1. Make sure the human solution is working and pushed as a branch to the corresponding arena. E.g. [gigachad](https://github.com/CodeClash-ai/RobotRumble/tree/human/entropicdrifter/gigachad) for RobotRumble.
10+
2. Then, in your configuration, simply specify one of the players as a `dummy` agent, with `branch_init` set to the branch name, such as:
11+
12+
```yaml
13+
players:
14+
- agent: dummy
15+
branch_init: human/entropicdrifter/gigachad
16+
name: gigachad
17+
```

‎configs/ablations/vs_human/RobotRumble__claude-sonnet-4-20250514__vs__seven-of-nine__r15__s250.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ game:
88
players:
99
- agent: mini
1010
name: claude-sonnet-4-20250514
11-
branch_init: python
11+
branch_init: starter/python
1212
config:
1313
agent: !include mini/default.yaml
1414
model:
@@ -19,7 +19,7 @@ players:
1919
max_tokens: 4096
2020
- agent: dummy
2121
name: seven-of-nine
22-
branch_init: human/seven-of-nine
22+
branch_init: human/entropicdrifter/seven-of-nine
2323
prompts:
2424
game_description: |
2525
You are a software developer ({{player_id}}) competing in a coding game called RobotRumble.

configs/ablations/vs_human/RobotRumble_gigachad.yaml renamed to configs/ablations/vs_human/RobotRumble__claude-sonnet-4-5-20250929__vs__gigachad__r15__s250.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ game:
66
players:
77
- agent: mini
88
name: claude-sonnet-4-5-20250929
9-
branch_init: python
9+
branch_init: starter/python
1010
config:
1111
agent: !include mini/default.yaml
1212
model:
@@ -17,7 +17,7 @@ players:
1717
max_tokens: 4096
1818
- agent: dummy
1919
name: gigachad
20-
branch_init: human/gigachad
20+
branch_init: human/entropicdrifter/gigachad
2121
prompts:
2222
game_description: |
2323
You are a software developer ({{player_id}}) competing in a coding game called RobotRumble.

‎configs/ablations/vs_human/RobotRumble__claude-sonnet-4-5-20250929__vs__seven-of-nine__r15__s250.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ game:
88
players:
99
- agent: mini
1010
name: claude-sonnet-4-5-20250929
11-
branch_init: python
11+
branch_init: starter/python
1212
config:
1313
agent: !include mini/default.yaml
1414
model:
@@ -19,7 +19,7 @@ players:
1919
max_tokens: 4096
2020
- agent: dummy
2121
name: seven-of-nine
22-
branch_init: human/seven-of-nine
22+
branch_init: human/entropicdrifter/seven-of-nine
2323
prompts:
2424
game_description: |
2525
You are a software developer ({{player_id}}) competing in a coding game called RobotRumble.

0 commit comments

Comments
 (0)