diff --git a/.env.example b/.env.example index ff2ffc36..8e0a4dd0 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,9 @@ OPENAI_API_KEY= GEMINI_API_KEY= XAI_API_KEY= DASHSCOPE_API_KEY= + +# Optional: Claude via a custom Anthropic-compatible endpoint (mini_anthropic_model.AnthropicModel). +# Used by the *_llama.yaml ladder configs. Set all three to route Claude through your endpoint. +LLAMA_API_KEY= +LLAMA_BASE_URL= +LLAMA_MODEL= diff --git a/.gitignore b/.gitignore index 58c9c682..37fe0ba7 100644 --- a/.gitignore +++ b/.gitignore @@ -233,3 +233,6 @@ docs/visualization/*.jsonl docs/visualization/*.png docs/visualization/code_org docs/visualization/elo_plots + +# Per-model llama endpoint configs — kept local only (obscures internal API details) +configs/mini/models/ diff --git a/codeclash/agents/mini_anthropic_model.py b/codeclash/agents/mini_anthropic_model.py new file mode 100644 index 00000000..9e6432f9 --- /dev/null +++ b/codeclash/agents/mini_anthropic_model.py @@ -0,0 +1,287 @@ +"""Custom mini-swe-agent model class for Claude served through a custom Anthropic-compatible +base URL, with explicit per-token cost tracking. + +mini-swe-agent's default LitellmModel converts everything to the OpenAI API format. Behind +some proxies/gateways that drops the `cache_control` markers, disabling prompt caching and +making Claude calls slow and expensive. This class speaks the native Anthropic API instead +(via the `anthropic` SDK against a configurable `base_url`), so caching is preserved, and it +computes request cost from its own `cost:` block rather than relying on a litellm price registry. + +To use it, point a player's `model` block at +`model_class: codeclash.agents.mini_anthropic_model.AnthropicModel` and provide the API key, +base URL, and model name (see the `*_env` config fields, which keep endpoint-specific values in +the environment rather than in committed configs). Requires the optional `anthropic` dependency +(`uv pip install -e '.[llama]'`). See configs/ablations/ladder/robotrumble_llama.yaml. +""" + +import json +import logging +import os +import time +from types import SimpleNamespace +from typing import Any + +import anthropic +from jinja2 import StrictUndefined, Template +from minisweagent.models import GLOBAL_MODEL_STATS +from minisweagent.models.utils.actions_toolcall import parse_toolcall_actions +from minisweagent.models.utils.retry import retry +from pydantic import BaseModel + +logger = logging.getLogger("anthropic_model") + +# Map Anthropic stop reasons to OpenAI-style finish_reasons so that finish_reason-based +# format_error_templates (e.g. the truncation branch) behave the same as for litellm models. +_ANTHROPIC_FINISH_REASON = { + "max_tokens": "length", + "tool_use": "tool_calls", + "end_turn": "stop", + "stop_sequence": "stop", +} + + +def _finish_reason_from_anthropic(stop_reason: str | None) -> str | None: + return _ANTHROPIC_FINISH_REASON.get(stop_reason, stop_reason) + + +ANTHROPIC_BASH_TOOL = { + "name": "bash", + "description": "Execute a bash command", + "input_schema": { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "The bash command to execute", + } + }, + "required": ["command"], + }, +} + + +class CostPerToken(BaseModel): + """Per-token costs in USD.""" + + input: float = 0.0 + output: float = 0.0 + cache_creation_input: float = 0.0 + cache_read_input: float = 0.0 + + +class AnthropicModelConfig(BaseModel): + model_name: str + """Anthropic model name, e.g. `claude-sonnet-4-5-20250929`. Overridden by `model_name_env` + if that names a set environment variable (so endpoint-specific model ids stay out of configs).""" + model_name_env: str | None = None + """Optional env var to read the model name from, overriding `model_name` when set.""" + model_kwargs: dict[str, Any] = {} + """Additional arguments passed to the API.""" + drop_none_model_kwargs: bool = True + """Drop all model_kwargs that are None. + This is so we can easily recursively merge this config with other configs targeting litellm. + """ + max_tokens: int = 16384 + """Maximum number of output tokens.""" + base_url: str | None = None + """Custom base URL for the Anthropic API (e.g. for proxies). Overridden by `base_url_env`.""" + base_url_env: str | None = None + """Optional env var to read the base URL from, overriding `base_url` when set (so endpoint + URLs stay out of configs).""" + api_key_env: str = "ANTHROPIC_API_KEY" + """Environment variable name for the API key.""" + cost: CostPerToken = CostPerToken() + """Per-token costs in USD for computing request cost from usage.""" + format_error_template: str = "{{ error }}" + """Template used when the LM's output is not in the expected format.""" + observation_template: str = ( + "{% if output.exception_info %}{{output.exception_info}}\n{% endif %}" + "{{output.returncode}}\n\n{{output.output}}" + ) + """Template used to render the observation after executing an action.""" + + +class AnthropicModel: + abort_exceptions: list[type[Exception]] = [ + anthropic.BadRequestError, + anthropic.AuthenticationError, + anthropic.PermissionDeniedError, + anthropic.NotFoundError, + KeyboardInterrupt, + ] + + def __init__(self, *, config_class: type = AnthropicModelConfig, **kwargs): + self.config = config_class(**kwargs) + # Resolve endpoint-specific values from the environment so they stay out of configs. + if self.config.model_name_env: + model_name = os.getenv(self.config.model_name_env, "") + if not model_name: + raise ValueError(f"Set the {self.config.model_name_env} environment variable to the model name.") + self.config.model_name = model_name + if self.config.base_url_env: + base_url = os.getenv(self.config.base_url_env, "") + if not base_url: + raise ValueError(f"Set the {self.config.base_url_env} environment variable to the API base URL.") + self.config.base_url = base_url + api_key = os.getenv(self.config.api_key_env, "") + if not api_key: + raise ValueError(f"API key not found. Set the {self.config.api_key_env} environment variable.") + client_kwargs: dict[str, Any] = {"api_key": api_key} + if self.config.base_url: + client_kwargs["base_url"] = self.config.base_url + self.client = anthropic.Anthropic(**client_kwargs) + + @staticmethod + def _set_cache_control_on_last_message(messages: list[dict]) -> list[dict]: + """Add cache_control to the last block of the last message.""" + import copy + + messages = copy.deepcopy(messages) + if not messages: + return messages + last = messages[-1] + content = last["content"] + if content is None: + last["cache_control"] = {"type": "ephemeral"} + elif isinstance(content, str): + last["content"] = [{"type": "text", "text": content, "cache_control": {"type": "ephemeral"}}] + else: + content[-1]["cache_control"] = {"type": "ephemeral"} + return messages + + @staticmethod + def _strip_display_text(content: list[dict] | str) -> list[dict] | str: + """Remove the 'text' key we add to non-text blocks for interactive display + and 'caller' (None) that model_dump() emits from newer SDK versions.""" + if not isinstance(content, list): + return content + return [ + { + k: v + for k, v in block.items() + if not (k == "text" and block.get("type") != "text") and not (k == "caller" and v is None) + } + for block in content + ] + + def _query(self, messages: list[dict], **kwargs): + assert messages[0]["role"] == "system" + api_messages = self._set_cache_control_on_last_message( + [{"role": m["role"], "content": self._strip_display_text(m["content"])} for m in messages[1:]] + ) + extra_model_kwargs = self.config.model_kwargs | kwargs + if self.config.drop_none_model_kwargs: + extra_model_kwargs = {k: v for k, v in extra_model_kwargs.items() if v is not None} + return self.client.messages.create( + model=self.config.model_name, + max_tokens=self.config.max_tokens, + system=messages[0]["content"], + messages=api_messages, + tools=[ANTHROPIC_BASH_TOOL], + **extra_model_kwargs, + ) + + def query(self, messages: list[dict[str, str]], **kwargs) -> dict: + for attempt in retry(logger=logger, abort_exceptions=self.abort_exceptions): + with attempt: + response = self._query(messages, **kwargs) + actions = self._parse_actions(response) + cost_output = self._calculate_cost(response) + GLOBAL_MODEL_STATS.add(cost_output["cost"]) + content = [] + for block in response.content: + d = block.model_dump() + if block.type == "tool_use": + d["text"] = f"```\n{block.input.get('command', json.dumps(block.input))}\n```" + content.append(d) + return { + "role": "assistant", + "content": content, + "extra": { + "actions": actions, + "response": response.model_dump(), + **cost_output, + "timestamp": time.time(), + }, + } + + def _parse_actions(self, response: anthropic.types.Message) -> list[dict]: + tool_calls = [ + SimpleNamespace( + id=block.id, + function=SimpleNamespace(name=block.name, arguments=json.dumps(block.input)), + ) + for block in response.content + if block.type == "tool_use" + ] + return parse_toolcall_actions( + tool_calls, + format_error_template=self.config.format_error_template, + template_kwargs={"finish_reason": _finish_reason_from_anthropic(response.stop_reason)}, + ) + + def _calculate_cost(self, response: anthropic.types.Message) -> dict[str, float]: + usage = response.usage + c = self.config.cost + return { + "cost": ( + usage.input_tokens * c.input + + usage.output_tokens * c.output + + getattr(usage, "cache_creation_input_tokens", 0) * c.cache_creation_input + + getattr(usage, "cache_read_input_tokens", 0) * c.cache_read_input + ) + } + + def format_message(self, **kwargs) -> dict: + return kwargs + + def format_observation_messages( + self, message: dict, outputs: list[dict], template_vars: dict | None = None + ) -> list[dict]: + """Format execution outputs as a single user message with tool_result blocks (Anthropic format).""" + actions = message.get("extra", {}).get("actions", []) + not_executed = {"output": "", "returncode": -1, "exception_info": "action was not executed"} + padded_outputs = outputs + [not_executed] * (len(actions) - len(outputs)) + tool_results = [] + extras = [] + for action, output in zip(actions, padded_outputs): + content = Template(self.config.observation_template, undefined=StrictUndefined).render( + output=output, **(template_vars or {}) + ) + tool_results.append( + { + "type": "tool_result", + "tool_use_id": action["tool_call_id"], + "content": content, + "text": content, + } + ) + extras.append( + { + "raw_output": output.get("output", ""), + "returncode": output.get("returncode"), + "timestamp": time.time(), + "exception_info": output.get("exception_info"), + **output.get("extra", {}), + } + ) + return [ + { + "role": "user", + "content": tool_results, + "extra": {"observations": extras}, + } + ] + + def get_template_vars(self, **kwargs) -> dict[str, Any]: + return self.config.model_dump() + + def serialize(self) -> dict: + return { + "info": { + "config": { + "model": self.config.model_dump(mode="json"), + "model_type": f"{self.__class__.__module__}.{self.__class__.__name__}", + }, + } + } diff --git a/codeclash/cli/ladder.py b/codeclash/cli/ladder.py index da4135bb..82b2acac 100644 --- a/codeclash/cli/ladder.py +++ b/codeclash/cli/ladder.py @@ -17,6 +17,43 @@ logger = get_logger("ladder") + +def _resolve_ladder_rules(ladder_rules: dict, rounds: int) -> tuple[float, int]: + """Validate the optional ``ladder_rules`` block and return ``(min_round_win_fraction, win_last_k)``. + + Defaults reproduce the historical behavior: win a strict majority of rounds + (``min_round_win_fraction=0.5``) AND win the final round (``win_last_k=1``). + """ + min_round_win_fraction = ladder_rules.get("min_round_win_fraction", 0.5) + win_last_k = ladder_rules.get("win_last_k", 1) + + # win_last_k: number of trailing rounds the player must win (1 == just the final round). + if isinstance(win_last_k, bool) or not isinstance(win_last_k, int): + typer.echo(f"ladder_rules.win_last_k must be an integer, got {win_last_k!r}.") + raise typer.Exit(1) + if win_last_k < 1: + typer.echo( + f"ladder_rules.win_last_k must be >= 1, got {win_last_k}. Use 1 to require winning only the final round." + ) + raise typer.Exit(1) + if win_last_k > rounds: + typer.echo(f"ladder_rules.win_last_k ({win_last_k}) cannot exceed tournament.rounds ({rounds}).") + raise typer.Exit(1) + + # min_round_win_fraction: player must win strictly more than this fraction of rounds. + if isinstance(min_round_win_fraction, bool) or not isinstance(min_round_win_fraction, (int, float)): + typer.echo(f"ladder_rules.min_round_win_fraction must be a number, got {min_round_win_fraction!r}.") + raise typer.Exit(1) + if not 0 <= min_round_win_fraction < 1: + typer.echo( + f"ladder_rules.min_round_win_fraction must be in [0, 1), got {min_round_win_fraction}. " + "0.5 requires a strict majority; 0 drops the majority requirement." + ) + raise typer.Exit(1) + + return float(min_round_win_fraction), win_last_k + + ladder_app = typer.Typer( no_args_is_help=True, add_completion=False, @@ -101,9 +138,16 @@ def run( config["tournament"]["rounds"], config["game"]["sims_per_round"], ) + min_round_win_fraction, win_last_k = _resolve_ladder_rules(config.get("ladder_rules", {}), rounds) timestamp = time.strftime("%y%m%d%H%M%S") del config["player"] del config["ladder"] + config.pop("ladder_rules", None) + + print( + f"Ladder advancement rule: win > {min_round_win_fraction:.0%} of {rounds} rounds " + f"and win the last {win_last_k} round(s)." + ) ladder_folder = f"LadderTournament.{config['game']['name']}.r{rounds}.s{sims}.{timestamp}" player["branch"] = ladder_folder parent_dir = LOCAL_LOG_DIR / getpass.getuser() / ladder_folder @@ -143,12 +187,22 @@ def run( metadata = yaml.safe_load(f) round_winners = [r["winner"] for r in metadata["round_stats"].values()] - # Player must have won majority of rounds and the last round to continue ladder + # Advancement rule (configurable via `ladder_rules`): win strictly more than + # `min_round_win_fraction` of rounds AND win the last `win_last_k` rounds. player_wins = sum(1 for w in round_winners if w == player["name"]) - player_won_last = round_winners[-1] == player["name"] - - if not player_wins > len(round_winners) // 2 or not player_won_last: - # If player lost tournament, ladder challenge ends + won_majority = player_wins > len(round_winners) * min_round_win_fraction + won_last_k = all(w == player["name"] for w in round_winners[-win_last_k:]) + + if not won_majority or not won_last_k: + # Player failed the advancement rule; the ladder challenge ends here. + print("=" * 10) + print( + f"{player['name']} did not clear {opponent['name']} " + f"(rank {opponent_rank}/{len(ladder)}): won {player_wins}/{len(round_winners)} rounds " + f"(needed > {min_round_win_fraction:.0%}), last {win_last_k} round(s) won: {won_last_k}.\n" + "Ladder challenge ends." + ) + print("=" * 10) break print("=" * 10) diff --git a/configs/ablations/ladder/README.md b/configs/ablations/ladder/README.md index ce76dc63..99d5e6c4 100644 --- a/configs/ablations/ladder/README.md +++ b/configs/ablations/ladder/README.md @@ -11,3 +11,27 @@ For instance, for RobotRumble, we created a ladder by doing the following steps: You can follow these steps to create your own "CC:" ladder. The tricky part is typically finding a large collection of human solutions for a particular arena. We've typically found that googling for online leaderboards or awesome- repositories (e.g. [BattleSnake](https://github.com/BattlesnakeOfficial/awesome-battlesnake)) is a good strategy. + +## Config layout + +Each arena has a few kinds of config in this folder: + +- `make_.yaml` — the round-robin used to **build** the ladder (`ladder make`), running PvP tournaments across all pairs of human bots to rank them. +- `.yaml` — the **run** config (`ladder run`): a climber ascends the ranked ladder rung by rung until it loses. +- `__.yaml` — per-model run configs (e.g. `battlesnake__opus_4_8.yaml`) that swap in a specific model via `model: !include mini/models/llama_.yaml`. +- `rungs/.yaml` — the ranked opponent list (worst first, strongest last), shared by both `.yaml` and every `__.yaml` through `ladder: !include ablations/ladder/rungs/.yaml`. Edit the ladder in this one file and every config for that arena picks it up. + +## Ladder advancement rule (`ladder_rules`) + +Each run config carries a `ladder_rules` block controlling what it takes to clear a rung and continue climbing: + +```yaml +ladder_rules: + min_round_win_fraction: 0.5 # must win strictly more than this fraction of rounds + win_last_k: 1 # ...and must win the last K rounds (1 == just the final round) +``` + +The defaults shown above reproduce the historical behavior (strict majority of rounds **and** win the final round); the block is optional and falls back to these values if omitted. Validation: + +- `win_last_k` must be an integer with `1 <= win_last_k <= tournament.rounds`. +- `min_round_win_fraction` must be a number in `[0, 1)`; `0` drops the majority requirement (e.g. `min_round_win_fraction: 0` + `win_last_k: 1` means "just win the final round"). diff --git a/configs/ablations/ladder/battlesnake.yaml b/configs/ablations/ladder/battlesnake.yaml index e4e40c77..689a5d68 100644 --- a/configs/ablations/ladder/battlesnake.yaml +++ b/configs/ablations/ladder/battlesnake.yaml @@ -4,6 +4,9 @@ # rung until it loses. Run: uv run codeclash ladder run configs/ablations/ladder/battlesnake.yaml tournament: rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) game: name: BattleSnake sims_per_round: 250 @@ -26,104 +29,4 @@ player: prompts: game_description: |- BattleSnake ladder -ladder: -- agent: dummy - branch_init: human/pambrose/pambrose-kotlin -- agent: dummy - branch_init: human/Nettogrof/nessegrev-julia -- agent: dummy - branch_init: human/Nettogrof/nessegrev-java -- agent: dummy - branch_init: human/csauve/bookworm -- agent: dummy - branch_init: human/coreyja/improbable-irene -- agent: dummy - branch_init: human/graeme-hill/snakebot -- agent: dummy - branch_init: human/coreyja/devious-devin -- agent: dummy - branch_init: human/m-schier/kreuzotter -- agent: dummy - branch_init: human/nbw/nbw-crystal -- agent: dummy - branch_init: human/Xe/since -- agent: dummy - branch_init: human/ccSnake2018/ccsnake -- agent: dummy - branch_init: human/coreyja/bombastic-bob -- agent: dummy - branch_init: human/coreyja/coreyja-rs -- agent: dummy - branch_init: human/coreyja/jump-flooding -- agent: dummy - branch_init: human/zacpez/scape-goat -- agent: dummy - branch_init: human/tim-hub/awesome-snake -- agent: dummy - branch_init: human/rdbrck/btas -- agent: dummy - branch_init: human/Spenca/vulture-snake -- agent: dummy - branch_init: human/moxuz/pinky-snek -- agent: dummy - branch_init: human/coreyja/amphibious-arthur -- agent: dummy - branch_init: human/OliverMKing/astar-snake -- agent: dummy - branch_init: human/nbw/nbw-ruby -- agent: dummy - branch_init: human/coreyja/eremetic-eric -- agent: dummy - branch_init: human/coreyja/gigantic-george -- agent: dummy - branch_init: human/Flipez/flipez-crystal -- agent: dummy - branch_init: human/jackisherwood/battlesnake-elon -- agent: dummy - branch_init: human/MorganConrad/tantilla -- agent: dummy - branch_init: human/ChaelCodes/cornelius -- agent: dummy - branch_init: human/joshhartmann11/battlejake2019 -- agent: dummy - branch_init: human/coreyja/famished-frank -- agent: dummy - branch_init: human/kentmacdonald2/beames -- agent: dummy - branch_init: human/TheApX/hungry -- agent: dummy - branch_init: human/xtagon/nagini -- agent: dummy - branch_init: human/joshhartmann11/battlejake -- agent: dummy - branch_init: human/tyrelh/tyrelh-python -- agent: dummy - branch_init: human/zakwht/zakwht-2018 -- agent: dummy - branch_init: human/rdbrck/bountysnake2018 -- agent: dummy - branch_init: human/JerryKott/jerrykott-2017 -- agent: dummy - branch_init: human/altersaddle/untimely-neglected-wearable -- agent: dummy - branch_init: human/MorganConrad/sisiutl -- agent: dummy - branch_init: human/tyrelh/tyrelh-2018 -- agent: dummy - branch_init: human/noahspriggs/tr-8r -- agent: dummy - branch_init: human/woofers/woofers-java -- agent: dummy - branch_init: human/Petah/project-z -- agent: dummy - branch_init: human/tbgiles/feisty-snake -- agent: dummy - branch_init: human/hirethissnake/sneaky-snake -- agent: dummy - branch_init: human/aleksiy325/snek-two -- agent: dummy - branch_init: human/tyrelh/tyrelh-2019 -- agent: dummy - branch_init: human/jhawthorn/snek -- agent: dummy - branch_init: human/smallsco/robosnake +ladder: !include ablations/ladder/rungs/battlesnake.yaml diff --git a/configs/ablations/ladder/battlesnake__gemini_3_5_flash.yaml b/configs/ablations/ladder/battlesnake__gemini_3_5_flash.yaml new file mode 100644 index 00000000..f5f2eca2 --- /dev/null +++ b/configs/ablations/ladder/battlesnake__gemini_3_5_flash.yaml @@ -0,0 +1,26 @@ +# BattleSnake ladder climbed by gemini-3-5-flash. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/battlesnake__gemini_3_5_flash.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: BattleSnake + sims_per_round: 250 + args: + width: 11 + height: 11 + browser: false +player: + agent: mini + name: gemini-3-5-flash + branch_init: human/pambrose/pambrose-kotlin + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_gemini_3_5_flash.yaml + push: True +prompts: + game_description: |- + BattleSnake ladder +ladder: !include ablations/ladder/rungs/battlesnake.yaml diff --git a/configs/ablations/ladder/battlesnake__gpt_5_5.yaml b/configs/ablations/ladder/battlesnake__gpt_5_5.yaml new file mode 100644 index 00000000..44c02b13 --- /dev/null +++ b/configs/ablations/ladder/battlesnake__gpt_5_5.yaml @@ -0,0 +1,26 @@ +# BattleSnake ladder climbed by gpt-5-5. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/battlesnake__gpt_5_5.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: BattleSnake + sims_per_round: 250 + args: + width: 11 + height: 11 + browser: false +player: + agent: mini + name: gpt-5-5 + branch_init: human/pambrose/pambrose-kotlin + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_gpt_5_5.yaml + push: True +prompts: + game_description: |- + BattleSnake ladder +ladder: !include ablations/ladder/rungs/battlesnake.yaml diff --git a/configs/ablations/ladder/battlesnake__opus_4_7.yaml b/configs/ablations/ladder/battlesnake__opus_4_7.yaml new file mode 100644 index 00000000..7bca867e --- /dev/null +++ b/configs/ablations/ladder/battlesnake__opus_4_7.yaml @@ -0,0 +1,26 @@ +# BattleSnake ladder climbed by opus-4-7. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/battlesnake__opus_4_7.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: BattleSnake + sims_per_round: 250 + args: + width: 11 + height: 11 + browser: false +player: + agent: mini + name: opus-4-7 + branch_init: human/pambrose/pambrose-kotlin + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_opus_4_7.yaml + push: True +prompts: + game_description: |- + BattleSnake ladder +ladder: !include ablations/ladder/rungs/battlesnake.yaml diff --git a/configs/ablations/ladder/battlesnake__opus_4_8.yaml b/configs/ablations/ladder/battlesnake__opus_4_8.yaml new file mode 100644 index 00000000..9c4e8d76 --- /dev/null +++ b/configs/ablations/ladder/battlesnake__opus_4_8.yaml @@ -0,0 +1,26 @@ +# BattleSnake ladder climbed by opus-4-8. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/battlesnake__opus_4_8.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: BattleSnake + sims_per_round: 250 + args: + width: 11 + height: 11 + browser: false +player: + agent: mini + name: opus-4-8 + branch_init: human/pambrose/pambrose-kotlin + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_opus_4_8.yaml + push: True +prompts: + game_description: |- + BattleSnake ladder +ladder: !include ablations/ladder/rungs/battlesnake.yaml diff --git a/configs/ablations/ladder/battlesnake__sonnet_5.yaml b/configs/ablations/ladder/battlesnake__sonnet_5.yaml new file mode 100644 index 00000000..63c50e89 --- /dev/null +++ b/configs/ablations/ladder/battlesnake__sonnet_5.yaml @@ -0,0 +1,26 @@ +# BattleSnake ladder climbed by sonnet-5. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/battlesnake__sonnet_5.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: BattleSnake + sims_per_round: 250 + args: + width: 11 + height: 11 + browser: false +player: + agent: mini + name: sonnet-5 + branch_init: human/pambrose/pambrose-kotlin + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_sonnet_5.yaml + push: True +prompts: + game_description: |- + BattleSnake ladder +ladder: !include ablations/ladder/rungs/battlesnake.yaml diff --git a/configs/ablations/ladder/battlesnake_llama_smoke.yaml b/configs/ablations/ladder/battlesnake_llama_smoke.yaml new file mode 100644 index 00000000..c0390dd6 --- /dev/null +++ b/configs/ablations/ladder/battlesnake_llama_smoke.yaml @@ -0,0 +1,30 @@ +# Cheap smoke to validate the llama-endpoint wiring before a full ladder run: +# Opus 4.8 (native Anthropic API via AnthropicModel) climbs a 2-rung BattleSnake ladder. +# 1 round, 10 sims — just enough to confirm the model authenticates, caches, and edits code. +# Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/battlesnake_llama_smoke.yaml -c +tournament: + rounds: 1 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: BattleSnake + sims_per_round: 10 + args: + width: 11 + height: 11 + browser: false +player: + agent: mini + name: opus-4-8 + branch_init: human/moxuz/pinky-snek + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_opus_4_8.yaml + push: True +prompts: + game_description: "BattleSnake ladder smoke over the llama endpoint." +ladder: +- {agent: dummy, name: pinky, branch_init: human/moxuz/pinky-snek} # simple heuristic (port) +- {agent: dummy, name: tr8r, branch_init: human/noahspriggs/tr-8r} # 2016 winner (port) diff --git a/configs/ablations/ladder/corewar.yaml b/configs/ablations/ladder/corewar.yaml index 2c2539e9..864e50aa 100644 --- a/configs/ablations/ladder/corewar.yaml +++ b/configs/ablations/ladder/corewar.yaml @@ -1,5 +1,8 @@ tournament: rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) game: name: CoreWar sims_per_round: 2000 @@ -18,532 +21,4 @@ player: prompts: game_description: |- Core War ladder -ladder: -- agent: dummy - branch_init: human/pspace -- agent: dummy - branch_init: human/validate -- agent: dummy - branch_init: human/dwarf -- agent: dummy - branch_init: human/smoothnoodlemap -- agent: dummy - branch_init: human/smoothnoodlemap6 -- agent: dummy - branch_init: human/returnofthelivingdead -- agent: dummy - branch_init: human/notepaper -- agent: dummy - branch_init: human/vagabond -- agent: dummy - branch_init: human/genocide -- agent: dummy - branch_init: human/paratroopsv21 -- agent: dummy - branch_init: human/trinity -- agent: dummy - branch_init: human/precipice -- agent: dummy - branch_init: human/hydra -- agent: dummy - branch_init: human/flypaper30 -- agent: dummy - branch_init: human/gammapaper30 -- agent: dummy - branch_init: human/flashpaper37 -- agent: dummy - branch_init: human/flashpaper -- agent: dummy - branch_init: human/fastfoodv21 -- agent: dummy - branch_init: human/twilightpitsv60 -- agent: dummy - branch_init: human/0stormbringer -- agent: dummy - branch_init: human/backstabber -- agent: dummy - branch_init: human/impurge -- agent: dummy - branch_init: human/requestv20 -- agent: dummy - branch_init: human/griffin2 -- agent: dummy - branch_init: human/imprimis6 -- agent: dummy - branch_init: human/killerinstinct -- agent: dummy - branch_init: human/crimp -- agent: dummy - branch_init: human/crimp2 -- agent: dummy - branch_init: human/bscannersliveinvain -- agent: dummy - branch_init: human/charonv70 -- agent: dummy - branch_init: human/nomuckingabout -- agent: dummy - branch_init: human/leprechaun1b -- agent: dummy - branch_init: human/charonv81 -- agent: dummy - branch_init: human/keystonet13 -- agent: dummy - branch_init: human/rave -- agent: dummy - branch_init: human/sphinxv28 -- agent: dummy - branch_init: human/hordesofmicrowarriors -- agent: dummy - branch_init: human/ncdecoy -- agent: dummy - branch_init: human/agony31 -- agent: dummy - branch_init: human/medusasv7x -- agent: dummy - branch_init: human/sj4a -- agent: dummy - branch_init: human/capskeyisstuck -- agent: dummy - branch_init: human/thermite10 -- agent: dummy - branch_init: human/ttti -- agent: dummy - branch_init: human/agony51 -- agent: dummy - branch_init: human/stasis -- agent: dummy - branch_init: human/leprechaunonspeed -- agent: dummy - branch_init: human/blur88 -- agent: dummy - branch_init: human/winterwerewolf3 -- agent: dummy - branch_init: human/lucky3 -- agent: dummy - branch_init: human/cannonade -- agent: dummy - branch_init: human/kitchensinkii -- agent: dummy - branch_init: human/seventyfive -- agent: dummy - branch_init: human/fatexpansionv -- agent: dummy - branch_init: human/irongate -- agent: dummy - branch_init: human/snake -- agent: dummy - branch_init: human/leapfrog -- agent: dummy - branch_init: human/chimerav35 -- agent: dummy - branch_init: human/leviathan -- agent: dummy - branch_init: human/pacman -- agent: dummy - branch_init: human/heremscimitar -- agent: dummy - branch_init: human/elementaldust2 -- agent: dummy - branch_init: human/foggyswamp -- agent: dummy - branch_init: human/armorya5 -- agent: dummy - branch_init: human/beholderseye17 -- agent: dummy - branch_init: human/phq -- agent: dummy - branch_init: human/abomination -- agent: dummy - branch_init: human/steppingstone -- agent: dummy - branch_init: human/agonyii -- agent: dummy - branch_init: human/twister -- agent: dummy - branch_init: human/evoltmp88 -- agent: dummy - branch_init: human/gothik -- agent: dummy - branch_init: human/quiz -- agent: dummy - branch_init: human/aeka -- agent: dummy - branch_init: human/vamp02b -- agent: dummy - branch_init: human/replicant -- agent: dummy - branch_init: human/gemoftheocean -- agent: dummy - branch_init: human/thermiteii -- agent: dummy - branch_init: human/blur -- agent: dummy - branch_init: human/blur2 -- agent: dummy - branch_init: human/flurry -- agent: dummy - branch_init: human/mirage2 -- agent: dummy - branch_init: human/nightofthelivingdead -- agent: dummy - branch_init: human/mirage15 -- agent: dummy - branch_init: human/soldieroffortune -- agent: dummy - branch_init: human/win -- agent: dummy - branch_init: human/icedragon -- agent: dummy - branch_init: human/onebite -- agent: dummy - branch_init: human/myvamp37 -- agent: dummy - branch_init: human/bluefunk -- agent: dummy - branch_init: human/tornado30 -- agent: dummy - branch_init: human/mason20 -- agent: dummy - branch_init: human/bayonet -- agent: dummy - branch_init: human/tnt -- agent: dummy - branch_init: human/zygote -- agent: dummy - branch_init: human/stalker -- agent: dummy - branch_init: human/julietandpaper -- agent: dummy - branch_init: human/thenextstep88 -- agent: dummy - branch_init: human/chameleon -- agent: dummy - branch_init: human/stoninc -- agent: dummy - branch_init: human/claw -- agent: dummy - branch_init: human/myvamp54 -- agent: dummy - branch_init: human/infiltrator -- agent: dummy - branch_init: human/yogibear -- agent: dummy - branch_init: human/grilledoctopus05 -- agent: dummy - branch_init: human/intotheunknown -- agent: dummy - branch_init: human/probe -- agent: dummy - branch_init: human/torcht18 -- agent: dummy - branch_init: human/damageincorporated -- agent: dummy - branch_init: human/lithium -- agent: dummy - branch_init: human/bluefunk3 -- agent: dummy - branch_init: human/labomba -- agent: dummy - branch_init: human/sneakyb2 -- agent: dummy - branch_init: human/hazyshadeii -- agent: dummy - branch_init: human/blizzard -- agent: dummy - branch_init: human/jackintheboxii -- agent: dummy - branch_init: human/recon2 -- agent: dummy - branch_init: human/curseoftheundead -- agent: dummy - branch_init: human/bloodlust -- agent: dummy - branch_init: human/eternalexile -- agent: dummy - branch_init: human/sprawlingchaos -- agent: dummy - branch_init: human/zooom -- agent: dummy - branch_init: human/vampsareback02 -- agent: dummy - branch_init: human/jinx -- agent: dummy - branch_init: human/pendulum -- agent: dummy - branch_init: human/nosferatu -- agent: dummy - branch_init: human/boysarebackintown -- agent: dummy - branch_init: human/discord -- agent: dummy - branch_init: human/jackinthebox -- agent: dummy - branch_init: human/barrage -- agent: dummy - branch_init: human/perseus -- agent: dummy - branch_init: human/kusanagi -- agent: dummy - branch_init: human/simple88v2 -- agent: dummy - branch_init: human/excalibur -- agent: dummy - branch_init: human/carmilla -- agent: dummy - branch_init: human/bpanamax -- agent: dummy - branch_init: human/oblivion -- agent: dummy - branch_init: human/hector2 -- agent: dummy - branch_init: human/xenosmilus -- agent: dummy - branch_init: human/macromagic -- agent: dummy - branch_init: human/whitemist -- agent: dummy - branch_init: human/fireandice -- agent: dummy - branch_init: human/grendelsrevenge -- agent: dummy - branch_init: human/herbalavenger -- agent: dummy - branch_init: human/unpit -- agent: dummy - branch_init: human/dust07 -- agent: dummy - branch_init: human/alladinscave -- agent: dummy - branch_init: human/hazylazyc11 -- agent: dummy - branch_init: human/bigitalshot -- agent: dummy - branch_init: human/shottonothing -- agent: dummy - branch_init: human/hazylazy -- agent: dummy - branch_init: human/valkyrie -- agent: dummy - branch_init: human/enigma -- agent: dummy - branch_init: human/lithobolia -- agent: dummy - branch_init: human/electrichead -- agent: dummy - branch_init: human/arrow -- agent: dummy - branch_init: human/blade -- agent: dummy - branch_init: human/vanquisher -- agent: dummy - branch_init: human/unpitq -- agent: dummy - branch_init: human/sputnik -- agent: dummy - branch_init: human/forgottenlore -- agent: dummy - branch_init: human/returnofvanquisher -- agent: dummy - branch_init: human/behemot -- agent: dummy - branch_init: human/impfinityv4g1 -- agent: dummy - branch_init: human/torment -- agent: dummy - branch_init: human/falconv03 -- agent: dummy - branch_init: human/borg -- agent: dummy - branch_init: human/thehistorian -- agent: dummy - branch_init: human/nightterrors -- agent: dummy - branch_init: human/hellfire -- agent: dummy - branch_init: human/revivalfire -- agent: dummy - branch_init: human/timescape10 -- agent: dummy - branch_init: human/forgottenlore2 -- agent: dummy - branch_init: human/electricrazor -- agent: dummy - branch_init: human/freighttrain -- agent: dummy - branch_init: human/digitalis2003 -- agent: dummy - branch_init: human/kryptonite -- agent: dummy - branch_init: human/riseofthedragon -- agent: dummy - branch_init: human/bluecandle -- agent: dummy - branch_init: human/rosebud -- agent: dummy - branch_init: human/slimetest -- agent: dummy - branch_init: human/quicksilver -- agent: dummy - branch_init: human/stormkeeper -- agent: dummy - branch_init: human/ompega -- agent: dummy - branch_init: human/nemesis -- agent: dummy - branch_init: human/fixed -- agent: dummy - branch_init: human/evolcap66 -- agent: dummy - branch_init: human/retroq -- agent: dummy - branch_init: human/devilish202 -- agent: dummy - branch_init: human/sunset -- agent: dummy - branch_init: human/blacken -- agent: dummy - branch_init: human/nighttrain -- agent: dummy - branch_init: human/diehard -- agent: dummy - branch_init: human/bulldozed -- agent: dummy - branch_init: human/revengeofthepapers -- agent: dummy - branch_init: human/uninvited -- agent: dummy - branch_init: human/disharmonious -- agent: dummy - branch_init: human/bitethebullet -- agent: dummy - branch_init: human/vain -- agent: dummy - branch_init: human/luca -- agent: dummy - branch_init: human/jade -- agent: dummy - branch_init: human/recycledbits -- agent: dummy - branch_init: human/spiritualblackdimension -- agent: dummy - branch_init: human/themystery -- agent: dummy - branch_init: human/unrequitedlove -- agent: dummy - branch_init: human/borgir -- agent: dummy - branch_init: human/gremlin -- agent: dummy - branch_init: human/gigolo -- agent: dummy - branch_init: human/ironicimps -- agent: dummy - branch_init: human/stylizedeuphoria -- agent: dummy - branch_init: human/ziggy -- agent: dummy - branch_init: human/impishv02 -- agent: dummy - branch_init: human/thunderstrike -- agent: dummy - branch_init: human/eccentric -- agent: dummy - branch_init: human/hullabaloo -- agent: dummy - branch_init: human/safetyinnumbers -- agent: dummy - branch_init: human/mandragora -- agent: dummy - branch_init: human/gargantuan -- agent: dummy - branch_init: human/elvenking -- agent: dummy - branch_init: human/npaperii -- agent: dummy - branch_init: human/hullab3loo -- agent: dummy - branch_init: human/reepicheep -- agent: dummy - branch_init: human/halcyon -- agent: dummy - branch_init: human/olivia -- agent: dummy - branch_init: human/neith -- agent: dummy - branch_init: human/numb -- agent: dummy - branch_init: human/returnofthependragon -- agent: dummy - branch_init: human/cinammon -- agent: dummy - branch_init: human/combatra -- agent: dummy - branch_init: human/armadillo -- agent: dummy - branch_init: human/simplicity -- agent: dummy - branch_init: human/kosmos -- agent: dummy - branch_init: human/azathoth -- agent: dummy - branch_init: human/danceoffallenangels -- agent: dummy - branch_init: human/returnofthejedimp -- agent: dummy - branch_init: human/blowrag -- agent: dummy - branch_init: human/artofcorewar -- agent: dummy - branch_init: human/silking -- agent: dummy - branch_init: human/goldeneye -- agent: dummy - branch_init: human/dawn -- agent: dummy - branch_init: human/sonofvain -- agent: dummy - branch_init: human/blackknight -- agent: dummy - branch_init: human/thefugitive -- agent: dummy - branch_init: human/frothandfizzle -- agent: dummy - branch_init: human/snowscan -- agent: dummy - branch_init: human/rust -- agent: dummy - branch_init: human/lastjudgement -- agent: dummy - branch_init: human/pdqscan -- agent: dummy - branch_init: human/mercenary -- agent: dummy - branch_init: human/dawn2 -- agent: dummy - branch_init: human/firestorm -- agent: dummy - branch_init: human/defensive -- agent: dummy - branch_init: human/burningmetal -- agent: dummy - branch_init: human/chainlockv02a -- agent: dummy - branch_init: human/decoysignal -- agent: dummy - branch_init: human/cloudburst -- agent: dummy - branch_init: human/mascafe -- agent: dummy - branch_init: human/devilstick -- agent: dummy - branch_init: human/unheardof -- agent: dummy - branch_init: human/returnofthefugitive -- agent: dummy - branch_init: human/silkworm -- agent: dummy - branch_init: human/maelstrom -- agent: dummy - branch_init: human/forjohn -- agent: dummy - branch_init: human/toxic +ladder: !include ablations/ladder/rungs/corewar.yaml diff --git a/configs/ablations/ladder/corewar__gemini_3_5_flash.yaml b/configs/ablations/ladder/corewar__gemini_3_5_flash.yaml new file mode 100644 index 00000000..ef5f1220 --- /dev/null +++ b/configs/ablations/ladder/corewar__gemini_3_5_flash.yaml @@ -0,0 +1,22 @@ +# CoreWar ladder climbed by gemini-3-5-flash. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/corewar__gemini_3_5_flash.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: CoreWar + sims_per_round: 2000 +player: + agent: mini + name: gemini-3-5-flash + branch_init: human/pspace + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_gemini_3_5_flash.yaml + push: True +prompts: + game_description: |- + Core War ladder +ladder: !include ablations/ladder/rungs/corewar.yaml diff --git a/configs/ablations/ladder/corewar__gpt_5_5.yaml b/configs/ablations/ladder/corewar__gpt_5_5.yaml new file mode 100644 index 00000000..ee3c5186 --- /dev/null +++ b/configs/ablations/ladder/corewar__gpt_5_5.yaml @@ -0,0 +1,22 @@ +# CoreWar ladder climbed by gpt-5-5. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/corewar__gpt_5_5.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: CoreWar + sims_per_round: 2000 +player: + agent: mini + name: gpt-5-5 + branch_init: human/pspace + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_gpt_5_5.yaml + push: True +prompts: + game_description: |- + Core War ladder +ladder: !include ablations/ladder/rungs/corewar.yaml diff --git a/configs/ablations/ladder/corewar__opus_4_7.yaml b/configs/ablations/ladder/corewar__opus_4_7.yaml new file mode 100644 index 00000000..a3991526 --- /dev/null +++ b/configs/ablations/ladder/corewar__opus_4_7.yaml @@ -0,0 +1,22 @@ +# CoreWar ladder climbed by opus-4-7. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/corewar__opus_4_7.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: CoreWar + sims_per_round: 2000 +player: + agent: mini + name: opus-4-7 + branch_init: human/pspace + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_opus_4_7.yaml + push: True +prompts: + game_description: |- + Core War ladder +ladder: !include ablations/ladder/rungs/corewar.yaml diff --git a/configs/ablations/ladder/corewar__opus_4_8.yaml b/configs/ablations/ladder/corewar__opus_4_8.yaml new file mode 100644 index 00000000..dc0ef353 --- /dev/null +++ b/configs/ablations/ladder/corewar__opus_4_8.yaml @@ -0,0 +1,22 @@ +# CoreWar ladder climbed by opus-4-8. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/corewar__opus_4_8.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: CoreWar + sims_per_round: 2000 +player: + agent: mini + name: opus-4-8 + branch_init: human/pspace + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_opus_4_8.yaml + push: True +prompts: + game_description: |- + Core War ladder +ladder: !include ablations/ladder/rungs/corewar.yaml diff --git a/configs/ablations/ladder/corewar__sonnet_5.yaml b/configs/ablations/ladder/corewar__sonnet_5.yaml new file mode 100644 index 00000000..a429c6b5 --- /dev/null +++ b/configs/ablations/ladder/corewar__sonnet_5.yaml @@ -0,0 +1,22 @@ +# CoreWar ladder climbed by sonnet-5. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/corewar__sonnet_5.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: CoreWar + sims_per_round: 2000 +player: + agent: mini + name: sonnet-5 + branch_init: human/pspace + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_sonnet_5.yaml + push: True +prompts: + game_description: |- + Core War ladder +ladder: !include ablations/ladder/rungs/corewar.yaml diff --git a/configs/ablations/ladder/robotrumble.yaml b/configs/ablations/ladder/robotrumble.yaml index 5f5bf17f..52298a69 100644 --- a/configs/ablations/ladder/robotrumble.yaml +++ b/configs/ablations/ladder/robotrumble.yaml @@ -1,5 +1,8 @@ tournament: rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) game: name: RobotRumble sims_per_round: 250 @@ -20,120 +23,4 @@ player: prompts: game_description: |- RobotRumble ladder -ladder: -- agent: dummy - branch_init: human/anton/anton3000 -- agent: dummy - branch_init: human/happysquid/test -- agent: dummy - branch_init: human/anton/wallifier -- agent: dummy - branch_init: human/ldang/nessy -- agent: dummy - branch_init: human/ldang/nemo -- agent: dummy - branch_init: human/navster8/bash-brothers -- agent: dummy - branch_init: human/aaoutkine/dark-knight -- agent: dummy - branch_init: human/mountain/neuralbot1-1h -- agent: dummy - branch_init: human/sivecano/clouded-mind -- agent: dummy - branch_init: human/mountain/neuralbot2-6h -- agent: dummy - branch_init: human/kalkin/artemis -- agent: dummy - branch_init: human/kalkin/artemis2 -- agent: dummy - branch_init: human/navster8/maginot-line -- agent: dummy - branch_init: human/jiricodes/jiricodes-bot -- agent: dummy - branch_init: human/sbasu3/meek-bot -- agent: dummy - branch_init: human/essickmango/fruity-test -- agent: dummy - branch_init: human/tabaxi3k/charles -- agent: dummy - branch_init: human/devchris/first_test -- agent: dummy - branch_init: human/aaa/jippty5 -- agent: dummy - branch_init: human/jay0jayjay/naivestarter -- agent: dummy - branch_init: human/luisa/luisasrobot -- agent: dummy - branch_init: human/luisa/baselinegere -- agent: dummy - branch_init: human/anton/anton4000 -- agent: dummy - branch_init: human/aayyad/testbot -- agent: dummy - branch_init: human/edward/flail -- agent: dummy - branch_init: human/mousetail/genetic-robot -- agent: dummy - branch_init: human/kalkin/maxad -- agent: dummy - branch_init: human/mjburgess/rule99 -- agent: dummy - branch_init: human/ketza/bob -- agent: dummy - branch_init: human/suddenlyseals/control-center -- agent: dummy - branch_init: human/aaoutkine/school-bot -- agent: dummy - branch_init: human/thesmilingturtl/naivefaa -- agent: dummy - branch_init: human/mario31313/alpha_13 -- agent: dummy - branch_init: human/underscore/bot1 -- agent: dummy - branch_init: human/lanity/sivuy -- agent: dummy - branch_init: human/mee42/follow-bot -- agent: dummy - branch_init: human/anton/om-om -- agent: dummy - branch_init: human/aaoutkine/silo34 -- agent: dummy - branch_init: human/mountain/neuralbot4-3h -- agent: dummy - branch_init: human/ketza/arthur -- agent: dummy - branch_init: human/mkap/test -- agent: dummy - branch_init: human/essickmango/pickle-up -- agent: dummy - branch_init: human/wolfsleuth/simple -- agent: dummy - branch_init: human/gerenuk/gere-ape -- agent: dummy - branch_init: human/clay/diag-lattice -- agent: dummy - branch_init: human/atl15/centerrr -- agent: dummy - branch_init: human/jammyliu/sixty-nine-line -- agent: dummy - branch_init: human/mitch84/walk_retreat -- agent: dummy - branch_init: human/tabaxi3k/black-magic-1 -- agent: dummy - branch_init: human/devchris/black_magic -- agent: dummy - branch_init: human/mitch84/retreat_walk2 -- agent: dummy - branch_init: human/mitch84/crw_preempt -- agent: dummy - branch_init: human/entropicdrifter/glommer -- agent: dummy - branch_init: human/mousetail/coward-bot -- agent: dummy - branch_init: human/entropicdrifter/glommerv2 -- agent: dummy - branch_init: human/entropicdrifter/we-are-borg -- agent: dummy - branch_init: human/entropicdrifter/seven-of-nine -- agent: dummy - branch_init: human/entropicdrifter/gigachad +ladder: !include ablations/ladder/rungs/robotrumble.yaml diff --git a/configs/ablations/ladder/robotrumble__gemini_3_5_flash.yaml b/configs/ablations/ladder/robotrumble__gemini_3_5_flash.yaml new file mode 100644 index 00000000..673edb08 --- /dev/null +++ b/configs/ablations/ladder/robotrumble__gemini_3_5_flash.yaml @@ -0,0 +1,24 @@ +# RobotRumble ladder climbed by gemini-3-5-flash. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/robotrumble__gemini_3_5_flash.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: RobotRumble + sims_per_round: 250 + args: + raw: false +player: + agent: mini + name: gemini-3-5-flash + branch_init: human/anton/anton3000 + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_gemini_3_5_flash.yaml + push: True +prompts: + game_description: |- + RobotRumble ladder +ladder: !include ablations/ladder/rungs/robotrumble.yaml diff --git a/configs/ablations/ladder/robotrumble__gpt_5_5.yaml b/configs/ablations/ladder/robotrumble__gpt_5_5.yaml new file mode 100644 index 00000000..cb0e0eb5 --- /dev/null +++ b/configs/ablations/ladder/robotrumble__gpt_5_5.yaml @@ -0,0 +1,24 @@ +# RobotRumble ladder climbed by gpt-5-5. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/robotrumble__gpt_5_5.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: RobotRumble + sims_per_round: 250 + args: + raw: false +player: + agent: mini + name: gpt-5-5 + branch_init: human/anton/anton3000 + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_gpt_5_5.yaml + push: True +prompts: + game_description: |- + RobotRumble ladder +ladder: !include ablations/ladder/rungs/robotrumble.yaml diff --git a/configs/ablations/ladder/robotrumble__opus_4_7.yaml b/configs/ablations/ladder/robotrumble__opus_4_7.yaml new file mode 100644 index 00000000..54e73ba6 --- /dev/null +++ b/configs/ablations/ladder/robotrumble__opus_4_7.yaml @@ -0,0 +1,24 @@ +# RobotRumble ladder climbed by opus-4-7. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/robotrumble__opus_4_7.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: RobotRumble + sims_per_round: 250 + args: + raw: false +player: + agent: mini + name: opus-4-7 + branch_init: human/anton/anton3000 + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_opus_4_7.yaml + push: True +prompts: + game_description: |- + RobotRumble ladder +ladder: !include ablations/ladder/rungs/robotrumble.yaml diff --git a/configs/ablations/ladder/robotrumble__opus_4_8.yaml b/configs/ablations/ladder/robotrumble__opus_4_8.yaml new file mode 100644 index 00000000..83a49437 --- /dev/null +++ b/configs/ablations/ladder/robotrumble__opus_4_8.yaml @@ -0,0 +1,24 @@ +# RobotRumble ladder climbed by opus-4-8. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/robotrumble__opus_4_8.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: RobotRumble + sims_per_round: 250 + args: + raw: false +player: + agent: mini + name: opus-4-8 + branch_init: human/anton/anton3000 + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_opus_4_8.yaml + push: True +prompts: + game_description: |- + RobotRumble ladder +ladder: !include ablations/ladder/rungs/robotrumble.yaml diff --git a/configs/ablations/ladder/robotrumble__sonnet_5.yaml b/configs/ablations/ladder/robotrumble__sonnet_5.yaml new file mode 100644 index 00000000..79cf31e0 --- /dev/null +++ b/configs/ablations/ladder/robotrumble__sonnet_5.yaml @@ -0,0 +1,24 @@ +# RobotRumble ladder climbed by sonnet-5. Requires LLAMA_API_KEY in .env. +# uv run codeclash ladder run configs/ablations/ladder/robotrumble__sonnet_5.yaml +tournament: + rounds: 5 +ladder_rules: + min_round_win_fraction: 0.5 # advance only on a strict majority of rounds + win_last_k: 1 # ...and win the last K rounds (K=1 == just the final round) +game: + name: RobotRumble + sims_per_round: 250 + args: + raw: false +player: + agent: mini + name: sonnet-5 + branch_init: human/anton/anton3000 + config: + agent: !include mini/default.yaml + model: !include mini/models/llama_sonnet_5.yaml + push: True +prompts: + game_description: |- + RobotRumble ladder +ladder: !include ablations/ladder/rungs/robotrumble.yaml diff --git a/configs/ablations/ladder/rungs/battlesnake.yaml b/configs/ablations/ladder/rungs/battlesnake.yaml new file mode 100644 index 00000000..5f527933 --- /dev/null +++ b/configs/ablations/ladder/rungs/battlesnake.yaml @@ -0,0 +1,100 @@ +- agent: dummy + branch_init: human/pambrose/pambrose-kotlin +- agent: dummy + branch_init: human/Nettogrof/nessegrev-julia +- agent: dummy + branch_init: human/Nettogrof/nessegrev-java +- agent: dummy + branch_init: human/csauve/bookworm +- agent: dummy + branch_init: human/coreyja/improbable-irene +- agent: dummy + branch_init: human/graeme-hill/snakebot +- agent: dummy + branch_init: human/coreyja/devious-devin +- agent: dummy + branch_init: human/m-schier/kreuzotter +- agent: dummy + branch_init: human/nbw/nbw-crystal +- agent: dummy + branch_init: human/Xe/since +- agent: dummy + branch_init: human/ccSnake2018/ccsnake +- agent: dummy + branch_init: human/coreyja/bombastic-bob +- agent: dummy + branch_init: human/coreyja/coreyja-rs +- agent: dummy + branch_init: human/coreyja/jump-flooding +- agent: dummy + branch_init: human/zacpez/scape-goat +- agent: dummy + branch_init: human/tim-hub/awesome-snake +- agent: dummy + branch_init: human/rdbrck/btas +- agent: dummy + branch_init: human/Spenca/vulture-snake +- agent: dummy + branch_init: human/moxuz/pinky-snek +- agent: dummy + branch_init: human/coreyja/amphibious-arthur +- agent: dummy + branch_init: human/OliverMKing/astar-snake +- agent: dummy + branch_init: human/nbw/nbw-ruby +- agent: dummy + branch_init: human/coreyja/eremetic-eric +- agent: dummy + branch_init: human/coreyja/gigantic-george +- agent: dummy + branch_init: human/Flipez/flipez-crystal +- agent: dummy + branch_init: human/jackisherwood/battlesnake-elon +- agent: dummy + branch_init: human/MorganConrad/tantilla +- agent: dummy + branch_init: human/ChaelCodes/cornelius +- agent: dummy + branch_init: human/joshhartmann11/battlejake2019 +- agent: dummy + branch_init: human/coreyja/famished-frank +- agent: dummy + branch_init: human/kentmacdonald2/beames +- agent: dummy + branch_init: human/TheApX/hungry +- agent: dummy + branch_init: human/xtagon/nagini +- agent: dummy + branch_init: human/joshhartmann11/battlejake +- agent: dummy + branch_init: human/tyrelh/tyrelh-python +- agent: dummy + branch_init: human/zakwht/zakwht-2018 +- agent: dummy + branch_init: human/rdbrck/bountysnake2018 +- agent: dummy + branch_init: human/JerryKott/jerrykott-2017 +- agent: dummy + branch_init: human/altersaddle/untimely-neglected-wearable +- agent: dummy + branch_init: human/MorganConrad/sisiutl +- agent: dummy + branch_init: human/tyrelh/tyrelh-2018 +- agent: dummy + branch_init: human/noahspriggs/tr-8r +- agent: dummy + branch_init: human/woofers/woofers-java +- agent: dummy + branch_init: human/Petah/project-z +- agent: dummy + branch_init: human/tbgiles/feisty-snake +- agent: dummy + branch_init: human/hirethissnake/sneaky-snake +- agent: dummy + branch_init: human/aleksiy325/snek-two +- agent: dummy + branch_init: human/tyrelh/tyrelh-2019 +- agent: dummy + branch_init: human/jhawthorn/snek +- agent: dummy + branch_init: human/smallsco/robosnake diff --git a/configs/ablations/ladder/rungs/corewar.yaml b/configs/ablations/ladder/rungs/corewar.yaml new file mode 100644 index 00000000..0afc23ad --- /dev/null +++ b/configs/ablations/ladder/rungs/corewar.yaml @@ -0,0 +1,528 @@ +- agent: dummy + branch_init: human/pspace +- agent: dummy + branch_init: human/validate +- agent: dummy + branch_init: human/dwarf +- agent: dummy + branch_init: human/smoothnoodlemap +- agent: dummy + branch_init: human/smoothnoodlemap6 +- agent: dummy + branch_init: human/returnofthelivingdead +- agent: dummy + branch_init: human/notepaper +- agent: dummy + branch_init: human/vagabond +- agent: dummy + branch_init: human/genocide +- agent: dummy + branch_init: human/paratroopsv21 +- agent: dummy + branch_init: human/trinity +- agent: dummy + branch_init: human/precipice +- agent: dummy + branch_init: human/hydra +- agent: dummy + branch_init: human/flypaper30 +- agent: dummy + branch_init: human/gammapaper30 +- agent: dummy + branch_init: human/flashpaper37 +- agent: dummy + branch_init: human/flashpaper +- agent: dummy + branch_init: human/fastfoodv21 +- agent: dummy + branch_init: human/twilightpitsv60 +- agent: dummy + branch_init: human/0stormbringer +- agent: dummy + branch_init: human/backstabber +- agent: dummy + branch_init: human/impurge +- agent: dummy + branch_init: human/requestv20 +- agent: dummy + branch_init: human/griffin2 +- agent: dummy + branch_init: human/imprimis6 +- agent: dummy + branch_init: human/killerinstinct +- agent: dummy + branch_init: human/crimp +- agent: dummy + branch_init: human/crimp2 +- agent: dummy + branch_init: human/bscannersliveinvain +- agent: dummy + branch_init: human/charonv70 +- agent: dummy + branch_init: human/nomuckingabout +- agent: dummy + branch_init: human/leprechaun1b +- agent: dummy + branch_init: human/charonv81 +- agent: dummy + branch_init: human/keystonet13 +- agent: dummy + branch_init: human/rave +- agent: dummy + branch_init: human/sphinxv28 +- agent: dummy + branch_init: human/hordesofmicrowarriors +- agent: dummy + branch_init: human/ncdecoy +- agent: dummy + branch_init: human/agony31 +- agent: dummy + branch_init: human/medusasv7x +- agent: dummy + branch_init: human/sj4a +- agent: dummy + branch_init: human/capskeyisstuck +- agent: dummy + branch_init: human/thermite10 +- agent: dummy + branch_init: human/ttti +- agent: dummy + branch_init: human/agony51 +- agent: dummy + branch_init: human/stasis +- agent: dummy + branch_init: human/leprechaunonspeed +- agent: dummy + branch_init: human/blur88 +- agent: dummy + branch_init: human/winterwerewolf3 +- agent: dummy + branch_init: human/lucky3 +- agent: dummy + branch_init: human/cannonade +- agent: dummy + branch_init: human/kitchensinkii +- agent: dummy + branch_init: human/seventyfive +- agent: dummy + branch_init: human/fatexpansionv +- agent: dummy + branch_init: human/irongate +- agent: dummy + branch_init: human/snake +- agent: dummy + branch_init: human/leapfrog +- agent: dummy + branch_init: human/chimerav35 +- agent: dummy + branch_init: human/leviathan +- agent: dummy + branch_init: human/pacman +- agent: dummy + branch_init: human/heremscimitar +- agent: dummy + branch_init: human/elementaldust2 +- agent: dummy + branch_init: human/foggyswamp +- agent: dummy + branch_init: human/armorya5 +- agent: dummy + branch_init: human/beholderseye17 +- agent: dummy + branch_init: human/phq +- agent: dummy + branch_init: human/abomination +- agent: dummy + branch_init: human/steppingstone +- agent: dummy + branch_init: human/agonyii +- agent: dummy + branch_init: human/twister +- agent: dummy + branch_init: human/evoltmp88 +- agent: dummy + branch_init: human/gothik +- agent: dummy + branch_init: human/quiz +- agent: dummy + branch_init: human/aeka +- agent: dummy + branch_init: human/vamp02b +- agent: dummy + branch_init: human/replicant +- agent: dummy + branch_init: human/gemoftheocean +- agent: dummy + branch_init: human/thermiteii +- agent: dummy + branch_init: human/blur +- agent: dummy + branch_init: human/blur2 +- agent: dummy + branch_init: human/flurry +- agent: dummy + branch_init: human/mirage2 +- agent: dummy + branch_init: human/nightofthelivingdead +- agent: dummy + branch_init: human/mirage15 +- agent: dummy + branch_init: human/soldieroffortune +- agent: dummy + branch_init: human/win +- agent: dummy + branch_init: human/icedragon +- agent: dummy + branch_init: human/onebite +- agent: dummy + branch_init: human/myvamp37 +- agent: dummy + branch_init: human/bluefunk +- agent: dummy + branch_init: human/tornado30 +- agent: dummy + branch_init: human/mason20 +- agent: dummy + branch_init: human/bayonet +- agent: dummy + branch_init: human/tnt +- agent: dummy + branch_init: human/zygote +- agent: dummy + branch_init: human/stalker +- agent: dummy + branch_init: human/julietandpaper +- agent: dummy + branch_init: human/thenextstep88 +- agent: dummy + branch_init: human/chameleon +- agent: dummy + branch_init: human/stoninc +- agent: dummy + branch_init: human/claw +- agent: dummy + branch_init: human/myvamp54 +- agent: dummy + branch_init: human/infiltrator +- agent: dummy + branch_init: human/yogibear +- agent: dummy + branch_init: human/grilledoctopus05 +- agent: dummy + branch_init: human/intotheunknown +- agent: dummy + branch_init: human/probe +- agent: dummy + branch_init: human/torcht18 +- agent: dummy + branch_init: human/damageincorporated +- agent: dummy + branch_init: human/lithium +- agent: dummy + branch_init: human/bluefunk3 +- agent: dummy + branch_init: human/labomba +- agent: dummy + branch_init: human/sneakyb2 +- agent: dummy + branch_init: human/hazyshadeii +- agent: dummy + branch_init: human/blizzard +- agent: dummy + branch_init: human/jackintheboxii +- agent: dummy + branch_init: human/recon2 +- agent: dummy + branch_init: human/curseoftheundead +- agent: dummy + branch_init: human/bloodlust +- agent: dummy + branch_init: human/eternalexile +- agent: dummy + branch_init: human/sprawlingchaos +- agent: dummy + branch_init: human/zooom +- agent: dummy + branch_init: human/vampsareback02 +- agent: dummy + branch_init: human/jinx +- agent: dummy + branch_init: human/pendulum +- agent: dummy + branch_init: human/nosferatu +- agent: dummy + branch_init: human/boysarebackintown +- agent: dummy + branch_init: human/discord +- agent: dummy + branch_init: human/jackinthebox +- agent: dummy + branch_init: human/barrage +- agent: dummy + branch_init: human/perseus +- agent: dummy + branch_init: human/kusanagi +- agent: dummy + branch_init: human/simple88v2 +- agent: dummy + branch_init: human/excalibur +- agent: dummy + branch_init: human/carmilla +- agent: dummy + branch_init: human/bpanamax +- agent: dummy + branch_init: human/oblivion +- agent: dummy + branch_init: human/hector2 +- agent: dummy + branch_init: human/xenosmilus +- agent: dummy + branch_init: human/macromagic +- agent: dummy + branch_init: human/whitemist +- agent: dummy + branch_init: human/fireandice +- agent: dummy + branch_init: human/grendelsrevenge +- agent: dummy + branch_init: human/herbalavenger +- agent: dummy + branch_init: human/unpit +- agent: dummy + branch_init: human/dust07 +- agent: dummy + branch_init: human/alladinscave +- agent: dummy + branch_init: human/hazylazyc11 +- agent: dummy + branch_init: human/bigitalshot +- agent: dummy + branch_init: human/shottonothing +- agent: dummy + branch_init: human/hazylazy +- agent: dummy + branch_init: human/valkyrie +- agent: dummy + branch_init: human/enigma +- agent: dummy + branch_init: human/lithobolia +- agent: dummy + branch_init: human/electrichead +- agent: dummy + branch_init: human/arrow +- agent: dummy + branch_init: human/blade +- agent: dummy + branch_init: human/vanquisher +- agent: dummy + branch_init: human/unpitq +- agent: dummy + branch_init: human/sputnik +- agent: dummy + branch_init: human/forgottenlore +- agent: dummy + branch_init: human/returnofvanquisher +- agent: dummy + branch_init: human/behemot +- agent: dummy + branch_init: human/impfinityv4g1 +- agent: dummy + branch_init: human/torment +- agent: dummy + branch_init: human/falconv03 +- agent: dummy + branch_init: human/borg +- agent: dummy + branch_init: human/thehistorian +- agent: dummy + branch_init: human/nightterrors +- agent: dummy + branch_init: human/hellfire +- agent: dummy + branch_init: human/revivalfire +- agent: dummy + branch_init: human/timescape10 +- agent: dummy + branch_init: human/forgottenlore2 +- agent: dummy + branch_init: human/electricrazor +- agent: dummy + branch_init: human/freighttrain +- agent: dummy + branch_init: human/digitalis2003 +- agent: dummy + branch_init: human/kryptonite +- agent: dummy + branch_init: human/riseofthedragon +- agent: dummy + branch_init: human/bluecandle +- agent: dummy + branch_init: human/rosebud +- agent: dummy + branch_init: human/slimetest +- agent: dummy + branch_init: human/quicksilver +- agent: dummy + branch_init: human/stormkeeper +- agent: dummy + branch_init: human/ompega +- agent: dummy + branch_init: human/nemesis +- agent: dummy + branch_init: human/fixed +- agent: dummy + branch_init: human/evolcap66 +- agent: dummy + branch_init: human/retroq +- agent: dummy + branch_init: human/devilish202 +- agent: dummy + branch_init: human/sunset +- agent: dummy + branch_init: human/blacken +- agent: dummy + branch_init: human/nighttrain +- agent: dummy + branch_init: human/diehard +- agent: dummy + branch_init: human/bulldozed +- agent: dummy + branch_init: human/revengeofthepapers +- agent: dummy + branch_init: human/uninvited +- agent: dummy + branch_init: human/disharmonious +- agent: dummy + branch_init: human/bitethebullet +- agent: dummy + branch_init: human/vain +- agent: dummy + branch_init: human/luca +- agent: dummy + branch_init: human/jade +- agent: dummy + branch_init: human/recycledbits +- agent: dummy + branch_init: human/spiritualblackdimension +- agent: dummy + branch_init: human/themystery +- agent: dummy + branch_init: human/unrequitedlove +- agent: dummy + branch_init: human/borgir +- agent: dummy + branch_init: human/gremlin +- agent: dummy + branch_init: human/gigolo +- agent: dummy + branch_init: human/ironicimps +- agent: dummy + branch_init: human/stylizedeuphoria +- agent: dummy + branch_init: human/ziggy +- agent: dummy + branch_init: human/impishv02 +- agent: dummy + branch_init: human/thunderstrike +- agent: dummy + branch_init: human/eccentric +- agent: dummy + branch_init: human/hullabaloo +- agent: dummy + branch_init: human/safetyinnumbers +- agent: dummy + branch_init: human/mandragora +- agent: dummy + branch_init: human/gargantuan +- agent: dummy + branch_init: human/elvenking +- agent: dummy + branch_init: human/npaperii +- agent: dummy + branch_init: human/hullab3loo +- agent: dummy + branch_init: human/reepicheep +- agent: dummy + branch_init: human/halcyon +- agent: dummy + branch_init: human/olivia +- agent: dummy + branch_init: human/neith +- agent: dummy + branch_init: human/numb +- agent: dummy + branch_init: human/returnofthependragon +- agent: dummy + branch_init: human/cinammon +- agent: dummy + branch_init: human/combatra +- agent: dummy + branch_init: human/armadillo +- agent: dummy + branch_init: human/simplicity +- agent: dummy + branch_init: human/kosmos +- agent: dummy + branch_init: human/azathoth +- agent: dummy + branch_init: human/danceoffallenangels +- agent: dummy + branch_init: human/returnofthejedimp +- agent: dummy + branch_init: human/blowrag +- agent: dummy + branch_init: human/artofcorewar +- agent: dummy + branch_init: human/silking +- agent: dummy + branch_init: human/goldeneye +- agent: dummy + branch_init: human/dawn +- agent: dummy + branch_init: human/sonofvain +- agent: dummy + branch_init: human/blackknight +- agent: dummy + branch_init: human/thefugitive +- agent: dummy + branch_init: human/frothandfizzle +- agent: dummy + branch_init: human/snowscan +- agent: dummy + branch_init: human/rust +- agent: dummy + branch_init: human/lastjudgement +- agent: dummy + branch_init: human/pdqscan +- agent: dummy + branch_init: human/mercenary +- agent: dummy + branch_init: human/dawn2 +- agent: dummy + branch_init: human/firestorm +- agent: dummy + branch_init: human/defensive +- agent: dummy + branch_init: human/burningmetal +- agent: dummy + branch_init: human/chainlockv02a +- agent: dummy + branch_init: human/decoysignal +- agent: dummy + branch_init: human/cloudburst +- agent: dummy + branch_init: human/mascafe +- agent: dummy + branch_init: human/devilstick +- agent: dummy + branch_init: human/unheardof +- agent: dummy + branch_init: human/returnofthefugitive +- agent: dummy + branch_init: human/silkworm +- agent: dummy + branch_init: human/maelstrom +- agent: dummy + branch_init: human/forjohn +- agent: dummy + branch_init: human/toxic diff --git a/configs/ablations/ladder/rungs/robotrumble.yaml b/configs/ablations/ladder/rungs/robotrumble.yaml new file mode 100644 index 00000000..8cdae2af --- /dev/null +++ b/configs/ablations/ladder/rungs/robotrumble.yaml @@ -0,0 +1,116 @@ +- agent: dummy + branch_init: human/anton/anton3000 +- agent: dummy + branch_init: human/happysquid/test +- agent: dummy + branch_init: human/anton/wallifier +- agent: dummy + branch_init: human/ldang/nessy +- agent: dummy + branch_init: human/ldang/nemo +- agent: dummy + branch_init: human/navster8/bash-brothers +- agent: dummy + branch_init: human/aaoutkine/dark-knight +- agent: dummy + branch_init: human/mountain/neuralbot1-1h +- agent: dummy + branch_init: human/sivecano/clouded-mind +- agent: dummy + branch_init: human/mountain/neuralbot2-6h +- agent: dummy + branch_init: human/kalkin/artemis +- agent: dummy + branch_init: human/kalkin/artemis2 +- agent: dummy + branch_init: human/navster8/maginot-line +- agent: dummy + branch_init: human/jiricodes/jiricodes-bot +- agent: dummy + branch_init: human/sbasu3/meek-bot +- agent: dummy + branch_init: human/essickmango/fruity-test +- agent: dummy + branch_init: human/tabaxi3k/charles +- agent: dummy + branch_init: human/devchris/first_test +- agent: dummy + branch_init: human/aaa/jippty5 +- agent: dummy + branch_init: human/jay0jayjay/naivestarter +- agent: dummy + branch_init: human/luisa/luisasrobot +- agent: dummy + branch_init: human/luisa/baselinegere +- agent: dummy + branch_init: human/anton/anton4000 +- agent: dummy + branch_init: human/aayyad/testbot +- agent: dummy + branch_init: human/edward/flail +- agent: dummy + branch_init: human/mousetail/genetic-robot +- agent: dummy + branch_init: human/kalkin/maxad +- agent: dummy + branch_init: human/mjburgess/rule99 +- agent: dummy + branch_init: human/ketza/bob +- agent: dummy + branch_init: human/suddenlyseals/control-center +- agent: dummy + branch_init: human/aaoutkine/school-bot +- agent: dummy + branch_init: human/thesmilingturtl/naivefaa +- agent: dummy + branch_init: human/mario31313/alpha_13 +- agent: dummy + branch_init: human/underscore/bot1 +- agent: dummy + branch_init: human/lanity/sivuy +- agent: dummy + branch_init: human/mee42/follow-bot +- agent: dummy + branch_init: human/anton/om-om +- agent: dummy + branch_init: human/aaoutkine/silo34 +- agent: dummy + branch_init: human/mountain/neuralbot4-3h +- agent: dummy + branch_init: human/ketza/arthur +- agent: dummy + branch_init: human/mkap/test +- agent: dummy + branch_init: human/essickmango/pickle-up +- agent: dummy + branch_init: human/wolfsleuth/simple +- agent: dummy + branch_init: human/gerenuk/gere-ape +- agent: dummy + branch_init: human/clay/diag-lattice +- agent: dummy + branch_init: human/atl15/centerrr +- agent: dummy + branch_init: human/jammyliu/sixty-nine-line +- agent: dummy + branch_init: human/mitch84/walk_retreat +- agent: dummy + branch_init: human/tabaxi3k/black-magic-1 +- agent: dummy + branch_init: human/devchris/black_magic +- agent: dummy + branch_init: human/mitch84/retreat_walk2 +- agent: dummy + branch_init: human/mitch84/crw_preempt +- agent: dummy + branch_init: human/entropicdrifter/glommer +- agent: dummy + branch_init: human/mousetail/coward-bot +- agent: dummy + branch_init: human/entropicdrifter/glommerv2 +- agent: dummy + branch_init: human/entropicdrifter/we-are-borg +- agent: dummy + branch_init: human/entropicdrifter/seven-of-nine +- agent: dummy + branch_init: human/entropicdrifter/gigachad diff --git a/configs/mini/litellm_custom_model_config.yaml b/configs/mini/litellm_custom_model_config.yaml index 97c7b32a..673d154a 100644 --- a/configs/mini/litellm_custom_model_config.yaml +++ b/configs/mini/litellm_custom_model_config.yaml @@ -47,5 +47,28 @@ ] } ] + }, + "gemini-3-5-flash-fair": { + "litellm_provider": "openai", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_tokens": 65535, + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 9e-06, + "output_cost_per_reasoning_token": 9e-06, + "cache_read_input_token_cost": 1.5e-07, + "source": "miscellaneous" + }, + "gpt-5-5-genai-responses": { + "litellm_provider": "openai", + "mode": "responses", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "cache_read_input_token_cost": 5e-07, + "source": "miscellaneous" } } diff --git a/pyproject.toml b/pyproject.toml index 2d3a0027..1a420f81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dev = [ "ruff", "Werkzeug", "frozen-flask", + "anthropic", ] aws = [ "boto3", diff --git a/uv.lock b/uv.lock index b6ff80b2..c7d039ac 100644 --- a/uv.lock +++ b/uv.lock @@ -143,6 +143,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "anthropic" +version = "0.116.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/a2/d31f14e28d49bae983a3634e38dfb4b31c50110b5e403596c5c6a20b23f8/anthropic-0.116.0.tar.gz", hash = "sha256:5fc248fbb9fe03ef686f8a774f81586bca31a043260aab88b387ea3660f4a396", size = 949149, upload-time = "2026-07-02T19:08:10.534Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/dd/2a1e81cf1b163acc340afc4ec74ed1d86f5eed1a809fabdeed3e0997b346/anthropic-0.116.0-py3-none-any.whl", hash = "sha256:6c0a7698e8d652455da3499978279bb2588c7264d0a35be3666009a4258c8256", size = 956896, upload-time = "2026-07-02T19:08:08.756Z" }, +] + [[package]] name = "anyio" version = "4.12.0" @@ -372,6 +391,7 @@ aws = [ { name = "boto3" }, ] dev = [ + { name = "anthropic" }, { name = "frozen-flask" }, { name = "pre-commit" }, { name = "pytest" }, @@ -389,6 +409,7 @@ docs = [ [package.metadata] requires-dist = [ + { name = "anthropic", marker = "extra == 'dev'" }, { name = "boto3", marker = "extra == 'aws'" }, { name = "cdifflib" }, { name = "flask" }, @@ -570,6 +591,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] +[[package]] +name = "docstring-parser" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, +] + [[package]] name = "execnet" version = "2.1.2"