update elo to include include-round-0 flag for ladder construction

Author: john-b-yang

.claude/skills/create-arena-ladder/SKILL.md

@@ -104,9 +104,14 @@ Local validation is necessary but NOT sufficient — a local shim skips Docker,
    `GITHUB_TOKEN=$(gh auth token) uv run codeclash ladder make configs/ablations/ladder/make_<a>.yaml --workers N`
    → logs land under `logs/ladder/<A>/`. Fast arenas run on a laptop; big ones (e.g. SCML's
    ~1275 pairs) want an AWS box under `tmux`/`nohup`.
-4. Rank: `python -m codeclash.analysis.metrics.elo -d logs/ladder/<A> --output-dir assets/<a>_elo`
+4. Rank: `uv run python -m codeclash.analysis.metrics.elo -d logs/ladder/<A> --include-round-0 --output-dir assets/<a>_elo`
    → prints the Bradley-Terry/Elo order (weakest → strongest); that ordering IS the ladder.
-   Tip: run a cheap low-`sims` pilot first and eyeball that baselines sit near the bottom.
+   - **`--include-round-0` is REQUIRED for ladder construction.** `ladder make` uses
+     `tournament.rounds: 0`, so round 0 IS the match; without the flag every tournament is
+     dropped (round 0 is normally the excluded identical-codebases baseline) and the fit
+     crashes on an empty matrix. Do NOT pass it for normal multi-round PvP/climbing Elo.
+   - Use `uv run python`, not bare `python` — the analysis deps (matplotlib) live in the uv venv.
+   - Tip: run a cheap low-`sims` pilot first and eyeball that baselines sit near the bottom.
 
 ## Phase 5 — Assemble the ranked configs
 

codeclash/analysis/metrics/elo.py

@@ -40,6 +40,7 @@ def __init__(
         score_type: SCORING_TYPES = "per_round_tertiary",
         max_round: int = 15,
         only_specific_round: bool = False,
+        include_round_0: bool = False,
     ):
         """This class builds a win matrix from a log directory, it doesn't fit anything yet.
         It also adds a "ALL" game to the win matrix, which is the sum of all games.
@@ -62,6 +63,9 @@ def __init__(
 
         The `max_round` parameter controls the maximum number of rounds to include in the score calculation (default: 15).
         The `only_specific_round` parameter controls whether to only include the specific round (True) or all rounds up to max_round (False).
+        The `include_round_0` parameter controls whether round 0 is counted. In normal PvP/climbing
+        tournaments round 0 is the identical-codebases baseline and is excluded. For ladder
+        construction (`ladder make`, `tournament.rounds: 0`) round 0 IS the match, so set this True.
         """
         self.win_matrix: dict[str, dict[tuple[str, str], list[float]]] = defaultdict(
             lambda: defaultdict(lambda: [0.0, 0.0])
@@ -71,6 +75,7 @@ def __init__(
         self.score_type = score_type
         self.max_round = max_round
         self.only_specific_round = only_specific_round
+        self.include_round_0 = include_round_0
         self._samples: dict[str, dict[tuple[str, str], list[tuple[float, float]]]] = defaultdict(
             lambda: defaultdict(list)
         )
@@ -154,13 +159,19 @@ def _process_tournament(self, metadata_path: Path) -> None:
             return
 
         player_names = [p["name"] for p in players]
-        models = [p["config"]["model"]["model_name"].strip("@") for p in players]
+        models = []
+        for p in players:
+            try:
+                models.append(p["config"]["model"]["model_name"].strip("@"))
+            except KeyError:
+                # Ladder bots have no model config; identify by branch (flatten "/" to keep years distinct).
+                models.append(p["name"].removeprefix("human/").replace("/", "__"))
 
         # Aggregate scores for each round
         p1_round_scores = []
         p2_round_scores = []
         for idx, stats in metadata["round_stats"].items():
-            if idx == "0":
+            if idx == "0" and not self.include_round_0:
                 continue
 
             round_num = int(idx)
@@ -1547,6 +1558,12 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
     parser = argparse.ArgumentParser(description="Build win matrix and fit Bradley-Terry model")
     parser.add_argument("-d", "--log_dir", type=Path, default=LOCAL_LOG_DIR)
     parser.add_argument("--print-matrix", action="store_true", help="Print win matrix")
+    parser.add_argument(
+        "--include-round-0",
+        action="store_true",
+        help="Count round 0 (normally the excluded identical-codebases baseline). REQUIRED for "
+        "ladder construction (`ladder make` uses tournament.rounds: 0, so round 0 IS the match).",
+    )
     parser.add_argument(
         "-s",
         "--score-type",
@@ -1578,7 +1595,9 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
     args = parser.parse_args()
 
     builder = ScoreMatrixBuilder(
-        all_games_normalization_scheme=args.all_normalization_scheme, score_type=args.score_type
+        all_games_normalization_scheme=args.all_normalization_scheme,
+        score_type=args.score_type,
+        include_round_0=args.include_round_0,
     )
     builder.build(args.log_dir)
 
@@ -1632,22 +1651,26 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
             ).run()
         write_bootstrap_metrics_table(bootstrap_results, args.output_dir, game="ALL")
 
-    logger.info("Running EloVsMaxRounds analysis")
-    EloVsMaxRounds(
-        log_dir=args.log_dir,
-        max_rounds=15,
-        all_games_normalization_scheme=args.all_normalization_scheme,
-        score_type=args.score_type,
-        regularization=args.regularization,
-        output_dir=args.output_dir,
-    ).run()
-
-    logger.info("Running EloOnlyAtRound analysis")
-    EloOnlyAtRound(
-        log_dir=args.log_dir,
-        max_rounds=15,
-        all_games_normalization_scheme=args.all_normalization_scheme,
-        score_type=args.score_type,
-        regularization=args.regularization,
-        output_dir=args.output_dir,
-    ).run()
+    # Max-round analyses are multi-round-only; skip them for single-round ladder round-robins.
+    if not args.include_round_0:
+        logger.info("Running EloVsMaxRounds analysis")
+        EloVsMaxRounds(
+            log_dir=args.log_dir,
+            max_rounds=15,
+            all_games_normalization_scheme=args.all_normalization_scheme,
+            score_type=args.score_type,
+            regularization=args.regularization,
+            output_dir=args.output_dir,
+        ).run()
+
+        logger.info("Running EloOnlyAtRound analysis")
+        EloOnlyAtRound(
+            log_dir=args.log_dir,
+            max_rounds=15,
+            all_games_normalization_scheme=args.all_normalization_scheme,
+            score_type=args.score_type,
+            regularization=args.regularization,
+            output_dir=args.output_dir,
+        ).run()
+    else:
+        logger.info("Skipping EloVsMaxRounds / EloOnlyAtRound (ladder mode: single round-0 round-robin)")