llm as judge: updates to aggregate/get inst

Author: klieret

codeclash/analysis/llm_as_judge/aggregate_results.py

@@ -45,6 +45,7 @@ def aggregate_results_to_dataframe(input_dir: Path) -> pd.DataFrame:
                     # Extract instance metadata
                     instance = Instance.model_validate(instance_data["instance"])
                     model_name, opponent_model_name = instance.get_lm_name_self_opponent()
+                    current_round_win_rate, next_round_win_rate = instance.get_current_next_round_win_rate()
 
                     # Create a flat row with all information
                     row = {
@@ -55,6 +56,8 @@ def aggregate_results_to_dataframe(input_dir: Path) -> pd.DataFrame:
                         "round_number": instance.round_number,
                         "model_name": model_name,
                         "opponent_model_name": opponent_model_name,
+                        "current_round_win_rate": current_round_win_rate,
+                        "next_round_win_rate": next_round_win_rate,
                     }
 
                     # Add all evaluation results

codeclash/analysis/llm_as_judge/get_instances.py

@@ -4,6 +4,8 @@
 
 import argparse
 from pathlib import Path
+from collections import defaultdict
+from pyexpat import model
 
 from codeclash.analysis.llm_as_judge.utils import InstanceBatch, get_instances
 
@@ -13,8 +15,9 @@
     parser.add_argument("-o", "--output-file", type=Path, help="Path to the output file", default="instances.json")
     args = parser.parse_args()
 
-    SELECTED_ROUNDS = [1, 2, 3, 5, 10, 15]
+    SELECTED_ROUNDS = [1, 2, 3, 5, 10, 12, 14, 15]
     SELECTED_GAME = "BattleSnake"
+    NUMBER_OF_INDEPENDENT_GAMES = 3
 
     # first get all instances, then filter
     instances = sorted(get_instances(args.input_dir), key=lambda x: x.instance_id)
@@ -24,17 +27,24 @@
     instances = [instance for instance in instances if instance.round_number in SELECTED_ROUNDS]
     print(f"Filtered to {len(instances)} instances because of round number")
 
-    # OK, now it gest more complicated, because we only want to keep one
-    # instance per combination of game, lm, and round
-    unique = {}
+    grouped = defaultdict(list)
     for instance in instances:
-        key = (*instance.get_lm_name_self_opponent(), instance.round_number)
-        if key not in unique:
-            unique[key] = instance
-    instances = list(unique.values())
+        model_name_player, model_name_opponent = instance.get_lm_name_self_opponent()
+        if model_name_player == model_name_opponent:
+            continue
+        key = (model_name_player, model_name_opponent, instance.round_number)
+        grouped[key].append(instance)
+    
+    selected_instances = []
+    for key, instance_list in grouped.items():
+        if len(instance_list) < NUMBER_OF_INDEPENDENT_GAMES:
+            print(f"Warning: Only found {len(instance_list)} instances for {key}, need {NUMBER_OF_INDEPENDENT_GAMES}")
+        selected_count = min(len(instance_list), NUMBER_OF_INDEPENDENT_GAMES)
+        selected_instances.extend(instance_list[:selected_count])
+    
+    instances = selected_instances
     unique_models = set([instance.get_lm_name_self_opponent()[0] for instance in instances])
-    print(unique_models)
-    print(f"Filtered to {len(instances)} instances after deduplication for game repetitions for {len(unique_models)} unique model matchups")
+    print(f"Filtered to {len(instances)} instances after keeping {NUMBER_OF_INDEPENDENT_GAMES} for game repetitions for {len(unique_models)} unique model matchups")
 
     batch = InstanceBatch(instances=instances)
     args.output_file.write_text(batch.model_dump_json())

codeclash/analysis/llm_as_judge/utils.py

@@ -2,6 +2,7 @@
 import json
 from pathlib import Path
 
+from codeclash.analysis.metrics.elo import get_scores
 from pydantic import BaseModel
 
 
@@ -31,14 +32,33 @@ class Instance(BaseModel):
     @property
     def instance_id(self) -> str:
         return f"{self.tournament_name}__{self.player_name}__r{self.round_number}"
+    
+    @property
+    def tournament_path(self) -> Path:
+        return self.trajectory_path.parent.parent.parent
+    
+    @property
+    def metadata_path(self) -> Path:
+        return self.tournament_path / "metadata.json"
 
     def get_lm_name_self_opponent(self) -> tuple[str, str]:
-        metadata_path = self.trajectory_path.parent.parent.parent / "metadata.json"
-        metadata = json.loads(metadata_path.read_text())
+        metadata = json.loads(self.metadata_path.read_text())
         player_configs = metadata["config"]["players"]
         player_config = [pc for pc in player_configs if pc["name"] == self.player_name][0]
         other_player_config = [pc for pc in player_configs if pc["name"] != self.player_name][0]
         return player_config["config"]["model"]["model_name"].removeprefix("@"), other_player_config["config"]["model"]["model_name"].removeprefix("@")
+    
+    def get_current_next_round_win_rate(self) -> tuple[float | None, float | None]:
+        metadata = json.loads(self.metadata_path.read_text())
+        current_round_stats = metadata["round_stats"].get(str(self.round_number))
+        next_round_stats = metadata["round_stats"].get(str(self.round_number + 1))
+        current_win_rate = None
+        next_win_rate = None
+        if current_round_stats is not None:
+            current_win_rate = get_scores(current_round_stats).get(self.player_name)
+        if next_round_stats is not None:
+            next_win_rate = get_scores(next_round_stats).get(self.player_name)
+        return current_win_rate, next_win_rate
 
 
 class InstanceBatch(BaseModel):