Fix(analysis): Various fixes to llm as judge

Author: klieret

codeclash/analysis/llm_as_judge/big_questions.py

@@ -66,6 +66,7 @@ def pretty_print(self) -> None:
 
 class ModelConfig(BaseModel):
     model_name: str
+    model_class: str | None = None
     model_kwargs: dict[str, Any]
 
 
@@ -90,7 +91,7 @@ def extract_triple_backticks(text: str) -> str:
 class BigQuestions:
     def __init__(self, config: BigQuestionsConfig):
         self.config = config
-        self.model = get_model(config.model.model_name, config={"model_kwargs": config.model.model_kwargs})
+        self.model = get_model(config.model.model_name, config={"model_kwargs": config.model.model_kwargs, "model_class": config.model.model_class})
 
     @property
     def data_id(self) -> str:
@@ -145,20 +146,24 @@ def _get_messages(self, instance: Instance) -> list[dict[str, Any]]:
 
     def _should_skip(self, target_path: Path, instance: Instance) -> bool:
         if not target_path.exists():
+            logger.debug(f"Not skipping: {target_path} does not exist")
             return False
         content = target_path.read_text()
         if not content.strip():
+            logger.debug(f"Not skipping: {target_path} is empty")
             return False
         data = json.loads(content)
         if self.data_id not in data:
+            logger.debug(f"Not skipping: {self.data_id} not in {target_path}")
             return False
-        if instance.instance_id in data[self.data_id]:
-            return True
-        return False
+        if instance.instance_id not in data[self.data_id]:
+            logger.debug(f"Not skipping: {instance.instance_id} not in {target_path} under key {self.data_id}")
+            return False
+        return True
 
     def _save_response(self, target_path: Path, response_data: dict[str, Any], instance: Instance) -> None:
         # atomic write with file lock in case other analyses are also writing
-        with FileLock(target_path):
+        with FileLock(target_path.with_suffix(".lock")):
             # read again if changed in the meantime
             data = {}
             if target_path.exists():

codeclash/analysis/llm_as_judge/big_questions.yaml

@@ -209,6 +209,7 @@ instance_prompt: |
 
   {{ trajectory_message_str }}
 model:
-  model_name: gpt-5
+  model_name: 'openai/gpt-5'
+  # model_class: portkey
   model_kwargs:
     reasoning_effort: high

codeclash/analysis/llm_as_judge/get_instances.py

@@ -13,7 +13,7 @@
     parser.add_argument("-o", "--output-file", type=Path, help="Path to the output file", default="instances.json")
     args = parser.parse_args()
 
-    SELECTED_ROUNDS = [1, 5, 10, 15]
+    SELECTED_ROUNDS = [1, 2, 3, 5, 10, 15]
     SELECTED_GAME = "BattleSnake"
 
     # first get all instances, then filter
@@ -28,11 +28,13 @@
     # instance per combination of game, lm, and round
     unique = {}
     for instance in instances:
-        key = (instance.tournament_name, instance.get_lm_name(), instance.round_number)
+        key = (*instance.get_lm_name_self_opponent(), instance.round_number)
         if key not in unique:
             unique[key] = instance
     instances = list(unique.values())
-    print(f"Filtered to {len(instances)} instances after deduplication for game repetitions")
+    unique_models = set([instance.get_lm_name_self_opponent()[0] for instance in instances])
+    print(unique_models)
+    print(f"Filtered to {len(instances)} instances after deduplication for game repetitions for {len(unique_models)} unique model matchups")
 
     batch = InstanceBatch(instances=instances)
     args.output_file.write_text(batch.model_dump_json())

codeclash/analysis/llm_as_judge/utils.py

@@ -32,10 +32,13 @@ class Instance(BaseModel):
     def instance_id(self) -> str:
         return f"{self.tournament_name}__{self.player_name}__r{self.round_number}"
 
-    def get_lm_name(self) -> str:
+    def get_lm_name_self_opponent(self) -> tuple[str, str]:
         metadata_path = self.trajectory_path.parent.parent.parent / "metadata.json"
         metadata = json.loads(metadata_path.read_text())
-        return metadata["config"]["players"][self.player_name]["config"]["model"]["model_name"]
+        player_configs = metadata["config"]["players"]
+        player_config = [pc for pc in player_configs if pc["name"] == self.player_name][0]
+        other_player_config = [pc for pc in player_configs if pc["name"] != self.player_name][0]
+        return player_config["config"]["model"]["model_name"].removeprefix("@"), other_player_config["config"]["model"]["model_name"].removeprefix("@")
 
 
 class InstanceBatch(BaseModel):
@@ -47,10 +50,15 @@ def find_tournament_folders(input_dir: Path) -> list[Path]:
 
 
 def parse_trajectory_name(trajectory_path: Path) -> Instance:
+    try:
+        round_number = int(trajectory_path.name.removesuffix(".traj.json").split("_r")[1])
+    except:
+        print(trajectory_path)
+        raise
     return Instance(
         trajectory_path=trajectory_path,
         player_name=trajectory_path.parent.name,
-        round_number=int(trajectory_path.name.split(".")[0].split("_r")[1]),
+        round_number=round_number,
         tournament_name=trajectory_path.parent.parent.parent.name,
     )