Add submission checks

Author: john-b-yang

codeclash/games/battlecode/battlecode.py

@@ -7,6 +7,8 @@
 from codeclash.games.game import CodeGame, RoundStats
 
 BC_LOG = "sim.log"
+BC_FOLDER = "mysubmission"
+BC_TIE = "Reason: The winning team won arbitrarily (coin flip)."
 
 
 class BattleCodeGame(CodeGame):
@@ -25,7 +27,7 @@ def __init__(self, config, *, tournament_id: str, local_output_dir: Path):
 
     def execute_round(self, agents: list[Player]):
         for agent in agents:
-            src, dest = f"/{agent.name}/src/mysubmission/", str(DIR_WORK / "src" / agent.name)
+            src, dest = f"/{agent.name}/src/{BC_FOLDER}/", str(DIR_WORK / "src" / agent.name)
             self.environment.execute(f"cp -r {src} {dest}")
         random.shuffle(agents)  # Start position matters in BattleCode! Shuffle to be fair.
         args = [f"--p{idx + 1}-dir src --p{idx + 1} {agent.name}" for idx, agent in enumerate(agents)]
@@ -36,27 +38,34 @@ def execute_round(self, agents: list[Player]):
         assert response["returncode"] == 0, response
 
     def get_results(self, agents: list[Player], round_num: int, stats: RoundStats):
-        winners = []
         with open(self.log_round(round_num) / BC_LOG) as f:
             lines = f.read().strip().split("\n")
         # Get the third-to-last line which contains the winner info
-        winner_line = lines[-3] if len(lines) >= 3 else ""
+        assert len(lines) >= 3, "Log file does not contain enough lines to determine winner"
+        winner_line = lines[-3]
+        reason_line = lines[-2]
         self.logger.debug(f"Winner line: {winner_line}")
+        self.logger.debug(f"Reason line: {reason_line}")
         match = re.search(r"\s\((.*)\)\swins\s\(", winner_line)
-        if match:
+        if match and reason_line != BC_TIE:
             winner_key = match.group(1)
             self.logger.debug(f"Winner key from match: {winner_key}")
             # Map A/B to actual agent names (much closer to original code)
             winner = {"A": agents[0].name, "B": agents[1].name}.get(winner_key, RESULT_TIE)
-            winners.append(winner)
         else:
-            winners.append(RESULT_TIE)
+            winner = RESULT_TIE
 
-        stats.winner = max(set(winners), key=winners.count)
-        stats.scores = {agent.name: winners.count(agent.name) for agent in agents}
+        stats.winner = winner
+        stats.scores = {agent.name: (1 if agent.name == winner else 0) for agent in agents}
         for player, score in stats.scores.items():
             stats.player_stats[player].score = score
 
     def validate_code(self, agent: Player) -> tuple[bool, str | None]:
-        # TODO: implement more checks
+        if BC_FOLDER not in agent.environment.execute("ls src")["output"]:
+            return False, f"`{BC_FOLDER}` directory not found in `src/`"
+        if "bot.py" not in agent.environment.execute(f"ls src/{BC_FOLDER}")["output"]:
+            return False, "`bot.py` not found in `src/mysubmission/`"
+        bot_content = agent.environment.execute(f"cat src/{BC_FOLDER}/bot.py")["output"].splitlines()
+        if "def turn():" not in bot_content:
+            return False, "`turn()` function not found in `bot.py`"
         return True, None

codeclash/games/corewar/corewar.py

@@ -6,6 +6,7 @@
 from codeclash.games.game import CodeGame, RoundStats
 
 COREWAR_LOG = "sim.log"
+COREWAR_FILE = "warrior.red"
 
 
 class CoreWarGame(CodeGame):
@@ -22,7 +23,7 @@ def __init__(self, config, *, tournament_id: str, local_output_dir: Path):
                 self.run_cmd_round += f" -{arg} {val}"
 
     def execute_round(self, agents: list[Player]):
-        args = [f"/{agent.name}/warriors/warrior.red" for agent in agents]
+        args = [f"/{agent.name}/warriors/{COREWAR_FILE}" for agent in agents]
         cmd = (
             f"{self.run_cmd_round} {shlex.join(args)} "
             f"-r {self.game_config['sims_per_round']} "
@@ -66,5 +67,6 @@ def get_results(self, agents: list[Player], round_num: int, stats: RoundStats):
             stats.player_stats[player].score = score
 
     def validate_code(self, agent: Player) -> tuple[bool, str | None]:
-        # TODO: implement more checks
+        if COREWAR_FILE not in agent.environment.execute("ls warriors")["output"]:
+            return False, f"`{COREWAR_FILE}` not found in `warriors/`"
         return True, None

codeclash/games/game.py

@@ -30,11 +30,6 @@ def to_dict(self) -> dict[str, Any]:
 
 
 class RoundStats:
-    winner: str
-    round_num: int
-    scores: dict[str, float]
-    player_stats: dict[str, PlayerStats] | None = None
-
     def __init__(self, round_num: int, agents: list[Player]):
         self.winner = None
         self.round_num = round_num
@@ -215,9 +210,10 @@ def run_round(self, agents: list[Player], round_num: int) -> RoundStats:
         for agent in agents:
             is_valid, error = self.validate_code(agent)
             if not is_valid:
-                self.logger.warning(f"Agent {agent.name} failed verification: {error}")
+                self.logger.warning(f"Agent {agent.name} failed submission validation: {error}")
                 stats.player_stats[agent.name].invalid_reason = error
                 continue
+            self.logger.info(f"Agent {agent.name} passed submission validation")
             stats.player_stats[agent.name].valid_submit = True
             validated.append(agent)
 

codeclash/games/robocode/robocode.py

@@ -7,6 +7,7 @@
 from codeclash.utils.environment import assert_zero_exit_code, create_file_in_container
 
 RC_LOG = "scoreboard.txt"
+RC_FILE = Path("MyTank.java")
 
 
 class RoboCodeGame(CodeGame):
@@ -67,7 +68,7 @@ def execute_round(self, agents: list[Player]):
                 self.environment.execute(cmd)
 
         # Create .battle file
-        selected_robots = ",".join([f"{agent.name}.MyTank*" for agent in agents])
+        selected_robots = ",".join([f"{agent.name}.{RC_FILE.stem}*" for agent in agents])
         # Use timestamp for unique battle file name since rounds are managed by tournament
         battle_file = f"{self.game_id}-battle{int(time.time())}.battle"
         battle_content = f"""#Battle Properties
@@ -106,5 +107,18 @@ def get_results(self, agents: list[Player], round_num: int, stats: RoundStats):
             stats.player_stats[player].score = score
 
     def validate_code(self, agent: Player) -> tuple[bool, str | None]:
-        # TODO: implement more checks
+        if "robots" not in agent.environment.execute("ls")["output"]:
+            return False, "`robots/` directory not found in submission root"
+        if "custom" not in agent.environment.execute("ls robots")["output"]:
+            return False, "`robots/custom/` directory not found"
+        if str(RC_FILE) not in agent.environment.execute("ls robots/custom")["output"]:
+            return False, (
+                f"`{RC_FILE}` not found in `robots/custom/`. "
+                f"You can include additional files, but the primary tank logic must be in `robots/custom/{RC_FILE}`"
+            )
+        response = agent.environment.execute('javac -cp "libs/robocode.jar" robots/custom/*.java')
+        if response["returncode"] != 0:
+            return False, f"Compilation error:\n{response['output']}"
+        if f"{RC_FILE.stem}.class" not in agent.environment.execute("ls robots/custom")["output"]:
+            return False, f"`{RC_FILE.stem}.class` not found after compilation"
         return True, None

codeclash/games/robotrumble/robotrumble.py

@@ -62,5 +62,15 @@ def get_results(self, agents: list[Player], round_num: int, stats: RoundStats):
                 stats.player_stats[player].score = score
 
     def validate_code(self, agent: Player) -> tuple[bool, str | None]:
-        # TODO: implement more checks
+        if "robot.py" not in agent.environment.execute("ls")["output"]:
+            return False, "robot.py not found in the root directory"
+        if "def robot(state, unit):" not in agent.environment.execute("cat robot.py")["output"]:
+            return (
+                False,
+                "robot.py does not contain the required robot function. It should be defined as 'def robot(state, unit): ...'",
+            )
+        test_run_cmd = f"{self.run_cmd_round} robot.py robot.py -t 1"
+        test_run = agent.environment.execute(test_run_cmd)["output"]
+        if "Some errors occurred:" in test_run:
+            return False, f"Running robot.py (with `{test_run_cmd}`) resulted in errors:\n{test_run}"
         return True, None