Handle Bomberland validation timeouts

Author: Muhtasham

codeclash/arenas/bomberland/bomberland.py

@@ -71,27 +71,31 @@ def validate_code(self, agent: Player) -> tuple[bool, str | None]:
         if syntax_check["returncode"] != 0:
             return False, f"Python syntax error in `{self.submission}`:\n{syntax_check['output']}"
 
-        import_check = agent.environment.execute(
-            "python - <<'PY'\n"
-            "import importlib.util\n"
-            f"spec = importlib.util.spec_from_file_location('submission_agent', {self.submission!r})\n"
-            "module = importlib.util.module_from_spec(spec)\n"
-            "spec.loader.exec_module(module)\n"
-            "assert hasattr(module, 'next_actions'), 'next_actions callable not found'\n"
-            "assert callable(module.next_actions), 'next_actions must be callable'\n"
-            "state = {\n"
-            "    'connection': {'agent_id': 'Alice'},\n"
-            "    'agents': {'Alice': {'unit_ids': ['u0']}},\n"
-            "    'unit_state': {'u0': {'agent_id': 'Alice', 'hp': 3, 'coordinates': [1, 1]}},\n"
-            "    'entities': [],\n"
-            "    'world': {'width': 5, 'height': 5},\n"
-            "    'tick': 0,\n"
-            "}\n"
-            "result = module.next_actions(state)\n"
-            "assert result is None or isinstance(result, dict), 'next_actions must return a dict or None'\n"
-            "PY",
-            timeout=int(self._game_arg("validation_timeout")),
-        )
+        validation_timeout = int(self._game_arg("validation_timeout"))
+        try:
+            import_check = agent.environment.execute(
+                "python - <<'PY'\n"
+                "import importlib.util\n"
+                f"spec = importlib.util.spec_from_file_location('submission_agent', {self.submission!r})\n"
+                "module = importlib.util.module_from_spec(spec)\n"
+                "spec.loader.exec_module(module)\n"
+                "assert hasattr(module, 'next_actions'), 'next_actions callable not found'\n"
+                "assert callable(module.next_actions), 'next_actions must be callable'\n"
+                "state = {\n"
+                "    'connection': {'agent_id': 'Alice'},\n"
+                "    'agents': {'Alice': {'unit_ids': ['u0']}},\n"
+                "    'unit_state': {'u0': {'agent_id': 'Alice', 'hp': 3, 'coordinates': [1, 1]}},\n"
+                "    'entities': [],\n"
+                "    'world': {'width': 5, 'height': 5},\n"
+                "    'tick': 0,\n"
+                "}\n"
+                "result = module.next_actions(state)\n"
+                "assert result is None or isinstance(result, dict), 'next_actions must return a dict or None'\n"
+                "PY",
+                timeout=validation_timeout,
+            )
+        except subprocess.TimeoutExpired:
+            return False, f"`next_actions` validation exceeded {validation_timeout}s timeout"
         if import_check["returncode"] != 0:
             return False, f"Could not import or call `next_actions` from `{self.submission}`:\n{import_check['output']}"
 
@@ -137,6 +141,17 @@ def get_results(self, agents: list[Player], round_num: int, stats: RoundStats):
             for agent in agents:
                 stats.scores[agent.name] = CRASH_SCORE
                 stats.player_stats[agent.name].score = CRASH_SCORE
+                stats.details.append(
+                    json.dumps(
+                        {
+                            "player": agent.name,
+                            "score": CRASH_SCORE,
+                            "status": "error",
+                            "error": f"missing Bomberland result file: {result_file}",
+                        },
+                        sort_keys=True,
+                    )
+                )
             return
 
         with open(result_file) as f:

tests/arenas/test_bomberland.py

@@ -1,5 +1,6 @@
 import importlib.util
 import json
+import subprocess
 import time
 from pathlib import Path
 
@@ -119,6 +120,28 @@ def execute(self, cmd, cwd=None, timeout=None):
         assert error is None
         assert player.environment.timeouts[-1] == 9
 
+    def test_validation_timeout_invalidates_submission(self):
+        arena = BomberlandArena.__new__(BomberlandArena)
+        arena.submission = "bomberland_agent.py"
+        arena.config = {"game": {"name": "Bomberland", "args": {"validation_timeout": 3}}}
+
+        class TimeoutEnvironment(MockEnvironment):
+            def __init__(self):
+                super().__init__(
+                    files={"bomberland_agent.py": "def next_actions(game_state):\n    return {}\n"},
+                    command_outputs={"python -m py_compile bomberland_agent.py": {"output": "", "returncode": 0}},
+                )
+
+            def execute(self, cmd, cwd=None, timeout=None):
+                if cmd.startswith("python - <<'PY'"):
+                    raise subprocess.TimeoutExpired(cmd=cmd, timeout=timeout)
+                return super().execute(cmd, cwd=cwd, timeout=timeout)
+
+        valid, error = arena.validate_code(MockPlayer("Alice", TimeoutEnvironment()))
+
+        assert valid is False
+        assert error == "`next_actions` validation exceeded 3s timeout"
+
 
 class TestBomberlandResults:
     def test_parse_winner(self, tmp_log_dir):
@@ -178,6 +201,25 @@ def test_missing_player_uses_crash_score(self, tmp_log_dir):
         assert stats.winner == "Alice"
         assert stats.scores == {"Alice": -5.0, "Bob": CRASH_SCORE}
 
+    def test_missing_result_file_records_crash_details(self, tmp_log_dir):
+        arena = BomberlandArena.__new__(BomberlandArena)
+        arena.log_local = tmp_log_dir
+        arena.logger = type("Logger", (), {"error": lambda self, msg: None})()
+        (tmp_log_dir / "rounds" / "1").mkdir(parents=True)
+
+        agents = [MockPlayer("Alice"), MockPlayer("Bob")]
+        stats = RoundStats(round_num=1, agents=agents)
+
+        arena.get_results(agents, 1, stats)
+
+        assert stats.winner == RESULT_TIE
+        assert stats.scores == {"Alice": CRASH_SCORE, "Bob": CRASH_SCORE}
+        assert len(stats.details) == 2
+        detail = json.loads(stats.details[0])
+        assert detail["status"] == "error"
+        assert detail["score"] == CRASH_SCORE
+        assert "missing Bomberland result file" in detail["error"]
+
 
 class TestBomberlandExecution:
     def test_execute_round_uses_nested_game_args(self):