Handle ABIDES validation timeouts

Author: Muhtasham

codeclash/arenas/abides/abides.py

@@ -55,28 +55,32 @@ def validate_code(self, agent: Player) -> tuple[bool, str | None]:
         if syntax_check["returncode"] != 0:
             return False, f"Python syntax error in `{self.submission}`:\n{syntax_check['output']}"
 
-        import_check = agent.environment.execute(
-            "python - <<'PY'\n"
-            "import importlib.util\n"
-            f"spec = importlib.util.spec_from_file_location('submission_agent', {self.submission!r})\n"
-            "module = importlib.util.module_from_spec(spec)\n"
-            "spec.loader.exec_module(module)\n"
-            "assert hasattr(module, 'decide'), 'decide function not found'\n"
-            "assert callable(module.decide), 'decide must be callable'\n"
-            "observation = {\n"
-            "    'symbol': 'JPM',\n"
-            "    'cash': 10000000,\n"
-            "    'position': 0,\n"
-            "    'best_bid': None,\n"
-            "    'best_ask': None,\n"
-            "    'last_trade': 100000,\n"
-            "    'market_open': True,\n"
-            "}\n"
-            "result = module.decide(observation)\n"
-            "assert result is None or isinstance(result, (list, tuple, dict)), 'decide must return a list, tuple, dict, or None'\n"
-            "PY",
-            timeout=int(self._game_arg("validation_timeout")),
-        )
+        validation_timeout = int(self._game_arg("validation_timeout"))
+        try:
+            import_check = agent.environment.execute(
+                "python - <<'PY'\n"
+                "import importlib.util\n"
+                f"spec = importlib.util.spec_from_file_location('submission_agent', {self.submission!r})\n"
+                "module = importlib.util.module_from_spec(spec)\n"
+                "spec.loader.exec_module(module)\n"
+                "assert hasattr(module, 'decide'), 'decide function not found'\n"
+                "assert callable(module.decide), 'decide must be callable'\n"
+                "observation = {\n"
+                "    'symbol': 'JPM',\n"
+                "    'cash': 10000000,\n"
+                "    'position': 0,\n"
+                "    'best_bid': None,\n"
+                "    'best_ask': None,\n"
+                "    'last_trade': 100000,\n"
+                "    'market_open': True,\n"
+                "}\n"
+                "result = module.decide(observation)\n"
+                "assert result is None or isinstance(result, (list, tuple, dict)), 'decide must return a list, tuple, dict, or None'\n"
+                "PY",
+                timeout=validation_timeout,
+            )
+        except subprocess.TimeoutExpired:
+            return False, f"`decide` validation exceeded {validation_timeout}s timeout"
         if import_check["returncode"] != 0:
             return (
                 False,

tests/arenas/test_abides.py

@@ -1,4 +1,5 @@
 import json
+import subprocess
 from pathlib import Path
 
 from codeclash.arenas.abides.abides import CRASH_SCORE, ABIDESArena
@@ -109,6 +110,27 @@ def test_import_failure(self, mock_player_factory):
         assert valid is False
         assert "Could not import or call" in error
 
+    def test_validation_timeout_invalidates_submission(self):
+        arena = ABIDESArena.__new__(ABIDESArena)
+        arena.submission = "abides_agent.py"
+        arena.config = {"game": {"args": {"validation_timeout": 5}}}
+
+        class TimeoutEnvironment(MockEnvironment):
+            def execute(self, cmd, cwd=None, timeout=None):
+                if cmd.startswith("python - <<'PY'"):
+                    raise subprocess.TimeoutExpired(cmd=cmd, timeout=timeout)
+                return super().execute(cmd, cwd=cwd, timeout=timeout)
+
+        player = MockPlayer(
+            "Alice",
+            TimeoutEnvironment(files={"abides_agent.py": "def decide(observation):\n    return []\n"}),
+        )
+
+        valid, error = arena.validate_code(player)
+
+        assert valid is False
+        assert "`decide` validation exceeded 5s timeout" in error
+
 
 class TestABIDESResults:
     def test_parse_winner(self, tmp_log_dir):