Fix tests and make them much simpler

Author: klieret

codeclash/__init__.py

@@ -1,9 +1,12 @@
 __version__ = "0.0.0"
 
 # Optional viewer import - only if Flask dependencies are available
-try:
-    from . import viewer
+from pathlib import Path
 
-    __all__ = ["viewer"]
+try:
+    from . import viewer  # noqa: F401
 except ImportError:
-    __all__ = []
+    pass
+
+
+CONFIG_DIR = Path(__file__).resolve().parent.parent / "configs"

codeclash/agents/minisweagent.py

@@ -10,6 +10,7 @@
 from minisweagent.agents.default import AgentConfig, DefaultAgent
 from minisweagent.environments.docker import DockerEnvironment
 from minisweagent.models import get_model
+from minisweagent.models.test_models import DeterministicModel
 from minisweagent.run.utils.save import save_traj
 from rich.console import Console
 
@@ -69,7 +70,11 @@ def __init__(self, config: dict, environment: DockerEnvironment, game_context: G
         super().__init__(config, environment=environment, game_context=game_context)
 
     def run(self):
-        model = get_model(config=self.config["config"]["model"])
+        # temporary workaround around https://github.com/SWE-agent/mini-swe-agent/issues/477
+        if "DeterministicModel" not in self.config["config"]["model"].get("model_class", ""):
+            model = get_model(config=self.config["config"]["model"])
+        else:
+            model = DeterministicModel(outputs=self.config["config"]["model"]["outputs"])
         self.agent = ClashAgent(
             model=model,
             env=self.environment,

codeclash/tournaments/abstract.py

@@ -1,4 +1,5 @@
 import getpass
+import os
 import time
 import traceback
 from pathlib import Path
@@ -13,7 +14,6 @@ def __init__(self, config: dict, *, name: str, **kwargs):
         self.config: dict = config
         self.name: str = name
         self.tournament_id: str = f"{self.name}.{config['game']['name']}.{time.strftime('%y%m%d%H%M%S')}"
-        self.local_output_dir: Path = (DIR_LOGS / getpass.getuser() / self.tournament_id).resolve()
         self._metadata: dict = {
             "name": self.name,
             "tournament_id": self.tournament_id,
@@ -22,6 +22,13 @@ def __init__(self, config: dict, *, name: str, **kwargs):
         }
         self.logger = get_logger(self.name, log_path=self.local_output_dir / "tournament.log", emoji="🏆")
 
+    @property
+    def local_output_dir(self) -> Path:
+        base_dir = DIR_LOGS
+        if "PYTEST_CURRENT_TEST" in os.environ:
+            base_dir = Path("/tmp/codeclash")
+        return (base_dir / getpass.getuser() / self.tournament_id).resolve()
+
     def get_metadata(self) -> dict:
         return self._metadata
 

configs/models/instant_submit.yaml

@@ -0,0 +1,4 @@
+model_name: instant_submit
+model_class: minisweagent.models.test_models.DeterministicModel
+outputs:
+  - "echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT"

configs/test_configs/battlesnake_pvp_test.yaml

@@ -0,0 +1,20 @@
+game:
+  name: BattleSnake
+  sims_per_round: 1
+  args:
+    width: 11
+    height: 11
+    browser: false
+tournament:
+  rounds: 1
+  evaluate_matrix: false
+players:
+- agent: mini
+  name: p1
+  config:
+    agent: !include mini/default.yaml
+    model:  !include models/instant_submit.yaml
+- agent: dummy
+  name: p2
+prompts:
+  game_description: "asdf"

main.py

@@ -3,24 +3,24 @@
 
 import yaml
 
+from codeclash import CONFIG_DIR
 from codeclash.tournaments.pvp import PvpTournament
 from codeclash.utils.yaml_utils import resolve_includes
 
 
 def main(config_path: Path, *, cleanup: bool = False, push_agent: bool = False):
     yaml_content = config_path.read_text()
-    preprocessed_yaml = resolve_includes(yaml_content, base_dir=config_path.parent)
+    preprocessed_yaml = resolve_includes(yaml_content, base_dir=CONFIG_DIR)
     config = yaml.safe_load(preprocessed_yaml)
     training = PvpTournament(config, cleanup=cleanup, push_agent=push_agent)
     training.run()
 
 
-if __name__ == "__main__":
+def main_cli(argv: list[str] | None = None):
     parser = argparse.ArgumentParser(description="CodeClash")
     parser.add_argument(
         "config_path",
         type=Path,
-        default="configs/battlesnake.yaml",
         help="Path to the config file.",
     )
     parser.add_argument(
@@ -35,5 +35,9 @@ def main(config_path: Path, *, cleanup: bool = False, push_agent: bool = False):
         action="store_true",
         help="If set, push each agent's codebase to a new repository after running.",
     )
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     main(**vars(args))
+
+
+if __name__ == "__main__":
+    main_cli()

main_single_player.py

@@ -3,24 +3,24 @@
 
 import yaml
 
+from codeclash import CONFIG_DIR
 from codeclash.tournaments.single_player import SinglePlayerTraining
 from codeclash.utils.yaml_utils import resolve_includes
 
 
 def main(config_path: Path, cleanup: bool = False):
     yaml_content = config_path.read_text()
-    preprocessed_yaml = resolve_includes(yaml_content, base_dir=config_path.parent)
+    preprocessed_yaml = resolve_includes(yaml_content, base_dir=CONFIG_DIR)
     config = yaml.safe_load(preprocessed_yaml)
     training = SinglePlayerTraining(config, cleanup=cleanup)
     training.run()
 
 
-if __name__ == "__main__":
+def main_cli(argv: list[str] | None = None):
     parser = argparse.ArgumentParser(description="CodeClash")
     parser.add_argument(
         "config_path",
         type=Path,
-        default="configs/battlesnake.yaml",
         help="Path to the config file.",
     )
     parser.add_argument(
@@ -29,5 +29,9 @@ def main(config_path: Path, cleanup: bool = False):
         action="store_true",
         help="If set, do not clean up the game environment after running.",
     )
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     main(**vars(args))
+
+
+if __name__ == "__main__":
+    main_cli()

tests/test_integration.py

@@ -5,66 +5,10 @@
 using DeterministicModel instead of real LLM models.
 """
 
-import tempfile
-from pathlib import Path
-from unittest.mock import patch
+from codeclash import CONFIG_DIR
+from main import main_cli
 
-import yaml
-from minisweagent.models.test_models import DeterministicModel
 
-from main import main
-
-
-def test_main_battlesnake_integration():
-    """
-    Integration test for main.py with configs/battlesnake.yaml.
-    Success criterion: execution completes without exceptions.
-    """
-    # Create a temporary config file with DeterministicModel settings
-    config_path = "configs/battlesnake.yaml"
-
-    # Read the original config
-    with open(config_path) as f:
-        config = yaml.safe_load(f)
-
-    # Create a temporary directory for test artifacts
-    with tempfile.TemporaryDirectory() as temp_dir:
-        temp_config_path = Path(temp_dir) / "test_battlesnake.yaml"
-
-        # Reduce rounds to 1 for faster testing
-        config["tournament"]["rounds"] = 1
-
-        # Write the modified config
-        with open(temp_config_path, "w") as f:
-            yaml.dump(config, f)
-
-        def mock_get_agent(original_get_agent):
-            """Wrapper to replace agent models with DeterministicModel"""
-
-            def wrapper(config, game_context, environment):
-                agent = original_get_agent(config, game_context, environment)
-                print("In wrapper, got agent of type ", type(agent))
-
-                # Replace model if the agent has one (specifically for MiniSWEAgent)
-                if hasattr(agent, "agent") and hasattr(agent.agent, "model"):
-                    print(f"Replacing model for agent {agent.name}")
-                    # Create DeterministicModel with the specified command
-                    deterministic_model = DeterministicModel(
-                        outputs=["```bash\necho 'COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT'\n```"]
-                    )
-                    agent.agent.model = deterministic_model
-
-                return agent
-
-            return wrapper
-
-        # Import the get_agent function and patch it where it's used in the tournaments
-        from codeclash.agents import get_agent
-
-        # Run the main function with cleanup enabled
-        with patch(
-            "codeclash.tournaments.pvp.get_agent",
-            side_effect=mock_get_agent(get_agent),
-        ):
-            # This should complete without raising any exceptions
-            main(temp_config_path, cleanup=True, push_agent=False)
+def test_pvp_battlesnake():
+    config_path = CONFIG_DIR / "test_configs" / "battlesnake_pvp_test.yaml"
+    main_cli(["-c", str(config_path)])