diff --git a/test_bot/buggy_engine b/test_bot/buggy_engine deleted file mode 100644 index 2f37019f0..000000000 --- a/test_bot/buggy_engine +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/sh - -python3 test_bot/buggy_engine.py diff --git a/test_bot/buggy_engine.bat b/test_bot/buggy_engine.bat deleted file mode 100644 index 213dbeb92..000000000 --- a/test_bot/buggy_engine.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -python test_bot\buggy_engine.py diff --git a/test_bot/buggy_engine.py b/test_bot/buggy_engine.py index f397fcb09..e1dbf0397 100644 --- a/test_bot/buggy_engine.py +++ b/test_bot/buggy_engine.py @@ -2,6 +2,12 @@ import chess import time +import typing + +if typing.TYPE_CHECKING: + from test_bot.test_games import scholars_mate +else: + from test_games import scholars_mate assert input() == "uci" @@ -12,13 +18,12 @@ def send_command(command: str) -> None: send_command("id name Procrastinator") -send_command("id author MZH") +send_command("id author lichess-bot-devs") send_command("uciok") delay_performed = False just_started = True -scholars_mate = ["a2a3", "e7e5", "a3a4", "f8c5", "a4a5", "d8h4", "a5a6", "h4f2"] - +board = chess.Board() while True: command, *remaining = input().split() if command == "quit": diff --git a/test_bot/buggy_engine_macos b/test_bot/buggy_engine_macos deleted file mode 100755 index 572743a0a..000000000 --- a/test_bot/buggy_engine_macos +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -python3 test_bot/buggy_engine.py diff --git a/test_bot/homemade.py b/test_bot/homemade.py index cf8080848..dabed27c1 100644 --- a/test_bot/homemade.py +++ b/test_bot/homemade.py @@ -1,28 +1,25 @@ -"""Homemade engine using Stockfish (used in testing).""" +"""Homemade engine playing scholar's mate.""" from homemade import ExampleEngine import chess import chess.engine -import sys from lib.config import Configuration from lib import model from lib.lichess_types import OPTIONS_GO_EGTB_TYPE, COMMANDS_TYPE, MOVE +from test_bot.test_games import scholars_mate # ruff: noqa: ARG002 -platform = sys.platform -file_extension = ".exe" if platform == "win32" else "" - - -class Stockfish(ExampleEngine): - """A homemade engine that uses Stockfish.""" +class ScholarsMate(ExampleEngine): + """A homemade engine that plays the scholar's mate.""" def __init__(self, commands: COMMANDS_TYPE, options: OPTIONS_GO_EGTB_TYPE, stderr: int | None, draw_or_resign: Configuration, game: model.Game | None, **popen_args: str) -> None: - """Start Stockfish.""" + """Set up engine.""" super().__init__(commands, options, stderr, draw_or_resign, game, **popen_args) - self.engine = chess.engine.SimpleEngine.popen_uci(f"./TEMP/sf{file_extension}") def search(self, board: chess.Board, time_limit: chess.engine.Limit, ponder: bool, draw_offered: bool, root_moves: MOVE) -> chess.engine.PlayResult: - """Get a move using Stockfish.""" - return self.engine.play(board, time_limit) + """Get the next scholar's mate move.""" + move_number = len(board.move_stack) + move = board.parse_uci(scholars_mate[move_number]) + return chess.engine.PlayResult(move, None) diff --git a/test_bot/test_bot.py b/test_bot/test_bot.py index f9310e9fd..3364b3bdf 100644 --- a/test_bot/test_bot.py +++ b/test_bot/test_bot.py @@ -1,122 +1,33 @@ """Test lichess-bot.""" -import pytest -import zipfile -import requests import yaml import chess import chess.engine import threading import os import sys -import stat -import shutil -import tarfile import datetime import logging +import tempfile from multiprocessing import Manager from queue import Queue import test_bot.lichess from lib import config -from lib.timer import Timer, to_seconds, seconds +from lib.timer import Timer, seconds from lib.engine_wrapper import test_suffix from lib.lichess_types import CONFIG_DICT_TYPE if "pytest" not in sys.modules: sys.exit(f"The script {os.path.basename(__file__)} should only be run by pytest.") from lib import lichess_bot +from test_bot.test_games import scholars_mate -platform = sys.platform -archive_ext = "zip" if platform == "win32" else "tar" -file_extension = ".exe" if platform == "win32" else "" - -def download_sf() -> None: - """Download Stockfish 16.""" - stockfish_path = f"./TEMP/sf{file_extension}" - if os.path.exists(stockfish_path): - return - - windows_linux_mac = "windows" if platform == "win32" else ("macos" if platform == "darwin" else "ubuntu") - sf_base = f"stockfish-{windows_linux_mac}-x86-64-modern" - archive_link = f"https://github.com/official-stockfish/Stockfish/releases/download/sf_16/{sf_base}.{archive_ext}" - - response = requests.get(archive_link, allow_redirects=True) - response.raise_for_status() - archive_name = f"./TEMP/sf_zip.{archive_ext}" - with open(archive_name, "wb") as file: - file.write(response.content) - - if archive_ext == "zip": - with zipfile.ZipFile(archive_name, "r") as archive_ref: - archive_ref.extractall("./TEMP/") # noqa: S202 - else: - with tarfile.TarFile(archive_name, "r") as archive_ref: - archive_ref.extractall("./TEMP/", filter="data") - - exe_ext = ".exe" if platform == "win32" else "" - shutil.copyfile(f"./TEMP/stockfish/{sf_base}{exe_ext}", stockfish_path) - - if platform != "win32": - st = os.stat(stockfish_path) - os.chmod(stockfish_path, st.st_mode | stat.S_IEXEC) - - -def download_lc0() -> None: - """Download Leela Chess Zero 0.29.0.""" - if os.path.exists("./TEMP/lc0.exe"): - return - - response = requests.get("https://github.com/LeelaChessZero/lc0/releases/download/v0.29.0/lc0-v0.29.0-windows-cpu-dnnl.zip", - allow_redirects=True) - response.raise_for_status() - with open("./TEMP/lc0_zip.zip", "wb") as file: - file.write(response.content) - with zipfile.ZipFile("./TEMP/lc0_zip.zip", "r") as zip_ref: - zip_ref.extractall("./TEMP/") # noqa: S202 - - -def download_arasan() -> None: - """Download Arasan.""" - if os.path.exists(f"./TEMP/arasan{file_extension}"): - return - if platform == "win32": - response = requests.get("https://arasanchess.org/arasan24.1.zip", allow_redirects=True) - else: - response = requests.get("https://arasanchess.org/arasan-linux-binaries-24.2.2.tar.gz", allow_redirects=True) - response.raise_for_status() - with open(f"./TEMP/arasan.{archive_ext}", "wb") as file: - file.write(response.content) - if archive_ext == "zip": - with zipfile.ZipFile(f"./TEMP/arasan.{archive_ext}", "r") as archive_ref: - archive_ref.extractall("./TEMP/") # noqa: S202 - else: - with tarfile.TarFile(f"./TEMP/arasan.{archive_ext}", "r") as archive_ref: - archive_ref.extractall("./TEMP/", filter="data") - shutil.copyfile(f"./TEMP/arasanx-64{file_extension}", f"./TEMP/arasan{file_extension}") - if platform != "win32": - st = os.stat(f"./TEMP/arasan{file_extension}") - os.chmod(f"./TEMP/arasan{file_extension}", st.st_mode | stat.S_IEXEC) - - -os.makedirs("TEMP", exist_ok=True) logging_level = logging.DEBUG testing_log_file_name = None lichess_bot.logging_configurer(logging_level, testing_log_file_name, True) logger = logging.getLogger(__name__) -class TrivialEngine: - """A trivial engine that should be trivial to beat.""" - - def play(self, board: chess.Board, *_: object) -> chess.engine.PlayResult: - """Choose the first legal move.""" - return chess.engine.PlayResult(next(iter(board.legal_moves)), None) - - def quit(self) -> None: - """Do nothing.""" - - -def lichess_org_simulator(opponent_path: str | None, - move_queue: Queue[chess.Move | None], +def lichess_org_simulator(move_queue: Queue[chess.Move | None], board_queue: Queue[chess.Board], clock_queue: Queue[tuple[datetime.timedelta, datetime.timedelta, datetime.timedelta]], results: Queue[bool]) -> None: @@ -136,27 +47,17 @@ def lichess_org_simulator(opponent_path: str | None, wtime = start_time btime = start_time - engine = chess.engine.SimpleEngine.popen_uci(opponent_path) if opponent_path else TrivialEngine() - while not board.is_game_over(): + move_timer = Timer() if board.turn == chess.WHITE: - if not board.move_stack: - move = engine.play(board, chess.engine.Limit(time=1)) - else: - move_timer = Timer() - move = engine.play(board, - chess.engine.Limit(white_clock=to_seconds(wtime - seconds(2.0)), - white_inc=to_seconds(increment), - black_clock=to_seconds(btime), - black_inc=to_seconds(increment))) - wtime -= move_timer.time_since_reset() - wtime += increment - engine_move = move.move - if engine_move is None: - raise RuntimeError("Engine attempted to make null move.") + move_count = len(board.move_stack) + engine_move = board.parse_uci(scholars_mate[move_count]) board.push(engine_move) board_queue.put(board) clock_queue.put((wtime, btime, increment)) + if len(board.move_stack) > 1: + wtime -= move_timer.time_since_reset() + wtime += increment else: move_timer = Timer() while (bot_move := move_queue.get()) is None: @@ -171,12 +72,11 @@ def lichess_org_simulator(opponent_path: str | None, board_queue.put(board) clock_queue.put((wtime, btime, increment)) - engine.quit() outcome = board.outcome() results.put(outcome is not None and outcome.winner == chess.BLACK) -def run_bot(raw_config: CONFIG_DICT_TYPE, logging_level: int, opponent_path: str | None = None) -> bool: +def run_bot(raw_config: CONFIG_DICT_TYPE, logging_level: int) -> bool: """ Start lichess-bot test with a mocked version of the lichess.org site. @@ -201,7 +101,7 @@ def run_bot(raw_config: CONFIG_DICT_TYPE, logging_level: int, opponent_path: str lichess_bot.disable_restart() results: Queue[bool] = manager.Queue() - thr = threading.Thread(target=lichess_org_simulator, args=[opponent_path, move_queue, board_queue, clock_queue, results]) + thr = threading.Thread(target=lichess_org_simulator, args=[move_queue, board_queue, clock_queue, results]) thr.start() lichess_bot.start(li, user_profile, CONFIG, logging_level, testing_log_file_name, True, one_game=True) @@ -218,134 +118,75 @@ def run_bot(raw_config: CONFIG_DICT_TYPE, logging_level: int, opponent_path: str return result -@pytest.mark.timeout(180, method="thread") -def test_sf() -> None: +def test_uci() -> None: """Test lichess-bot with Stockfish (UCI).""" with open("./config.yml.default") as file: CONFIG = yaml.safe_load(file) - CONFIG["token"] = "" - CONFIG["engine"]["dir"] = "./TEMP/" - CONFIG["engine"]["name"] = f"sf{file_extension}" - CONFIG["engine"]["uci_options"]["Threads"] = 1 - CONFIG["pgn_directory"] = "TEMP/sf_game_record" - logger.info("Downloading Stockfish") - try: - download_sf() - except Exception: - logger.exception("Could not download the Stockfish chess engine") - pytest.skip("Could not download the Stockfish chess engine") - win = run_bot(CONFIG, logging_level) - logger.info("Finished Testing SF") - assert win - assert os.path.isfile(os.path.join(CONFIG["pgn_directory"], - "bo vs b - zzzzzzzz.pgn")) - -@pytest.mark.timeout(180, method="thread") -def test_lc0() -> None: - """Test lichess-bot with Leela Chess Zero (UCI).""" - if platform != "win32": - pytest.skip("Platform must be Windows.") + with tempfile.TemporaryDirectory() as temp: + CONFIG["token"] = "" + CONFIG["engine"]["dir"] = "test_bot" + CONFIG["engine"]["name"] = "uci_engine.py" + CONFIG["engine"]["interpreter"] = sys.executable + CONFIG["pgn_directory"] = os.path.join(temp, "uci_game_record") + CONFIG["engine"]["uci_options"] = {} + win = run_bot(CONFIG, logging_level) + logger.info("Finished Testing UCI") + assert win + assert os.path.isfile(os.path.join(CONFIG["pgn_directory"], + "bo vs b - zzzzzzzz.pgn")) + + +def test_xboard() -> None: + """Test lichess-bot with an XBoard engine.""" with open("./config.yml.default") as file: CONFIG = yaml.safe_load(file) - CONFIG["token"] = "" - CONFIG["engine"]["dir"] = "./TEMP/" - CONFIG["engine"]["working_dir"] = "./TEMP/" - CONFIG["engine"]["name"] = "lc0.exe" - CONFIG["engine"]["uci_options"]["Threads"] = 1 - CONFIG["engine"]["uci_options"].pop("Hash", None) - CONFIG["engine"]["uci_options"].pop("Move Overhead", None) - CONFIG["pgn_directory"] = "TEMP/lc0_game_record" - logger.info("Downloading LC0") - try: - download_lc0() - except Exception: - logger.exception("Could not download the LC0 chess engine") - pytest.skip("Could not download the LC0 chess engine") - win = run_bot(CONFIG, logging_level) - logger.info("Finished Testing LC0") - assert win - assert os.path.isfile(os.path.join(CONFIG["pgn_directory"], - "bo vs b - zzzzzzzz.pgn")) - -@pytest.mark.timeout(150, method="thread") -def test_arasan() -> None: - """Test lichess-bot with Arasan (XBoard).""" - if platform not in ("linux", "win32"): - pytest.skip("Platform must be Windows or Linux.") - with open("./config.yml.default") as file: - CONFIG = yaml.safe_load(file) - CONFIG["token"] = "" - CONFIG["engine"]["dir"] = "./TEMP/" - CONFIG["engine"]["working_dir"] = "./TEMP/" - CONFIG["engine"]["protocol"] = "xboard" - CONFIG["engine"]["name"] = f"arasan{file_extension}" - CONFIG["engine"]["ponder"] = False - CONFIG["pgn_directory"] = "TEMP/arasan_game_record" - logger.info("Downloading Arasan") - try: - download_arasan() - except Exception: - logger.exception("Could not download the Arasan chess engine") - pytest.skip("Could not download the Arasan chess engine") - win = run_bot(CONFIG, logging_level) - logger.info("Finished Testing Arasan") - assert win - assert os.path.isfile(os.path.join(CONFIG["pgn_directory"], - "bo vs b - zzzzzzzz.pgn")) + with tempfile.TemporaryDirectory() as temp: + CONFIG["token"] = "" + CONFIG["engine"]["dir"] = "test_bot" + CONFIG["engine"]["name"] = "xboard_engine.py" + CONFIG["engine"]["protocol"] = "xboard" + CONFIG["engine"]["interpreter"] = sys.executable + CONFIG["pgn_directory"] = os.path.join(temp, "lc0_game_record") + win = run_bot(CONFIG, logging_level) + logger.info("Finished Testing XBoard") + assert win + assert os.path.isfile(os.path.join(CONFIG["pgn_directory"], + "bo vs b - zzzzzzzz.pgn")) -@pytest.mark.timeout(180, method="thread") def test_homemade() -> None: - """Test lichess-bot with a homemade engine running Stockfish (Homemade).""" - try: - download_sf() - except Exception: - logger.exception("Could not download the Stockfish chess engine") - pytest.skip("Could not download the Stockfish chess engine") - + """Test lichess-bot with a homemade engine.""" with open("./config.yml.default") as file: CONFIG = yaml.safe_load(file) - CONFIG["token"] = "" - CONFIG["engine"]["name"] = f"Stockfish{test_suffix}" - CONFIG["engine"]["protocol"] = "homemade" - CONFIG["pgn_directory"] = "TEMP/homemade_game_record" - win = run_bot(CONFIG, logging_level) - logger.info("Finished Testing Homemade") - assert win - assert os.path.isfile(os.path.join(CONFIG["pgn_directory"], - "bo vs b - zzzzzzzz.pgn")) + + with tempfile.TemporaryDirectory() as temp: + CONFIG["token"] = "" + CONFIG["engine"]["name"] = f"ScholarsMate{test_suffix}" + CONFIG["engine"]["protocol"] = "homemade" + CONFIG["pgn_directory"] = os.path.join(temp, "homemade_game_record") + win = run_bot(CONFIG, logging_level) + logger.info("Finished Testing Homemade") + assert win + assert os.path.isfile(os.path.join(CONFIG["pgn_directory"], + "bo vs b - zzzzzzzz.pgn")) -@pytest.mark.timeout(60, method="thread") def test_buggy_engine() -> None: """Test lichess-bot with an engine that causes a timeout error within python-chess.""" with open("./config.yml.default") as file: CONFIG = yaml.safe_load(file) - CONFIG["token"] = "" - CONFIG["engine"]["dir"] = "test_bot" - - def engine_path(CONFIG: CONFIG_DICT_TYPE) -> str: - directory: str = CONFIG["engine"]["dir"] - name: str = CONFIG["engine"]["name"].removesuffix(".py") - path = os.path.join(directory, name) - if platform == "win32": - path += ".bat" - else: - if platform == "darwin": - path += "_macos" - st = os.stat(path) - os.chmod(path, st.st_mode | stat.S_IEXEC) - return path - - CONFIG["engine"]["name"] = "buggy_engine.py" - CONFIG["engine"]["interpreter"] = "python" if platform == "win32" else "python3" - CONFIG["engine"]["uci_options"] = {"go_commands": {"movetime": 100}} - CONFIG["pgn_directory"] = "TEMP/bug_game_record" - win = run_bot(CONFIG, logging_level, engine_path(CONFIG)) - logger.info("Finished Testing buggy engine") - assert win - assert os.path.isfile(os.path.join(CONFIG["pgn_directory"], - "bo vs b - zzzzzzzz.pgn")) + with tempfile.TemporaryDirectory() as temp: + CONFIG["token"] = "" + CONFIG["engine"]["dir"] = "test_bot" + CONFIG["engine"]["name"] = "buggy_engine.py" + CONFIG["engine"]["interpreter"] = sys.executable + CONFIG["engine"]["uci_options"] = {"go_commands": {"movetime": 100}} + CONFIG["pgn_directory"] = os.path.join(temp, "bug_game_record") + win = run_bot(CONFIG, logging_level) + logger.info("Finished Testing Buggy Engine") + assert win + assert os.path.isfile(os.path.join(CONFIG["pgn_directory"], + "bo vs b - zzzzzzzz.pgn")) diff --git a/test_bot/test_games.py b/test_bot/test_games.py new file mode 100644 index 000000000..b8158336e --- /dev/null +++ b/test_bot/test_games.py @@ -0,0 +1,3 @@ +"""Games for testing.""" + +scholars_mate = ["a2a3", "e7e5", "a3a4", "f8c5", "a4a5", "d8h4", "a5a6", "h4f2"] diff --git a/test_bot/uci_engine.py b/test_bot/uci_engine.py new file mode 100644 index 000000000..cb7fbc67a --- /dev/null +++ b/test_bot/uci_engine.py @@ -0,0 +1,42 @@ +"""An engine mimics a UCI engine.""" + +import chess +import typing + +if typing.TYPE_CHECKING: + from test_bot.test_games import scholars_mate +else: + from test_games import scholars_mate + +assert input() == "uci" + + +def send_command(command: str) -> None: + """Send UCI commands to lichess-bot without output buffering.""" + print(command, flush=True) # noqa: T201 (print() found) + + +send_command("id name UCI_Test_Bot") +send_command("id author lichess-bot-devs") +send_command("uciok") + +board = chess.Board() +while True: + command, *remaining = input().split() + if command == "quit": + break + elif command == "isready": + send_command("readyok") + elif command == "position": + spec_type, *remaining = remaining + assert spec_type == "startpos" + board = chess.Board() + if remaining: + moves_label, *move_list = remaining + assert moves_label == "moves" + for move in move_list: + board.push_uci(move) + elif command == "go": + move_count = len(board.move_stack) + move = scholars_mate[move_count] + send_command(f"bestmove {move}") diff --git a/test_bot/xboard_engine.py b/test_bot/xboard_engine.py new file mode 100644 index 000000000..2c341a5d4 --- /dev/null +++ b/test_bot/xboard_engine.py @@ -0,0 +1,36 @@ +"""An engine mimics an XBoard engine.""" + +import chess +import typing + +if typing.TYPE_CHECKING: + from test_bot.test_games import scholars_mate +else: + from test_games import scholars_mate + +assert input() == "xboard" +assert input() == "protover 2" + + +def send_command(command: str) -> None: + """Send UCI commands to lichess-bot without output buffering.""" + print(command, flush=True) # noqa: T201 (print() found) + + +send_command('feature myname="XBoard Test Bot" ping=1 setboard=1 usermove=1 done=1') + +board = chess.Board() +while True: + command, *remaining = input().split() + if command == "quit": + break + elif command == "ping": + send_command(f"pong {''.join(remaining)}") + elif command == "new": + board = chess.Board() + elif command == "usermove": + board.push_xboard("".join(remaining)) + move_count = len(board.move_stack) + move = scholars_mate[move_count] + send_command(f"move {move}") + board.push_xboard(move)