test_evaluate.py

"""Tests for evaluation framework: EvalTask, EvalResult, BenchmarkResults, run_benchmark."""

from __future__ import annotations

from unittest.mock import AsyncMock, MagicMock

import pytest

from hawk.evaluate import BenchmarkResults, EvalResult, EvalTask, run_benchmark, run_benchmark_async


class TestEvalTask:
    """Tests for EvalTask dataclass."""

    def test_defaults(self) -> None:
        task = EvalTask(name="t1", prompt="What is 2+2?")
        assert task.name == "t1"
        assert task.prompt == "What is 2+2?"
        assert task.category == "general"
        assert task.expected_tools == []
        assert task.validate is None
        assert task.max_turns == 10

    def test_custom_values(self) -> None:
        task = EvalTask(
            name="weather",
            prompt="What's the weather?",
            category="tools",
            expected_tools=["get_weather"],
            validate=lambda r: "sunny" in r.response,
            max_turns=5,
        )
        assert task.category == "tools"
        assert task.expected_tools == ["get_weather"]
        assert task.max_turns == 5


class TestEvalResult:
    """Tests for EvalResult dataclass."""

    def test_defaults(self) -> None:
        result = EvalResult(task_name="t1", success=True, duration_ms=100.0)
        assert result.tokens_in == 0
        assert result.tokens_out == 0
        assert result.turns_taken == 0
        assert result.error is None

    def test_with_error(self) -> None:
        result = EvalResult(task_name="t1", success=False, duration_ms=50.0, error="timeout")
        assert result.success is False
        assert result.error == "timeout"


class TestBenchmarkResults:
    """Tests for BenchmarkResults aggregation."""

    def test_empty(self) -> None:
        br = BenchmarkResults()
        assert br.total_tasks == 0
        assert br.passed == 0
        assert br.failed == 0
        assert br.pass_rate == 0.0
        assert br.avg_duration_ms == 0.0
        assert br.total_tokens == 0

    def test_all_passed(self) -> None:
        br = BenchmarkResults(
            results=[
                EvalResult(
                    task_name="t1", success=True, duration_ms=100.0, tokens_in=10, tokens_out=5
                ),
                EvalResult(
                    task_name="t2", success=True, duration_ms=200.0, tokens_in=20, tokens_out=10
                ),
            ]
        )
        assert br.total_tasks == 2
        assert br.passed == 2
        assert br.failed == 0
        assert br.pass_rate == 1.0
        assert br.avg_duration_ms == 150.0
        assert br.total_tokens == 45

    def test_mixed_results(self) -> None:
        br = BenchmarkResults(
            results=[
                EvalResult(task_name="t1", success=True, duration_ms=100.0),
                EvalResult(task_name="t2", success=False, duration_ms=200.0, error="fail"),
            ]
        )
        assert br.passed == 1
        assert br.failed == 1
        assert br.pass_rate == 0.5

    def test_by_category(self) -> None:
        # by_category splits on "/" — names without "/" get "general"
        br = BenchmarkResults(
            results=[
                EvalResult(task_name="math/add", success=True, duration_ms=100.0),
                EvalResult(task_name="math/mul", success=True, duration_ms=100.0),
                EvalResult(task_name="general/weather", success=False, duration_ms=100.0),
            ]
        )
        cats = br.by_category()
        assert len(cats["math"]) == 2
        assert len(cats["general"]) == 1

    def test_summary(self) -> None:
        br = BenchmarkResults(
            results=[
                EvalResult(
                    task_name="t1", success=True, duration_ms=100.0, tokens_in=10, tokens_out=5
                ),
            ]
        )
        summary = br.summary()
        assert "1/1 passed" in summary
        assert "100ms" in summary
        assert "15" in summary  # total tokens

    def test_summary_with_failures(self) -> None:
        br = BenchmarkResults(
            results=[
                EvalResult(task_name="t1", success=True, duration_ms=100.0),
                EvalResult(task_name="t2", success=False, duration_ms=50.0, error="bad output"),
            ]
        )
        summary = br.summary()
        assert "1/2 passed" in summary
        assert "Failures:" in summary
        assert "t2" in summary
        assert "bad output" in summary


def _make_mock_agent(response: str = "ok", tokens_in: int = 10, tokens_out: int = 5) -> MagicMock:
    """Create a mock agent with chat and reset methods."""
    agent = MagicMock()
    resp = MagicMock()
    resp.response = response
    resp.tokens_in = tokens_in
    resp.tokens_out = tokens_out
    resp.turns_taken = 1
    resp.duration = "1.0s"
    agent.chat.return_value = resp
    return agent


class TestRunBenchmark:
    """Tests for run_benchmark."""

    def test_single_task(self) -> None:
        agent = _make_mock_agent()
        tasks = [EvalTask(name="t1", prompt="hello")]
        results = run_benchmark(agent, tasks)
        assert results.total_tasks == 1
        assert results.passed == 1

    def test_multiple_tasks(self) -> None:
        agent = _make_mock_agent()
        tasks = [
            EvalTask(name="t1", prompt="hello"),
            EvalTask(name="t2", prompt="world"),
        ]
        results = run_benchmark(agent, tasks)
        assert results.total_tasks == 2
        assert agent.chat.call_count == 2

    def test_multiple_runs(self) -> None:
        agent = _make_mock_agent()
        tasks = [EvalTask(name="t1", prompt="hello")]
        results = run_benchmark(agent, tasks, runs=3)
        assert results.total_tasks == 3
        assert agent.chat.call_count == 3

    def test_reset_between_tasks(self) -> None:
        agent = _make_mock_agent()
        tasks = [
            EvalTask(name="t1", prompt="hello"),
            EvalTask(name="t2", prompt="world"),
        ]
        run_benchmark(agent, tasks, reset_between_tasks=True)
        assert agent.reset.call_count == 2

    def test_no_reset(self) -> None:
        agent = _make_mock_agent()
        tasks = [
            EvalTask(name="t1", prompt="hello"),
            EvalTask(name="t2", prompt="world"),
        ]
        run_benchmark(agent, tasks, reset_between_tasks=False)
        agent.reset.assert_not_called()

    def test_validation_pass(self) -> None:
        agent = _make_mock_agent(response="The temperature is 72F")
        tasks = [
            EvalTask(
                name="t1",
                prompt="weather?",
                validate=lambda r: "temperature" in r.response,
            )
        ]
        results = run_benchmark(agent, tasks)
        assert results.passed == 1

    def test_validation_fail(self) -> None:
        agent = _make_mock_agent(response="ok")
        tasks = [
            EvalTask(
                name="t1",
                prompt="weather?",
                validate=lambda r: "temperature" in r.response,
            )
        ]
        results = run_benchmark(agent, tasks)
        assert results.passed == 0
        assert results.failed == 1

    def test_agent_exception(self) -> None:
        agent = MagicMock()
        agent.chat.side_effect = RuntimeError("agent crashed")
        tasks = [EvalTask(name="t1", prompt="hello")]
        results = run_benchmark(agent, tasks)
        assert results.passed == 0
        assert results.failed == 1
        assert results.results[0].error == "agent crashed"

    def test_category_in_task_name(self) -> None:
        agent = _make_mock_agent()
        tasks = [EvalTask(name="add", prompt="2+2", category="math")]
        results = run_benchmark(agent, tasks)
        assert results.results[0].task_name == "math/add"


@pytest.mark.asyncio
class TestRunBenchmarkAsync:
    """Tests for run_benchmark_async."""

    async def test_single_task(self) -> None:
        agent = AsyncMock()
        agent.reset = MagicMock()  # reset is sync; AsyncMock would leak a coroutine
        resp = MagicMock()
        resp.response = "ok"
        resp.tokens_in = 10
        resp.tokens_out = 5
        resp.turns_taken = 1
        resp.duration = "1.0s"
        agent.chat.return_value = resp
        tasks = [EvalTask(name="t1", prompt="hello")]
        results = await run_benchmark_async(agent, tasks)
        assert results.total_tasks == 1
        assert results.passed == 1

    async def test_multiple_runs(self) -> None:
        agent = AsyncMock()
        agent.reset = MagicMock()  # reset is sync; AsyncMock would leak a coroutine
        resp = MagicMock()
        resp.response = "ok"
        resp.tokens_in = 10
        resp.tokens_out = 5
        resp.turns_taken = 1
        resp.duration = "1.0s"
        agent.chat.return_value = resp
        tasks = [EvalTask(name="t1", prompt="hello")]
        results = await run_benchmark_async(agent, tasks, runs=2)
        assert results.total_tasks == 2

    async def test_validation_fail(self) -> None:
        agent = AsyncMock()
        agent.reset = MagicMock()  # reset is sync; AsyncMock would leak a coroutine
        resp = MagicMock()
        resp.response = "no match"
        resp.tokens_in = 10
        resp.tokens_out = 5
        resp.turns_taken = 1
        resp.duration = "1.0s"
        agent.chat.return_value = resp
        tasks = [
            EvalTask(
                name="t1",
                prompt="test",
                validate=lambda r: "target" in r.response,
            )
        ]
        results = await run_benchmark_async(agent, tasks)
        assert results.failed == 1

    async def test_agent_exception(self) -> None:
        agent = AsyncMock()
        agent.reset = MagicMock()  # reset is sync; AsyncMock would leak a coroutine
        agent.chat.side_effect = RuntimeError("async crash")
        tasks = [EvalTask(name="t1", prompt="hello")]
        results = await run_benchmark_async(agent, tasks)
        assert results.failed == 1
        assert results.results[0].error == "async crash"