claude-code-chat-browser/tests/test_reduce_baselines.py at 93cb445d5ab41e9b37ccd4dc83ea359538fb18ad · cppalliance/claude-code-chat-browser · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
"""Tests for scripts/reduce_baselines.py."""

from __future__ import annotations

import json

import pytest

from scripts.check_benchmark_regression import BenchmarkDataError
from scripts.reduce_baselines import reduce_baselines


def _write_raw(path, benchmarks: list[dict], *, machine: str = "Linux") -> None:
    path.write_text(
        json.dumps(
            {
                "machine_info": {"system": machine},
                "benchmarks": benchmarks,
            },
            indent=2,
        ),
        encoding="utf-8",
    )


def test_reduce_baselines_writes_gated_groups_only(tmp_path) -> None:
    raw = tmp_path / "raw.json"
    out = tmp_path / "baselines.json"
    _write_raw(
        raw,
        [
            {"group": "parse", "name": "test_parse_session_medium", "stats": {"mean": 0.002}},
            {"group": "parse", "name": "test_parse_session_small", "stats": {"mean": 0.0001}},
            {"group": "cache", "name": "test_cache_warm_hit", "stats": {"mean": 1e-05}},
        ],
    )

    output = reduce_baselines(raw, out)

    assert output["machine"] == "Linux"
    assert "test_parse_session_medium" in output["groups"]["parse"]
    assert "test_parse_session_small" in output["groups"]["parse"]
    assert "cache" not in output["groups"]


def test_reduce_baselines_applies_slack(tmp_path) -> None:
    raw = tmp_path / "raw.json"
    out = tmp_path / "baselines.json"
    _write_raw(
        raw,
        [{"group": "parse", "name": "test_parse_session_medium", "stats": {"mean": 0.002}}],
    )

    reduce_baselines(raw, out, slack=1.5)
    data = json.loads(out.read_text(encoding="utf-8"))

    assert data["groups"]["parse"]["test_parse_session_medium"] == pytest.approx(0.003)


def test_reduce_baselines_rejects_missing_benchmarks_key(tmp_path) -> None:
    raw = tmp_path / "raw.json"
    raw.write_text("{}", encoding="utf-8")

    with pytest.raises(BenchmarkDataError, match="'benchmarks' array"):
        reduce_baselines(raw, tmp_path / "out.json")


def test_reduce_baselines_cli_rejects_non_positive_slack(tmp_path) -> None:
    from scripts.reduce_baselines import main

    raw = tmp_path / "raw.json"
    _write_raw(
        raw,
        [{"group": "parse", "name": "test_parse_session_small", "stats": {"mean": 0.0001}}],
    )

    with pytest.raises(SystemExit) as exc_info:
        main([str(raw), str(tmp_path / "out.json"), "--slack", "0"])
    assert exc_info.value.code == 2


def test_reduce_baselines_uses_peak_bytes_extra_info(tmp_path) -> None:
    raw = tmp_path / "raw.json"
    out = tmp_path / "baselines.json"
    _write_raw(
        raw,
        [
            {
                "group": "parse",
                "name": "test_parse_large_peak_memory",
                "stats": {"mean": 0.05},
                "extra_info": {"peak_bytes": 10_000_000},
            }
        ],
    )

    output = reduce_baselines(raw, out)

    assert output["groups"]["parse"]["test_parse_large_peak_memory"] == 10_000_000.0


def test_reduce_baselines_machine_info_non_dict(tmp_path) -> None:
    raw = tmp_path / "raw.json"
    raw.write_text(
        json.dumps(
            {
                "machine_info": "not-a-dict",
                "benchmarks": [
                    {
                        "group": "parse",
                        "name": "test_parse_session_medium",
                        "stats": {"mean": 0.002},
                    }
                ],
            }
        ),
        encoding="utf-8",
    )

    output = reduce_baselines(raw, tmp_path / "out.json")

    assert output["machine"] is None