Add CLI output validation and ScenarioGroup edge cases

Author: Евгений Блинов

tests/cli/test_scenario_cli.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 import subprocess
 import sys
 import textwrap
@@ -54,7 +55,19 @@ def test_cli_outputs_worst(self) -> None:
 
     def test_cli_output_has_s_suffix(self) -> None:
         proc = run_script(scenario_script())
-        assert 's\n' in proc.stdout or proc.stdout.rstrip().endswith('s')
+        # Each value line (mean/best/worst) must end with 's'
+        lines = proc.stdout.strip().splitlines()
+        for line in lines[1:]:  # skip 'benchmark: bench' header
+            assert line.endswith('s'), f'line does not end with s: {line!r}'
+
+    def test_cli_exact_output_format(self) -> None:
+        proc = run_script(scenario_script())
+        lines = proc.stdout.strip().splitlines()
+        assert len(lines) == 4
+        assert lines[0] == 'benchmark: bench'
+        assert re.match(r'^mean:  \d+\.\d{6}s$', lines[1]), f'unexpected format: {lines[1]!r}'
+        assert re.match(r'^best:  \d+\.\d{6}s$', lines[2]), f'unexpected format: {lines[2]!r}'
+        assert re.match(r'^worst: \d+\.\d{6}s$', lines[3]), f'unexpected format: {lines[3]!r}'
 
     def test_cli_exit_code_0_by_default(self) -> None:
         proc = run_script(scenario_script())

tests/cli/test_scenario_group_cli.py

@@ -105,6 +105,37 @@ def test_help_mentions_number(self) -> None:
         combined = proc.stdout + proc.stderr
         assert 'number' in combined.lower()
 
+    def test_help_mentions_max_mean(self) -> None:
+        proc = run_script(group_script(), '--help')
+        combined = proc.stdout + proc.stderr
+        assert '--max-mean' in combined
+
+    def test_help_does_not_run_benchmark(self) -> None:
+        proc = run_script(group_script(), '--help')
+        assert 'benchmark:' not in proc.stdout
+
+
+def empty_group_script() -> str:
+    return textwrap.dedent(f'''
+        import sys
+        sys.path.insert(0, {str(__import__('pathlib').Path(__file__).parent.parent.parent)!r})
+        from microbenchmark import ScenarioGroup
+
+        group = ScenarioGroup()
+        group.cli()
+    ''')
+
+
+class TestScenarioGroupCliEmptyGroup:
+    def test_empty_group_exits_0(self) -> None:
+        proc = run_script(empty_group_script())
+        assert proc.returncode == 0
+
+    def test_empty_group_no_output(self) -> None:
+        proc = run_script(empty_group_script())
+        assert proc.stdout == ''
+        assert proc.stderr == ''
+
 
 def single_scenario_script() -> str:
     return textwrap.dedent(f'''

tests/units/test_scenario.py

@@ -243,6 +243,31 @@ def fn() -> None:
             s.run()
         assert counter[0] == 3
 
+    def test_run_exception_during_warmup_propagates(self) -> None:
+        counter = [0]
+
+        def fn() -> None:
+            counter[0] += 1
+            if counter[0] == 2:
+                raise RuntimeError('fail in warmup')
+
+        s = Scenario(fn, name='s', number=5)
+        with pytest.raises(RuntimeError, match='fail in warmup'):
+            s.run(warmup=3)
+        assert counter[0] == 2  # stopped at 2nd warmup call
+
+    def test_run_number_one(self) -> None:
+        tick = [0]
+
+        def fake_timer() -> float:
+            tick[0] += 1
+            return float(tick[0])
+
+        s = Scenario(lambda: None, name='s', number=1, timer=fake_timer)
+        result = s.run()
+        assert len(result.durations) == 1
+        assert result.durations[0] == pytest.approx(1.0)  # end(2) - start(1) = 1
+
 
 class TestScenarioAdd:
     def test_add_scenario_returns_group(self) -> None:

tests/units/test_scenario_group.py

@@ -32,38 +32,59 @@ def test_scenario_plus_scenario(self) -> None:
         s1, s2 = make_scenario('s1'), make_scenario('s2')
         group = s1 + s2
         assert isinstance(group, ScenarioGroup)
-        assert len(group.run()) == 2
+        results = group.run()
+        assert len(results) == 2
+        assert results[0].scenario is s1
+        assert results[1].scenario is s2
 
     def test_group_plus_scenario(self) -> None:
         s1, s2, s3 = make_scenario('s1'), make_scenario('s2'), make_scenario('s3')
         group = ScenarioGroup(s1, s2) + s3
         assert isinstance(group, ScenarioGroup)
-        assert len(group.run()) == 3
+        results = group.run()
+        assert len(results) == 3
+        assert results[0].scenario is s1
+        assert results[1].scenario is s2
+        assert results[2].scenario is s3
 
     def test_scenario_plus_group(self) -> None:
         s1, s2, s3 = make_scenario('s1'), make_scenario('s2'), make_scenario('s3')
         group = s1 + ScenarioGroup(s2, s3)
         assert isinstance(group, ScenarioGroup)
-        assert len(group.run()) == 3
+        results = group.run()
+        assert len(results) == 3
+        assert results[0].scenario is s1
+        assert results[1].scenario is s2
+        assert results[2].scenario is s3
 
     def test_group_plus_group(self) -> None:
         s1, s2, s3 = make_scenario('s1'), make_scenario('s2'), make_scenario('s3')
         group = ScenarioGroup(s1) + ScenarioGroup(s2, s3)
         assert isinstance(group, ScenarioGroup)
-        assert len(group.run()) == 3
+        results = group.run()
+        assert len(results) == 3
+        assert results[0].scenario is s1
+        assert results[1].scenario is s2
+        assert results[2].scenario is s3
 
     def test_triple_sum_is_flat(self) -> None:
         s1, s2, s3 = make_scenario('s1'), make_scenario('s2'), make_scenario('s3')
         group = s1 + s2 + s3
         assert isinstance(group, ScenarioGroup)
-        assert len(group.run()) == 3
+        results = group.run()
+        assert len(results) == 3
+        assert results[0].scenario is s1
+        assert results[1].scenario is s2
+        assert results[2].scenario is s3
 
     def test_add_returns_new_group(self) -> None:
         s1, s2 = make_scenario('s1'), make_scenario('s2')
         g = ScenarioGroup(s1)
         new_g = g + s2
         assert new_g is not g
         assert len(g._scenarios) == 1  # original not mutated
+        assert new_g._scenarios[0] is s1
+        assert new_g._scenarios[1] is s2
 
     def test_add_unknown_type_returns_not_implemented(self) -> None:
         g = ScenarioGroup()
@@ -124,6 +145,19 @@ def test_empty_group_run_with_warmup(self) -> None:
         g = ScenarioGroup()
         assert g.run(warmup=10) == []
 
+    def test_run_negative_warmup_acts_as_zero(self) -> None:
+        counter = [0]
+
+        def fn() -> None:
+            counter[0] += 1
+
+        s = Scenario(fn, name='s', number=3)
+        g = ScenarioGroup(s)
+        results = g.run(warmup=-5)
+        assert len(results) == 1
+        assert len(results[0].durations) == 3
+        assert counter[0] == 3  # range(-5) == empty, so no warmup calls
+
     def test_run_returns_benchmark_results(self) -> None:
         s = make_scenario()
         g = ScenarioGroup(s)