feat: add strict assertion types for test framework

Author: nikolay-e

tests/cases/diff/regression/regression_003_large_config_fragmentation.yaml

@@ -157,4 +157,5 @@ assertions:
 options:
   max_fragments: 8
   max_files: 3
+  max_fragments_per_file: 3
   min_score: 90

tests/cases/diff/regression/regression_008_enrichment_bounded.yaml

@@ -0,0 +1,39 @@
+# Regression: context enrichment should be bounded
+# A small change should not pull in massive amounts of context.
+
+initial:
+  src/utils/helpers.py: |
+    def format_date(date_str):
+        parts = date_str.split("-")
+        return f"{parts[2]}/{parts[1]}/{parts[0]}"
+
+    def format_currency(amount, currency="USD"):
+        if currency == "USD":
+            return f"${amount:.2f}"
+        return f"{amount:.2f} {currency}"
+
+changed:
+  src/utils/helpers.py: |
+    def format_date(date_str):
+        parts = date_str.split("-")
+        return f"{parts[2]}.{parts[1]}.{parts[0]}"
+
+    def format_currency(amount, currency="USD"):
+        if currency == "USD":
+            return f"${amount:.2f}"
+        return f"{amount:.2f} {currency}"
+
+assertions:
+  must_include_files:
+  - helpers.py
+
+  must_include_content:
+  - format_date
+
+  must_not_include:
+  - GARBAGE_REG_008_A
+
+options:
+  max_enrichment: 20.0
+  min_recall: 1.0
+  max_noise_rate: 0.05

tests/cases/diff/regression/regression_009_single_file_fragmentation.yaml

@@ -0,0 +1,55 @@
+# Regression: single-file change should not produce excessive per-file fragments
+# When only one function changes in a file, the file should appear as 1-2 fragments.
+
+initial:
+  src/auth/login.py: |
+    from hashlib import sha256
+
+    def hash_password(password):
+        return sha256(password.encode()).hexdigest()
+
+    def verify_password(password, password_hash):
+        return hash_password(password) == password_hash
+
+    def create_session(user_id):
+        import uuid
+        return str(uuid.uuid4())
+
+    def validate_session(session_id):
+        return len(session_id) == 36
+
+changed:
+  src/auth/login.py: |
+    from hashlib import sha256
+    import secrets
+
+    def hash_password(password, salt=None):
+        if salt is None:
+            salt = secrets.token_hex(16)
+        return f"{salt}:{sha256((salt + password).encode()).hexdigest()}"
+
+    def verify_password(password, password_hash):
+        salt, hashed = password_hash.split(":")
+        return hash_password(password, salt) == password_hash
+
+    def create_session(user_id):
+        import uuid
+        return str(uuid.uuid4())
+
+    def validate_session(session_id):
+        return len(session_id) == 36
+
+assertions:
+  must_include_files:
+  - login.py
+
+  must_include_content:
+  - hash_password
+  - verify_password
+
+  must_not_include:
+  - GARBAGE_REG_009_A
+
+options:
+  max_fragments_per_file: 5
+  min_recall: 1.0

tests/framework/loader.py

@@ -74,6 +74,11 @@ def _parse_yaml_test(data: dict, source_file: Path | None = None) -> YamlTestCas
         must_not_include_files=must_not_include_files,
         max_fragments=options.get("max_fragments", data.get("max_fragments")),
         max_files=options.get("max_files", data.get("max_files")),
+        max_fragments_per_file=options.get("max_fragments_per_file", data.get("max_fragments_per_file")),
+        max_enrichment=options.get("max_enrichment", data.get("max_enrichment")),
+        min_recall=options.get("min_recall", data.get("min_recall")),
+        max_noise_rate=options.get("max_noise_rate", data.get("max_noise_rate")),
+        max_context_tokens=options.get("max_context_tokens", data.get("max_context_tokens")),
         commit_message=options.get("commit_message", data.get("commit_message", "Update files")),
         min_budget=options.get("min_budget", data.get("min_budget")),
         add_garbage_files=options.get("add_garbage", data.get("add_garbage_files", True)),

tests/framework/types.py

@@ -25,6 +25,11 @@ class YamlTestCase:
     must_not_include_files: list[str] = field(default_factory=list)
     max_fragments: int | None = None
     max_files: int | None = None
+    max_fragments_per_file: int | None = None
+    max_enrichment: float | None = None
+    min_recall: float | None = None
+    max_noise_rate: float | None = None
+    max_context_tokens: int | None = None
     commit_message: str = "Update files"
     min_budget: int | None = None
     add_garbage_files: bool = True

tests/test_yaml_diff.py

@@ -1,3 +1,4 @@
+from collections import Counter
 from pathlib import Path
 
 import pytest
@@ -24,13 +25,26 @@ def test_cases_loaded():
 MIN_INDIVIDUAL_SCORE = 10.0
 
 
+def _count_fragments_per_file(context: dict) -> dict[str, int]:
+    counts: Counter[str] = Counter()
+    for frag in context.get("fragments", []):
+        path = frag.get("path", "")
+        if path:
+            counts[path] += 1
+    return dict(counts)
+
+
 @pytest.mark.parametrize("case", ALL_CASES, ids=lambda c: c.id)
 def test_diff_yaml(yaml_test_runner: YamlTestRunner, case: YamlTestCase, record_property, request):
     if case.xfail:
         request.node.add_marker(pytest.mark.xfail(reason=case.xfail, strict=True))
     context = yaml_test_runner.run_test_case(case)
     breakdown = yaml_test_runner.score_test_case(context, case)
 
+    fragments = context.get("fragments", [])
+    frag_count = len(fragments)
+    unique_files = len({f.get("path", "") for f in fragments if f.get("path")})
+
     record_property("score", breakdown.score)
     record_property("recall", round(breakdown.recall * 100, 1))
     record_property("noise_rate", round(breakdown.noise_rate * 100, 1))
@@ -39,12 +53,41 @@ def test_diff_yaml(yaml_test_runner: YamlTestRunner, case: YamlTestCase, record_
     record_property("enrichment", round(breakdown.enrichment * 100))
     record_property("diff_tokens", breakdown.diff_tokens)
     record_property("context_tokens", breakdown.context_tokens)
+    record_property("fragment_count", frag_count)
+    record_property("unique_files", unique_files)
 
     effective_min = case.min_score if case.min_score is not None else MIN_INDIVIDUAL_SCORE
     assert breakdown.score >= effective_min, f"[{case.id}] score {breakdown.score:.1f}% below minimum {effective_min}%"
     if case.must_include_files:
         assert breakdown.diff_covered, f"[{case.id}] diff lines not covered by context"
 
+    if case.max_fragments_per_file is not None:
+        per_file = _count_fragments_per_file(context)
+        for path, count in per_file.items():
+            assert (
+                count <= case.max_fragments_per_file
+            ), f"[{case.id}] {path} has {count} fragments, exceeds max_fragments_per_file={case.max_fragments_per_file}"
+
+    if case.max_enrichment is not None:
+        assert (
+            breakdown.enrichment <= case.max_enrichment
+        ), f"[{case.id}] enrichment {breakdown.enrichment:.1f}x exceeds max_enrichment={case.max_enrichment}"
+
+    if case.min_recall is not None:
+        assert (
+            breakdown.recall >= case.min_recall
+        ), f"[{case.id}] recall {breakdown.recall:.1%} below min_recall={case.min_recall:.0%}"
+
+    if case.max_noise_rate is not None:
+        assert (
+            breakdown.noise_rate <= case.max_noise_rate
+        ), f"[{case.id}] noise_rate {breakdown.noise_rate:.1%} exceeds max_noise_rate={case.max_noise_rate:.0%}"
+
+    if case.max_context_tokens is not None:
+        assert (
+            breakdown.context_tokens <= case.max_context_tokens
+        ), f"[{case.id}] {breakdown.context_tokens} context tokens exceeds max_context_tokens={case.max_context_tokens}"
+
 
 @pytest.mark.parametrize(
     "case",