fix(proof): enforce EvidenceRule.test_id/must_pass in proof runner (#729) (#802)

* fix(proof): enforce EvidenceRule.test_id/must_pass in proof runner (#729)

A requirement is no longer SATISFIED by any green pytest run — each
must_pass evidence rule is enforced by a pytest run scoped to its
test_id (-k), and a named test that was never written is a FAILED
obligation.

- EvidenceRule gains an owning gate field, stamped at capture,
  persisted in the ledger; legacy rows derive it from the test_id
  prefix convention
- gates.run/_run_pytest accept a test_selector; with a selector,
  "no tests matched" (exit 5) is a failure, not an empty suite
- _run_gate enforces pytest-style rules on every runnable gate: SEC
  runs ruff AND its test_sec_* rules (codex review finding)
- UNIT stub now defines test_unit_<slug> to match its rule, and stub
  slugs share obligations.slugify with rules (was [:50] vs [:60])
- drop unused pytest import in tests/test_new_feature.py (pre-existing
  ruff failure)

* review: guard empty gate checks, warn on unenforceable rules, pin gate=None no-op

Addresses claude-review findings on #802: explicit FAILED when gates.run
returns no checks, per-requirement warning for rules with no resolvable
gate, docstring note on per-rule subprocesses, and a test pinning that
gate=None rules fall back to whole-suite behavior.

* review: harden enforcement per CodeRabbit findings

- invalid persisted gate values fall back to test_id prefix derivation
  instead of crashing requirement load
- SKIPPED/ERROR pytest checks fail enforcement (pytest unavailable is
  not proof)
- must_pass rules without a pytest-style test_id fail loudly instead of
  vanishing silently

Author: frankbria

codeframe/core/gates.py

@@ -287,6 +287,7 @@ def run(
     gates: Optional[list[str]] = None,
     verbose: bool = False,
     auto_install_deps: bool = True,
+    test_selector: Optional[str] = None,
 ) -> GateResult:
     """Run verification gates.
 
@@ -295,6 +296,8 @@ def run(
         gates: Specific gates to run (None = all available)
         verbose: Whether to capture full output
         auto_install_deps: Whether to auto-install missing dependencies before test gates (default: True)
+        test_selector: Optional pytest ``-k`` keyword expression; only applies to
+            the pytest gate. With a selector, "no tests matched" is a failure.
 
     Returns:
         GateResult with all check results
@@ -358,7 +361,7 @@ def run(
     # Run each gate
     for gate_name in gates:
         if gate_name == "pytest":
-            check = _run_pytest(repo_path, verbose)
+            check = _run_pytest(repo_path, verbose, test_selector=test_selector)
         elif gate_name == "ruff":
             check = _run_ruff(repo_path, verbose)
         elif gate_name == "mypy":
@@ -476,8 +479,10 @@ def _detect_available_gates(repo_path: Path) -> list[str]:
     return gates
 
 
-def _run_pytest(repo_path: Path, verbose: bool = False) -> GateCheck:
-    """Run pytest."""
+def _run_pytest(
+    repo_path: Path, verbose: bool = False, test_selector: Optional[str] = None
+) -> GateCheck:
+    """Run pytest, optionally scoped to a ``-k`` keyword expression."""
     import time
 
     start = time.time()
@@ -496,6 +501,8 @@ def _run_pytest(repo_path: Path, verbose: bool = False) -> GateCheck:
             cmd = ["uv", "run", "pytest", "-v", "--tb=short"]
         else:
             cmd = ["pytest", "-v", "--tb=short"]
+        if test_selector:
+            cmd += ["-k", test_selector]
 
         result = subprocess.run(
             cmd,
@@ -535,9 +542,12 @@ def _run_pytest(repo_path: Path, verbose: bool = False) -> GateCheck:
             status = GateStatus.FAILED
         elif result.returncode == 5:
             # Exit code 5: no tests collected
-            # Check if this is a clean "no tests" or an error during collection
+            # With an explicit selector, "nothing matched" means the named
+            # test doesn't exist — that is a failure, not an empty suite.
             output_lower = output.lower()
-            if "error" in output_lower or "importerror" in output_lower or "modulenotfounderror" in output_lower:
+            if test_selector:
+                status = GateStatus.FAILED
+            elif "error" in output_lower or "importerror" in output_lower or "modulenotfounderror" in output_lower:
                 # Collection error disguised as "no tests collected"
                 status = GateStatus.FAILED
             elif "no tests ran" in output_lower or "collected 0 items" in output_lower:

codeframe/core/proof/ledger.py

@@ -190,12 +190,32 @@ def _obligations_from_json(raw: str) -> list[Obligation]:
 
 
 def _evidence_rules_to_json(rules: list[EvidenceRule]) -> str:
-    return json.dumps([{"test_id": r.test_id, "must_pass": r.must_pass} for r in rules])
+    return json.dumps([
+        {"test_id": r.test_id, "must_pass": r.must_pass, "gate": r.gate.value if r.gate else None}
+        for r in rules
+    ])
 
 
 def _evidence_rules_from_json(raw: str) -> list[EvidenceRule]:
+    from codeframe.core.proof.obligations import gate_from_test_id
+
+    def _resolve_gate(d: dict) -> Optional[Gate]:
+        # Legacy rows lack "gate"; invalid persisted values (schema drift,
+        # manual edits) also fall back to test_id prefix derivation.
+        try:
+            return Gate(d["gate"])
+        except (KeyError, ValueError):
+            return gate_from_test_id(d["test_id"])
+
     data = json.loads(raw)
-    return [EvidenceRule(test_id=d["test_id"], must_pass=d.get("must_pass", True)) for d in data]
+    return [
+        EvidenceRule(
+            test_id=d["test_id"],
+            must_pass=d.get("must_pass", True),
+            gate=_resolve_gate(d),
+        )
+        for d in data
+    ]
 
 
 def _waiver_to_json(waiver: Optional[Waiver]) -> Optional[str]:

codeframe/core/proof/models.py

@@ -110,6 +110,7 @@ class EvidenceRule:
 
     test_id: str
     must_pass: bool = True
+    gate: Optional[Gate] = None  # owning gate; None on legacy rows without a derivable prefix
 
 
 @dataclass

codeframe/core/proof/obligations.py

@@ -83,21 +83,38 @@ def get_obligations(glitch_type: GlitchType) -> list[Obligation]:
     return [Obligation(gate=g) for g in gates]
 
 
+TEST_ID_PREFIXES: dict[Gate, str] = {
+    Gate.UNIT: "test_unit_",
+    Gate.CONTRACT: "test_contract_",
+    Gate.E2E: "test_e2e_",
+    Gate.VISUAL: "test_visual_",
+    Gate.A11Y: "test_a11y_",
+    Gate.PERF: "test_perf_",
+    Gate.SEC: "test_sec_",
+    Gate.DEMO: "test_demo_",
+    Gate.MANUAL: "manual_check_",
+}
+
+
+def gate_from_test_id(test_id: str) -> Gate | None:
+    """Derive the owning gate from a test_id prefix (for legacy persisted rules)."""
+    for gate, prefix in TEST_ID_PREFIXES.items():
+        if test_id.startswith(prefix):
+            return gate
+    return None
+
+
+def slugify(text: str) -> str:
+    """Create a safe identifier from text.
+
+    Shared by evidence rules and stub generation so a generated stub's
+    function name matches the rule's test_id exactly.
+    """
+    slug = re.sub(r"[^a-z0-9]+", "_", text.lower().strip())[:60].strip("_")
+    return slug or "unnamed"
+
+
 def suggest_evidence_rules(gate: Gate, description: str) -> list[EvidenceRule]:
     """Generate starter evidence rules for an obligation gate."""
-    # Create a sensible test ID from the description
-    slug = re.sub(r"[^a-z0-9]+", "_", description.lower().strip())[:60].strip("_")
-
-    prefix_map = {
-        Gate.UNIT: "test_unit_",
-        Gate.CONTRACT: "test_contract_",
-        Gate.E2E: "test_e2e_",
-        Gate.VISUAL: "test_visual_",
-        Gate.A11Y: "test_a11y_",
-        Gate.PERF: "test_perf_",
-        Gate.SEC: "test_sec_",
-        Gate.DEMO: "test_demo_",
-        Gate.MANUAL: "manual_check_",
-    }
-    prefix = prefix_map.get(gate, "test_")
-    return [EvidenceRule(test_id=f"{prefix}{slug}", must_pass=True)]
+    prefix = TEST_ID_PREFIXES.get(gate, "test_")
+    return [EvidenceRule(test_id=f"{prefix}{slugify(description)}", must_pass=True, gate=gate)]

codeframe/core/proof/runner.py

@@ -9,12 +9,13 @@
 import logging
 import uuid
 from datetime import datetime, timezone
-from typing import Optional
+from typing import Optional, Sequence
 
 from codeframe.core.proof import ledger
 from codeframe.core.proof.evidence import attach_evidence
 from codeframe.core.proof.models import (
     PROOF_CONFIG_FILENAME,
+    EvidenceRule,
     Gate,
     GateOutcome,
     ProofRun,
@@ -80,12 +81,26 @@ def _load_proof_config(workspace: Workspace) -> tuple[Optional[set[Gate]], str]:
 }
 
 
-def _run_gate(workspace: Workspace, gate: Gate) -> tuple[GateOutcome, str]:
+def _run_gate(
+    workspace: Workspace,
+    gate: Gate,
+    rules: Sequence[EvidenceRule] = (),
+) -> tuple[GateOutcome, str]:
     """Execute a single gate and return (outcome, output).
 
     Uses core/gates.py for gates that have direct tool support. Gates without
     an automated runner return UNVERIFIABLE — the obligation could not be
     checked, which is distinct from running and failing.
+
+    For pytest-backed gates, each ``must_pass`` evidence rule is enforced
+    individually: pytest runs scoped to the rule's ``test_id`` (via ``-k``),
+    and a named test that doesn't exist is a FAILED obligation — a green
+    whole-suite run proves nothing about a test that was never written.
+    Rules with ``must_pass=False`` are informational only.
+
+    Each enforced rule deliberately gets its own pytest subprocess — do not
+    collapse them into one ``-k "a or b"`` run; per-rule exit codes are what
+    distinguish "named test missing" from "collected but failing".
     """
     core_gate_name = _GATE_TO_CORE.get(gate)
     if not core_gate_name:
@@ -95,13 +110,61 @@ def _run_gate(workspace: Workspace, gate: Gate) -> tuple[GateOutcome, str]:
         )
 
     try:
-        from codeframe.core.gates import run as run_gates
-        result = run_gates(workspace, gates=[core_gate_name], verbose=False)
-        output_parts = [
-            f"{check.name}: {check.status.value}" for check in result.checks
-        ]
-        outcome = GateOutcome.PASSED if result.passed else GateOutcome.FAILED
-        return outcome, "\n".join(output_parts)
+        from codeframe.core import gates as core_gates
+
+        # Only pytest-style test_ids can be enforced by a scoped pytest run;
+        # e.g. SEC's test_sec_* rules are pytest tests even though the SEC
+        # gate's own runner is ruff.
+        enforced = [r for r in rules if r.must_pass and r.test_id.startswith("test_")]
+        unenforceable = [r for r in rules if r.must_pass and not r.test_id.startswith("test_")]
+
+        lines: list[str] = []
+        all_passed = True
+
+        for rule in enforced:
+            result = core_gates.run(
+                workspace,
+                gates=["pytest"],
+                verbose=False,
+                test_selector=rule.test_id,
+            )
+            check = result.checks[0] if result.checks else None
+            if check is None:
+                lines.append(f"{rule.test_id}: FAILED — no gate check returned")
+                all_passed = False
+            elif check.exit_code == 5:
+                lines.append(f"{rule.test_id}: FAILED — named test missing (not collected)")
+                all_passed = False
+            elif check.status == core_gates.GateStatus.PASSED:
+                lines.append(f"{rule.test_id}: passed")
+            else:
+                # SKIPPED (pytest unavailable) and ERROR (timeout) are not
+                # proof — enforcement needs a positive pass, unlike the
+                # whole-suite path where SKIPPED counts as passing.
+                lines.append(f"{rule.test_id}: FAILED ({check.status.value})")
+                all_passed = False
+
+        # A must_pass rule we cannot enforce must not silently count as
+        # satisfied — that is the exact bug this module exists to prevent.
+        for rule in unenforceable:
+            lines.append(f"{rule.test_id}: FAILED — must_pass rule has no pytest-style test_id")
+            all_passed = False
+
+        # Run the gate's own runner unless it is pytest and the enforced rules
+        # already covered it (scoped runs replace the whole-suite run).
+        if core_gate_name != "pytest" or not enforced:
+            result = core_gates.run(workspace, gates=[core_gate_name], verbose=False)
+            lines.extend(
+                f"{check.name}: {check.status.value}" for check in result.checks
+            )
+            all_passed = all_passed and result.passed
+
+        for rule in rules:
+            if not rule.must_pass:
+                lines.append(f"{rule.test_id}: informational (must_pass=False, not enforced)")
+
+        outcome = GateOutcome.PASSED if all_passed else GateOutcome.FAILED
+        return outcome, "\n".join(lines)
     except Exception as exc:
         logger.warning("Gate %s failed to run: %s", gate.value, exc)
         return GateOutcome.FAILED, str(exc)
@@ -183,6 +246,13 @@ def run_proof(
 
         req_results: list[tuple[Gate, GateOutcome]] = []
 
+        unresolved = [r.test_id for r in req.evidence_rules if r.gate is None]
+        if unresolved:
+            logger.warning(
+                "REQ %s: %d evidence rule(s) with no resolvable gate are not enforced: %s",
+                req.id, len(unresolved), unresolved,
+            )
+
         for obl in req.obligations:
             # Apply gate filter
             if gate_filter and obl.gate != gate_filter:
@@ -192,8 +262,9 @@ def run_proof(
             if enabled_gates is not None and obl.gate not in enabled_gates:
                 continue
 
-            # Run the gate
-            outcome, output = _run_gate(workspace, obl.gate)
+            # Run the gate, enforcing this requirement's evidence rules for it
+            gate_rules = [r for r in req.evidence_rules if r.gate == obl.gate]
+            outcome, output = _run_gate(workspace, obl.gate, gate_rules)
 
             # Write artifact
             artifact_path = artifact_dir / f"{req.id}_{obl.gate.value}_{run_id}.txt"

codeframe/core/proof/stubs.py

@@ -12,7 +12,7 @@
 import pytest
 
 
-def test_{slug}():
+def test_unit_{slug}():
     """Proves: {description}"""
     # Arrange
     # TODO: Set up test data
@@ -128,10 +128,13 @@ def test_sec_{slug}():
 
 
 def _slugify(text: str) -> str:
-    """Create a safe identifier from text."""
-    import re
-    slug = re.sub(r"[^a-z0-9]+", "_", text.lower().strip())[:50].strip("_")
-    return slug or "unnamed"
+    """Create a safe identifier from text.
+
+    Delegates to obligations.slugify so stub function names always match the
+    evidence-rule test_ids that enforce them (issue #729).
+    """
+    from codeframe.core.proof.obligations import slugify
+    return slugify(text)
 
 
 def generate_stubs(req: Requirement) -> dict[Gate, str]:

tests/cli/test_proof_commands.py

@@ -212,7 +212,7 @@ def test_run_mixed_fail_and_unverifiable_exits_one(self, mock_run_gate, ws):
             ),
         )
 
-        def _outcome(_ws, gate):
+        def _outcome(_ws, gate, _rules=()):
             if gate == Gate.UNIT:
                 return (GateOutcome.FAILED, "assertion failed")
             return (GateOutcome.UNVERIFIABLE, "cannot verify")