Add layered admissibility scoring

ProfRandom92 · web-flow · commit 3d9c6fc04cb8 · 2026-05-19T02:16:46.000-07:00
Add layered admissibility scoring

- Add deterministic AdmissibilityScorer with layer scores and overall admissibility score.
- Add stable JSON-safe score serialization and fixture integration tests.
- Document v1 scoring rules, determinism guarantees, and non-goals.

Validation reported in PR: targeted scorer, comparator, validator, fixture tests, full pytest, and npm run check passed.
diff --git a/docs/LAYERED_ADMISSIBILITY_SCORE_v1.md b/docs/LAYERED_ADMISSIBILITY_SCORE_v1.md
@@ -0,0 +1,70 @@
+# Layered Admissibility Score v1
+
+## Purpose
+
+`layered-admissibility-score-v1` adds a deterministic scoring layer on top of `ContractValidator` results. It converts fixture-level pass/fail outcomes into explicit, serializable layer scores and an overall admissibility score.
+
+## Score fields
+
+The scorer emits:
+
+- `structural_score`
+- `relational_score`
+- `operational_score`
+- `governance_score`
+- `overall_admissibility_score`
+- `expected_admissible`
+- `observed_admissible`
+- `passed_contracts`
+- `failed_contracts`
+- `failure_labels`
+- `layer_scores` (per-layer contract lists, labels, and score)
+
+## Layer scoring rules
+
+For each layer (`structural`, `relational`, `operational`, `governance`):
+
+- Score is `passed_contract_count / total_contract_count` in that layer.
+- If a layer has no contracts in the input result set, that layer score is `1.0`.
+- Passed/failed contract IDs are sorted deterministically.
+- Failure labels are sorted, unique, and derived from non-null `failure_label` values.
+
+## Overall scoring rule
+
+`overall_admissibility_score` is the unweighted arithmetic mean of the four layer scores:
+
+- structural
+- relational
+- operational
+- governance
+
+`observed_admissible` is true only when every `ValidationResult.passed` is true.
+
+`expected_admissible` defaults to `observed_admissible` unless an explicit override is provided.
+
+## Determinism guarantees
+
+- No randomness.
+- No clock/time dependencies.
+- No external API/network calls.
+- Stable sorted outputs for contract IDs and failure labels.
+- `to_dict` produces JSON-compatible structures with tuple fields serialized as lists.
+
+## Non-goals
+
+- No learned weighting.
+- No LLM judges.
+- No embeddings.
+- No fuzzy matching.
+- No semantic equivalence.
+
+## How this connects
+
+- **ContractValidator:** consumes `ValidationResult` objects produced by contract validation.
+- **Positive/negative fixtures:** scores both `coding_workflow_pr_review_v1` and `coding_workflow_pr_review_degraded_v1` deterministically.
+- **Future degradation curves:** provides stable primitives for trajectory/degradation analysis across fixture families.
+
+## Prototype caveat
+
+- v1 uses unweighted averages only.
+- Future versions may add explicit configured weights, but not learned weights.
diff --git a/src/validation/admissibility_scorer.py b/src/validation/admissibility_scorer.py
@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from src.validation.contract_validator import Layer, ValidationResult
+
+
+@dataclass(frozen=True, slots=True)
+class LayerScore:
+    layer: Layer
+    passed_contracts: tuple[str, ...]
+    failed_contracts: tuple[str, ...]
+    failure_labels: tuple[str, ...]
+    score: float
+
+
+@dataclass(frozen=True, slots=True)
+class AdmissibilityScore:
+    structural_score: float
+    relational_score: float
+    operational_score: float
+    governance_score: float
+    overall_admissibility_score: float
+    expected_admissible: bool
+    observed_admissible: bool
+    passed_contracts: tuple[str, ...]
+    failed_contracts: tuple[str, ...]
+    failure_labels: tuple[str, ...]
+    layer_scores: tuple[LayerScore, ...]
+
+
+class AdmissibilityScorer:
+    _LAYER_ORDER: tuple[Layer, ...] = (
+        Layer.STRUCTURAL,
+        Layer.RELATIONAL,
+        Layer.OPERATIONAL,
+        Layer.GOVERNANCE,
+    )
+
+    def score(self, results: list[ValidationResult], expected_admissible: bool | None = None) -> AdmissibilityScore:
+        observed_admissible = all(result.passed for result in results)
+        effective_expected = observed_admissible if expected_admissible is None else expected_admissible
+
+        passed_contracts = tuple(sorted(result.contract_id for result in results if result.passed))
+        failed_contracts = tuple(sorted(result.contract_id for result in results if not result.passed))
+        failure_labels = tuple(sorted({result.failure_label for result in results if result.failure_label is not None}))
+
+        layer_scores: list[LayerScore] = []
+        score_by_layer: dict[Layer, float] = {}
+
+        for layer in self._LAYER_ORDER:
+            layer_results = [result for result in results if result.layer == layer]
+            passed_in_layer = tuple(sorted(result.contract_id for result in layer_results if result.passed))
+            failed_in_layer = tuple(sorted(result.contract_id for result in layer_results if not result.passed))
+            labels_in_layer = tuple(sorted({result.failure_label for result in layer_results if result.failure_label is not None}))
+            total_contracts = len(layer_results)
+            layer_score = 1.0 if total_contracts == 0 else len(passed_in_layer) / total_contracts
+            score_by_layer[layer] = layer_score
+            layer_scores.append(
+                LayerScore(
+                    layer=layer,
+                    passed_contracts=passed_in_layer,
+                    failed_contracts=failed_in_layer,
+                    failure_labels=labels_in_layer,
+                    score=layer_score,
+                )
+            )
+
+        overall_admissibility_score = sum(score_by_layer[layer] for layer in self._LAYER_ORDER) / len(self._LAYER_ORDER)
+
+        return AdmissibilityScore(
+            structural_score=score_by_layer[Layer.STRUCTURAL],
+            relational_score=score_by_layer[Layer.RELATIONAL],
+            operational_score=score_by_layer[Layer.OPERATIONAL],
+            governance_score=score_by_layer[Layer.GOVERNANCE],
+            overall_admissibility_score=overall_admissibility_score,
+            expected_admissible=effective_expected,
+            observed_admissible=observed_admissible,
+            passed_contracts=passed_contracts,
+            failed_contracts=failed_contracts,
+            failure_labels=failure_labels,
+            layer_scores=tuple(layer_scores),
+        )
+
+    def to_dict(self, score: AdmissibilityScore) -> dict[str, object]:
+        return {
+            "structural_score": score.structural_score,
+            "relational_score": score.relational_score,
+            "operational_score": score.operational_score,
+            "governance_score": score.governance_score,
+            "overall_admissibility_score": score.overall_admissibility_score,
+            "expected_admissible": score.expected_admissible,
+            "observed_admissible": score.observed_admissible,
+            "passed_contracts": list(score.passed_contracts),
+            "failed_contracts": list(score.failed_contracts),
+            "failure_labels": list(score.failure_labels),
+            "layer_scores": [
+                {
+                    "layer": layer_score.layer.value,
+                    "passed_contracts": list(layer_score.passed_contracts),
+                    "failed_contracts": list(layer_score.failed_contracts),
+                    "failure_labels": list(layer_score.failure_labels),
+                    "score": layer_score.score,
+                }
+                for layer_score in score.layer_scores
+            ],
+        }
diff --git a/tests/test_admissibility_scorer.py b/tests/test_admissibility_scorer.py
@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from src.validation.admissibility_scorer import AdmissibilityScorer
+from src.validation.contract_validator import ContractType, ContractValidator, Layer, ValidationResult
+
+
+def _result(contract_id: str, layer: Layer, passed: bool, failure_label: str | None = None) -> ValidationResult:
+    return ValidationResult(
+        contract_id=contract_id,
+        layer=layer,
+        contract_type=ContractType.ORDERING,
+        passed=passed,
+        severity="high",
+        failure_label=failure_label,
+        deterministic_evidence={},
+    )
+
+
+def _load_json(path: Path) -> dict[str, object]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _fixture_validation_results(fixture_root: Path) -> list[ValidationResult]:
+    original = {
+        **_load_json(fixture_root / "original/trace.json"),
+        **_load_json(fixture_root / "original/state.json"),
+        "dependency_graph": _load_json(fixture_root / "original/dependency_graph.json"),
+    }
+    reconstructed = {
+        **_load_json(fixture_root / "reconstructed/trace.json"),
+        **_load_json(fixture_root / "reconstructed/state.json"),
+        "dependency_graph": _load_json(fixture_root / "reconstructed/dependency_graph.json"),
+    }
+    contracts = [_load_json(path) for path in sorted((fixture_root / "original/contracts").glob("*.json"))]
+    return ContractValidator().validate_contracts(original=original, reconstructed=reconstructed, contracts=contracts)
+
+
+def test_all_contracts_pass_score_is_one() -> None:
+    score = AdmissibilityScorer().score(
+        [
+            _result("op_a", Layer.OPERATIONAL, True),
+            _result("rel_a", Layer.RELATIONAL, True),
+        ]
+    )
+
+    assert score.overall_admissibility_score == 1.0
+    assert score.observed_admissible is True
+    assert score.failed_contracts == ()
+
+
+def test_failed_relational_contract_reduces_relational_and_overall_score() -> None:
+    score = AdmissibilityScorer().score(
+        [
+            _result("rel_a", Layer.RELATIONAL, True),
+            _result("rel_b", Layer.RELATIONAL, False, "REL_FAIL"),
+        ]
+    )
+
+    assert score.relational_score == 0.5
+    assert score.overall_admissibility_score == 0.875
+    assert score.observed_admissible is False
+
+
+def test_failed_operational_contract_reduces_operational_score() -> None:
+    score = AdmissibilityScorer().score([_result("op_a", Layer.OPERATIONAL, False, "OP_FAIL")])
+
+    assert score.operational_score == 0.0
+    assert score.overall_admissibility_score == 0.75
+
+
+def test_empty_results_are_admissible_with_all_scores_one() -> None:
+    score = AdmissibilityScorer().score([])
+
+    assert score.structural_score == 1.0
+    assert score.relational_score == 1.0
+    assert score.operational_score == 1.0
+    assert score.governance_score == 1.0
+    assert score.overall_admissibility_score == 1.0
+    assert score.observed_admissible is True
+
+
+def test_failure_labels_are_sorted_unique() -> None:
+    score = AdmissibilityScorer().score(
+        [
+            _result("a", Layer.RELATIONAL, False, "Z_LABEL"),
+            _result("b", Layer.OPERATIONAL, False, "A_LABEL"),
+            _result("c", Layer.GOVERNANCE, False, "A_LABEL"),
+        ]
+    )
+
+    assert score.failure_labels == ("A_LABEL", "Z_LABEL")
+
+
+def test_passed_and_failed_contracts_are_sorted() -> None:
+    score = AdmissibilityScorer().score(
+        [
+            _result("c", Layer.RELATIONAL, True),
+            _result("a", Layer.RELATIONAL, False, "X"),
+            _result("b", Layer.OPERATIONAL, True),
+        ]
+    )
+
+    assert score.passed_contracts == ("b", "c")
+    assert score.failed_contracts == ("a",)
+
+
+def test_to_dict_is_stable_and_json_compatible() -> None:
+    scorer = AdmissibilityScorer()
+    score = scorer.score([_result("b", Layer.OPERATIONAL, True), _result("a", Layer.RELATIONAL, False, "REL_FAIL")])
+
+    as_dict_first = scorer.to_dict(score)
+    as_dict_second = scorer.to_dict(score)
+
+    assert as_dict_first == as_dict_second
+    assert isinstance(as_dict_first["passed_contracts"], list)
+    assert isinstance(as_dict_first["failed_contracts"], list)
+    assert isinstance(as_dict_first["failure_labels"], list)
+    assert isinstance(as_dict_first["layer_scores"], list)
+
+
+def test_expected_admissible_override() -> None:
+    score = AdmissibilityScorer().score([_result("rel_a", Layer.RELATIONAL, True)], expected_admissible=False)
+
+    assert score.expected_admissible is False
+    assert score.observed_admissible is True
+
+
+def test_scores_positive_fixture_contract_results() -> None:
+    results = _fixture_validation_results(Path("fixtures/coding_workflow_pr_review_v1"))
+    score = AdmissibilityScorer().score(results)
+
+    assert score.observed_admissible is True
+    assert score.overall_admissibility_score == 1.0
+
+
+def test_scores_negative_fixture_contract_results() -> None:
+    results = _fixture_validation_results(Path("fixtures/coding_workflow_pr_review_degraded_v1"))
+    score = AdmissibilityScorer().score(results)
+
+    assert score.observed_admissible is False
+    assert score.relational_score < 1.0
+    assert score.operational_score < 1.0
+    assert "POLICY_ORDER_BROKEN" in score.failure_labels
+    assert "RECOVERY_PATH_INVALID" in score.failure_labels
+    assert "CAUSAL_DEPENDENCY_LOSS" in score.failure_labels
+    assert "INVARIANT_VIOLATION" in score.failure_labels
