Keep deterministic fallback output evidence-aware (#8)

* fix: soften unsupported certainty in fallback summaries

* test: keep fallback language evidence-aware

* docs: define the supported assessment generation boundary

* test: keep public assessment calls provenance-aware

Author: simaba

docs/assessment-architecture.md

@@ -0,0 +1,26 @@
+# Assessment Generation Boundary
+
+## Supported entry point
+
+Use `src.engine.run_assessment()` for every application, CLI, test, or integration call that produces an assessment.
+
+```python
+from src.engine import run_assessment
+
+result = run_assessment(project, mode="dmaic", audience="pm")
+```
+
+The engine delegates to `src.assessment_service.run_assessment_with_provenance()`. Every returned `AssessmentResult` records one of two modes:
+
+- `llm`: a live model response was parsed into the required assessment structure.
+- `deterministic_fallback`: a predictable local starter assessment was used, with a safe reason explaining why.
+
+Set `require_llm=True` when deterministic fallback would be misleading; the call then raises a concise `AssessmentGenerationError` instead of returning fallback output.
+
+## Boundary rule
+
+The phase module contains low-level prompt, parsing, and deterministic-fallback helpers. It is not an application-facing generation API. New code must not call phase-level model-generation helpers directly, because that would bypass provenance and strict-mode behavior.
+
+## Review rule
+
+A regression test verifies that the supported engine is wired to the provenance-aware service and does not import or call the legacy phase-level generator. This preserves the intended public behavior while allowing the lower-level helper module to be refactored separately.

src/assessment_service.py

@@ -32,14 +32,39 @@ def _generation_note(mode: str, reason: str | None = None) -> str:
     return f"Generation note: deterministic fallback. {suffix}"
 
 
+def _soften_fallback_certainty(result: AssessmentResult) -> AssessmentResult:
+    """Keep deterministic starter output from overstating unvalidated inputs."""
+    define_items = [
+        replace(
+            item,
+            statement=item.statement.replace(
+                "Problem confirmed:", "Reported problem statement:"
+            ),
+        )
+        for item in result.dmaic_structure.get("define", [])
+    ]
+    dmaic_structure = {**result.dmaic_structure, "define": define_items}
+    return replace(
+        result,
+        cleaned_problem_statement=result.cleaned_problem_statement.replace(
+            "has a measurable performance gap:",
+            "has a reported performance concern:",
+        ),
+        role_summary=result.role_summary.replace(
+            "has a confirmed performance gap", "has a reported performance concern"
+        ),
+        dmaic_structure=dmaic_structure,
+    )
+
+
 def _fallback(
     project: ProjectInput,
     mode: str,
     audience: str,
     reason: str,
 ) -> AssessmentResult:
     """Return a deterministic result that records why it was used."""
-    result = _deterministic_fallback(project, mode, audience)
+    result = _soften_fallback_certainty(_deterministic_fallback(project, mode, audience))
     return replace(
         result,
         role_summary=f"{_generation_note('deterministic_fallback', reason)}\n\n{result.role_summary}",

tests/test_assessment_api_boundary.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import inspect
+
+import src.engine as engine
+from src.assessment_service import run_assessment_with_provenance
+
+
+def test_engine_delegates_to_provenance_aware_generation_service() -> None:
+    source = inspect.getsource(engine)
+
+    assert engine.run_assessment.__module__ == "src.engine"
+    assert "run_assessment_with_provenance" in source
+    assert "run_llm_assessment" not in source
+
+
+def test_engine_uses_the_supported_generation_service() -> None:
+    result = engine.run_assessment
+
+    assert callable(result)
+    assert callable(run_assessment_with_provenance)

tests/test_assessment_provenance.py

@@ -30,6 +30,20 @@ def test_missing_api_key_is_disclosed_as_deterministic_fallback(monkeypatch) ->
     assert result.role_summary.startswith("Generation note: deterministic fallback.")
 
 
+def test_fallback_does_not_present_unvalidated_input_as_confirmed(monkeypatch) -> None:
+    monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+
+    result = run_assessment(project(), mode="dmaic", audience="executive")
+
+    assert "confirmed performance gap" not in result.role_summary
+    assert "reported performance concern" in result.role_summary
+    assert "measurable performance gap" not in result.cleaned_problem_statement
+    assert "reported performance concern" in result.cleaned_problem_statement
+    assert result.dmaic_structure["define"][0].statement.startswith(
+        "Reported problem statement:"
+    )
+
+
 def test_require_llm_fails_instead_of_silently_falling_back(monkeypatch) -> None:
     monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)