fix(eval): smoke_eval resolves models via tiers + sonnet-5 baseline

benchmarks/BASELINES.md

@@ -105,3 +105,30 @@ Five questions chosen for maximum discrimination in CI (lowest cost, highest sig
 With working RAG (treated condition): 5/5 correct → faithfulness 100%, accuracy 100%.
 With broken RAG (baseline condition): 2/5 hallucinated + 3/5 partial → faithfulness 60%, accuracy 0%.
 Both outcomes are far from the gates (95% / 85%), making the smoke set decisive.
+
+## Sonnet-5 Judge Baseline (2026-07-10 — model tiers rollout)
+
+`smoke_eval.py` now resolves its models via the attune tier contract
+(`attune_author.model_tiers`): answers = capable tier, judge = premium
+tier. The rag-gate CI pin (`ATTUNE_MODEL_PREMIUM=claude-sonnet-5`) makes
+CI runs sonnet-5-judged; previously the models were hardcoded
+(sonnet-4-6 answers / opus-4-6 judge) and the pin was decorative.
+
+First sonnet-5/sonnet-5 run (local, dev key, 2026-07-10 — recorded for
+specs/fable-model-tiers task 10 and specs/rag-gate-accuracy-baseline in
+the attune workspace repo):
+
+| Metric | Score | Gate | Result |
+| :----- | ----: | ---: | :----- |
+| Faithfulness (hard) | 100.0% | ≥95% | ✅ PASS |
+| Strict Accuracy (advisory) | 60.0% | ≥85% | ⚠️ advisory warn (was 40% under sonnet-4-6/opus-4-6) |
+
+Per-question (treated): q2 correct, q9 correct, q12 correct,
+q27 partial, q39 partial. The enum (q27) and dataclass (q39) questions
+remain the accuracy stragglers — q39 template regeneration is the
+standing remediation candidate.
+
+Claude 5 API notes baked into `smoke_eval.py` by this run: the
+`temperature` param is rejected ("deprecated for this model", 400), and
+responses may lead with a ThinkingBlock — read the first TEXT block,
+never `content[0].text`.

benchmarks/smoke_eval.py

@@ -31,6 +31,8 @@
 from anthropic import Anthropic
 from dotenv import load_dotenv
 
+from attune_author.model_tiers import resolve_model
+
 load_dotenv()
 
 # ---------------------------------------------------------------------------
@@ -41,8 +43,14 @@
 BENCH = REPO / "benchmarks" / "hallucination-v0.3.9"
 
 SMOKE_QUESTION_IDS = {2, 9, 12, 27, 39}
-ANSWER_MODEL_ID = "claude-sonnet-4-6"
-JUDGE_MODEL_ID = "claude-opus-4-6"
+# Models resolve via the attune tier contract so the rag-gate CI pin
+# (ATTUNE_MODEL_PREMIUM=claude-sonnet-5, guard-stepped) actually reaches
+# the eval — previously these were hardcoded (sonnet-4-6 / opus-4-6) and
+# the pin was decorative. Judge = premium tier (CI pins sonnet-5 to keep
+# the $5/$20 ATTUNE_CI_EVAL_KEY caps valid); answers = capable tier.
+# specs/fable-model-tiers tasks 8/10 (attune workspace repo).
+ANSWER_MODEL_ID = resolve_model("capable")
+JUDGE_MODEL_ID = resolve_model("premium")
 
 FAITHFULNESS_GATE = 0.95  # end_user persona — HARD gate (drives exit code)
 ACCURACY_GATE = 0.85  # developer persona — ADVISORY on smoke; full-25 enforces
@@ -151,16 +159,28 @@ def build_treated_context(baseline: str, feature: str) -> str:
 _JSON_RE = re.compile(r"\{[\s\S]*\}")
 
 
+def _first_text(response) -> str:
+    """First TEXT block — Claude 5 responses can lead with a ThinkingBlock
+    (no ``.text``) even without requesting thinking. Hit live 2026-07-10."""
+    for block in response.content or []:
+        text = getattr(block, "text", None)
+        if text:
+            return text
+    return ""
+
+
 def get_answer(client: Anthropic, context: str, question: str) -> str:
     user = f"{context}\n\n---\n\n# Question\n\n{question}"
+    # No explicit temperature: Claude 5 models reject the param
+    # ("`temperature` is deprecated for this model", 400) — hit live
+    # 2026-07-10 on the first sonnet-5 baseline run.
     r = client.messages.create(
         model=ANSWER_MODEL_ID,
         max_tokens=1024,
-        temperature=0,
         system=ANSWER_SYSTEM,
         messages=[{"role": "user", "content": user}],
     )
-    return r.content[0].text
+    return _first_text(r)
 
 
 def judge_answer(client: Anthropic, question: str, truth: str, answer: str) -> dict:
@@ -172,11 +192,10 @@ def judge_answer(client: Anthropic, question: str, truth: str, answer: str) -> d
     r = client.messages.create(
         model=JUDGE_MODEL_ID,
         max_tokens=512,
-        temperature=0,
         system=JUDGE_SYSTEM,
         messages=[{"role": "user", "content": user}],
     )
-    raw = r.content[0].text
+    raw = _first_text(r)
     m = _JSON_RE.search(raw)
     if not m:
         return {"verdict": "error", "reasoning": f"No JSON in judge response: {raw[:200]}"}