diff --git a/benchmarks/BASELINES.md b/benchmarks/BASELINES.md index f7036d6..ad5cc68 100644 --- a/benchmarks/BASELINES.md +++ b/benchmarks/BASELINES.md @@ -105,3 +105,30 @@ Five questions chosen for maximum discrimination in CI (lowest cost, highest sig With working RAG (treated condition): 5/5 correct → faithfulness 100%, accuracy 100%. With broken RAG (baseline condition): 2/5 hallucinated + 3/5 partial → faithfulness 60%, accuracy 0%. Both outcomes are far from the gates (95% / 85%), making the smoke set decisive. + +## Sonnet-5 Judge Baseline (2026-07-10 — model tiers rollout) + +`smoke_eval.py` now resolves its models via the attune tier contract +(`attune_author.model_tiers`): answers = capable tier, judge = premium +tier. The rag-gate CI pin (`ATTUNE_MODEL_PREMIUM=claude-sonnet-5`) makes +CI runs sonnet-5-judged; previously the models were hardcoded +(sonnet-4-6 answers / opus-4-6 judge) and the pin was decorative. + +First sonnet-5/sonnet-5 run (local, dev key, 2026-07-10 — recorded for +specs/fable-model-tiers task 10 and specs/rag-gate-accuracy-baseline in +the attune workspace repo): + +| Metric | Score | Gate | Result | +| :----- | ----: | ---: | :----- | +| Faithfulness (hard) | 100.0% | ≥95% | ✅ PASS | +| Strict Accuracy (advisory) | 60.0% | ≥85% | ⚠️ advisory warn (was 40% under sonnet-4-6/opus-4-6) | + +Per-question (treated): q2 correct, q9 correct, q12 correct, +q27 partial, q39 partial. The enum (q27) and dataclass (q39) questions +remain the accuracy stragglers — q39 template regeneration is the +standing remediation candidate. + +Claude 5 API notes baked into `smoke_eval.py` by this run: the +`temperature` param is rejected ("deprecated for this model", 400), and +responses may lead with a ThinkingBlock — read the first TEXT block, +never `content[0].text`. diff --git a/benchmarks/smoke_eval.py b/benchmarks/smoke_eval.py index b3d683a..4d02da6 100644 --- a/benchmarks/smoke_eval.py +++ b/benchmarks/smoke_eval.py @@ -31,6 +31,8 @@ from anthropic import Anthropic from dotenv import load_dotenv +from attune_author.model_tiers import resolve_model + load_dotenv() # --------------------------------------------------------------------------- @@ -41,8 +43,14 @@ BENCH = REPO / "benchmarks" / "hallucination-v0.3.9" SMOKE_QUESTION_IDS = {2, 9, 12, 27, 39} -ANSWER_MODEL_ID = "claude-sonnet-4-6" -JUDGE_MODEL_ID = "claude-opus-4-6" +# Models resolve via the attune tier contract so the rag-gate CI pin +# (ATTUNE_MODEL_PREMIUM=claude-sonnet-5, guard-stepped) actually reaches +# the eval — previously these were hardcoded (sonnet-4-6 / opus-4-6) and +# the pin was decorative. Judge = premium tier (CI pins sonnet-5 to keep +# the $5/$20 ATTUNE_CI_EVAL_KEY caps valid); answers = capable tier. +# specs/fable-model-tiers tasks 8/10 (attune workspace repo). +ANSWER_MODEL_ID = resolve_model("capable") +JUDGE_MODEL_ID = resolve_model("premium") FAITHFULNESS_GATE = 0.95 # end_user persona — HARD gate (drives exit code) ACCURACY_GATE = 0.85 # developer persona — ADVISORY on smoke; full-25 enforces @@ -151,16 +159,28 @@ def build_treated_context(baseline: str, feature: str) -> str: _JSON_RE = re.compile(r"\{[\s\S]*\}") +def _first_text(response) -> str: + """First TEXT block — Claude 5 responses can lead with a ThinkingBlock + (no ``.text``) even without requesting thinking. Hit live 2026-07-10.""" + for block in response.content or []: + text = getattr(block, "text", None) + if text: + return text + return "" + + def get_answer(client: Anthropic, context: str, question: str) -> str: user = f"{context}\n\n---\n\n# Question\n\n{question}" + # No explicit temperature: Claude 5 models reject the param + # ("`temperature` is deprecated for this model", 400) — hit live + # 2026-07-10 on the first sonnet-5 baseline run. r = client.messages.create( model=ANSWER_MODEL_ID, max_tokens=1024, - temperature=0, system=ANSWER_SYSTEM, messages=[{"role": "user", "content": user}], ) - return r.content[0].text + return _first_text(r) def judge_answer(client: Anthropic, question: str, truth: str, answer: str) -> dict: @@ -172,11 +192,10 @@ def judge_answer(client: Anthropic, question: str, truth: str, answer: str) -> d r = client.messages.create( model=JUDGE_MODEL_ID, max_tokens=512, - temperature=0, system=JUDGE_SYSTEM, messages=[{"role": "user", "content": user}], ) - raw = r.content[0].text + raw = _first_text(r) m = _JSON_RE.search(raw) if not m: return {"verdict": "error", "reasoning": f"No JSON in judge response: {raw[:200]}"}