Add Grok 4.5 to PolicyBench (#120)

* Add Grok 4.5: model card, thinking budget, app metadata

Onboarding gauntlet passed the forced tool contract 3/3 and 16/16 with
14,225 tokens on the full-scenario probe, so grok-4.5 gets an explicit
card with the thinking-class budget and grok-4.3's 420s timeout.

Harness fix uncovered by the first run round: litellm rejects
max_completion_tokens for xai models, so the thinking-budget branch
sends the same 16,384 budget as max_tokens for xai/ ids; grok-4.3
keeps the flat family budget (no card).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Publish 24-model board with Grok 4.5 (dashboard-data-20260710)

100-scenario supervised run: 1,984/1,984 rows, zero errors, $7.27.
Grok 4.5 debuts at 80.9% within $1 (#5 of 24). 304 audit cases
re-judged under the decisive-diagnosis contract (0 hedged, 0
reference-suspect); all 23 incumbent modelStats byte-identical to
dashboard-data-20260709b.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Promote Grok 4.5 to xAI's frontier flagship

The scenario explorer's frontier filter takes one flagship per
provider with fallbacks; Grok 4.5 leads with 4.3 retained for older
frozen bundles, mirroring the Sol/5.5 pattern.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: MaxGhenis

annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv

@@ -2,7 +2,7 @@ import { DEFAULT_VERSION_ID } from "../lib/dataVersionsRuntime";
 import type { BenchData, CountryCode } from "../types";
 import SiteHeader, { type HeaderNavItem } from "./SiteHeader";
 
-const SNAPSHOT_DATE_LABEL = "Snapshot 2026-07-09";
+const SNAPSHOT_DATE_LABEL = "Snapshot 2026-07-10";
 
 export default function Hero({
   selectedView,

app/src/components/Hero.tsx

@@ -1,9 +1,9 @@
 {
   "version": 1,
   "repo": "PolicyEngine/policybench",
-  "tag": "dashboard-data-20260709b",
+  "tag": "dashboard-data-20260710",
   "asset": "dashboard-data.json",
-  "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260709b/dashboard-data.json",
-  "sha256": "cdb99bfb3e5181803b2dd4229cfcf23e9bb31a4208d7130bcf7e73f939edc0e5",
-  "bytes": 64159293
+  "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260710/dashboard-data.json",
+  "sha256": "fd77509de2011c6c81e2dc3e7d2c62db3b076c1ee00e248c9008ac3ab0f40482",
+  "bytes": 66366891
 }

app/src/data.artifact.json

@@ -5,7 +5,7 @@
     {
       "id": "1.1",
       "label": "1.1",
-      "description": "Corrected ground truth (policyengine-us 1.755.4); every model at unconfigured provider defaults - 23 models",
+      "description": "Corrected ground truth (policyengine-us 1.755.4); every model at unconfigured provider defaults - 24 models",
       "snapshotLabel": null,
       "artifact": {
         "pointer": "live"

app/src/data.versions.json

@@ -11,6 +11,7 @@ export const MODEL_ORDER = [
   "claude-sonnet-4.6",
   "claude-haiku-4.5",
   "grok-4.3",
+  "grok-4.5",
   "grok-build-0.1",
   "gpt-5.6-sol",
   "gpt-5.6-terra",
@@ -38,6 +39,7 @@ export const MODEL_LABELS: Record<string, string> = {
   "claude-sonnet-5": "Claude Sonnet 5",
   "claude-sonnet-4.6": "Claude Sonnet 4.6",
   "grok-4.3": "Grok 4.3",
+  "grok-4.5": "Grok 4.5",
   "grok-build-0.1": "Grok Build 0.1",
   "gpt-5.6-sol": "GPT-5.6 Sol",
   "gpt-5.6-terra": "GPT-5.6 Terra",
@@ -111,7 +113,7 @@ export function orderModels(models: Iterable<string>): string[] {
 const FRONTIER_MODEL_GROUPS = [
   ["claude-fable-5", "claude-opus-4.8"],
   ["gpt-5.6-sol", "gpt-5.5"],
-  ["grok-4.3"],
+  ["grok-4.5", "grok-4.3"],
   ["gemini-3.1-pro-preview"],
   ["deepseek-v4-pro"],
   ["kimi-k2.6"],

app/src/modelMeta.ts

@@ -76,7 +76,7 @@ describe("model metadata", () => {
     const models = [
       "claude-fable-5",
       "gpt-5.6-sol",
-      "grok-4.3",
+      "grok-4.5",
       "gemini-3.1-pro-preview",
       "deepseek-v4-pro",
       "kimi-k2.6",

app/tests/modelMeta.test.ts

@@ -177,6 +177,7 @@ The current default non-Claude model set is:
 
 ```bash
 grok-4.3
+grok-4.5
 grok-build-0.1
 gpt-5.6-sol
 gpt-5.6-terra

docs/runbook.md

@@ -20,8 +20,8 @@
   ],
   "live_dashboard_artifact": {
     "asset": "dashboard-data.json",
-    "bytes": 64159293,
-    "derivation": "Additive fold of complete GPT-5.6 Sol, Terra, and Luna runs into the 20-model v1.1 board: 45,632 prediction rows across 23 models and 1,984 canonical outputs. All 489 wrong rows across 239 new model-output cases were independently audited; verdict validation found 0 missing, hedged, or reference-suspect cases after adjudication. The live annotation file preserves all 5,351 incumbent row annotations and adds the 489 GPT-5.6 annotations. Incumbent predictions, metrics, and audit annotations are unchanged. The pre-existing scenario_109 SNAP reference explanation now describes the January-September and October-December parameter periods correctly; its reference value and every score are unchanged.",
+    "bytes": 66366891,
+    "derivation": "Adds xAI Grok 4.5 to the 23-model v1.1 board (24 models, 47,616 prediction rows). Onboarded via the probe gauntlet (tool contract, thinking-class budget); 100-scenario supervised sync run, 1,984/1,984 rows, $7.27. 304 audit cases re-judged under the decisive-diagnosis contract (0 hedged, 0 reference-suspect; 6,031 row annotations). All 23 incumbent modelStats are byte-identical to dashboard-data-20260709b.",
     "notes": "Dataset version remains v1.1 with the corrected PolicyEngine references. GPT-5.6 usage and exact reconstructed cost include cache-write prompt tokens; claude-fable-5 retains its prior artifact-level usage values.",
     "population_weights": {
       "sha256": "0b3b96b8247477631510371b9c3b021a18a535e8ebbf8c73a8aa67cff3110cd4",
@@ -32,9 +32,9 @@
       "policyengine_us_version": "1.755.4",
       "policyengine_version": "4.16.1"
     },
-    "sha256": "cdb99bfb3e5181803b2dd4229cfcf23e9bb31a4208d7130bcf7e73f939edc0e5",
-    "tag": "dashboard-data-20260709b",
-    "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260709b/dashboard-data.json"
+    "sha256": "fd77509de2011c6c81e2dc3e7d2c62db3b076c1ee00e248c9008ac3ab0f40482",
+    "tag": "dashboard-data-20260710",
+    "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260710/dashboard-data.json"
   },
   "live_dashboard_note": "Two artifacts are tracked. published_dashboard_artifact pins the manuscript's frozen record: it equals the combined export of the source run data.json files listed under source_run_artifacts, byte for byte. live_dashboard_artifact is the release asset the committed pointer app/src/data.artifact.json references; it starts from the frozen export and may gain additive updates (injected metrics, newly benchmarked models) without changing the frozen models' scores. Every publish-dashboard run must update live_dashboard_artifact (tag, sha256, bytes, derivation) in the same commit as the pointer.",
   "model_response_date": "2026-06-12 to 2026-06-13",

paper/snapshot/20260501/manifest.json

@@ -41,6 +41,7 @@
     "claude-sonnet-4.6": "claude-sonnet-4-6",
     "claude-haiku-4.5": "claude-haiku-4-5-20251001",
     "grok-4.3": "xai/grok-4.3",
+    "grok-4.5": "xai/grok-4.5",
     "grok-build-0.1": "xai/grok-build-0.1",
     **GPT_56_MODELS,
     "gpt-5.5": "gpt-5.5",

policybench/config.py

@@ -521,7 +521,11 @@ def _completion_controls(
                 cap=GEMINI_MAX_COMPLETION_TOKENS_CAP,
             )
         }
-    if model_id.startswith("xai/"):
+    card = card_for(model_id)
+    # A thinking-budget card outranks the legacy xai/ family branch:
+    # grok-4.5 reasons by default (14k tokens on the 16-variable probe),
+    # while grok-4.3 has no card and keeps the flat family budget.
+    if model_id.startswith("xai/") and not (card is not None and card.thinking_budget):
         if include_explanations:
             base_tokens = EXPLANATION_MAX_COMPLETION_TOKENS
         else:
@@ -531,7 +535,6 @@ def _completion_controls(
                 base_tokens, variables, include_explanations
             )
         }
-    card = card_for(model_id)
     if card is not None and card.thinking_budget:
         # Reasoning-by-default models bill reasoning against the same
         # completion budget as the answer, and the reasoning spend does not
@@ -540,15 +543,17 @@ def _completion_controls(
         # explanation path got in #101, or default-effort reasoning exhausts
         # the small dynamic budget before any output. Unused headroom is
         # free.
-        base_tokens = THINKING_CLAUDE_MAX_COMPLETION_TOKENS_CAP
-        return {
-            "max_completion_tokens": _completion_token_budget(
-                base_tokens,
-                variables,
-                include_explanations,
-                cap=THINKING_CLAUDE_MAX_COMPLETION_TOKENS_CAP,
-            )
-        }
+        budget = _completion_token_budget(
+            THINKING_CLAUDE_MAX_COMPLETION_TOKENS_CAP,
+            variables,
+            include_explanations,
+            cap=THINKING_CLAUDE_MAX_COMPLETION_TOKENS_CAP,
+        )
+        # xAI rejects max_completion_tokens outright (litellm
+        # UnsupportedParamsError); it takes the same budget as max_tokens.
+        if model_id.startswith("xai/"):
+            return {"max_tokens": budget}
+        return {"max_completion_tokens": budget}
     if model_id.startswith("gpt-5"):
         if include_explanations:
             base_tokens = EXPLANATION_MAX_COMPLETION_TOKENS