scripts: exhaustive-grid verification; witnessed 3M/3M on simple n=3

exhaustive_examples slices the complete (op, a, b) grid arithmetically
so each shard materializes only its own range; load/generate_answers
split out of run() so mismatch triage (automatic unbatched re-run,
separating model errors from batching artifacts) reuses the loaded
model.  modal_compile.exhaustive_remote runs one shard per B200,
pulling the model from the published Hub repo — the artifact of
record, not a build byproduct.

Witnessed 2026-07-26: physicsrob/torchwright-calculator-simple-
max-digits-3, every A op B with both operands in [0, 999] and op in
{+,-,*} — 3,000,000/3,000,000 exact, 8 shards, ~25 min wall.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: physicsrob

modal_compile.py

@@ -160,6 +160,62 @@ def accuracy_remote(
     return {"bundle": dirname, "n": n, "seed": seed, "mismatches": n_bad}
 
 
+@app.function(gpu="B200", cpu=8, memory=65536, timeout=7200)
+def exhaustive_remote(
+    repo_id: str,
+    shard: int,
+    n_shards: int,
+    max_value: int = 999,
+    batch_size: int = 8192,
+) -> dict:
+    """One shard of the exhaustive grid, straight from the published Hub repo.
+
+    Every ``A op B`` with both operands in ``[0, max_value]`` and each
+    op, split op-major across ``n_shards`` workers.  A mismatch is
+    immediately re-run unbatched to separate a model error from a
+    batching/padding artifact before it lands in the report.
+    """
+    from scripts.verify_bundle_accuracy import (
+        exhaustive_examples,
+        expected,
+        generate_answers,
+        load,
+    )
+
+    total = 3 * (max_value + 1) ** 2
+    start = shard * total // n_shards
+    stop = (shard + 1) * total // n_shards
+    model, tok = load(repo_id)
+    results = generate_answers(
+        model,
+        tok,
+        exhaustive_examples(start, stop, max_value=max_value),
+        batch_size=batch_size,
+        max_new_tokens=10,
+    )
+    mismatches = []
+    for a, op, b, out in results:
+        want = expected(a, op, b)
+        if out != want:
+            single = generate_answers(
+                model, tok, [(a, op, b)], batch_size=1, max_new_tokens=10
+            )[0][3]
+            mismatches.append(
+                {"expr": f"{a}{op}{b}", "batched": out, "single": single, "want": want}
+            )
+            print(
+                f"[shard {shard}] MISMATCH {a}{op}{b}: batched={out!r} "
+                f"single={single!r} expected={want}",
+                flush=True,
+            )
+    print(
+        f"[shard {shard}] SUMMARY {len(results) - len(mismatches)}/{len(results)} "
+        f"correct ({start}..{stop})",
+        flush=True,
+    )
+    return {"shard": shard, "n": len(results), "mismatches": mismatches}
+
+
 @app.function(cpu=4, memory=8192, timeout=7200, volumes={BUNDLE_ROOT: bundles})
 def card_remote(dirname: str) -> str:
     """Regenerate a volume bundle's README without recompiling.

scripts/verify_bundle_accuracy.py

@@ -22,9 +22,14 @@
 
 import argparse
 from collections import Counter
+from typing import TYPE_CHECKING, Any, cast
 
 import numpy as np
 
+if TYPE_CHECKING:
+    from transformers import PreTrainedModel
+    from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
 Example = tuple[int, str, int]
 
 
@@ -49,15 +54,28 @@ def expected(a: int, op: str, b: int) -> str:
     return str(f(a, b))
 
 
-def run(
-    path: str,
-    examples: list[Example],
-    *,
-    batch_size: int = 250,
-    device: str = "cuda",
-    max_new_tokens: int = 32,
-) -> list[tuple[int, str, int, str]]:
-    """Batched greedy decode; returns ``(a, op, b, model_answer)`` rows."""
+def exhaustive_examples(
+    start: int, stop: int, max_value: int = 999, ops: str = "+-*"
+) -> list[Example]:
+    """Examples ``[start, stop)`` of the exhaustive ``(op, a, b)`` grid.
+
+    The grid is op-major then row-major: index
+    ``i = (op_idx * (max_value+1) + a) * (max_value+1) + b`` — computed
+    arithmetically so a shard materializes only its own slice.
+    """
+    m = max_value + 1
+    out: list[Example] = []
+    for i in range(start, stop):
+        op_idx, rem = divmod(i, m * m)
+        a, b = divmod(rem, m)
+        out.append((a, ops[op_idx], b))
+    return out
+
+
+def load(
+    path: str, device: str = "cuda"
+) -> tuple["PreTrainedModel", "PreTrainedTokenizerBase"]:
+    """Load a bundle (local dir or Hub repo id) in strict fp32."""
     import torch
     from transformers import AutoModelForCausalLM, AutoTokenizer
 
@@ -68,15 +86,28 @@ def run(
     tok.pad_token = tok.eos_token
     tok.padding_side = "left"
     model = AutoModelForCausalLM.from_pretrained(path, torch_dtype=torch.float32)
-    model = model.eval().to(device)
+    return cast("PreTrainedModel", model.eval().to(device)), tok
 
+
+def generate_answers(
+    model: "PreTrainedModel",
+    tok: "PreTrainedTokenizerBase",
+    examples: list[Example],
+    *,
+    batch_size: int = 250,
+    max_new_tokens: int = 32,
+) -> list[tuple[int, str, int, str]]:
+    """Batched greedy decode; returns ``(a, op, b, model_answer)`` rows."""
+    import torch
+
+    device = model.device
     results: list[tuple[int, str, int, str]] = []
     with torch.no_grad():
         for start in range(0, len(examples), batch_size):
             chunk = examples[start : start + batch_size]
             prompts = [f"{a}{op}{b}\n" for a, op, b in chunk]
             enc = tok(prompts, return_tensors="pt", padding=True).to(device)
-            g = model.generate(
+            g = cast("Any", model).generate(
                 **enc,
                 max_new_tokens=max_new_tokens,
                 do_sample=False,
@@ -93,6 +124,21 @@ def run(
     return results
 
 
+def run(
+    path: str,
+    examples: list[Example],
+    *,
+    batch_size: int = 250,
+    device: str = "cuda",
+    max_new_tokens: int = 32,
+) -> list[tuple[int, str, int, str]]:
+    """Load-and-generate convenience over :func:`load` / :func:`generate_answers`."""
+    model, tok = load(path, device)
+    return generate_answers(
+        model, tok, examples, batch_size=batch_size, max_new_tokens=max_new_tokens
+    )
+
+
 def report(results: list[tuple[int, str, int, str]]) -> int:
     """Print the verdict breakdown; returns the mismatch count."""
     by_op: Counter[str] = Counter()