Move unbacked-parity regression policy into the benchmark; widen MobileBert (#189813)

Summary:
The periodic inductor_huggingface_unbacked_parity job intermittently reds on
MobileBertForMaskedLM with no real regression behind it. Its backed and unbacked
compiled times are ~equal and land in a bimodal distribution independently, so
the backed-vs-unbacked diff is ~+-2% autotune/measurement jitter -- centered at
zero over 16 independent runs, and a pre-onset build reproduces the same
variance, so it is not a code regression.

Move the regression policy out of the CI shell script and into the benchmark
(_run_compare_backed_unbacked in benchmarks/dynamo/common.py), next to the
comparison itself, as a per-model threshold table (MobileBertForMaskedLM = 3.0%,
others 1.0%). The benchmark now emits a canonical UNBACKED_PARITY_REGRESSION /
UNBACKED_PARITY_OK verdict line, and .ci/pytorch/test.sh detects that line
instead of re-deriving the threshold in bash.

X-link: pytorch/pytorch#189813
Approved by: https://github.com/bobrenjc93

Reviewed By: jeanschmidt

Differential Revision: D112002277

fbshipit-source-id: 8753da02d79cbf776ab1c82a00339a1c18d7c7dd

Author: williamwen42

userbenchmark/dynamo/dynamobench/common.py

@@ -4027,6 +4027,21 @@ def main(runner, original_dir=None, args=None):
                 process_entry(0, runner, original_dir, args)
 
 
+# Per-model tolerance (percent) for the backed-vs-unbacked compiled-time parity
+# check. Most models track backed vs unbacked to within ~0.5%. MobileBertForMaskedLM
+# has bimodal compiled timings (backed and unbacked land in different modes
+# independently), so its backed-vs-unbacked diff is ~+-2% run-to-run jitter with no
+# directional regression; it gets a wider band so the noise doesn't red the periodic
+# job while a genuine regression is still caught.
+UNBACKED_PARITY_THRESHOLDS = {"MobileBertForMaskedLM": 3.0}
+DEFAULT_UNBACKED_PARITY_THRESHOLD = 1.0
+
+
+def _unbacked_parity_diff_pct(backed_ms, unbacked_ms):
+    """Percent by which the unbacked compiled time exceeds the backed one."""
+    return (unbacked_ms - backed_ms) / backed_ms * 100
+
+
 def _run_compare_backed_unbacked(runner, args):
     """Run backed and unbacked per-model, alternating, and compare speedup."""
     import re
@@ -4045,7 +4060,7 @@ def print_comparison(all_results):
             b_ms = modes.get("backed_ms")
             u_ms = modes.get("unbacked_ms")
             if b_ms is not None and u_ms is not None:
-                ms_diff_pct = (u_ms - b_ms) / b_ms * 100
+                ms_diff_pct = _unbacked_parity_diff_pct(b_ms, u_ms)
                 print(
                     f"  {name:<40s} {b_ms:>10.3f} {u_ms:>11.3f} {ms_diff_pct:>+7.1f}%",
                     flush=True,
@@ -4158,7 +4173,7 @@ def print_comparison(all_results):
         ):
             b_ms = all_results[model]["backed_ms"]
             u_ms = all_results[model]["unbacked_ms"]
-            ms_diff_pct = (u_ms - b_ms) / b_ms * 100
+            ms_diff_pct = _unbacked_parity_diff_pct(b_ms, u_ms)
             print(
                 f"  => diff: {ms_diff_pct:+.1f}% ({b_ms:.3f} ms vs {u_ms:.3f} ms)",
                 flush=True,
@@ -4175,6 +4190,27 @@ def print_comparison(all_results):
 
     print_comparison(all_results)
 
+    # Emit a canonical regression verdict using per-model thresholds. The CI
+    # wrapper (.ci/pytorch/test.sh) detects the UNBACKED_PARITY_REGRESSION line
+    # rather than re-deriving the threshold, so the policy lives here as data.
+    regressions = []
+    for name, modes in all_results.items():
+        b_ms = modes.get("backed_ms")
+        u_ms = modes.get("unbacked_ms")
+        if b_ms is not None and u_ms is not None:
+            diff_pct = _unbacked_parity_diff_pct(b_ms, u_ms)
+            threshold = UNBACKED_PARITY_THRESHOLDS.get(
+                name, DEFAULT_UNBACKED_PARITY_THRESHOLD
+            )
+            if diff_pct > threshold:
+                regressions.append(
+                    f"{name}: +{diff_pct:.2f}% > {threshold:.1f}% threshold"
+                )
+    if regressions:
+        print(f"UNBACKED_PARITY_REGRESSION: {', '.join(regressions)}", flush=True)
+    else:
+        print("UNBACKED_PARITY_OK: no per-model regressions", flush=True)
+
 
 def write_csv_when_exception(args, name: str, status: str, device=None):
     print(status)