Skip to content

Commit 100f1fd

Browse files
kmbandyclaude
andcommitted
chore(calib): env-gated heavy-FT / act-order debug instrumentation
ML8_DEBUG_HEAVY (per-round heavy-FT Y_SNR trajectory) + ML8_DEBUG_ACTORDER (diag-H conditioning) prints in batched_gptq; off unless the env var is set. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent e87d715 commit 100f1fd

1 file changed

Lines changed: 29 additions & 0 deletions

File tree

scripts/calibration/batched_gptq.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,15 @@ def batched_gptq_quantize(
412412
torch.cuda.empty_cache()
413413
importance = H_orig.diagonal(dim1=-2, dim2=-1).mean(0) # [K] (shared H ⇒ same per e)
414414
perm = torch.argsort(importance, descending=True)
415+
if os.environ.get("ML8_DEBUG_ACTORDER"):
416+
_imp = importance.float().clamp_min(0)
417+
_s = _imp.sort(descending=True).values
418+
_tot = _s.sum().clamp_min(1e-30)
419+
_top5 = (_s[:max(1, K // 20)].sum() / _tot).item() * 100.0
420+
_cv = (_imp.std() / _imp.mean().clamp_min(1e-30)).item()
421+
_mm = (_imp.max() / _imp.min().clamp_min(1e-30)).item()
422+
print(f"[actorder-dbg] K={K} diagH: cv={_cv:.3f} max/min={_mm:.2f} "
423+
f"top5%_share={_top5:.1f}% (uniform=5%)", flush=True)
415424
gidx_orig = torch.arange(K, device=dev) // group_size
416425
Hp = H_orig[:, perm][:, :, perm] # [E,K,K] permuted
417426
Hinv_p = torch.empty((E, K, K), device=dev, dtype=torch.float32)
@@ -455,7 +464,21 @@ def _reassign(cents, scls):
455464
orig_f = W_stack.float() # ORIGINAL weights = tune target
456465
gidx_f = torch.arange(K, device=dev) // group_size
457466
cent = centroids_all.clone(); scl = scales_all.clone()
467+
# Env-gated heavy-FT dissection (ML8_DEBUG_HEAVY): per-round trajectory of
468+
# the frozen-index tune loss AND the shipped (post-reassign) output-space
469+
# Y_SNR. Forks H1-vs-overfit: tune-loss drops but post-reassign Y_SNR flat
470+
# ⇒ tune gain discarded by reassign/error-prop (objective mismatch); Y_SNR
471+
# climbs on calib but PPL doesn't ⇒ overfit. Zero cost when unset.
472+
_dbg_heavy = os.environ.get("ML8_DEBUG_HEAVY")
473+
def _ydb(Qx): # output-space SNR (dB), mean over E
474+
_dy = orig_f - Qx
475+
_ey = torch.einsum("eij,ejk,eik->e", _dy, H_orig, _dy).clamp_min(1e-30)
476+
_sy = torch.einsum("eij,ejk,eik->e", orig_f, H_orig, orig_f).clamp_min(1e-30)
477+
return (10.0 * torch.log10(_sy / _ey)).mean().item()
478+
if _dbg_heavy:
479+
print(f"[heavy-dbg] pre-heavy (straight-sweep) Y_SNR={_ydb(Q):.4f} dB", flush=True)
458480
for _r in range(heavy_rounds):
481+
_l0 = None
459482
with torch.enable_grad():
460483
cp = cent.detach().requires_grad_(True)
461484
sp = scl.detach().requires_grad_(True)
@@ -476,10 +499,16 @@ def _reassign(cents, scls):
476499
loss = (tmp * d).sum()
477500
else:
478501
loss = torch.einsum("eij,ejk,eik->e", d, H_orig, d).sum()
502+
if _dbg_heavy and _s == 0:
503+
_l0 = float(loss.detach())
479504
opt.zero_grad(); loss.backward(); opt.step()
480505
cent = cp.detach(); scl = sp.detach()
481506
cent_use = snap_to_e4m3(cent) if snap_centroids == "e4m3" else cent
482507
indices, Q = _reassign(cent_use, scl) # frozen-index tune → reassign
508+
if _dbg_heavy:
509+
print(f"[heavy-dbg] round {_r}: tune_loss {_l0:.4e} -> {float(loss.detach()):.4e} "
510+
f"(frozen-idx, {heavy_steps} steps) | post-reassign Y_SNR={_ydb(Q):.4f} dB",
511+
flush=True)
483512
if dev.type == "cuda":
484513
torch.cuda.empty_cache()
485514
centroids_all = snap_to_e4m3(cent) if snap_centroids == "e4m3" else cent

0 commit comments

Comments
 (0)