feat(metal): MTP confidence-capture instrument — LTHN_MTP_CONF + calibration force mode (#359)

Per drafted token, the drafter's softmax probability of its own greedy pick (host pass over the already-CPU-visible logits row, suppression mirrored) rides AssistantDraftBlockResult.Probs; the greedy generate loop appends one JSONL cycle line pairing probs with the verified accepted-prefix length and carry offset. LTHN_MTP_CONF_FORCE=1 bypasses the low-accept bail, deep bootstrap, and rate-exit verdicts so calibration samples the full regime distribution (the gate's economics select against exactly the cycles the curve needs); both levers off = byte-identical behaviour, zero extra work. Receipts: 26B 370 cycles/1836 drafts + 12B 250/1245 — reliability monotone (top bin 91%/89% match), cumprod truncation at theta=0.40 retains 99.2%/99.1% of accepted tokens for 8-9% draft work saved on both drafters; 40%/30% of cycles accept the full K=4 with median end-of-block cumprod 0.995/0.979 (the cap raise is the bigger lever, especially at depth: 83% acceptance at 16K).

Co-Authored-By: Virgil <virgil@lethean.io>

Author: Snider

go/engine/metal/assistant_load.go

@@ -120,9 +120,12 @@ type AssistantDraftStepResult struct {
 }
 
 // AssistantDraftBlockResult is a chained native assistant proposal block.
+// Probs is nil unless the LTHN_MTP_CONF capture is armed (mtp_conf_diag.go):
+// per drafted position, the drafter's softmax probability of its own pick.
 type AssistantDraftBlockResult struct {
 	Tokens []int32
 	Hidden []byte
+	Probs  []float32
 }
 
 // AssistantVerifyResult reports target-side verification of a proposed
@@ -1210,6 +1213,10 @@ func (pair *AssistantPair) draftBlockFromSessionWithSuppress(target *ArchSession
 	} else {
 		tokens = target.mtpDraftTokenScratch(maxDraftTokens)
 	}
+	var confProbs []float32
+	if mtpConfEnabled() {
+		confProbs = make([]float32, 0, maxDraftTokens)
+	}
 	fused := pair.fusedDraft()
 	if fused != nil {
 		if err := fused.loadKV(targetKVs); err != nil {
@@ -1263,6 +1270,9 @@ func (pair *AssistantPair) draftBlockFromSessionWithSuppress(target *ArchSession
 				nativeTraceLog(core.Sprintf("mtp-diag draft step %d: tok=%d fwd=%.1fms head=%.1fms greedy=%.1fms normed{%s}\n",
 					len(tokens), tok, fwdMs, headMs, greedyMs, mtpDiagBF16Stats(normed)))
 			}
+			if confProbs != nil {
+				confProbs = append(confProbs, mtpConfProb(logits, tok, suppress))
+			}
 			stepToken, currentHidden = tok, nextHidden
 		} else {
 			projectedOut := target.mtpProjectionScratch(pair.Assistant.Arch.Hidden * bf16Size)
@@ -1279,6 +1289,9 @@ func (pair *AssistantPair) draftBlockFromSessionWithSuppress(target *ArchSession
 				nativeTraceLog(core.Sprintf("mtp-diag draft step %d: tok=%d projected{%s} logits{%s}\n",
 					len(tokens), step.Token, mtpDiagBF16Stats(projected), mtpDiagBF16Stats(logitsOut)))
 			}
+			if confProbs != nil {
+				confProbs = append(confProbs, mtpConfProb(logitsOut, step.Token, suppress))
+			}
 			stepToken, currentHidden = step.Token, step.Hidden
 		}
 		tokens = append(tokens, stepToken)
@@ -1287,7 +1300,7 @@ func (pair *AssistantPair) draftBlockFromSessionWithSuppress(target *ArchSession
 	if !copyTokens {
 		target.mtpDraftTokens = tokens
 	}
-	return AssistantDraftBlockResult{Tokens: tokens, Hidden: currentHidden}, nil
+	return AssistantDraftBlockResult{Tokens: tokens, Hidden: currentHidden, Probs: confProbs}, nil
 }
 
 func (pair *AssistantPair) draftBlockSampledFromSessionWithSuppress(target *ArchSession, lastToken int32, maxDraftTokens int, copyTokens bool, params model.SampleParams, sampler *model.Sampler) (AssistantDraftBlockResult, error) {
@@ -1765,6 +1778,13 @@ func (pair *AssistantPair) GenerateFromSessionEach(target *ArchSession, promptID
 			nativeTraceLog(core.Sprintf("mtp-diag verify %d: draft=%.1fms verify=%.1fms last=%d block=%v accepted=%v replacement=%d allAccepted=%v\n",
 				result.TargetVerifyCalls, diagDraftMs, total-diagDraftMs, lastToken, block, verify.AcceptedTokens, verify.ReplacementToken, verify.AllAccepted))
 		}
+		if draft.Probs != nil {
+			off := 0
+			if carryPresent {
+				off = 1
+			}
+			mtpConfRecordCycle(posBeforeVerify, off, draft.Probs, verify.AcceptedCount)
+		}
 		emitStart := 0
 		if carryPresent && len(verify.AcceptedTokens) > 0 && verify.AcceptedTokens[0] == carryLead {
 			emitStart = 1
@@ -1801,7 +1821,7 @@ func (pair *AssistantPair) GenerateFromSessionEach(target *ArchSession, promptID
 		if verify.AllAccepted {
 			lowAcceptStreak = 0
 			carryLead = -1
-			if !wasProbing && reng.needsDeepBootstrap(target.pos, len(result.Tokens), maxNew) {
+			if !mtpConfForce && !wasProbing && reng.needsDeepBootstrap(target.pos, len(result.Tokens), maxNew) {
 				if mtpDiagForTest {
 					nativeTraceLog(core.Sprintf("mtp-diag reengage deep-bootstrap: pos=%d emitted=%d\n", target.pos, len(result.Tokens)))
 				}
@@ -1819,7 +1839,7 @@ func (pair *AssistantPair) GenerateFromSessionEach(target *ArchSession, promptID
 			} else {
 				bail = reng.engagedCycle(newDrafts, time.Since(cycleT0).Seconds(), len(result.Tokens))
 			}
-			if bail {
+			if bail && !mtpConfForce {
 				if done, perr := runPlainStretch(carryLead); perr != nil {
 					return result, perr
 				} else if done {
@@ -1842,7 +1862,7 @@ func (pair *AssistantPair) GenerateFromSessionEach(target *ArchSession, promptID
 		}
 		// Give up on drafting only after the drafter has stayed weak for several
 		// consecutive blocks — one near-tie block is transient, not a mismatched pair.
-		if !stopped && len(result.Tokens) < maxNew && lowAcceptStreak >= nativeAssistantLowAcceptPatience {
+		if !mtpConfForce && !stopped && len(result.Tokens) < maxNew && lowAcceptStreak >= nativeAssistantLowAcceptPatience {
 			if mtpReengageDisabled {
 				if err := nativeAssistantFinishLowAcceptFromTargetCache(target, &result, replacement, maxNew, eosID, suppress, yield); err != nil {
 					return result, err
@@ -1864,7 +1884,7 @@ func (pair *AssistantPair) GenerateFromSessionEach(target *ArchSession, promptID
 		if stopped {
 			break
 		}
-		if !wasProbing && reng.needsDeepBootstrap(target.pos, len(result.Tokens), maxNew) {
+		if !mtpConfForce && !wasProbing && reng.needsDeepBootstrap(target.pos, len(result.Tokens), maxNew) {
 			if mtpDiagForTest {
 				nativeTraceLog(core.Sprintf("mtp-diag reengage deep-bootstrap: pos=%d emitted=%d\n", target.pos, len(result.Tokens)))
 			}
@@ -1882,7 +1902,7 @@ func (pair *AssistantPair) GenerateFromSessionEach(target *ArchSession, promptID
 		} else {
 			bail = reng.engagedCycle(newDrafts+1, time.Since(cycleT0).Seconds(), len(result.Tokens))
 		}
-		if bail {
+		if bail && !mtpConfForce {
 			if done, perr := runPlainStretch(replacement); perr != nil {
 				return result, perr
 			} else if done {
@@ -2026,7 +2046,7 @@ func (pair *AssistantPair) GenerateSampledFromSessionEach(target *ArchSession, p
 		if verify.AllAccepted {
 			lowAcceptStreak = 0
 			carryLead = -1
-			if !wasProbing && reng.needsDeepBootstrap(target.pos, len(result.Tokens), maxNew) {
+			if !mtpConfForce && !wasProbing && reng.needsDeepBootstrap(target.pos, len(result.Tokens), maxNew) {
 				if mtpDiagForTest {
 					nativeTraceLog(core.Sprintf("mtp-diag reengage deep-bootstrap: pos=%d emitted=%d\n", target.pos, len(result.Tokens)))
 				}
@@ -2044,7 +2064,7 @@ func (pair *AssistantPair) GenerateSampledFromSessionEach(target *ArchSession, p
 			} else {
 				bail = reng.engagedCycle(newDrafts, time.Since(cycleT0).Seconds(), len(result.Tokens))
 			}
-			if bail {
+			if bail && !mtpConfForce {
 				if done, perr := runPlainStretch(carryLead); perr != nil {
 					return result, perr
 				} else if done {
@@ -2082,7 +2102,7 @@ func (pair *AssistantPair) GenerateSampledFromSessionEach(target *ArchSession, p
 		}
 		// One weak block is a transient near-tie, not a mismatched pair — only fall
 		// back to plain target decode after several consecutive weak blocks.
-		if !stopped && len(result.Tokens) < maxNew && lowAcceptStreak >= nativeAssistantLowAcceptPatience {
+		if !mtpConfForce && !stopped && len(result.Tokens) < maxNew && lowAcceptStreak >= nativeAssistantLowAcceptPatience {
 			if mtpReengageDisabled {
 				var err error
 				history, err = nativeAssistantFinishLowAcceptSampledFromTargetCache(target, &result, replacement, maxNew, stopTokens, sampler, params, history, yield)
@@ -2107,7 +2127,7 @@ func (pair *AssistantPair) GenerateSampledFromSessionEach(target *ArchSession, p
 		if stopped {
 			break
 		}
-		if !wasProbing && reng.needsDeepBootstrap(target.pos, len(result.Tokens), maxNew) {
+		if !mtpConfForce && !wasProbing && reng.needsDeepBootstrap(target.pos, len(result.Tokens), maxNew) {
 			if mtpDiagForTest {
 				nativeTraceLog(core.Sprintf("mtp-diag reengage deep-bootstrap: pos=%d emitted=%d\n", target.pos, len(result.Tokens)))
 			}
@@ -2125,7 +2145,7 @@ func (pair *AssistantPair) GenerateSampledFromSessionEach(target *ArchSession, p
 		} else {
 			bail = reng.engagedCycle(newDrafts+1, time.Since(cycleT0).Seconds(), len(result.Tokens))
 		}
-		if bail {
+		if bail && !mtpConfForce {
 			if done, perr := runPlainStretch(replacement); perr != nil {
 				return result, perr
 			} else if done {

go/engine/metal/mtp_conf_diag.go

@@ -0,0 +1,123 @@
+// SPDX-Licence-Identifier: EUPL-1.2
+
+//go:build darwin && arm64
+
+package native
+
+import (
+	"math"
+	"os"
+	"strconv"
+	"sync"
+
+	core "dappco.re/go"
+)
+
+// LTHN_MTP_CONF=<path> — the #359 instrument: append one JSON line per MTP
+// verify cycle pairing the drafter's per-position self-confidence (softmax
+// probability of its own greedy pick) with the target's accepted prefix
+// length. The reliability curve derived from this capture is where the
+// confidence-scheduling threshold θ comes from — DSpark's 0.4 is calibrated
+// to their trained confidence head on Qwen3, not to a gemma4 drafter's raw
+// softmax, so we measure our own. Diag-only: when unset the draft loop takes
+// zero extra work (no softmax, no allocation, no I/O). Greedy lane only —
+// the sampled draft lane is not instrumented.
+var mtpConfCapturePath = os.Getenv("LTHN_MTP_CONF")
+
+func mtpConfEnabled() bool { return mtpConfCapturePath != "" }
+
+// LTHN_MTP_CONF_FORCE=1 — calibration mode (greedy lane): with the capture
+// armed, draft every cycle by bypassing the low-accept bail and the deep
+// bootstrap. The re-engagement gate's economics select AGAINST the very
+// cycles the curve needs (it stops drafting exactly where the drafter is
+// weak), so an unforced capture oversamples the good regimes. Never set on a
+// serving run — it deliberately trades tok/s for unbiased coverage.
+var mtpConfForce = os.Getenv("LTHN_MTP_CONF_FORCE") == "1" && mtpConfCapturePath != ""
+
+// mtpConfProb returns softmax probability of tokID over a host bf16 logits
+// row, mirroring draftGreedyTokenWithSuppress's iteration exactly: suppressed
+// ids are excluded from the distribution (the greedy pick was made without
+// them, so its confidence must be too). Two passes, f64 accumulation.
+func mtpConfProb(logits []byte, tokID int32, suppressed []int32) float32 {
+	vocab := len(logits) / bf16Size
+	if vocab == 0 || tokID < 0 || int(tokID) >= vocab {
+		return 0
+	}
+	maxV := float32(math.Inf(-1))
+	for id := range vocab {
+		if nativeAssistantSuppressed(int32(id), suppressed) {
+			continue
+		}
+		if v := bf16ToF32(logits[id*bf16Size], logits[id*bf16Size+1]); v > maxV {
+			maxV = v
+		}
+	}
+	var sum float64
+	for id := range vocab {
+		if nativeAssistantSuppressed(int32(id), suppressed) {
+			continue
+		}
+		v := bf16ToF32(logits[id*bf16Size], logits[id*bf16Size+1])
+		sum += math.Exp(float64(v - maxV))
+	}
+	if sum <= 0 {
+		return 0
+	}
+	tok := bf16ToF32(logits[tokID*bf16Size], logits[tokID*bf16Size+1])
+	return float32(math.Exp(float64(tok-maxV)) / sum)
+}
+
+// mtpConfSink is the process-lifetime append writer. Open is lazy so serving
+// binaries that never draft pay nothing; a failed open disables capture for
+// the rest of the process (a diag lever must never break generation).
+type mtpConfSink struct {
+	mu   sync.Mutex
+	f    *os.File
+	dead bool
+}
+
+var mtpConfOut mtpConfSink
+
+// mtpConfRecordCycle appends one cycle line:
+//
+//	{"pos":18432,"carry":1,"probs":[0.9531,0.8125,0.4023,0.1201],"accepted":3}
+//
+// pos is the target position before verify; carry is the block offset (a
+// pending lead token verified ahead of the draft), so drafted position k is
+// prefix-accepted iff k+carry < accepted. Labels are derived at analysis
+// time; the line stores raw facts only.
+func mtpConfRecordCycle(pos, carry int, probs []float32, accepted int) {
+	mtpConfOut.mu.Lock()
+	defer mtpConfOut.mu.Unlock()
+	if mtpConfOut.dead {
+		return
+	}
+	if mtpConfOut.f == nil {
+		f, err := os.OpenFile(mtpConfCapturePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
+		if err != nil {
+			mtpConfOut.dead = true
+			nativeTraceLog(core.Sprintf("mtp-conf: capture disabled, open %s: %v\n", mtpConfCapturePath, err))
+			return
+		}
+		mtpConfOut.f = f
+	}
+	buf := make([]byte, 0, 64+12*len(probs))
+	buf = append(buf, `{"pos":`...)
+	buf = strconv.AppendInt(buf, int64(pos), 10)
+	buf = append(buf, `,"carry":`...)
+	buf = strconv.AppendInt(buf, int64(carry), 10)
+	buf = append(buf, `,"probs":[`...)
+	for i, p := range probs {
+		if i > 0 {
+			buf = append(buf, ',')
+		}
+		buf = strconv.AppendFloat(buf, float64(p), 'f', 4, 32)
+	}
+	buf = append(buf, `],"accepted":`...)
+	buf = strconv.AppendInt(buf, int64(accepted), 10)
+	buf = append(buf, '}', '\n')
+	if _, err := mtpConfOut.f.Write(buf); err != nil {
+		mtpConfOut.dead = true
+		nativeTraceLog(core.Sprintf("mtp-conf: capture disabled, write: %v\n", err))
+	}
+}

go/engine/metal/mtp_conf_diag_test.go

@@ -0,0 +1,93 @@
+// SPDX-Licence-Identifier: EUPL-1.2
+
+//go:build darwin && arm64
+
+package native
+
+import (
+	"math"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func mtpConfTestRow(logits []float32) []byte {
+	row := make([]byte, len(logits)*bf16Size)
+	for i, v := range logits {
+		h := f32ToBF16(v)
+		row[i*bf16Size] = byte(h)
+		row[i*bf16Size+1] = byte(h >> 8)
+	}
+	return row
+}
+
+// TestMTPConfProb pins the softmax probability against a hand-computed
+// distribution on a 4-token row (all values exactly representable in bf16).
+func TestMTPConfProb(t *testing.T) {
+	row := mtpConfTestRow([]float32{2, 1, 0, -1})
+	var sum float64
+	for _, v := range []float64{2, 1, 0, -1} {
+		sum += math.Exp(v - 2)
+	}
+	want := float32(math.Exp(0) / sum)
+	got := mtpConfProb(row, 0, nil)
+	if math.Abs(float64(got-want)) > 1e-6 {
+		t.Fatalf("prob(top) = %v, want %v", got, want)
+	}
+	wantLow := float32(math.Exp(-3) / sum)
+	if gotLow := mtpConfProb(row, 3, nil); math.Abs(float64(gotLow-wantLow)) > 1e-6 {
+		t.Fatalf("prob(bottom) = %v, want %v", gotLow, wantLow)
+	}
+	if bad := mtpConfProb(row, 4, nil); bad != 0 {
+		t.Fatalf("prob(out of range) = %v, want 0", bad)
+	}
+}
+
+// TestMTPConfProbSuppressed pins that suppressed ids are excluded from the
+// distribution, mirroring draftGreedyTokenWithSuppress: with the top token
+// suppressed, the runner-up's probability is over the remaining support.
+func TestMTPConfProbSuppressed(t *testing.T) {
+	row := mtpConfTestRow([]float32{2, 1, 0, -1})
+	var sum float64
+	for _, v := range []float64{1, 0, -1} {
+		sum += math.Exp(v - 1)
+	}
+	want := float32(math.Exp(0) / sum)
+	got := mtpConfProb(row, 1, []int32{0})
+	if math.Abs(float64(got-want)) > 1e-6 {
+		t.Fatalf("prob(runner-up, top suppressed) = %v, want %v", got, want)
+	}
+}
+
+// TestMTPConfRecordCycle pins the JSONL line format byte-exactly and that the
+// sink appends across calls.
+func TestMTPConfRecordCycle(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "conf.jsonl")
+	savedPath := mtpConfCapturePath
+	mtpConfCapturePath = path
+	mtpConfOut.mu.Lock()
+	savedF, savedDead := mtpConfOut.f, mtpConfOut.dead
+	mtpConfOut.f, mtpConfOut.dead = nil, false
+	mtpConfOut.mu.Unlock()
+	defer func() {
+		mtpConfOut.mu.Lock()
+		if mtpConfOut.f != nil {
+			mtpConfOut.f.Close()
+		}
+		mtpConfOut.f, mtpConfOut.dead = savedF, savedDead
+		mtpConfOut.mu.Unlock()
+		mtpConfCapturePath = savedPath
+	}()
+
+	mtpConfRecordCycle(100, 1, []float32{0.5, 0.25}, 2)
+	mtpConfRecordCycle(104, 0, []float32{0.9990}, 0)
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read capture: %v", err)
+	}
+	want := `{"pos":100,"carry":1,"probs":[0.5000,0.2500],"accepted":2}` + "\n" +
+		`{"pos":104,"carry":0,"probs":[0.9990],"accepted":0}` + "\n"
+	if string(data) != want {
+		t.Fatalf("capture = %q, want %q", data, want)
+	}
+}