chore: WIP staging — replication hasSubs, persist buf pool, cgo WaitForTrust, regen-plain brace escape

- pkg/registry/replication.go: hasSubs() helper lets replicaPushLoop
  skip the 50-100 MB snapshot build when no replicas are attached.
- pkg/registry/server/server_persist.go + wal.go: reuse the snapshot
  bytes.Buffer across save ticks via a sync.Pool pre-grown to 128 MB,
  eliminating the ~1 GB live growSlice heap that drove GC STW pauses.
- sdk/cgo/bindings.go: expose PilotWaitForTrust for foreign-language
  SDKs that need a blocking handshake completion primitive.
- web/scripts/regen-plain.mjs: escape { } when rendering Astro plain
  mirrors, so JSON examples with curly braces don't trip JSX parsing.

Author: teovl

pkg/registry/replication.go

@@ -36,6 +36,17 @@ func newReplicationManager() *replicationManager {
 	}
 }
 
+// hasSubs reports whether any replication subscriber is currently
+// connected. Used by replicaPushLoop to skip building the full
+// snapshot (which can be 50-100 MB and allocate 1 GB/sec at 170k
+// nodes) when no replicas are attached — most rendezvous deployments
+// run without replicas in steady state.
+func (rm *replicationManager) hasSubs() bool {
+	rm.mu.Lock()
+	defer rm.mu.Unlock()
+	return len(rm.subs) > 0
+}
+
 // addSub registers a connection as a replication subscriber.
 func (rm *replicationManager) addSub(conn net.Conn) {
 	rm.mu.Lock()

pkg/registry/server/server_persist.go

@@ -3,6 +3,7 @@
 package server
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
@@ -11,6 +12,7 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 
 	"github.com/TeoSlayer/pilotprotocol/internal/fsutil"
@@ -20,6 +22,19 @@ import (
 	"github.com/TeoSlayer/pilotprotocol/pkg/urlvalidate"
 )
 
+// flushSaveBufPool reuses the bytes buffer that backs the snapshot JSON
+// across save ticks. Pre-grown to 128 MB so the first save at fleet
+// scale (~50-100 MB JSON) doesn't grow it further. After the first
+// save the pool always returns a buffer at peak capacity, so subsequent
+// saves do zero allocation in the encode path. Eliminates the ~1 GB
+// live `bytes.growSlice` heap that was driving GC STW pauses.
+var flushSaveBufPool = sync.Pool{
+	New: func() interface{} {
+		b := make([]byte, 0, 128*1024*1024)
+		return &b
+	},
+}
+
 // rawNodeCopy holds raw node fields copied under RLock (no encoding).
 // base64/time.Format happens outside the lock to minimize lock hold time.
 type rawNodeCopy struct {
@@ -383,22 +398,47 @@ func (s *Server) flushSave() error {
 	}
 	s.auditMu.Unlock()
 
-	// Compute checksum: marshal once without checksum (omitempty omits it), hash,
-	// then inject the checksum into the JSON without a second marshal.
+	// Compute checksum: encode once without checksum (omitempty omits it), hash,
+	// then inject the checksum into the JSON without a second encode.
+	//
+	// 2026-05-14: switched from json.Marshal (one-shot allocate) to json.Encoder
+	// writing into a pooled bytes.Buffer. At 170k+ active nodes the snapshot is
+	// ~50-100 MB and was the dominant heap allocator (~1 GB live, GC pressure
+	// causing kernel UDP drops during STW pauses). The pooled buffer is reused
+	// across save ticks — the underlying slice grows once and stays at peak,
+	// no per-tick allocation thereafter.
 	snap.Checksum = ""
-	data, err := json.Marshal(snap)
-	if err != nil {
-		slog.Error("registry save marshal error", "err", err)
-		return fmt.Errorf("marshal snapshot: %w", err)
+	bp := flushSaveBufPool.Get().(*[]byte)
+	buf := bytes.NewBuffer((*bp)[:0])
+	enc := json.NewEncoder(buf)
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(snap); err != nil {
+		*bp = buf.Bytes()[:0]
+		flushSaveBufPool.Put(bp)
+		slog.Error("registry save encode error", "err", err)
+		return fmt.Errorf("encode snapshot: %w", err)
+	}
+	data := buf.Bytes()
+	// json.Encoder.Encode appends a newline; drop it to match prior Marshal output.
+	if len(data) > 0 && data[len(data)-1] == '\n' {
+		data = data[:len(data)-1]
 	}
 	hash := sha256.Sum256(data)
 	checksum := hex.EncodeToString(hash[:])
-	// Insert "checksum":"<hex>" before the closing brace. json.Marshal of a struct
-	// always produces a JSON object ending with '}' (no trailing whitespace).
+	// Insert "checksum":"<hex>" before the closing brace. json.Encoder of a struct
+	// always produces a JSON object ending with '}' (after newline trim).
 	if len(data) == 0 || data[len(data)-1] != '}' {
-		return fmt.Errorf("marshal snapshot: unexpected JSON format (expected trailing '}')")
+		*bp = buf.Bytes()[:0]
+		flushSaveBufPool.Put(bp)
+		return fmt.Errorf("encode snapshot: unexpected JSON format (expected trailing '}')")
 	}
 	data = append(data[:len(data)-1], []byte(`,"checksum":"`+checksum+`"}`)...)
+	defer func() {
+		// Return the (possibly grown) underlying buffer to the pool. AtomicWrite
+		// has copied data to disk by this point, so it is safe to release.
+		*bp = data[:0]
+		flushSaveBufPool.Put(bp)
+	}()
 
 	// Persist to disk atomically
 	if s.storePath != "" {

pkg/registry/server/wal/wal.go

@@ -289,6 +289,13 @@ type StoreCallbacks struct {
 	Push func([]byte)
 	// IncSaveFailures increments the save-failure metric counter.
 	IncSaveFailures func()
+	// HasReplicaSubs reports whether any replication subscriber is
+	// currently attached. replicaPushLoop uses this to skip the full
+	// snapshot build when no replicas are listening — at fleet scale
+	// the snapshot allocates ~1 GB/sec, so skipping is a large win
+	// for deployments that run without replicas. May be nil; nil is
+	// treated as "always push" for backward compat.
+	HasReplicaSubs func() bool
 }
 
 // Store manages the WAL file, save channels, standby flag, and replication
@@ -464,8 +471,10 @@ func (st *Store) replicaPushLoop() {
 			dirty = true
 		case <-ticker.C:
 			if dirty {
-				if data := st.cbs.SnapshotJSON(); data != nil {
-					st.cbs.Push(data)
+				if st.cbs.HasReplicaSubs == nil || st.cbs.HasReplicaSubs() {
+					if data := st.cbs.SnapshotJSON(); data != nil {
+						st.cbs.Push(data)
+					}
 				}
 				dirty = false
 			}
@@ -476,7 +485,7 @@ func (st *Store) replicaPushLoop() {
 				dirty = true
 			default:
 			}
-			if dirty {
+			if dirty && (st.cbs.HasReplicaSubs == nil || st.cbs.HasReplicaSubs()) {
 				if data := st.cbs.SnapshotJSON(); data != nil {
 					st.cbs.Push(data)
 				}

sdk/cgo/bindings.go

@@ -143,6 +143,19 @@ func PilotHandshake(h C.uint64_t, nodeID C.uint32_t, justification *C.char) *C.c
 	return okJSON(r)
 }
 
+//export PilotWaitForTrust
+func PilotWaitForTrust(h C.uint64_t, nodeID C.uint32_t, timeoutMs C.uint32_t) *C.char {
+	d, err := driverFromHandle(h)
+	if err != nil {
+		return errJSON(err)
+	}
+	r, err := d.WaitForTrust(uint32(nodeID), uint32(timeoutMs))
+	if err != nil {
+		return errJSON(err)
+	}
+	return okJSON(r)
+}
+
 //export PilotApproveHandshake
 func PilotApproveHandshake(h C.uint64_t, nodeID C.uint32_t) *C.char {
 	d, err := driverFromHandle(h)

web/scripts/regen-plain.mjs

@@ -244,7 +244,9 @@ function escapeHtml(s) {
   return String(s)
     .replace(/&/g, '&')
     .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;');
+    .replace(/>/g, '&gt;')
+    .replace(/\{/g, '&#123;')
+    .replace(/\}/g, '&#125;');
 }
 
 function renderBlock(block) {