fix(daemon): asymmetric-crypto + outside-window self-recovery

Two related rc3 bugs surfaced live against the list-agents public agent
on 2026-05-11. Both have the same shape: one side of an established
tunnel ends up with state the other side can't help it recover from,
and the only path back was a manual daemon restart. After these fixes
the daemon self-heals in under 5 s without operator action.

Bug 1 — asymmetric crypto loss has no in-band recovery
  When a peer drops its Crypto for us (e.g. via DecryptFailDropGrace
  after a relay-buffered stale-frame burst) but keeps the same X25519
  ephemeral, every PILA it retransmits arrives at us with
  keyChanged=false. The duplicate-PILA branch in HandleAuthFrame
  cleared pendingRekey and returned silently; the peer's loop never
  got our reply and was stuck until its X25519 changed (i.e. a daemon
  restart). Fix: in that branch, also call SendKeyExchangeToNode when
  InboundDecryptStale(peerNodeID) is true. The constant
  KeyExchangeReplyStaleThreshold (6 s) was already present with a
  doc comment describing exactly this behaviour — the wiring was
  missing. Pinned by TestAsymmetricRecoveryRepliesOnDuplicatePILA
  WhenStale and TestAsymmetricRecoveryRateLimitsRepliesWhenHealthy.

Bug 2 — outside-replay-window divergence has no recovery
  Sustained ErrOutsideWindow rejections (peer's send counter has
  drifted far behind our window's high-water-mark) used to be logged
  and ignored. Triggering rekey on every occurrence storms because
  the relay re-delivers early-counter frames after every rotate, so
  the original code deliberately did nothing — at the cost of
  leaving the receiver wedged.

  Fix mirrors the existing AEAD-fail-grace pattern:
    - new OutsideWindowCount field on Crypto, incremented in
      envelope.DecryptFrame on each ErrOutsideWindow, reset on any
      successful decrypt;
    - new OutsideWindowDropThreshold (30) and OutsideWindowDropGrace
      (3 s) constants;
    - new Store.ShouldDropOnOutsideWindow(peerNodeID, c) gate;
    - tunnel.go's ErrOutsideWindow case now calls the gate and on
      trip does CompareAndDrop + maybeRequestRekey.
  Combined with the Bug 1 fix, both sides reset cleanly. Pinned by
  TestOutsideWindowCountIncrementsAndResets, TestShouldDropOnOutside
  WindowGatesOnThresholdAndGrace, and TestOutsideWindowDropEndToEnd.

Also adds TestDialRetryBudgetCleansUpOrphanSynSent pinning the
existing dial-retry-budget cleanup contract, useful when triaging
"stuck SYN_SENT" reports.

Author: teovl

pkg/daemon/envelope/envelope.go

@@ -155,6 +155,15 @@ func DecryptFrame(store *keyexchange.Store, data []byte) DecryptResult {
 		err := ErrReplay
 		if recvCounter < maxN && maxN-recvCounter >= keyexchange.ReplayWindowSize {
 			err = ErrOutsideWindow
+			// Track consecutive outside-window rejections. A sustained
+			// burst means the peer's send counter has diverged from our
+			// window's high-water-mark too far for in-band recovery —
+			// only a fresh key exchange resets both sides. The caller
+			// (L7) consults ShouldDropOnOutsideWindow to enforce the
+			// threshold + grace gate. Mirrors DecryptFailCount.
+			c.ReplayMu.Lock()
+			c.OutsideWindowCount++
+			c.ReplayMu.Unlock()
 		}
 		return DecryptResult{
 			PeerNodeID:   peerNodeID,
@@ -183,11 +192,14 @@ func DecryptFrame(store *keyexchange.Store, data []byte) DecryptResult {
 		}
 	}
 
-	// Successful decrypt — reset the AEAD failure counter.
+	// Successful decrypt — reset both fault counters under one lock.
 	c.ReplayMu.Lock()
 	if c.DecryptFailCount != 0 {
 		c.DecryptFailCount = 0
 	}
+	if c.OutsideWindowCount != 0 {
+		c.OutsideWindowCount = 0
+	}
 	c.ReplayMu.Unlock()
 
 	return DecryptResult{

pkg/daemon/envelope/zz_outside_window_recovery_test.go

@@ -0,0 +1,188 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package envelope_test
+
+// Regression tests for the outside-replay-window stuck-state bug
+// observed in v1.10.0-rc3 against list-agents on 2026-05-11:
+//
+// list-agents' MaxRecvNonce for our node stayed at 8518 while our
+// outbound counter sat at ~400 after a fresh key exchange. Every frame
+// we sent was rejected as ErrOutsideWindow. The current code path at
+// tunnel.go ErrOutsideWindow case logs and returns — no recovery.
+// Tunnel half-dead until a daemon restart on either side.
+//
+// The fix tracks consecutive outside-window rejections on the Crypto
+// (mirroring DecryptFailCount) and surfaces a ShouldDropOnOutsideWindow
+// gate that the L7 handler uses to drop + rekey. These tests exercise
+// the keyexchange/envelope half of the fix; the tunnel.go wiring is
+// covered by integration tests on the daemon.
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/TeoSlayer/pilotprotocol/pkg/daemon/envelope"
+	"github.com/TeoSlayer/pilotprotocol/pkg/daemon/keyexchange"
+)
+
+// TestOutsideWindowCountIncrementsAndResets pins the bookkeeping: each
+// ErrOutsideWindow bumps OutsideWindowCount; a successful decrypt
+// resets it to 0. This is the input ShouldDropOnOutsideWindow consumes.
+func TestOutsideWindowCountIncrementsAndResets(t *testing.T) {
+	t.Parallel()
+	s := newPeerSetup(t)
+
+	// Stash a frame at counter=1.
+	stale, err := envelope.EncryptFrame(s.localStore, s.peerID, []byte("stale"))
+	if err != nil {
+		t.Fatalf("encrypt stale: %v", err)
+	}
+
+	// Advance peer MaxRecvNonce past ReplayWindowSize+1 by encrypting and
+	// delivering fresh frames so stale (counter=1) lands outside the window.
+	advance := keyexchange.ReplayWindowSize + 8
+	for i := 0; i < advance; i++ {
+		f, err := envelope.EncryptFrame(s.localStore, s.peerID, []byte("advance"))
+		if err != nil {
+			t.Fatalf("encrypt advance #%d: %v", i, err)
+		}
+		if r := envelope.DecryptFrame(s.peerStore, f[4:]); r.Err != nil {
+			t.Fatalf("advance decrypt #%d: %v", i, r.Err)
+		}
+	}
+
+	c := s.peerStore.Get(s.localID)
+	c.ReplayMu.Lock()
+	pre := c.OutsideWindowCount
+	c.ReplayMu.Unlock()
+	if pre != 0 {
+		t.Fatalf("OutsideWindowCount=%d before any outside-window event; want 0", pre)
+	}
+
+	// Now deliver the stale frame.
+	if r := envelope.DecryptFrame(s.peerStore, stale[4:]); !errors.Is(r.Err, envelope.ErrOutsideWindow) {
+		t.Fatalf("stale decrypt err=%v; want ErrOutsideWindow", r.Err)
+	}
+	c.ReplayMu.Lock()
+	post := c.OutsideWindowCount
+	c.ReplayMu.Unlock()
+	if post != 1 {
+		t.Fatalf("OutsideWindowCount=%d after one outside-window event; want 1", post)
+	}
+
+	// Successful decrypt resets to 0.
+	f, _ := envelope.EncryptFrame(s.localStore, s.peerID, []byte("fresh"))
+	if r := envelope.DecryptFrame(s.peerStore, f[4:]); r.Err != nil {
+		t.Fatalf("fresh decrypt: %v", r.Err)
+	}
+	c.ReplayMu.Lock()
+	post2 := c.OutsideWindowCount
+	c.ReplayMu.Unlock()
+	if post2 != 0 {
+		t.Fatalf("OutsideWindowCount=%d after successful decrypt; want 0", post2)
+	}
+}
+
+// TestShouldDropOnOutsideWindowGatesOnThresholdAndGrace pins the
+// recovery trigger: the gate must require BOTH the count threshold
+// AND the grace age to elapse. This is what the L7 handler uses to
+// decide whether to drop crypto and request a rekey.
+func TestShouldDropOnOutsideWindowGatesOnThresholdAndGrace(t *testing.T) {
+	t.Parallel()
+	s := newPeerSetup(t)
+
+	c := s.peerStore.Get(s.localID)
+	if c == nil {
+		t.Fatalf("no Crypto for local on peer side")
+	}
+
+	// Below threshold: never drop, regardless of age.
+	c.ReplayMu.Lock()
+	c.OutsideWindowCount = keyexchange.OutsideWindowDropThreshold - 1
+	c.ReplayMu.Unlock()
+	// Force age past grace so the only failing check is the threshold.
+	backdateCreatedAt(c, time.Now().Add(-2*keyexchange.OutsideWindowDropGrace))
+	if s.peerStore.ShouldDropOnOutsideWindow(s.localID, c) {
+		t.Fatalf("dropped at count<threshold")
+	}
+
+	// At threshold but within grace: don't drop yet.
+	c.ReplayMu.Lock()
+	c.OutsideWindowCount = keyexchange.OutsideWindowDropThreshold
+	c.ReplayMu.Unlock()
+	backdateCreatedAt(c, time.Now()) // fresh
+	if s.peerStore.ShouldDropOnOutsideWindow(s.localID, c) {
+		t.Fatalf("dropped within grace period")
+	}
+
+	// Threshold met AND grace elapsed: drop.
+	backdateCreatedAt(c, time.Now().Add(-2*keyexchange.OutsideWindowDropGrace))
+	if !s.peerStore.ShouldDropOnOutsideWindow(s.localID, c) {
+		t.Fatalf("did not drop at threshold + past grace")
+	}
+}
+
+// TestOutsideWindowDropEndToEnd is the smoking-gun reproduction of the
+// list-agents bug: simulate 30+ consecutive outside-window rejections
+// over the grace period, then assert the gate signals drop. Prior to
+// the fix the count never moved (no recovery path); now it climbs and
+// the gate trips, letting the L7 handler tear down the diverged Crypto.
+func TestOutsideWindowDropEndToEnd(t *testing.T) {
+	t.Parallel()
+	s := newPeerSetup(t)
+
+	// Capture N+1 stale frames before advancing peer's MaxRecvNonce so
+	// that each one lands outside the window when delivered later.
+	const stales = keyexchange.OutsideWindowDropThreshold + 1
+	staleFrames := make([][]byte, stales)
+	for i := 0; i < stales; i++ {
+		f, err := envelope.EncryptFrame(s.localStore, s.peerID, []byte("stale"))
+		if err != nil {
+			t.Fatalf("encrypt stale #%d: %v", i, err)
+		}
+		staleFrames[i] = f
+	}
+
+	// Advance peer's window past all stale counters.
+	advance := keyexchange.ReplayWindowSize + stales + 8
+	for i := 0; i < advance; i++ {
+		f, err := envelope.EncryptFrame(s.localStore, s.peerID, []byte("advance"))
+		if err != nil {
+			t.Fatalf("encrypt advance #%d: %v", i, err)
+		}
+		if r := envelope.DecryptFrame(s.peerStore, f[4:]); r.Err != nil {
+			t.Fatalf("advance decrypt #%d: %v", i, r.Err)
+		}
+	}
+
+	// Backdate Crypto so the grace gate is satisfied at the moment we
+	// hit the count threshold below.
+	c := s.peerStore.Get(s.localID)
+	backdateCreatedAt(c, time.Now().Add(-2*keyexchange.OutsideWindowDropGrace))
+
+	// Deliver the stale frames. Each one should produce ErrOutsideWindow
+	// and bump the counter. ShouldDropOnOutsideWindow should flip true
+	// once threshold is reached.
+	for i, f := range staleFrames {
+		r := envelope.DecryptFrame(s.peerStore, f[4:])
+		if !errors.Is(r.Err, envelope.ErrOutsideWindow) {
+			t.Fatalf("stale #%d err=%v; want ErrOutsideWindow", i, r.Err)
+		}
+	}
+
+	if !s.peerStore.ShouldDropOnOutsideWindow(s.localID, c) {
+		c.ReplayMu.Lock()
+		count := c.OutsideWindowCount
+		c.ReplayMu.Unlock()
+		t.Fatalf("BUG-2: ShouldDropOnOutsideWindow=false after %d "+
+			"consecutive ErrOutsideWindow (count=%d, threshold=%d)",
+			stales, count, keyexchange.OutsideWindowDropThreshold)
+	}
+}
+
+// backdateCreatedAt rewrites the CreatedAt timestamp on a Crypto so
+// age-gated tests don't have to sleep through the real grace period.
+func backdateCreatedAt(c *keyexchange.Crypto, when time.Time) {
+	c.CreatedAt = when
+}

pkg/daemon/keyexchange/crypto.go

@@ -44,6 +44,22 @@ const SalvageMaxAge = 5 * time.Second
 // L5 concept) and Crypto.DecryptFailCount (per-peer state).
 const DecryptFailDropThreshold = 5
 
+// OutsideWindowDropThreshold is how many consecutive outside-replay-window
+// rejections cause us to drop the peer's Crypto and trigger a fresh key
+// exchange. Tuned conservatively: ReplayWindowSize=256 means an in-spec
+// burst of late relay-buffered frames can produce a few outside-window
+// rejections normally. 30 consecutive without any successful decrypt is
+// strong evidence the peer's send counter is far behind our window
+// max-water-mark and no in-band recovery exists. Reset to 0 on success.
+const OutsideWindowDropThreshold = 30
+
+// OutsideWindowDropGrace mirrors DecryptFailDropGrace but for the
+// outside-window path. Reuses the same 3-second value: a freshly
+// installed Crypto can legitimately see a small flurry of stale
+// in-flight frames at low counters; only after the new state has had
+// time to drain those should we treat the persistence as divergence.
+const OutsideWindowDropGrace = 3 * time.Second
+
 // DecryptFailDropGrace is the minimum age a Crypto must reach before
 // repeated decrypt failures can drop it. Set above the typical relay RTT
 // (≤200 ms) plus a comfortable margin: stale ciphertext from before the
@@ -97,6 +113,18 @@ type Crypto struct {
 	// decrypt.
 	DecryptFailCount int
 
+	// OutsideWindowCount tracks consecutive frames rejected because their
+	// counter was more than ReplayWindowSize behind MaxRecvNonce. A few
+	// of these are normal (late relay-buffered frames). A sustained burst
+	// means the peer's send counter and our receive window have diverged
+	// far enough that no in-band recovery is possible — the only signal
+	// the peer has to know there's a problem is silence, and the only
+	// fix is a fresh key exchange. Once this counter exceeds
+	// OutsideWindowDropThreshold AND the Crypto is older than
+	// OutsideWindowDropGrace, the daemon drops this Crypto and requests
+	// a rekey. Reset to 0 on any successful decrypt.
+	OutsideWindowCount int
+
 	// CreatedAt is when this Crypto was installed. Used by L5's
 	// handleAuthKeyExchange to decide between preserving existing state
 	// (fresh handshake retransmit/reply within seconds) and resetting it

pkg/daemon/keyexchange/handle.go

@@ -141,8 +141,20 @@ func (m *Manager) HandleAuthFrame(data []byte, from *net.UDPAddr, fromRelay bool
 		m.SendKeyExchangeToNode(peerNodeID)
 		m.ClearPendingRekey(peerNodeID)
 	} else {
-		// Already had a session and peer didn't rotate — clear any
-		// outstanding pending state so we don't keep retrying.
+		// Already had a session and peer didn't rotate. If we have NO
+		// recent successful decrypt from this peer, our previous reply
+		// PILA may have been lost OR the peer dropped its crypto for us
+		// (e.g. via the DecryptFailDropGrace path after a relay-buffered
+		// stale-frame burst). In either case the peer needs our PILA to
+		// re-derive the shared secret — silence here is the asymmetric-
+		// crypto deadlock observed against list-agents on 2026-05-11.
+		// KeyExchangeReplyStaleThreshold (6 s) gates this so an active
+		// session with regular inbound traffic cannot trigger a reply
+		// loop. Don't reinstall locally — preserves our nonce counter
+		// and replay window for in-flight encrypted traffic.
+		if m.InboundDecryptStale(peerNodeID) {
+			m.SendKeyExchangeToNode(peerNodeID)
+		}
 		m.ClearPendingRekey(peerNodeID)
 	}
 	return true

pkg/daemon/keyexchange/store.go

@@ -179,6 +179,36 @@ func (s *Store) ShouldDropOnDecryptFail(peerNodeID uint32, c *Crypto) bool {
 	return current == c
 }
 
+// ShouldDropOnOutsideWindow mirrors ShouldDropOnDecryptFail for the
+// outside-replay-window divergence path. Returns true when the peer's
+// OutsideWindowCount has reached OutsideWindowDropThreshold AND the
+// Crypto is older than OutsideWindowDropGrace AND it is still the
+// currently-installed entry for peerNodeID.
+//
+// The caller (L7 handleEncrypted) is responsible for dropping the
+// Crypto via CompareAndDrop and triggering a fresh key exchange when
+// this returns true. This is the symmetric-state recovery for the
+// case where our receive-window's high-water-mark has drifted so far
+// past the peer's actual send counter that no in-band signal exists.
+// Reproduces the rc3 list-agents bug on 2026-05-11 where MaxRecvNonce
+// stayed at 8518 while the sender's outbound counter sat at ~400.
+func (s *Store) ShouldDropOnOutsideWindow(peerNodeID uint32, c *Crypto) bool {
+	if c == nil {
+		return false
+	}
+	c.ReplayMu.Lock()
+	rejections := c.OutsideWindowCount
+	c.ReplayMu.Unlock()
+	if rejections < OutsideWindowDropThreshold {
+		return false
+	}
+	if time.Since(c.CreatedAt) < OutsideWindowDropGrace {
+		return false
+	}
+	current := s.Get(peerNodeID)
+	return current == c
+}
+
 // RecordSalvage stashes a plaintext send into the per-peer ring buffer.
 // On a subsequent peer-initiated rekey, DrainSalvage will hand back the
 // entries; the caller re-encrypts with the new key and re-sends —