fix(daemon): in-window replay recovery, rekey relay fallback, FIN-not-RST keepalive reap

Three independent fleet-wedge fixes that compose into the rc7 release.

1. In-window replay-collision recovery (`pkg/daemon/keyexchange/`,
   `pkg/daemon/envelope/`, `pkg/daemon/tunnel.go`).

   When a peer restarts but its persisted X25519 identity carries
   over, the new daemon's send counter resets to 1 while our
   MaxRecvNonce still sits at the pre-restart watermark. Every
   subsequent frame from the peer lands at an already-marked counter
   in the replay bitmap and is dropped BEFORE AEAD-Open runs, so
   neither DecryptFailCount nor OutsideWindowCount increment. Pre-
   rc7, no recovery path engaged and the session wedged
   indefinitely (the 2026-05-11 list-agents bug).

   Fix: track ReplayCount per Crypto, gate via ShouldDropOnReplay
   (threshold=30, grace=3s, CompareAndDrop-guarded). On trip, drop
   the wedged Crypto + request rekey. Storm-safe — the grace window
   prevents the v1.9.1 rekey→fresh-session→relay-replay→rekey loop.

2. Rekey relay-fallback (`pkg/daemon/keyexchange/loop.go`,
   `pkg/daemon/tunnel.go`).

   When the local NAT remaps our external port across a daemon
   restart, peers' cached endpoints go stale; rekey PILA goes out
   fine, peer's PILA reply lands in a black hole. The blackhole
   heuristic needs BlackholeMissesRequired=3 observations × 4s = 12s
   to engage, but rekey gives up at 24s — too late for the existing
   routing flip to kick in. Fix: PreRetransmitHook fires once per
   retransmit; tunnel.go installs a hook that flips the peer to
   relay after RekeyRelayFallbackAfter=2 attempts. Existing
   ResetPendingRekeyAttempts handles the budget reset.

3. FIN-not-RST keepalive reap (`pkg/daemon/daemon.go`).

   When KeepaliveUnacked reached 3, the reaper constructed a raw RST
   inline and forced StateClosed. The RST poisoned the peer's
   session table for that conn_id, cascading into "connection
   refused" on the peer's next outbound. Observed fleet-wide on
   2026-05-11 as bursts of N successful sends followed by total
   loss of reachability. Fix: extract keepaliveSweep; replace the
   raw RST with CloseConnection() (proper FIN with retx tracking,
   transitions to StateFinWait).

Tests:
- 5 envelope unit tests (bookkeeping, isolation, gate semantics,
  end-to-end, storm-safety) in zz_replay_recovery_test.go
- 1 daemon unit test (TestSustainedReplayTriggersRekey) pinning the
  tunnel.go ErrReplay→drop+rekey wiring, plus updated
  TestReplayNeverTriggersRekey comment acknowledging the
  persistent-identity case
- 3 daemon unit tests for the rekey relay-fallback hook
  (threshold flip, end-to-end via tick, below-threshold no-flip)
- 3 daemon unit tests for the FIN-not-RST keepalive reap
  (FIN-not-RST regression, probe path unaffected, non-established
  conns skipped)
- 1 real end-to-end test (`tests/zz_rc6_peer_restart_recovery_test.go`)
  spinning up beacon + registry + two daemons on loopback,
  exercising the full peer-restart recovery cycle through the
  driver API — no internal hooks, no white-box manipulation

Author: teovl

pkg/daemon/daemon.go

@@ -4339,61 +4339,78 @@ func (d *Daemon) idleSweepLoop() {
 			// resolveCache and hostnameCache: strictly TTL-gated on read; evict after 2× TTL.
 			d.reapCaches()
 
-			// Send keepalive probes to connections idle beyond keepalive interval.
-			// Dead-peer detection: if 3 consecutive probes go unanswered, RST + close.
-			idle := d.ports.IdleConnections(keepaliveInterval)
-			for _, conn := range idle {
-				conn.Mu.Lock()
-				st := conn.State
-				sendSeq := conn.SendSeq
-				recvAck := conn.RecvAck
-				kaUnacked := conn.KeepaliveUnacked
-				conn.Mu.Unlock()
-				if st != StateEstablished {
-					continue
-				}
-				if kaUnacked >= 3 {
-					slog.Warn("dead peer detected (3 keepalives unanswered), sending RST",
-						"conn_id", conn.ID, "remote_addr", conn.RemoteAddr, "remote_port", conn.RemotePort)
-					rst := &protocol.Packet{
-						Version:  protocol.Version,
-						Flags:    protocol.FlagRST,
-						Protocol: protocol.ProtoStream,
-						Src:      conn.LocalAddr,
-						Dst:      conn.RemoteAddr,
-						SrcPort:  conn.LocalPort,
-						DstPort:  conn.RemotePort,
-					}
-					d.tunnels.Send(conn.RemoteAddr.Node, rst)
-					conn.Mu.Lock()
-					conn.State = StateClosed
-					conn.Mu.Unlock()
-					conn.CloseRecvBuf()
-					d.ports.RemoveConnection(conn.ID)
-					d.publishEvent("conn.dead_peer", map[string]interface{}{
-						"remote_addr": conn.RemoteAddr.String(), "remote_port": conn.RemotePort,
-						"local_port": conn.LocalPort, "conn_id": conn.ID,
-					})
-					continue
-				}
-				conn.Mu.Lock()
-				conn.KeepaliveUnacked++
-				conn.Mu.Unlock()
-				probe := &protocol.Packet{
-					Version:  protocol.Version,
-					Flags:    protocol.FlagACK,
-					Protocol: protocol.ProtoStream,
-					Src:      conn.LocalAddr,
-					Dst:      conn.RemoteAddr,
-					SrcPort:  conn.LocalPort,
-					DstPort:  conn.RemotePort,
-					Seq:      sendSeq,
-					Ack:      recvAck,
-					Window:   conn.RecvWindow(),
-				}
-				d.tunnels.Send(conn.RemoteAddr.Node, probe)
-			}
+			// Keepalive probes + dead-peer detection. See keepaliveSweep
+			// for details. Extracted into its own method so tests can
+			// drive a single tick without standing up the full ticker.
+			d.keepaliveSweep(keepaliveInterval)
+		}
+	}
+}
+
+// keepaliveSweep runs one iteration of the keepalive-probe / dead-peer
+// detection loop. For each connection idle beyond `keepaliveInterval`:
+//
+//   - If it's in StateEstablished and 3 prior probes already went
+//     unanswered, close it politely via CloseConnection (sends FIN
+//     with retx tracking, transitions to StateFinWait). This is
+//     intentionally NOT a RST: an idle StateEstablished connection
+//     is the expected steady-state after a successful request/reply
+//     cycle (pilot's pseudo-TCP has no half-close API), so RST'ing
+//     it poisons the peer's session table for that conn_id and
+//     cascades into "dial: connection refused" on the peer's next
+//     outbound to us. Observed fleet-wide on 2026-05-11 as bursts
+//     of N successful sends followed by total loss of reachability
+//     for a peer; root cause was the prior inline-RST keepalive
+//     reaper. CloseConnection's FIN lets the peer cleanly transition
+//     FIN-ACK → StateTimeWait → reap with no poisoning.
+//
+//   - Otherwise, increment KeepaliveUnacked and send a probe packet
+//     (FlagACK only). The probe doubles as a NAT-keepalive: it bumps
+//     the consumer-NAT idle timer for our external mapping every
+//     keepaliveInterval, holding the inbound path open between
+//     application-driven sends.
+//
+// Exposed (unexported, same-package) so the white-box tests in
+// zz_keepalive_fin_not_rst_test.go can drive a single tick without
+// the ticker-loop overhead.
+func (d *Daemon) keepaliveSweep(keepaliveInterval time.Duration) {
+	idle := d.ports.IdleConnections(keepaliveInterval)
+	for _, conn := range idle {
+		conn.Mu.Lock()
+		st := conn.State
+		sendSeq := conn.SendSeq
+		recvAck := conn.RecvAck
+		kaUnacked := conn.KeepaliveUnacked
+		conn.Mu.Unlock()
+		if st != StateEstablished {
+			continue
+		}
+		if kaUnacked >= 3 {
+			slog.Info("idle peer reaped (3 keepalives unanswered, sending FIN)",
+				"conn_id", conn.ID, "remote_addr", conn.RemoteAddr, "remote_port", conn.RemotePort)
+			d.publishEvent("conn.dead_peer", map[string]interface{}{
+				"remote_addr": conn.RemoteAddr.String(), "remote_port": conn.RemotePort,
+				"local_port": conn.LocalPort, "conn_id": conn.ID,
+			})
+			d.CloseConnection(conn)
+			continue
+		}
+		conn.Mu.Lock()
+		conn.KeepaliveUnacked++
+		conn.Mu.Unlock()
+		probe := &protocol.Packet{
+			Version:  protocol.Version,
+			Flags:    protocol.FlagACK,
+			Protocol: protocol.ProtoStream,
+			Src:      conn.LocalAddr,
+			Dst:      conn.RemoteAddr,
+			SrcPort:  conn.LocalPort,
+			DstPort:  conn.RemotePort,
+			Seq:      sendSeq,
+			Ack:      recvAck,
+			Window:   conn.RecvWindow(),
 		}
+		d.tunnels.Send(conn.RemoteAddr.Node, probe)
 	}
 }
 

pkg/daemon/envelope/envelope.go

@@ -164,6 +164,20 @@ func DecryptFrame(store *keyexchange.Store, data []byte) DecryptResult {
 			c.ReplayMu.Lock()
 			c.OutsideWindowCount++
 			c.ReplayMu.Unlock()
+		} else {
+			// In-window replay collision. Symmetric counterpart to
+			// OutsideWindowCount on the *other* side of the window:
+			// peer's counter is INSIDE [max-window, max] but at a
+			// position we've already marked. Sustained collisions mean
+			// the peer's send counter reset (peer restarted with a
+			// persistent X25519 identity, so no PILA was negotiated)
+			// and every frame they now produce lands on a bit we
+			// already set. Recovery is structurally identical to
+			// outside-window: gate via ShouldDropOnReplay (threshold +
+			// grace), then drop the Crypto and trigger a fresh exchange.
+			c.ReplayMu.Lock()
+			c.ReplayCount++
+			c.ReplayMu.Unlock()
 		}
 		return DecryptResult{
 			PeerNodeID:   peerNodeID,
@@ -192,14 +206,17 @@ func DecryptFrame(store *keyexchange.Store, data []byte) DecryptResult {
 		}
 	}
 
-	// Successful decrypt — reset both fault counters under one lock.
+	// Successful decrypt — reset all three fault counters under one lock.
 	c.ReplayMu.Lock()
 	if c.DecryptFailCount != 0 {
 		c.DecryptFailCount = 0
 	}
 	if c.OutsideWindowCount != 0 {
 		c.OutsideWindowCount = 0
 	}
+	if c.ReplayCount != 0 {
+		c.ReplayCount = 0
+	}
 	c.ReplayMu.Unlock()
 
 	return DecryptResult{