Skip to content

Commit 5b500d5

Browse files
teovlclaude
andcommitted
fix(transport): dual-route key-exchange for dual-NAT convergence
When two peers are both behind NAT (e.g. Mac home-NAT ↔ GCP VM stateful conntrack), the direct PILA key-exchange frame never lands, and the tunnel only reconverges after slow blackhole detection flips the peer to relay mode — measured 28s–3min on the canonical Mac↔VM rig, far longer than the dial/send timeouts, so send-file/send-message time out and the crypto state desyncs. sendKeyExchangeToNode now ALSO pushes the key-exchange via the beacon relay whenever the peer is not yet relay-flagged and a beacon is available. The relay copy converges in ~1 RTT. It is a no-op once the peer is relay-flagged (the primary send already went via relay), and relayProbeLoop keeps probing direct so a genuine direct path still upgrades the peer out of relay. Best-effort: a failed relay copy falls back to the existing slow path. Adds routing.SendRelayFrame (forced-relay send primitive, ignores the per-peer relay flag and blackhole heuristic) and the ClearRekeyGaveUp / ClearLastRekeyReq rekey-state shims. Verified on the canonical Mac↔VM dual-NAT rig: - G2 liveness: idle 5min (and 90min) then small msg arrives, no reset. - Small msg ACK in ~0.42s (was 28s–3min). - 64KB send-file byte-perfect (sha256 match), incl. from a cold daemon restart (fresh in-memory peer table) — tunnel re-converges in ~12s. - No regressions: 0 panics, 0 relay-copy failures on either end. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 82649a7 commit 5b500d5

2 files changed

Lines changed: 76 additions & 0 deletions

File tree

pkg/daemon/routing/writeframe.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,3 +82,33 @@ func (m *Manager) WriteFrame(nodeID uint32, addr *net.UDPAddr, frame []byte, cou
8282
_, _ = m.HandleSendError(nodeID, err)
8383
return SendOutcome{}, err
8484
}
85+
86+
// SendRelayFrame sends a frame to a peer via the beacon relay path
87+
// unconditionally — ignoring the per-peer relay flag and the blackhole
88+
// heuristic. Used by the key-exchange convergence path to guarantee a
89+
// dual-NAT peer receives the PILA via relay even before its relay flag
90+
// has been set. Without this, two peers both behind NAT only reconverge
91+
// after slow blackhole detection flips the direct path to relay
92+
// (measured 28s–3min on the Mac↔GCP-VM rig). Returns ErrNoAddress if no
93+
// beacon is configured. Does not bump the per-peer outbound-send
94+
// timestamp (this is an out-of-band copy, not the primary path).
95+
func (m *Manager) SendRelayFrame(nodeID uint32, frame []byte) error {
96+
m.mu.RLock()
97+
bAddr := m.beaconAddr
98+
sock := m.sock
99+
m.mu.RUnlock()
100+
if bAddr == nil {
101+
return ErrNoAddress
102+
}
103+
if sock == nil {
104+
return fmt.Errorf("routing: socket not set")
105+
}
106+
// MsgRelay: [0x05][senderNodeID(4)][destNodeID(4)][frame...]
107+
msg := make([]byte, 1+4+4+len(frame))
108+
msg[0] = protocol.BeaconMsgRelay
109+
binary.BigEndian.PutUint32(msg[1:5], m.localNodeID())
110+
binary.BigEndian.PutUint32(msg[5:9], nodeID)
111+
copy(msg[9:], frame)
112+
_, err := sock.Send(msg, bAddr)
113+
return err
114+
}

pkg/daemon/tunnel.go

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,25 @@ func (tm *TunnelManager) pruneRekeyBudgetLocked(now time.Time) bool {
407407
return len(tm.lastRekeyReq) < maxRekeyRequesters
408408
}
409409

410+
// ClearRekeyGaveUp lifts the per-peer give-up cooldown so a fresh rekey
411+
// cycle can start immediately. Thin shim over keyexchange.Manager so the
412+
// IPC layer (pkg/daemon/ipc.go handlePreferDirect) can reach it without
413+
// importing the keyexchange package directly.
414+
func (tm *TunnelManager) ClearRekeyGaveUp(peerNodeID uint32) {
415+
tm.kx.ClearRekeyGaveUp(peerNodeID)
416+
}
417+
418+
// ClearLastRekeyReq drops the per-peer rate-limit timestamp recorded by
419+
// maybeRequestRekey. After a forced reset (pilotctl prefer-direct) the
420+
// next "encrypted packet but no key" event must be allowed to fire a
421+
// fresh PILA immediately — without this, the 3-second gate can silently
422+
// swallow the first packet that would otherwise re-establish the tunnel.
423+
func (tm *TunnelManager) ClearLastRekeyReq(peerNodeID uint32) {
424+
tm.rekeyMu.Lock()
425+
delete(tm.lastRekeyReq, peerNodeID)
426+
tm.rekeyMu.Unlock()
427+
}
428+
410429
// maybeRequestRekey conditionally sends a key-exchange to a peer that sent us
411430
// an encrypted packet we can't decrypt. Rate-limited per peer. Returns true if
412431
// we actually sent one.
@@ -1419,6 +1438,33 @@ func (tm *TunnelManager) deriveSecret(peerPubKeyBytes []byte) (*peerCrypto, erro
14191438
// BOOTSTRAP-EXCEPTION marker — moved with the function).
14201439
func (tm *TunnelManager) sendKeyExchangeToNode(peerNodeID uint32) {
14211440
tm.kx.SendKeyExchangeToNode(peerNodeID)
1441+
1442+
// Dual-NAT convergence fix: when a peer is not (yet) relay-flagged but
1443+
// a beacon is available, ALSO push the key-exchange via relay. For two
1444+
// peers both behind NAT the direct PILA never lands, and the tunnel
1445+
// only reconverges after slow blackhole detection flips the path to
1446+
// relay — measured 28s–3min on the Mac↔GCP-VM rig, far longer than the
1447+
// dial/send timeouts. The relay copy converges in ~1 RTT. This is a
1448+
// no-op once the peer is relay-flagged (the primary send already went
1449+
// via relay), and relayProbeLoop keeps probing direct so a genuine
1450+
// direct path still upgrades the peer out of relay. Best-effort: a
1451+
// failed relay copy just falls back to the existing slow path.
1452+
if tm.routing.IsRelayPeer(peerNodeID) || tm.routing.BeaconAddr() == nil {
1453+
return
1454+
}
1455+
var frame []byte
1456+
if tm.kx.HasIdentity() {
1457+
frame = tm.kx.BuildAuthFrame()
1458+
}
1459+
if frame == nil {
1460+
frame = tm.kx.BuildUnauthFrame()
1461+
}
1462+
if frame == nil {
1463+
return
1464+
}
1465+
if err := tm.routing.SendRelayFrame(peerNodeID, frame); err != nil {
1466+
slog.Debug("kx relay-copy send failed", "peer_node_id", peerNodeID, "error", err)
1467+
}
14221468
}
14231469

14241470
// markPendingRekey is the legacy shim for keyexchange.Manager.MarkPendingRekey.

0 commit comments

Comments
 (0)