daemon: rx-watchdog also detects the partial (outbound-dial) wedge (#387)

The v1.12.5 watchdog catches a FULL inbound wedge (PktsRecv fully stalls)
and is production-proven. But 2026-07-15 the overlay went dark in a way it
missed: pilot-director and list-agents both failed with dial timeouts while
PktsRecv kept *trickling* forward from a couple of peer keepalives — so the
rx-silence timer reset every tick and the watchdog stayed quiet. Only a
manual daemon restart cleared it (its re-registration re-punched the NAT
mapping).

Add an outbound signal: DialConnection records a success (clears the
counter) or a full-budget timeout (increments consecutiveDialTimeouts). The
watchdog now treats dialTimeouts >= dialWedgeThreshold (4 back-to-back dial
failures, zero successes between — implausible unless our own send path is
dead) as a wedge, even while rx trickles: rx progress no longer counts as
healthy in that state. It runs the same soft-recovery (beacon + registry
re-register) and, if it persists, the same guarded hard-exit → supervisor
respawn. The soft path resets the dial counter so the next real dial
re-tests, avoiding a stale-counter false exit when no dials have happened.

Co-authored-by: Teodor Calin <teodor@vulturelabs.io>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: 3 people

pkg/daemon/daemon.go

@@ -350,6 +350,21 @@ type Daemon struct {
 	// signature) from "whole network unreachable" (restart just loops).
 	lastRegistryOKNano atomic.Int64
 
+	// lastDialOKNano / consecutiveDialTimeouts feed the rx watchdog's
+	// PARTIAL-wedge detector. A daemon can be "up" with rx trickling —
+	// a couple of keepalive packets from existing peers keep PktsRecv
+	// advancing, so the rx-silence check stays quiet — yet be unable to
+	// open a single new outbound data-exchange connection: every
+	// DialConnection returns ErrDialTimeout. Observed 2026-07-15, the
+	// overlay went dark (pilot-director + list-agents both unreachable)
+	// and the watchdog did NOT fire because the keepalive trickle reset
+	// its silence timer every tick; only a manual daemon restart cleared
+	// it. consecutiveDialTimeouts counts back-to-back dial timeouts
+	// (reset to 0 on any successful dial); a run of them to distinct
+	// peers means our outbound path is wedged, not that one peer is dead.
+	lastDialOKNano          atomic.Int64
+	consecutiveDialTimeouts atomic.Uint64
+
 	startTime       time.Time
 	stopCh          chan struct{}         // closed on Stop() to signal goroutines
 	beaconSelection *beaconSelectionState // multi-beacon discovery state
@@ -3674,6 +3689,10 @@ func (d *Daemon) dialConnectionLocked(ctx context.Context, dstAddr protocol.Addr
 				if st == StateEstablished {
 					d.startRetxLoop(conn)
 				}
+				// A completed handshake proves the outbound path is alive —
+				// clears the rx-watchdog's partial-wedge signal.
+				d.lastDialOKNano.Store(time.Now().UnixNano())
+				d.consecutiveDialTimeouts.Store(0)
 				return conn, nil
 			}
 			if st == StateClosed {
@@ -3720,6 +3739,10 @@ func (d *Daemon) dialConnectionLocked(ctx context.Context, dstAddr protocol.Addr
 				if relayActivatedHere && !d.tunnels.IsRelayPinned(dstAddr.Node) {
 					d.tunnels.SetRelayPeer(dstAddr.Node, false)
 				}
+				// Full direct+relay retry budget exhausted with no SYN-ACK.
+				// Feeds the rx-watchdog's partial-wedge detector: a run of
+				// these to distinct peers means our outbound is wedged.
+				d.consecutiveDialTimeouts.Add(1)
 				return nil, protocol.ErrDialTimeout
 			}
 			// Resend SYN (uses relay if relayActive)

pkg/daemon/rxwatchdog.go

@@ -55,6 +55,16 @@ const (
 	// nothing to send) never triggers.
 	rxSilenceMinSentDelta = 30
 
+	// dialWedgeThreshold: consecutive DialConnection timeouts (each a full
+	// direct+relay retry budget with no SYN-ACK) that mark the outbound
+	// path as wedged. A run this long means separate dials to (typically)
+	// distinct peers all failed — implausible unless our own send path is
+	// dead, not the peers. This is the PARTIAL wedge the rx-silence check
+	// misses: rx keeps trickling (keepalives) so silence never trips, but
+	// no new connection can be opened. 4 = at least four back-to-back
+	// user-visible dial failures with zero successes in between.
+	dialWedgeThreshold = 4
+
 	// rxWatchdogSoftMax is how many consecutive soft recoveries run
 	// before the watchdog considers the hard exit.
 	rxWatchdogSoftMax = 3
@@ -136,10 +146,18 @@ func (d *Daemon) rxWatchdogTick(st *rxWatchdogState, now time.Time) (action rxWa
 
 	recv := atomic.LoadUint64(&d.tunnels.PktsRecv)
 	sent := atomic.LoadUint64(&d.tunnels.PktsSent)
+	dialTimeouts := d.consecutiveDialTimeouts.Load()
+
+	rxProgressed := recv != st.recvSeen
+	// PARTIAL wedge: the outbound path is dead — a run of DialConnection
+	// timeouts to (typically) distinct peers with zero successes between
+	// them — even though rx is still trickling. rx progress alone no
+	// longer counts as healthy while this holds.
+	dialWedged := dialTimeouts >= dialWedgeThreshold
 
-	if recv != st.recvSeen {
+	if rxProgressed && !dialWedged {
 		if st.softAttempts > 0 {
-			slog.Info("inbound path recovered",
+			slog.Info("transport recovered",
 				"silent_for", now.Sub(st.lastProgress).Truncate(time.Second).String(),
 				"soft_attempts", st.softAttempts)
 			d.publishEvent("tunnel.rx_recovered", map[string]any{
@@ -154,12 +172,31 @@ func (d *Daemon) rxWatchdogTick(st *rxWatchdogState, now time.Time) (action rxWa
 		return rxActionProgress
 	}
 
+	// Keep the rx-silence measure honest even when we don't reset the soft
+	// counter: if rx trickled forward (dial-wedged case), advance the
+	// baseline so `silence` reflects real inbound stall, not the trickle.
+	if rxProgressed {
+		st.recvSeen = recv
+		st.sentAtProgress = sent
+		st.lastProgress = now
+	}
+
 	silence := now.Sub(st.lastProgress)
 	sentDelta := sent - st.sentAtProgress
-	if silence < rxSilenceThreshold || sentDelta < rxSilenceMinSentDelta {
+	rxSilentWedge := silence >= rxSilenceThreshold && sentDelta >= rxSilenceMinSentDelta
+
+	if !rxSilentWedge && !dialWedged {
 		return rxActionNone
 	}
 
+	wedgeReason := "inbound-silent"
+	if dialWedged {
+		wedgeReason = "outbound-dial-wedged"
+		if rxSilentWedge {
+			wedgeReason = "inbound-silent+outbound-dial-wedged"
+		}
+	}
+
 	// rawRxAge separates "socket path dead" (old) from "session layer
 	// dead" (fresh — datagrams arrive but nothing decrypts/delivers).
 	rawRxAge := time.Duration(0)
@@ -169,24 +206,36 @@ func (d *Daemon) rxWatchdogTick(st *rxWatchdogState, now time.Time) (action rxWa
 
 	if st.softAttempts < rxWatchdogSoftMax {
 		st.softAttempts++
-		slog.Warn("inbound path silent while transmitting — attempting soft recovery",
+		slog.Warn("transport wedged while transmitting — attempting soft recovery",
+			"reason", wedgeReason,
 			"silent_for", silence.Truncate(time.Second).String(),
 			"pkts_sent_during_silence", sentDelta,
+			"dial_timeouts", dialTimeouts,
 			"raw_rx_age", rawRxAge.Truncate(time.Second).String(),
 			"attempt", st.softAttempts,
 			"max_soft_attempts", rxWatchdogSoftMax)
 		d.publishEvent("tunnel.rx_silence", map[string]any{
+			"reason":                   wedgeReason,
 			"silent_for_seconds":       int64(silence.Seconds()),
 			"pkts_sent_during_silence": sentDelta,
+			"dial_timeouts":            dialTimeouts,
 			"raw_rx_age_seconds":       int64(rawRxAge.Seconds()),
 			"attempt":                  st.softAttempts,
 		})
-		// The beacon's discover reply is itself an inbound datagram, so
-		// this doubles as an active probe of the rx path.
+		// Re-register with the beacon (re-punches our NAT mapping; the
+		// discover reply also doubles as an active inbound probe) and the
+		// registry. This is what a manual restart effectively did to clear
+		// the partial wedge.
 		d.tunnels.RegisterWithBeacon()
 		if d.regConn != nil {
 			d.reRegister()
 		}
+		// Reset the dial counter so the next real dial re-tests the path:
+		// if recovery worked the next dial succeeds (counter stays 0); if
+		// not, failures re-accumulate to threshold and softAttempts climbs
+		// toward the hard exit. Avoids a stale counter forcing an exit when
+		// no new dials have happened to confirm the wedge persists.
+		d.consecutiveDialTimeouts.Store(0)
 		return rxActionSoftRecover
 	}
 
@@ -212,15 +261,19 @@ func (d *Daemon) rxWatchdogTick(st *rxWatchdogState, now time.Time) (action rxWa
 			"silent_for", silence.Truncate(time.Second).String(),
 			"uptime", uptime.Truncate(time.Second).String())
 	default:
-		slog.Error("inbound path wedged — exiting for supervisor respawn",
+		slog.Error("transport wedged — exiting for supervisor respawn",
+			"reason", wedgeReason,
 			"silent_for", silence.Truncate(time.Second).String(),
 			"pkts_sent_during_silence", sentDelta,
+			"dial_timeouts", dialTimeouts,
 			"raw_rx_age", rawRxAge.Truncate(time.Second).String(),
 			"soft_attempts", st.softAttempts,
 			"exit_code", rxWedgeExitCode)
 		d.publishEvent("tunnel.rx_wedged_exit", map[string]any{
+			"reason":                   wedgeReason,
 			"silent_for_seconds":       int64(silence.Seconds()),
 			"pkts_sent_during_silence": sentDelta,
+			"dial_timeouts":            dialTimeouts,
 			"soft_attempts":            st.softAttempts,
 			"exit_code":                rxWedgeExitCode,
 		})

pkg/daemon/zz_rx_watchdog_test.go

@@ -173,3 +173,58 @@ func TestRxWatchdogHardExitWithheldWhenNeverProgressedOrYoung(t *testing.T) {
 		t.Fatalf("exit fired despite guards (code %d)", *exitCode)
 	}
 }
+
+// TestRxWatchdogPartialWedgeDialTimeouts pins the 2026-07-15 regression:
+// rx keeps trickling (PktsRecv advances every tick from keepalives) so the
+// rx-silence check never fires, but every outbound DialConnection times out
+// — the overlay is dark and the old watchdog stayed quiet. The dial-timeout
+// counter must now drive the same soft-recovery, and rx trickle must NOT be
+// treated as healthy while the outbound path is wedged.
+func TestRxWatchdogPartialWedgeDialTimeouts(t *testing.T) {
+	t.Parallel()
+	d := newRxWatchdogTestDaemon(t)
+	now := time.Now()
+
+	atomic.StoreUint64(&d.tunnels.PktsRecv, 50)
+	st := &rxWatchdogState{lastProgress: now, recvSeen: 49}
+	if got := d.rxWatchdogTick(st, now); got != rxActionProgress {
+		t.Fatalf("baseline healthy: action = %q, want %q", got, rxActionProgress)
+	}
+
+	d.consecutiveDialTimeouts.Store(dialWedgeThreshold)
+	atomic.AddUint64(&d.tunnels.PktsRecv, 3) // trickle: recv advanced
+	if got := d.rxWatchdogTick(st, now.Add(30*time.Second)); got != rxActionSoftRecover {
+		t.Fatalf("partial wedge (rx trickle + dial timeouts): action = %q, want %q", got, rxActionSoftRecover)
+	}
+	if c := d.consecutiveDialTimeouts.Load(); c != 0 {
+		t.Fatalf("dial counter after soft recovery = %d, want 0", c)
+	}
+
+	d.consecutiveDialTimeouts.Store(dialWedgeThreshold - 1)
+	atomic.AddUint64(&d.tunnels.PktsRecv, 3)
+	if got := d.rxWatchdogTick(st, now.Add(60*time.Second)); got != rxActionProgress {
+		t.Fatalf("post-recovery (dial timeouts below threshold): action = %q, want %q", got, rxActionProgress)
+	}
+}
+
+// TestRxWatchdogPartialWedgeHardExit verifies a persistent dial-wedge (rx
+// healthy, registry reachable) escalates to the supervisor-respawn exit.
+func TestRxWatchdogPartialWedgeHardExit(t *testing.T) {
+	d := newRxWatchdogTestDaemon(t)
+	exitCode := swapExitForTest(t)
+	now := time.Now()
+	d.lastRegistryOKNano.Store(now.UnixNano())
+
+	atomic.StoreUint64(&d.tunnels.PktsRecv, 100)
+	d.consecutiveDialTimeouts.Store(dialWedgeThreshold)
+	st := &rxWatchdogState{
+		lastProgress: now, recvSeen: 100, sentAtProgress: 0,
+		softAttempts: rxWatchdogSoftMax,
+	}
+	if got := d.rxWatchdogTick(st, now); got != rxActionExit {
+		t.Fatalf("persistent dial wedge: action = %q, want %q", got, rxActionExit)
+	}
+	if *exitCode != rxWedgeExitCode {
+		t.Fatalf("exit code = %d, want %d", *exitCode, rxWedgeExitCode)
+	}
+}