Make tunnel-handshake startup transient instead of fatal

Symptom: 'pilotctl ping <peer>' or any first send to a peer immediately
after a daemon restart could return 'send SYN: pending queue full for
node X: oldest packet dropped' and abort. The user sees a hard failure
even though the daemon is only 1-2 RTTs away from a working tunnel.

Root cause: tunnel.go appended the new packet to the pending queue
(success), then returned an error solely to surface that an older
packet had been dropped to make room. DialConnection's first-SYN
send-error path bailed on any error, so it never reached its 6-retry
loop — the SYN it just successfully queued never got retransmitted.

Changes:

* pkg/daemon/tunnel.go: define typed sentinel ErrPendingDropped and
  return it (wrapped) from sendEncryptedToNode when the new packet is
  queued but the oldest had to be dropped.

* pkg/daemon/daemon.go DialConnection: errors.Is(err, ErrPendingDropped)
  → log debug, fall through to the existing SYN retransmit loop.
  Treat any other Send error as fatal as before.

* cmd/pilotctl/main.go: classifyDaemonError() recognizes the
  pending-queue and dial-timeout patterns and emits an actionable hint
  ("tunnel handshake in progress — retry in a few seconds" /
   "no reply from peer; relay can take ~30s to converge after a beacon
    roll") instead of the generic 'check ping' boilerplate. Wired into
  the connect/send-message dial sites.

* pkg/beacon/server.go: drop a stray 'pi' typo on line 1 that was
  blocking the build (intentional gossip changes preserved).

Author: teovl

cmd/pilotctl/main.go

@@ -97,6 +97,32 @@ func fatalCode(code string, format string, args ...interface{}) {
 	os.Exit(1)
 }
 
+// classifyDaemonError inspects an error string from the daemon and, when it
+// recognizes a transient or operator-friendly failure mode, returns a more
+// actionable hint to the user. Returns "" if no specific guidance applies —
+// callers fall back to whatever default hint they had.
+//
+// Currently recognized patterns:
+//   - "pending queue full ... key exchange pending" — the encrypted tunnel
+//     to the peer is mid-handshake. The packet was queued; waiting a moment
+//     and retrying typically succeeds.
+//   - "dial timeout"/"dial: daemon: dial timeout" — the SYN went out but no
+//     SYN-ACK came back. Often means the relay path is mid-convergence
+//     after a beacon roll, or the peer is offline.
+func classifyDaemonError(err error) string {
+	if err == nil {
+		return ""
+	}
+	s := err.Error()
+	switch {
+	case strings.Contains(s, "pending queue full") || strings.Contains(s, "key exchange pending"):
+		return "tunnel handshake in progress — the packet was queued. Wait a few seconds and retry; the SYN will go out as soon as the encrypted session keys are derived."
+	case strings.Contains(s, "dial timeout"):
+		return "no reply from peer. Check reachability with `pilotctl ping <peer>`; if relay is fresh after a beacon roll it can take ~30s for endpoints to converge."
+	}
+	return ""
+}
+
 // fatalHint is like fatalCode but adds an actionable hint telling the user what to do next.
 func fatalHint(code, hint, format string, args ...interface{}) {
 	msg := fmt.Sprintf(format, args...)
@@ -2418,8 +2444,11 @@ func cmdConnect(args []string) {
 	if message != "" {
 		conn, err := d.DialAddr(target, port)
 		if err != nil {
-			fatalHint("connection_failed",
-				fmt.Sprintf("check that %s is reachable: pilotctl ping %s", target, target),
+			hint := classifyDaemonError(err)
+			if hint == "" {
+				hint = fmt.Sprintf("check that %s is reachable: pilotctl ping %s", target, target)
+			}
+			fatalHint("connection_failed", hint,
 				"cannot connect to %s port %d", target, port)
 		}
 		defer conn.Close()
@@ -2771,8 +2800,11 @@ func cmdSendFile(args []string) {
 
 	client, err := dataexchange.Dial(d, target)
 	if err != nil {
-		fatalHint("connection_failed",
-			fmt.Sprintf("check that %s is reachable: pilotctl ping %s", target, target),
+		hint := classifyDaemonError(err)
+		if hint == "" {
+			hint = fmt.Sprintf("check that %s is reachable: pilotctl ping %s", target, target)
+		}
+		fatalHint("connection_failed", hint,
 			"cannot connect to %s (data exchange port %d)", target, protocol.PortDataExchange)
 	}
 	defer client.Close()
@@ -2832,8 +2864,11 @@ func cmdSendMessage(args []string) {
 
 	client, err := dataexchange.Dial(d, target)
 	if err != nil {
-		fatalHint("connection_failed",
-			fmt.Sprintf("check that %s is reachable: pilotctl ping %s", target, target),
+		hint := classifyDaemonError(err)
+		if hint == "" {
+			hint = fmt.Sprintf("check that %s is reachable: pilotctl ping %s", target, target)
+		}
+		fatalHint("connection_failed", hint,
 			"cannot connect to %s (data exchange port %d)", target, protocol.PortDataExchange)
 	}
 	defer client.Close()

pkg/daemon/daemon.go

@@ -2412,8 +2412,18 @@ func (d *Daemon) dialConnectionLocked(ctx context.Context, dstAddr protocol.Addr
 	}
 
 	if err := d.tunnels.Send(dstAddr.Node, syn); err != nil {
-		d.ports.RemoveConnection(conn.ID)
-		return nil, fmt.Errorf("send SYN: %w", err)
+		// ErrPendingDropped means the SYN itself IS queued; only an older
+		// packet was dropped to make room. The tunnel is mid-handshake.
+		// Don't abort the dial — the SYN will go out as soon as the key
+		// exchange completes, and if not, the retry loop below will
+		// retransmit. Aborting here turns a transient handshake-startup
+		// condition into a hard "dial failed" for the user.
+		if !errors.Is(err, ErrPendingDropped) {
+			d.ports.RemoveConnection(conn.ID)
+			return nil, fmt.Errorf("send SYN: %w", err)
+		}
+		slog.Debug("dial: SYN queued during tunnel handshake (will retransmit)",
+			"peer_node_id", dstAddr.Node, "dst_port", dstPort)
 	}
 	conn.Mu.Lock()
 	conn.SendSeq++

pkg/daemon/tunnel.go

@@ -250,6 +250,19 @@ const maxPendingPerPeer = 64
 // maxPendingPeers limits the total number of peers with pending key exchanges.
 const maxPendingPeers = 256
 
+// ErrPendingDropped is returned by sendEncryptedToNode when the per-peer
+// pending queue was already at maxPendingPerPeer and the oldest queued
+// packet had to be dropped to make room for the new one. The CALLER's
+// packet is still queued — it will be sent as soon as key exchange
+// finishes — but an older packet was lost to back-pressure.
+//
+// Callers that distinguish this error from a hard failure can choose to
+// retry (the dial path does this; one of the SYN retransmits will land
+// after the queue drains). Surfacing it as a typed error also lets
+// pilotctl render a "tunnel handshaking" hint instead of an opaque
+// "send SYN: pending queue full" message.
+var ErrPendingDropped = errors.New("pending queue full: oldest queued packet dropped while key exchange pending")
+
 // RecvChSize is the capacity of the incoming packet channel.
 // Increased from 1024 to 8192 for 1M-node scale to prevent drops during
 // bursts (e.g., many peers sending simultaneously after a cron trigger).
@@ -1808,7 +1821,11 @@ func (tm *TunnelManager) SendTo(addr *net.UDPAddr, nodeID uint32, pkt *protocol.
 				"peer_node_id", nodeID,
 				"queue_len", qlen,
 				"limit", maxPendingPerPeer)
-			return fmt.Errorf("pending queue full for node %d: oldest packet dropped", nodeID)
+			// The new packet IS queued (line above appended it). What was
+			// dropped is the oldest packet, not this one. Callers that
+			// errors.Is(err, ErrPendingDropped) can treat this as transient
+			// — a SYN retransmit will succeed once the queue drains.
+			return fmt.Errorf("%w (peer_node_id=%d)", ErrPendingDropped, nodeID)
 		}
 		return nil // queued, will be sent encrypted after key exchange
 	}