Filter unreachable beacon addresses from registry-discovered list

Production registries return ALL registered beacons including those
running on a private VPC (e.g. GCP 10.128.0.x). When pickBeacon's hash
lands on one of those, every relay packet vanishes — off-VPC daemons
cannot reach RFC1918 addresses. Symptom is a tunnel that establishes
fine but where new dials hang and existing connections die at the
keepalive interval, requiring a daemon restart to resolve (until the
next refresh tick re-poisons the cache).

filterUnreachable drops RFC1918 / loopback / link-local / unspecified
literals from the discovered list before merging with bootstrap. The
operator-configured bootstrap list is preserved verbatim — intra-VPC
deployments can still pin a private beacon there. DNS hostnames are
kept since they may resolve to public addresses.

Also fix pilotctl ping conn.Read timeout: the dial was guarded by
perAttempt but the echo read was not, so a relay-degraded peer caused
ping to hang indefinitely instead of reporting timeout.

Author: teovl

cmd/pilotctl/main.go

@@ -4342,6 +4342,7 @@ func cmdPing(args []string) {
 		payload := fmt.Sprintf("ping-%d", i)
 		conn.Write([]byte(payload))
 
+		conn.SetReadDeadline(time.Now().Add(perAttempt))
 		buf := make([]byte, 1024)
 		n, err := conn.Read(buf)
 		conn.Close()

pkg/daemon/beacon_discovery.go

@@ -62,6 +62,13 @@ func fetchBeaconList(client beaconLister) ([]string, error) {
 			out = append(out, addr)
 		}
 	}
+	// Drop addresses that an off-VPC client cannot reach (RFC1918,
+	// loopback, link-local). The registry returns whatever beacons have
+	// registered, including ones running on a private VPC; if our hash
+	// lands on one of those, all relay traffic vanishes (silent black-
+	// hole). Bootstrap entries are NOT filtered — operators on the same
+	// VPC can still pin a private beacon there.
+	out = filterUnreachable(out)
 	// Sort to give a deterministic order even when the registry's map
 	// iteration produces a different order each call. mergeBeaconLists
 	// dedupes against bootstrap, but stable order keeps the hash-pick
@@ -130,7 +137,10 @@ func loadBeaconCache(identityPath string) ([]string, error) {
 	if err := json.Unmarshal(data, &entry); err != nil {
 		return nil, fmt.Errorf("parse beacon cache: %w", err)
 	}
-	return entry.Addrs, nil
+	// Filter unreachable addresses from disk too — a previous daemon may
+	// have persisted a list that included private VPC IPs before this
+	// fix was in place.
+	return filterUnreachable(entry.Addrs), nil
 }
 
 // beaconSelectionState tracks the daemon's beacon picks across refresh

pkg/daemon/beacon_select.go

@@ -4,9 +4,45 @@ package daemon
 
 import (
 	"hash/fnv"
+	"net"
 	"strings"
 )
 
+// isUnreachableBeaconHost reports whether host is an RFC1918 / loopback /
+// link-local / unspecified address — an address that a public-internet
+// daemon cannot reach. The registry's beacon_list endpoint can return
+// internal addresses for beacons running on the same VPC (e.g. GCP
+// 10.128.0.0/16); those are useless to off-VPC daemons and, if picked,
+// silently black-hole all relay traffic. Only IP literals are checked
+// — DNS hostnames are kept (they may resolve to public addresses).
+func isUnreachableBeaconHost(host string) bool {
+	ip := net.ParseIP(host)
+	if ip == nil {
+		return false
+	}
+	return ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsUnspecified()
+}
+
+// filterUnreachable removes addresses whose host part is an RFC1918 /
+// loopback / link-local literal. Used on the DISCOVERED list before
+// merging with bootstrap — the operator's bootstrap list is preserved
+// verbatim (intra-VPC deployments can still pin a private beacon there).
+func filterUnreachable(addrs []string) []string {
+	out := make([]string, 0, len(addrs))
+	for _, a := range addrs {
+		host, _, err := net.SplitHostPort(a)
+		if err != nil {
+			// Fall back to the whole string if there's no port.
+			host = a
+		}
+		if isUnreachableBeaconHost(host) {
+			continue
+		}
+		out = append(out, a)
+	}
+	return out
+}
+
 // Multi-beacon support
 //
 // A daemon may be configured with EITHER:

pkg/daemon/beacon_select_test.go

@@ -146,3 +146,51 @@ func TestPickBeaconStableAcrossSeparateListInstances(t *testing.T) {
 		}
 	}
 }
+
+// TestFilterUnreachableDropsPrivateAndLoopback covers the silent-blackhole
+// fix: discovered beacon lists from the registry can include addresses
+// running on a private VPC (e.g. GCP 10.128.0.x). When the daemon's
+// pickBeacon hash lands on one of those, all relay traffic vanishes
+// because off-VPC clients cannot reach it. filterUnreachable must drop
+// RFC1918, loopback, link-local and unspecified literals. DNS hostnames
+// are kept because they may resolve to public IPs.
+func TestFilterUnreachableDropsPrivateAndLoopback(t *testing.T) {
+	in := []string{
+		"34.71.57.205:9001",   // public — kept
+		"10.128.0.78:9001",    // private RFC1918 — dropped
+		"192.168.1.5:9001",    // private RFC1918 — dropped
+		"172.16.0.5:9001",     // private RFC1918 — dropped
+		"127.0.0.1:9001",      // loopback — dropped
+		"169.254.1.1:9001",    // link-local — dropped
+		"0.0.0.0:9001",        // unspecified — dropped
+		"beacon.example.com:9001", // DNS hostname — kept
+		"8.8.8.8:9001",        // public — kept
+	}
+	got := filterUnreachable(in)
+	want := []string{
+		"34.71.57.205:9001",
+		"beacon.example.com:9001",
+		"8.8.8.8:9001",
+	}
+	if len(got) != len(want) {
+		t.Fatalf("filtered len = %d (%v), want %d (%v)", len(got), got, len(want), want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("got[%d] = %q, want %q", i, got[i], want[i])
+		}
+	}
+}
+
+// TestFilterUnreachableEmptyInputs guards against panics on edge inputs.
+func TestFilterUnreachableEmptyInputs(t *testing.T) {
+	if got := filterUnreachable(nil); len(got) != 0 {
+		t.Fatalf("nil input -> %v, want empty", got)
+	}
+	if got := filterUnreachable([]string{}); len(got) != 0 {
+		t.Fatalf("empty input -> %v, want empty", got)
+	}
+	if got := filterUnreachable([]string{"malformed-no-port"}); len(got) != 1 {
+		t.Fatalf("malformed entry should be kept (host parses cleanly w/o port): got %v", got)
+	}
+}