From 980ab4244dd7eab399fcda8f258621b324b5924d Mon Sep 17 00:00:00 2001 From: Jonathan Nobels Date: Wed, 21 May 2025 15:27:32 -0400 Subject: [PATCH 001/263] VERSION.txt: this is v1.85.0 (#16042) Signed-off-by: Jonathan Nobels --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 6b4de0a42b03c..f288d11142d11 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.83.0 +1.85.0 From aa8bc23c496821dfa00771c9604fc4a71ead7d4c Mon Sep 17 00:00:00 2001 From: James 'zofrex' Sanderson Date: Thu, 22 May 2025 13:40:32 +0100 Subject: [PATCH 002/263] control/controlclient,health,tailcfg: refactor control health messages (#15839) * control/controlclient,health,tailcfg: refactor control health messages Updates tailscale/corp#27759 Signed-off-by: James Sanderson Signed-off-by: Paul Scott <408401+icio@users.noreply.github.com> Co-authored-by: Paul Scott <408401+icio@users.noreply.github.com> --- control/controlclient/auto.go | 7 +- control/controlclient/direct.go | 4 +- control/controlclient/map.go | 22 +++- control/controlclient/map_test.go | 42 ++++--- health/health.go | 191 ++++++++++++++++++++---------- health/health_test.go | 165 ++++++++++++++++++++++---- health/state.go | 28 ++++- health/warnings.go | 10 -- ipn/ipnlocal/local.go | 16 ++- tailcfg/tailcfg.go | 52 +++++++- tailcfg/tailcfg_test.go | 76 ++++++++++++ types/netmap/netmap.go | 4 +- 12 files changed, 495 insertions(+), 122 deletions(-) diff --git a/control/controlclient/auto.go b/control/controlclient/auto.go index e0168c19db6c0..e6335e54d251b 100644 --- a/control/controlclient/auto.go +++ b/control/controlclient/auto.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "tailscale.com/health" "tailscale.com/logtail/backoff" "tailscale.com/net/sockstats" "tailscale.com/tailcfg" @@ -198,7 +199,11 @@ func NewNoStart(opts Options) (_ *Auto, err error) { c.mapCtx, c.mapCancel = context.WithCancel(context.Background()) c.mapCtx = sockstats.WithSockStats(c.mapCtx, sockstats.LabelControlClientAuto, opts.Logf) - c.unregisterHealthWatch = opts.HealthTracker.RegisterWatcher(direct.ReportHealthChange) + c.unregisterHealthWatch = opts.HealthTracker.RegisterWatcher(func(c health.Change) { + if c.WarnableChanged { + direct.ReportWarnableChange(c.Warnable, c.UnhealthyState) + } + }) return c, nil } diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index ac799e2d916dc..2d6dc6e36c299 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -1623,9 +1623,9 @@ func postPingResult(start time.Time, logf logger.Logf, c *http.Client, pr *tailc return nil } -// ReportHealthChange reports to the control plane a change to this node's +// ReportWarnableChange reports to the control plane a change to this node's // health. w must be non-nil. us can be nil to indicate a healthy state for w. -func (c *Direct) ReportHealthChange(w *health.Warnable, us *health.UnhealthyState) { +func (c *Direct) ReportWarnableChange(w *health.Warnable, us *health.UnhealthyState) { if w == health.NetworkStatusWarnable || w == health.IPNStateWarnable || w == health.LoginStateWarnable { // We don't report these. These include things like the network is down // (in which case we can't report anyway) or the user wanted things diff --git a/control/controlclient/map.go b/control/controlclient/map.go index 3173040fe31d8..abfc5eb170c44 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -6,7 +6,10 @@ package controlclient import ( "cmp" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" + "io" "maps" "net" "reflect" @@ -828,6 +831,16 @@ func (ms *mapSession) sortedPeers() []tailcfg.NodeView { func (ms *mapSession) netmap() *netmap.NetworkMap { peerViews := ms.sortedPeers() + // Convert all ms.lastHealth to the new [netmap.NetworkMap.DisplayMessages]. + var msgs map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage + for _, h := range ms.lastHealth { + mak.Set(&msgs, tailcfg.DisplayMessageID("control-health-"+strhash(h)), tailcfg.DisplayMessage{ + Title: "Coordination server reports an issue", + Severity: tailcfg.SeverityMedium, + Text: "The coordination server is reporting a health issue: " + h, + }) + } + nm := &netmap.NetworkMap{ NodeKey: ms.publicNodeKey, PrivateKey: ms.privateNodeKey, @@ -842,7 +855,7 @@ func (ms *mapSession) netmap() *netmap.NetworkMap { SSHPolicy: ms.lastSSHPolicy, CollectServices: ms.collectServices, DERPMap: ms.lastDERPMap, - ControlHealth: ms.lastHealth, + DisplayMessages: msgs, TKAEnabled: ms.lastTKAInfo != nil && !ms.lastTKAInfo.Disabled, } @@ -868,5 +881,12 @@ func (ms *mapSession) netmap() *netmap.NetworkMap { if DevKnob.ForceProxyDNS() { nm.DNS.Proxied = true } + return nm } + +func strhash(h string) string { + s := sha256.New() + io.WriteString(s, h) + return hex.EncodeToString(s.Sum(nil)) +} diff --git a/control/controlclient/map_test.go b/control/controlclient/map_test.go index ccc57ae2b86a8..9abaae9236b6f 100644 --- a/control/controlclient/map_test.go +++ b/control/controlclient/map_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "net/netip" "reflect" "strings" @@ -1148,23 +1149,36 @@ func TestNetmapHealthIntegration(t *testing.T) { ht.GotStreamedMapResponse() nm := ms.netmapForResponse(&tailcfg.MapResponse{ - Health: []string{"Test message"}, + Health: []string{ + "Test message", + "Another message", + }, }) - ht.SetControlHealth(nm.ControlHealth) - - state := ht.CurrentState() - warning, ok := state.Warnings["control-health"] + ht.SetControlHealth(nm.DisplayMessages) - if !ok { - t.Fatal("no warning found in current state with code 'control-health'") - } - if got, want := warning.Title, "Coordination server reports an issue"; got != want { - t.Errorf("warning.Title = %q, want %q", got, want) + want := map[health.WarnableCode]health.UnhealthyState{ + "control-health-c0719e9a8d5d838d861dc6f675c899d2b309a3a65bb9fe6b11e5afcbf9a2c0b1": { + WarnableCode: "control-health-c0719e9a8d5d838d861dc6f675c899d2b309a3a65bb9fe6b11e5afcbf9a2c0b1", + Title: "Coordination server reports an issue", + Severity: health.SeverityMedium, + Text: "The coordination server is reporting a health issue: Test message", + }, + "control-health-1dc7017a73a3c55c0d6a8423e3813c7ab6562d9d3064c2ec6ac7822f61b1db9c": { + WarnableCode: "control-health-1dc7017a73a3c55c0d6a8423e3813c7ab6562d9d3064c2ec6ac7822f61b1db9c", + Title: "Coordination server reports an issue", + Severity: health.SeverityMedium, + Text: "The coordination server is reporting a health issue: Another message", + }, } - if got, want := warning.Severity, health.SeverityMedium; got != want { - t.Errorf("warning.Severity = %s, want %s", got, want) + + got := maps.Clone(ht.CurrentState().Warnings) + for k := range got { + if !strings.HasPrefix(string(k), "control-health") { + delete(got, k) + } } - if got, want := warning.Text, "The coordination server is reporting an health issue: Test message"; got != want { - t.Errorf("warning.Text = %q, want %q", got, want) + + if d := cmp.Diff(want, got); d != "" { + t.Fatalf("CurrentStatus().Warnings[\"control-health*\"] different than expected (-want +got)\n%s", d) } } diff --git a/health/health.go b/health/health.go index 1ec2bcc9b0dd1..6dbbf782cabce 100644 --- a/health/health.go +++ b/health/health.go @@ -88,34 +88,35 @@ type Tracker struct { // sysErr maps subsystems to their current error (or nil if the subsystem is healthy) // Deprecated: using Warnables should be preferred sysErr map[Subsystem]error - watchers set.HandleSet[func(*Warnable, *UnhealthyState)] // opt func to run if error state changes + watchers set.HandleSet[func(Change)] // opt func to run if error state changes timer tstime.TimerController latestVersion *tailcfg.ClientVersion // or nil checkForUpdates bool applyUpdates opt.Bool - inMapPoll bool - inMapPollSince time.Time - lastMapPollEndedAt time.Time - lastStreamedMapResponse time.Time - lastNoiseDial time.Time - derpHomeRegion int - derpHomeless bool - derpRegionConnected map[int]bool - derpRegionHealthProblem map[int]string - derpRegionLastFrame map[int]time.Time - derpMap *tailcfg.DERPMap // last DERP map from control, could be nil if never received one - lastMapRequestHeard time.Time // time we got a 200 from control for a MapRequest - ipnState string - ipnWantRunning bool - ipnWantRunningLastTrue time.Time // when ipnWantRunning last changed false -> true - anyInterfaceUp opt.Bool // empty means unknown (assume true) - controlHealth []string - lastLoginErr error - localLogConfigErr error - tlsConnectionErrors map[string]error // map[ServerName]error - metricHealthMessage *metrics.MultiLabelMap[metricHealthMessageLabel] + inMapPoll bool + inMapPollSince time.Time + lastMapPollEndedAt time.Time + lastStreamedMapResponse time.Time + lastNoiseDial time.Time + derpHomeRegion int + derpHomeless bool + derpRegionConnected map[int]bool + derpRegionHealthProblem map[int]string + derpRegionLastFrame map[int]time.Time + derpMap *tailcfg.DERPMap // last DERP map from control, could be nil if never received one + lastMapRequestHeard time.Time // time we got a 200 from control for a MapRequest + ipnState string + ipnWantRunning bool + ipnWantRunningLastTrue time.Time // when ipnWantRunning last changed false -> true + anyInterfaceUp opt.Bool // empty means unknown (assume true) + lastNotifiedControlMessages map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage // latest control messages processed, kept for change detection + controlMessages map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage // latest control messages received + lastLoginErr error + localLogConfigErr error + tlsConnectionErrors map[string]error // map[ServerName]error + metricHealthMessage *metrics.MultiLabelMap[metricHealthMessageLabel] } func (t *Tracker) now() time.Time { @@ -207,13 +208,15 @@ func unregister(w *Warnable) { // the program. type WarnableCode string -// A Warnable is something that we might want to warn the user about, or not. A Warnable is either -// in an healthy or unhealth state. A Warnable is unhealthy if the Tracker knows about a WarningState -// affecting the Warnable. -// In most cases, Warnables are components of the backend (for instance, "DNS" or "Magicsock"). -// Warnables are similar to the Subsystem type previously used in this package, but they provide -// a unique identifying code for each Warnable, along with more metadata that makes it easier for -// a GUI to display the Warnable in a user-friendly way. +// A Warnable is something that we might want to warn the user about, or not. A +// Warnable is either in a healthy or unhealthy state. A Warnable is unhealthy if +// the Tracker knows about a WarningState affecting the Warnable. +// +// In most cases, Warnables are components of the backend (for instance, "DNS" +// or "Magicsock"). Warnables are similar to the Subsystem type previously used +// in this package, but they provide a unique identifying code for each +// Warnable, along with more metadata that makes it easier for a GUI to display +// the Warnable in a user-friendly way. type Warnable struct { // Code is a string that uniquely identifies this Warnable across the entire Tailscale backend, // and can be mapped to a user-displayable localized string. @@ -409,12 +412,18 @@ func (t *Tracker) setUnhealthyLocked(w *Warnable, args Args) { prevWs := t.warnableVal[w] mak.Set(&t.warnableVal, w, ws) if !ws.Equal(prevWs) { + + change := Change{ + WarnableChanged: true, + Warnable: w, + UnhealthyState: w.unhealthyState(ws), + } for _, cb := range t.watchers { // If the Warnable has been unhealthy for more than its TimeToVisible, the callback should be // executed immediately. Otherwise, the callback should be enqueued to run once the Warnable // becomes visible. if w.IsVisible(ws, t.now) { - cb(w, w.unhealthyState(ws)) + cb(change) continue } @@ -427,7 +436,7 @@ func (t *Tracker) setUnhealthyLocked(w *Warnable, args Args) { // Check if the Warnable is still unhealthy, as it could have become healthy between the time // the timer was set for and the time it was executed. if t.warnableVal[w] != nil { - cb(w, w.unhealthyState(ws)) + cb(change) delete(t.pendingVisibleTimers, w) } }) @@ -460,8 +469,23 @@ func (t *Tracker) setHealthyLocked(w *Warnable) { delete(t.pendingVisibleTimers, w) } + change := Change{ + WarnableChanged: true, + Warnable: w, + } for _, cb := range t.watchers { - cb(w, nil) + cb(change) + } +} + +// notifyWatchersControlChangedLocked calls each watcher to signal that control +// health messages have changed (and should be fetched via CurrentState). +func (t *Tracker) notifyWatchersControlChangedLocked() { + change := Change{ + ControlHealthChanged: true, + } + for _, cb := range t.watchers { + cb(change) } } @@ -488,23 +512,57 @@ func (t *Tracker) AppendWarnableDebugFlags(base []string) []string { return ret } -// RegisterWatcher adds a function that will be called whenever the health state of any Warnable changes. -// If a Warnable becomes unhealthy or its unhealthy state is updated, the callback will be called with its -// current Representation. -// If a Warnable becomes healthy, the callback will be called with ws set to nil. -// The provided callback function will be executed in its own goroutine. The returned function can be used -// to unregister the callback. -func (t *Tracker) RegisterWatcher(cb func(w *Warnable, r *UnhealthyState)) (unregister func()) { - return t.registerSyncWatcher(func(w *Warnable, r *UnhealthyState) { - go cb(w, r) +// Change is used to communicate a change to health. This could either be due to +// a Warnable changing from health to unhealthy (or vice-versa), or because the +// health messages received from the control-plane have changed. +// +// Exactly one *Changed field will be true. +type Change struct { + // ControlHealthChanged indicates it was health messages from the + // control-plane server that changed. + ControlHealthChanged bool + + // WarnableChanged indicates it was a client Warnable which changed state. + WarnableChanged bool + // Warnable is whose health changed, as indicated in UnhealthyState. + Warnable *Warnable + // UnhealthyState is set if the changed Warnable is now unhealthy, or nil + // if Warnable is now healthy. + UnhealthyState *UnhealthyState +} + +// RegisterWatcher adds a function that will be called its own goroutine +// whenever the health state of any client [Warnable] or control-plane health +// messages changes. The returned function can be used to unregister the +// callback. +// +// If a client [Warnable] becomes unhealthy or its unhealthy state is updated, +// the callback will be called with WarnableChanged set to true and the Warnable +// and its UnhealthyState: +// +// go cb(Change{WarnableChanged: true, Warnable: w, UnhealthyState: us}) +// +// If a Warnable becomes healthy, the callback will be called with +// WarnableChanged set to true, the Warnable set, and UnhealthyState set to nil: +// +// go cb(Change{WarnableChanged: true, Warnable: w, UnhealthyState: nil}) +// +// If the health messages from the control-plane change, the callback will be +// called with ControlHealthChanged set to true. Recipients can fetch the set of +// control-plane health messages by calling [Tracker.CurrentState]: +// +// go cb(Change{ControlHealthChanged: true}) +func (t *Tracker) RegisterWatcher(cb func(Change)) (unregister func()) { + return t.registerSyncWatcher(func(c Change) { + go cb(c) }) } // registerSyncWatcher adds a function that will be called whenever the health -// state of any Warnable changes. The provided callback function will be -// executed synchronously. Call RegisterWatcher to register any callbacks that -// won't return from execution immediately. -func (t *Tracker) registerSyncWatcher(cb func(w *Warnable, r *UnhealthyState)) (unregister func()) { +// state changes. The provided callback function will be executed synchronously. +// Call RegisterWatcher to register any callbacks that won't return from +// execution immediately. +func (t *Tracker) registerSyncWatcher(cb func(c Change)) (unregister func()) { if t.nil() { return func() {} } @@ -512,7 +570,7 @@ func (t *Tracker) registerSyncWatcher(cb func(w *Warnable, r *UnhealthyState)) ( t.mu.Lock() defer t.mu.Unlock() if t.watchers == nil { - t.watchers = set.HandleSet[func(*Warnable, *UnhealthyState)]{} + t.watchers = set.HandleSet[func(Change)]{} } handle := t.watchers.Add(cb) if t.timer == nil { @@ -659,13 +717,15 @@ func (t *Tracker) updateLegacyErrorWarnableLocked(key Subsystem, err error) { } } -func (t *Tracker) SetControlHealth(problems []string) { +func (t *Tracker) SetControlHealth(problems map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage) { if t.nil() { return } t.mu.Lock() defer t.mu.Unlock() - t.controlHealth = problems + + t.controlMessages = problems + t.selfCheckLocked() } @@ -961,11 +1021,11 @@ func (t *Tracker) OverallError() error { return t.multiErrLocked() } -// Strings() returns a string array containing the Text of all Warnings -// currently known to the Tracker. These strings can be presented to the -// user, although ideally you would use the Code property on each Warning -// to show a localized version of them instead. -// This function is here for legacy compatibility purposes and is deprecated. +// Strings() returns a string array containing the Text of all Warnings and +// ControlHealth messages currently known to the Tracker. These strings can be +// presented to the user, although ideally you would use the Code property on +// each Warning to show a localized version of them instead. This function is +// here for legacy compatibility purposes and is deprecated. func (t *Tracker) Strings() []string { if t.nil() { return nil @@ -991,6 +1051,19 @@ func (t *Tracker) stringsLocked() []string { result = append(result, w.Text(ws.Args)) } } + + warnLen := len(result) + for _, c := range t.controlMessages { + if c.Title != "" && c.Text != "" { + result = append(result, c.Title+": "+c.Text) + } else if c.Title != "" { + result = append(result, c.Title) + } else if c.Text != "" { + result = append(result, c.Text) + } + } + sort.Strings(result[warnLen:]) + return result } @@ -1171,14 +1244,10 @@ func (t *Tracker) updateBuiltinWarnablesLocked() { t.setHealthyLocked(derpRegionErrorWarnable) } - if len(t.controlHealth) > 0 { - for _, s := range t.controlHealth { - t.setUnhealthyLocked(controlHealthWarnable, Args{ - ArgError: s, - }) - } - } else { - t.setHealthyLocked(controlHealthWarnable) + // Check if control health messages have changed + if !maps.EqualFunc(t.lastNotifiedControlMessages, t.controlMessages, tailcfg.DisplayMessage.Equal) { + t.lastNotifiedControlMessages = t.controlMessages + t.notifyWatchersControlChangedLocked() } if err := envknob.ApplyDiskConfigError(); err != nil { diff --git a/health/health_test.go b/health/health_test.go index aa39045817ce2..f609cfb1613b1 100644 --- a/health/health_test.go +++ b/health/health_test.go @@ -5,12 +5,14 @@ package health import ( "fmt" + "maps" "reflect" "slices" "strconv" "testing" "time" + "github.com/google/go-cmp/cmp" "tailscale.com/tailcfg" "tailscale.com/tstest" "tailscale.com/types/opt" @@ -25,6 +27,7 @@ func TestAppendWarnableDebugFlags(t *testing.T) { w := Register(&Warnable{ Code: WarnableCode(fmt.Sprintf("warnable-code-%d", i)), MapDebugFlag: fmt.Sprint(i), + Text: StaticMessage(""), }) defer unregister(w) if i%2 == 0 { @@ -114,7 +117,9 @@ func TestWatcher(t *testing.T) { becameUnhealthy := make(chan struct{}) becameHealthy := make(chan struct{}) - watcherFunc := func(w *Warnable, us *UnhealthyState) { + watcherFunc := func(c Change) { + w := c.Warnable + us := c.UnhealthyState if w != testWarnable { t.Fatalf("watcherFunc was called, but with an unexpected Warnable: %v, want: %v", w, testWarnable) } @@ -184,7 +189,9 @@ func TestSetUnhealthyWithTimeToVisible(t *testing.T) { becameUnhealthy := make(chan struct{}) becameHealthy := make(chan struct{}) - watchFunc := func(w *Warnable, us *UnhealthyState) { + watchFunc := func(c Change) { + w := c.Warnable + us := c.UnhealthyState if w != mw { t.Fatalf("watcherFunc was called, but with an unexpected Warnable: %v, want: %v", w, w) } @@ -457,21 +464,94 @@ func TestControlHealth(t *testing.T) { ht.SetIPNState("NeedsLogin", true) ht.GotStreamedMapResponse() - ht.SetControlHealth([]string{"Test message"}) - state := ht.CurrentState() - warning, ok := state.Warnings["control-health"] + baseWarns := ht.CurrentState().Warnings + baseStrs := ht.Strings() - if !ok { - t.Fatal("no warning found in current state with code 'control-health'") - } - if got, want := warning.Title, "Coordination server reports an issue"; got != want { - t.Errorf("warning.Title = %q, want %q", got, want) - } - if got, want := warning.Severity, SeverityMedium; got != want { - t.Errorf("warning.Severity = %s, want %s", got, want) - } - if got, want := warning.Text, "The coordination server is reporting an health issue: Test message"; got != want { - t.Errorf("warning.Text = %q, want %q", got, want) + ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "control-health-test": { + Title: "Control health message", + Text: "Extra help", + }, + "control-health-title": { + Title: "Control health title only", + }, + }) + + t.Run("Warnings", func(t *testing.T) { + wantWarns := map[WarnableCode]UnhealthyState{ + "control-health-test": { + WarnableCode: "control-health-test", + Severity: SeverityMedium, + Title: "Control health message", + Text: "Extra help", + }, + "control-health-title": { + WarnableCode: "control-health-title", + Severity: SeverityMedium, + Title: "Control health title only", + }, + } + state := ht.CurrentState() + gotWarns := maps.Clone(state.Warnings) + for k := range gotWarns { + if _, inBase := baseWarns[k]; inBase { + delete(gotWarns, k) + } + } + if diff := cmp.Diff(wantWarns, gotWarns); diff != "" { + t.Fatalf(`CurrentState().Warnings["control-health-*"] wrong (-want +got):\n%s`, diff) + } + }) + + t.Run("Strings()", func(t *testing.T) { + wantStrs := []string{ + "Control health message: Extra help", + "Control health title only", + } + var gotStrs []string + for _, s := range ht.Strings() { + if !slices.Contains(baseStrs, s) { + gotStrs = append(gotStrs, s) + } + } + if diff := cmp.Diff(wantStrs, gotStrs); diff != "" { + t.Fatalf(`Strings() wrong (-want +got):\n%s`, diff) + } + }) + + t.Run("tailscaled_health_messages", func(t *testing.T) { + var r usermetric.Registry + ht.SetMetricsRegistry(&r) + + got := ht.metricHealthMessage.Get(metricHealthMessageLabel{ + Type: MetricLabelWarning, + }).String() + want := strconv.Itoa( + 2 + // from SetControlHealth + len(baseStrs), + ) + if got != want { + t.Errorf("metricsHealthMessage.Get(warning) = %q, want %q", got, want) + } + }) +} + +func TestControlHealthNotifiesOnSet(t *testing.T) { + ht := Tracker{} + ht.SetIPNState("NeedsLogin", true) + ht.GotStreamedMapResponse() + + gotNotified := false + ht.registerSyncWatcher(func(_ Change) { + gotNotified = true + }) + + ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": {}, + }) + + if !gotNotified { + t.Errorf("watcher did not get called, want it to be called") } } @@ -480,12 +560,45 @@ func TestControlHealthNotifiesOnChange(t *testing.T) { ht.SetIPNState("NeedsLogin", true) ht.GotStreamedMapResponse() + ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-1": {}, + }) + gotNotified := false - ht.registerSyncWatcher(func(_ *Warnable, _ *UnhealthyState) { + ht.registerSyncWatcher(func(_ Change) { gotNotified = true }) - ht.SetControlHealth([]string{"Test message"}) + ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-2": {}, + }) + + if !gotNotified { + t.Errorf("watcher did not get called, want it to be called") + } +} + +func TestControlHealthNotifiesOnDetailsChange(t *testing.T) { + ht := Tracker{} + ht.SetIPNState("NeedsLogin", true) + ht.GotStreamedMapResponse() + + ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-1": { + Title: "Title", + }, + }) + + gotNotified := false + ht.registerSyncWatcher(func(_ Change) { + gotNotified = true + }) + + ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-1": { + Title: "Updated title", + }, + }) if !gotNotified { t.Errorf("watcher did not get called, want it to be called") @@ -498,16 +611,20 @@ func TestControlHealthNoNotifyOnUnchanged(t *testing.T) { ht.GotStreamedMapResponse() // Set up an existing control health issue - ht.SetControlHealth([]string{"Test message"}) + ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": {}, + }) // Now register our watcher gotNotified := false - ht.registerSyncWatcher(func(_ *Warnable, _ *UnhealthyState) { + ht.registerSyncWatcher(func(_ Change) { gotNotified = true }) // Send the same control health message again - should not notify - ht.SetControlHealth([]string{"Test message"}) + ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": {}, + }) if gotNotified { t.Errorf("watcher got called, want it to not be called") @@ -519,11 +636,13 @@ func TestControlHealthIgnoredOutsideMapPoll(t *testing.T) { ht.SetIPNState("NeedsLogin", true) gotNotified := false - ht.registerSyncWatcher(func(_ *Warnable, _ *UnhealthyState) { + ht.registerSyncWatcher(func(_ Change) { gotNotified = true }) - ht.SetControlHealth([]string{"Test message"}) + ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "control-health": {}, + }) state := ht.CurrentState() _, ok := state.Warnings["control-health"] diff --git a/health/state.go b/health/state.go index c06f6ef59c8ed..cf4f922d7563e 100644 --- a/health/state.go +++ b/health/state.go @@ -5,6 +5,8 @@ package health import ( "time" + + "tailscale.com/tailcfg" ) // State contains the health status of the backend, and is @@ -21,7 +23,8 @@ type State struct { } // UnhealthyState contains information to be shown to the user to inform them -// that a Warnable is currently unhealthy. +// that a [Warnable] is currently unhealthy or [tailcfg.DisplayMessage] is being +// sent from the control-plane. type UnhealthyState struct { WarnableCode WarnableCode Severity Severity @@ -98,11 +101,34 @@ func (t *Tracker) CurrentState() *State { wm[w.Code] = *w.unhealthyState(ws) } + for id, msg := range t.lastNotifiedControlMessages { + code := WarnableCode(id) + wm[code] = UnhealthyState{ + WarnableCode: code, + Severity: severityFromTailcfg(msg.Severity), + Title: msg.Title, + Text: msg.Text, + ImpactsConnectivity: msg.ImpactsConnectivity, + // TODO(tailscale/corp#27759): DependsOn? + } + } + return &State{ Warnings: wm, } } +func severityFromTailcfg(s tailcfg.DisplayMessageSeverity) Severity { + switch s { + case tailcfg.SeverityHigh: + return SeverityHigh + case tailcfg.SeverityLow: + return SeverityLow + default: + return SeverityMedium + } +} + // isEffectivelyHealthyLocked reports whether w is effectively healthy. // That means it's either actually healthy or it has a dependency that // that's unhealthy, so we should treat w as healthy to not spam users diff --git a/health/warnings.go b/health/warnings.go index 7a21f9695ff6d..3997e66b39ad0 100644 --- a/health/warnings.go +++ b/health/warnings.go @@ -238,16 +238,6 @@ var applyDiskConfigWarnable = Register(&Warnable{ }, }) -// controlHealthWarnable is a Warnable that warns the user that the coordination server is reporting an health issue. -var controlHealthWarnable = Register(&Warnable{ - Code: "control-health", - Title: "Coordination server reports an issue", - Severity: SeverityMedium, - Text: func(args Args) string { - return fmt.Sprintf("The coordination server is reporting an health issue: %v", args[ArgError]) - }, -}) - // warmingUpWarnableDuration is the duration for which the warmingUpWarnable is reported by the backend after the user // has changed ipnWantRunning to true from false. const warmingUpWarnableDuration = 5 * time.Second diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 468fd72eb59cf..d2f6c86f7f113 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -933,11 +933,15 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) { } } -func (b *LocalBackend) onHealthChange(w *health.Warnable, us *health.UnhealthyState) { - if us == nil { - b.logf("health(warnable=%s): ok", w.Code) - } else { - b.logf("health(warnable=%s): error: %s", w.Code, us.Text) +func (b *LocalBackend) onHealthChange(change health.Change) { + if change.WarnableChanged { + w := change.Warnable + us := change.UnhealthyState + if us == nil { + b.logf("health(warnable=%s): ok", w.Code) + } else { + b.logf("health(warnable=%s): error: %s", w.Code, us.Text) + } } // Whenever health changes, send the current health state to the frontend. @@ -5826,7 +5830,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { b.pauseOrResumeControlClientLocked() if nm != nil { - b.health.SetControlHealth(nm.ControlHealth) + b.health.SetControlHealth(nm.DisplayMessages) } else { b.health.SetControlHealth(nil) } diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 0a58d8f0cc229..7e2fa3ffc5f8e 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2028,7 +2028,7 @@ type MapResponse struct { // plane's perspective. A nil value means no change from the previous // MapResponse. A non-nil 0-length slice restores the health to good (no // known problems). A non-zero length slice are the list of problems that - // the control place sees. + // the control plane sees. // // Note that this package's type, due its use of a slice and omitempty, is // unable to marshal a zero-length non-nil slice. The control server needs @@ -2078,6 +2078,56 @@ type MapResponse struct { DefaultAutoUpdate opt.Bool `json:",omitempty"` } +// DisplayMessage represents a health state of the node from the control plane's +// perspective. It is deliberately similar to health.Warnable as both get +// converted into health.UnhealthyState to be sent to the GUI. +type DisplayMessage struct { + // Title is a string that the GUI uses as title for this message. The title + // should be short and fit in a single line. + Title string + + // Text is an extended string that the GUI will display to the user. + Text string + + // Severity is the severity of the DisplayMessage, which the GUI can use to + // determine how to display it. Maps to health.Severity. + Severity DisplayMessageSeverity + + // ImpactsConnectivity is whether the health problem will impact the user's + // ability to connect to the Internet or other nodes on the tailnet, which + // the GUI can use to determine how to display it. + ImpactsConnectivity bool `json:",omitempty"` +} + +// DisplayMessageID is a string that uniquely identifies the kind of health +// issue (e.g. "session-expired"). +type DisplayMessageID string + +// Equal returns true iff all fields are equal. +func (m DisplayMessage) Equal(o DisplayMessage) bool { + return m.Title == o.Title && + m.Text == o.Text && + m.Severity == o.Severity && + m.ImpactsConnectivity == o.ImpactsConnectivity +} + +// DisplayMessageSeverity represents how serious a [DisplayMessage] is. Analogous +// to health.Severity. +type DisplayMessageSeverity string + +const ( + // SeverityHigh is the highest severity level, used for critical errors that need immediate attention. + // On platforms where the client GUI can deliver notifications, a SeverityHigh message will trigger + // a modal notification. + SeverityHigh DisplayMessageSeverity = "high" + // SeverityMedium is used for errors that are important but not critical. This won't trigger a modal + // notification, however it will be displayed in a more visible way than a SeverityLow message. + SeverityMedium DisplayMessageSeverity = "medium" + // SeverityLow is used for less important notices that don't need immediate attention. The user will + // have to go to a Settings window, or another "hidden" GUI location to see these messages. + SeverityLow DisplayMessageSeverity = "low" +) + // ClientVersion is information about the latest client version that's available // for the client (and whether they're already running it). // diff --git a/tailcfg/tailcfg_test.go b/tailcfg/tailcfg_test.go index 079162a150191..60e86794a195c 100644 --- a/tailcfg/tailcfg_test.go +++ b/tailcfg/tailcfg_test.go @@ -878,3 +878,79 @@ func TestCheckTag(t *testing.T) { }) } } + +func TestDisplayMessageEqual(t *testing.T) { + base := DisplayMessage{ + Title: "title", + Text: "text", + Severity: SeverityHigh, + ImpactsConnectivity: false, + } + + type test struct { + name string + value DisplayMessage + wantEqual bool + } + + for _, test := range []test{ + { + name: "same", + value: DisplayMessage{ + Title: "title", + Text: "text", + Severity: SeverityHigh, + ImpactsConnectivity: false, + }, + wantEqual: true, + }, + { + name: "different-title", + value: DisplayMessage{ + Title: "different title", + Text: "text", + Severity: SeverityHigh, + ImpactsConnectivity: false, + }, + wantEqual: false, + }, + { + name: "different-text", + value: DisplayMessage{ + Title: "title", + Text: "different text", + Severity: SeverityHigh, + ImpactsConnectivity: false, + }, + wantEqual: false, + }, + { + name: "different-severity", + value: DisplayMessage{ + Title: "title", + Text: "text", + Severity: SeverityMedium, + ImpactsConnectivity: false, + }, + wantEqual: false, + }, + { + name: "different-impactsConnectivity", + value: DisplayMessage{ + Title: "title", + Text: "text", + Severity: SeverityHigh, + ImpactsConnectivity: true, + }, + wantEqual: false, + }, + } { + t.Run(test.name, func(t *testing.T) { + got := base.Equal(test.value) + + if got != test.wantEqual { + t.Errorf("Equal: got %t, want %t", got, test.wantEqual) + } + }) + } +} diff --git a/types/netmap/netmap.go b/types/netmap/netmap.go index c6250c49ce9c9..963f80a441ee4 100644 --- a/types/netmap/netmap.go +++ b/types/netmap/netmap.go @@ -54,12 +54,12 @@ type NetworkMap struct { // between updates and should not be modified. DERPMap *tailcfg.DERPMap - // ControlHealth are the list of health check problems for this + // DisplayMessages are the list of health check problems for this // node from the perspective of the control plane. // If empty, there are no known problems from the control plane's // point of view, but the node might know about its own health // check problems. - ControlHealth []string + DisplayMessages map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage // TKAEnabled indicates whether the tailnet key authority should be // enabled, from the perspective of the control plane. From 3ee4c60ff0257d11842523c1c59492345030dce2 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Thu, 22 May 2025 12:14:16 -0700 Subject: [PATCH 003/263] cmd/derper: fix mesh auth for DERP servers (#16061) To authenticate mesh keys, the DERP servers used a simple == comparison, which is susceptible to a side channel timing attack. By extracting the mesh key for a DERP server, an attacker could DoS it by forcing disconnects using derp.Client.ClosePeer. They could also enumerate the public Wireguard keys, IP addresses and ports for nodes connected to that DERP server. DERP servers configured without mesh keys deny all such requests. This patch also extracts the mesh key logic into key.DERPMesh, to prevent this from happening again. Security bulletin: https://tailscale.com/security-bulletins#ts-2025-003 Fixes tailscale/corp#28720 Signed-off-by: Simon Law --- cmd/derper/derper.go | 14 +--- cmd/derper/derper_test.go | 43 ---------- derp/derp_client.go | 8 +- derp/derp_server.go | 28 +++++-- derp/derp_test.go | 103 +++++++++++++++++++++++- derp/derphttp/derphttp_client.go | 2 +- derp/derphttp/derphttp_test.go | 10 ++- types/key/derp.go | 68 ++++++++++++++++ types/key/derp_test.go | 133 +++++++++++++++++++++++++++++++ 9 files changed, 338 insertions(+), 71 deletions(-) create mode 100644 types/key/derp.go create mode 100644 types/key/derp_test.go diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 3c6fda68c4d59..840de3fba671e 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -96,9 +96,6 @@ var ( var ( tlsRequestVersion = &metrics.LabelMap{Label: "version"} tlsActiveVersion = &metrics.LabelMap{Label: "version"} - - // Exactly 64 hexadecimal lowercase digits. - validMeshKey = regexp.MustCompile(`^[0-9a-f]{64}$`) ) const setecMeshKeyName = "meshkey" @@ -159,14 +156,6 @@ func writeNewConfig() config { return cfg } -func checkMeshKey(key string) (string, error) { - key = strings.TrimSpace(key) - if !validMeshKey.MatchString(key) { - return "", errors.New("key must contain exactly 64 hex digits") - } - return key, nil -} - func main() { flag.Parse() if *versionFlag { @@ -246,10 +235,9 @@ func main() { log.Printf("No mesh key configured for --dev mode") } else if meshKey == "" { log.Printf("No mesh key configured") - } else if key, err := checkMeshKey(meshKey); err != nil { + } else if err := s.SetMeshKey(meshKey); err != nil { log.Fatalf("invalid mesh key: %v", err) } else { - s.SetMeshKey(key) log.Println("DERP mesh key configured") } diff --git a/cmd/derper/derper_test.go b/cmd/derper/derper_test.go index 12686ce4eb5f3..6dce1fcdfebdd 100644 --- a/cmd/derper/derper_test.go +++ b/cmd/derper/derper_test.go @@ -138,46 +138,3 @@ func TestTemplate(t *testing.T) { t.Error("Output is missing debug info") } } - -func TestCheckMeshKey(t *testing.T) { - testCases := []struct { - name string - input string - want string - wantErr bool - }{ - { - name: "KeyOkay", - input: "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6", - want: "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6", - wantErr: false, - }, - { - name: "TrimKeyOkay", - input: " f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6 ", - want: "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6", - wantErr: false, - }, - { - name: "NotAKey", - input: "zzthisisnotakey", - want: "", - wantErr: true, - }, - } - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - k, err := checkMeshKey(tt.input) - if err != nil && !tt.wantErr { - t.Errorf("unexpected error: %v", err) - } - if k != tt.want && err == nil { - t.Errorf("want: %s doesn't match expected: %s", tt.want, k) - } - - }) - } - -} diff --git a/derp/derp_client.go b/derp/derp_client.go index 7a646fa517940..a9b92299c291f 100644 --- a/derp/derp_client.go +++ b/derp/derp_client.go @@ -30,7 +30,7 @@ type Client struct { logf logger.Logf nc Conn br *bufio.Reader - meshKey string + meshKey key.DERPMesh canAckPings bool isProber bool @@ -56,7 +56,7 @@ func (f clientOptFunc) update(o *clientOpt) { f(o) } // clientOpt are the options passed to newClient. type clientOpt struct { - MeshKey string + MeshKey key.DERPMesh ServerPub key.NodePublic CanAckPings bool IsProber bool @@ -66,7 +66,7 @@ type clientOpt struct { // access to join the mesh. // // An empty key means to not use a mesh key. -func MeshKey(key string) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.MeshKey = key }) } +func MeshKey(k key.DERPMesh) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.MeshKey = k }) } // IsProber returns a ClientOpt to pass to the DERP server during connect to // declare that this client is a a prober. @@ -182,7 +182,7 @@ type clientInfo struct { func (c *Client) sendClientKey() error { msg, err := json.Marshal(clientInfo{ Version: ProtocolVersion, - MeshKey: c.meshKey, + MeshKey: c.meshKey.String(), CanAckPings: c.canAckPings, IsProber: c.isProber, }) diff --git a/derp/derp_server.go b/derp/derp_server.go index abda9da73a6fc..6f86c3ea458cf 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -134,7 +134,7 @@ type Server struct { publicKey key.NodePublic logf logger.Logf memSys0 uint64 // runtime.MemStats.Sys at start (or early-ish) - meshKey string + meshKey key.DERPMesh limitedLogf logger.Logf metaCert []byte // the encoded x509 cert to send after LetsEncrypt cert+intermediate dupPolicy dupPolicy @@ -464,8 +464,13 @@ func genDroppedCounters() { // amongst themselves. // // It must be called before serving begins. -func (s *Server) SetMeshKey(v string) { - s.meshKey = v +func (s *Server) SetMeshKey(v string) error { + k, err := key.ParseDERPMesh(v) + if err != nil { + return err + } + s.meshKey = k + return nil } // SetVerifyClients sets whether this DERP server verifies clients through tailscaled. @@ -506,10 +511,10 @@ func (s *Server) SetTCPWriteTimeout(d time.Duration) { } // HasMeshKey reports whether the server is configured with a mesh key. -func (s *Server) HasMeshKey() bool { return s.meshKey != "" } +func (s *Server) HasMeshKey() bool { return !s.meshKey.IsZero() } // MeshKey returns the configured mesh key, if any. -func (s *Server) MeshKey() string { return s.meshKey } +func (s *Server) MeshKey() key.DERPMesh { return s.meshKey } // PrivateKey returns the server's private key. func (s *Server) PrivateKey() key.NodePrivate { return s.privateKey } @@ -1355,7 +1360,18 @@ func (c *sclient) requestMeshUpdate() { // isMeshPeer reports whether the client is a trusted mesh peer // node in the DERP region. func (s *Server) isMeshPeer(info *clientInfo) bool { - return info != nil && info.MeshKey != "" && info.MeshKey == s.meshKey + // Compare mesh keys in constant time to prevent timing attacks. + // Since mesh keys are a fixed length, we don’t need to be concerned + // about timing attacks on client mesh keys that are the wrong length. + // See https://github.com/tailscale/corp/issues/28720 + if info == nil || info.MeshKey == "" { + return false + } + k, err := key.ParseDERPMesh(info.MeshKey) + if err != nil { + return false + } + return s.meshKey.Equal(k) } // verifyClient checks whether the client is allowed to connect to the derper, diff --git a/derp/derp_test.go b/derp/derp_test.go index c5a92bafae1dd..0093ee2b15372 100644 --- a/derp/derp_test.go +++ b/derp/derp_test.go @@ -511,11 +511,13 @@ func (ts *testServer) close(t *testing.T) error { return nil } +const testMeshKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + func newTestServer(t *testing.T, ctx context.Context) *testServer { t.Helper() logf := logger.WithPrefix(t.Logf, "derp-server: ") s := NewServer(key.NewNode(), logf) - s.SetMeshKey("mesh-key") + s.SetMeshKey(testMeshKey) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -591,8 +593,12 @@ func newRegularClient(t *testing.T, ts *testServer, name string) *testClient { func newTestWatcher(t *testing.T, ts *testServer, name string) *testClient { return newTestClient(t, ts, name, func(nc net.Conn, priv key.NodePrivate, logf logger.Logf) (*Client, error) { + mk, err := key.ParseDERPMesh(testMeshKey) + if err != nil { + return nil, err + } brw := bufio.NewReadWriter(bufio.NewReader(nc), bufio.NewWriter(nc)) - c, err := NewClient(priv, nc, brw, logf, MeshKey("mesh-key")) + c, err := NewClient(priv, nc, brw, logf, MeshKey(mk)) if err != nil { return nil, err } @@ -1627,3 +1633,96 @@ func TestGetPerClientSendQueueDepth(t *testing.T) { }) } } + +func TestSetMeshKey(t *testing.T) { + for name, tt := range map[string]struct { + key string + want key.DERPMesh + wantErr bool + }{ + "clobber": { + key: testMeshKey, + wantErr: false, + }, + "invalid": { + key: "badf00d", + wantErr: true, + }, + } { + t.Run(name, func(t *testing.T) { + s := &Server{} + + err := s.SetMeshKey(tt.key) + if tt.wantErr { + if err == nil { + t.Fatalf("expected err") + } + return + } + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + want, err := key.ParseDERPMesh(tt.key) + if err != nil { + t.Fatal(err) + } + if !s.meshKey.Equal(want) { + t.Fatalf("got %v, want %v", s.meshKey, want) + } + }) + } +} + +func TestIsMeshPeer(t *testing.T) { + s := &Server{} + err := s.SetMeshKey(testMeshKey) + if err != nil { + t.Fatal(err) + } + for name, tt := range map[string]struct { + info *clientInfo + want bool + wantAllocs float64 + }{ + "nil": { + info: nil, + want: false, + wantAllocs: 0, + }, + "empty": { + info: &clientInfo{MeshKey: ""}, + want: false, + wantAllocs: 0, + }, + "invalid": { + info: &clientInfo{MeshKey: "invalid"}, + want: false, + wantAllocs: 2, // error message + }, + "mismatch": { + info: &clientInfo{MeshKey: "0badf00d00000000000000000000000000000000000000000000000000000000"}, + want: false, + wantAllocs: 1, + }, + "match": { + info: &clientInfo{MeshKey: testMeshKey}, + want: true, + wantAllocs: 1, + }, + } { + t.Run(name, func(t *testing.T) { + var got bool + allocs := testing.AllocsPerRun(1, func() { + got = s.isMeshPeer(tt.info) + }) + if got != tt.want { + t.Fatalf("got %t, want %t: info = %#v", got, tt.want, tt.info) + } + + if allocs != tt.wantAllocs && tt.want { + t.Errorf("%f allocations, want %f", allocs, tt.wantAllocs) + } + }) + } +} diff --git a/derp/derphttp/derphttp_client.go b/derp/derphttp/derphttp_client.go index faa218ca25f0a..8c42e9070252e 100644 --- a/derp/derphttp/derphttp_client.go +++ b/derp/derphttp/derphttp_client.go @@ -57,7 +57,7 @@ type Client struct { TLSConfig *tls.Config // optional; nil means default HealthTracker *health.Tracker // optional; used if non-nil only DNSCache *dnscache.Resolver // optional; nil means no caching - MeshKey string // optional; for trusted clients + MeshKey key.DERPMesh // optional; for trusted clients IsProber bool // optional; for probers to optional declare themselves as such // WatchConnectionChanges is whether the client wishes to subscribe to diff --git a/derp/derphttp/derphttp_test.go b/derp/derphttp/derphttp_test.go index cfb3676cda16f..8d02db922605b 100644 --- a/derp/derphttp/derphttp_test.go +++ b/derp/derphttp/derphttp_test.go @@ -212,6 +212,8 @@ func TestPing(t *testing.T) { } } +const testMeshKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + func newTestServer(t *testing.T, k key.NodePrivate) (serverURL string, s *derp.Server) { s = derp.NewServer(k, t.Logf) httpsrv := &http.Server{ @@ -224,7 +226,7 @@ func newTestServer(t *testing.T, k key.NodePrivate) (serverURL string, s *derp.S t.Fatal(err) } serverURL = "http://" + ln.Addr().String() - s.SetMeshKey("1234") + s.SetMeshKey(testMeshKey) go func() { if err := httpsrv.Serve(ln); err != nil { @@ -243,7 +245,11 @@ func newWatcherClient(t *testing.T, watcherPrivateKey key.NodePrivate, serverToW if err != nil { t.Fatal(err) } - c.MeshKey = "1234" + k, err := key.ParseDERPMesh(testMeshKey) + if err != nil { + t.Fatal(err) + } + c.MeshKey = k return } diff --git a/types/key/derp.go b/types/key/derp.go new file mode 100644 index 0000000000000..1fe690189c7be --- /dev/null +++ b/types/key/derp.go @@ -0,0 +1,68 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package key + +import ( + "crypto/subtle" + "encoding/hex" + "errors" + "fmt" + "strings" + + "go4.org/mem" + "tailscale.com/types/structs" +) + +var ErrInvalidMeshKey = errors.New("invalid mesh key") + +// DERPMesh is a mesh key, used for inter-DERP-node communication and for +// privileged DERP clients. +type DERPMesh struct { + _ structs.Incomparable // == isn't constant-time + k [32]byte // 64-digit hexadecimal numbers fit in 32 bytes +} + +// DERPMeshFromRaw32 parses a 32-byte raw value as a DERP mesh key. +func DERPMeshFromRaw32(raw mem.RO) DERPMesh { + if raw.Len() != 32 { + panic("input has wrong size") + } + var ret DERPMesh + raw.Copy(ret.k[:]) + return ret +} + +// ParseDERPMesh parses a DERP mesh key from a string. +// This function trims whitespace around the string. +// If the key is not a 64-digit hexadecimal number, ErrInvalidMeshKey is returned. +func ParseDERPMesh(key string) (DERPMesh, error) { + key = strings.TrimSpace(key) + if len(key) != 64 { + return DERPMesh{}, fmt.Errorf("%w: must be 64-digit hexadecimal number", ErrInvalidMeshKey) + } + decoded, err := hex.DecodeString(key) + if err != nil { + return DERPMesh{}, fmt.Errorf("%w: %v", ErrInvalidMeshKey, err) + } + return DERPMeshFromRaw32(mem.B(decoded)), nil +} + +// IsZero reports whether k is the zero value. +func (k DERPMesh) IsZero() bool { + return k.Equal(DERPMesh{}) +} + +// Equal reports whether k and other are the same key. +func (k DERPMesh) Equal(other DERPMesh) bool { + // Compare mesh keys in constant time to prevent timing attacks. + // Since mesh keys are a fixed length, we don’t need to be concerned + // about timing attacks on client mesh keys that are the wrong length. + // See https://github.com/tailscale/corp/issues/28720 + return subtle.ConstantTimeCompare(k.k[:], other.k[:]) == 1 +} + +// String returns k as a hex-encoded 64-digit number. +func (k DERPMesh) String() string { + return hex.EncodeToString(k.k[:]) +} diff --git a/types/key/derp_test.go b/types/key/derp_test.go new file mode 100644 index 0000000000000..b91cbbf8c4e01 --- /dev/null +++ b/types/key/derp_test.go @@ -0,0 +1,133 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package key + +import ( + "errors" + "testing" + + "go4.org/mem" +) + +func TestDERPMeshIsValid(t *testing.T) { + for name, tt := range map[string]struct { + input string + want string + wantErr error + }{ + "good": { + input: "0123456789012345678901234567890123456789012345678901234567890123", + want: "0123456789012345678901234567890123456789012345678901234567890123", + wantErr: nil, + }, + "hex": { + input: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + want: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + wantErr: nil, + }, + "uppercase": { + input: "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF", + want: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + wantErr: nil, + }, + "whitespace": { + input: " 0123456789012345678901234567890123456789012345678901234567890123 ", + want: "0123456789012345678901234567890123456789012345678901234567890123", + wantErr: nil, + }, + "short": { + input: "0123456789abcdef", + wantErr: ErrInvalidMeshKey, + }, + "long": { + input: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0", + wantErr: ErrInvalidMeshKey, + }, + } { + t.Run(name, func(t *testing.T) { + k, err := ParseDERPMesh(tt.input) + if !errors.Is(err, tt.wantErr) { + t.Errorf("err %v, want %v", err, tt.wantErr) + } + + got := k.String() + if got != tt.want && tt.wantErr == nil { + t.Errorf("got %q, want %q", got, tt.want) + } + + }) + } + +} + +func TestDERPMesh(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + str string + hex []byte + equal bool // are str and hex equal? + }{ + "zero": { + str: "0000000000000000000000000000000000000000000000000000000000000000", + hex: []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + equal: true, + }, + "equal": { + str: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + hex: []byte{ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + }, + equal: true, + }, + "unequal": { + str: "0badc0de00000000000000000000000000000000000000000000000000000000", + hex: []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + equal: false, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + k, err := ParseDERPMesh(tt.str) + if err != nil { + t.Fatal(err) + } + + // string representation should round-trip + s := k.String() + if s != tt.str { + t.Fatalf("string %s, want %s", s, tt.str) + } + + // if tt.equal, then tt.hex is intended to be equal + if k.k != [32]byte(tt.hex) && tt.equal { + t.Fatalf("decoded %x, want %x", k.k, tt.hex) + } + + h := DERPMeshFromRaw32(mem.B(tt.hex)) + if k.Equal(h) != tt.equal { + if tt.equal { + t.Fatalf("%v != %v", k, h) + } else { + t.Fatalf("%v == %v", k, h) + } + } + + }) + } +} From a05924a9e5018da6f64fd92eb9ba37e599cab567 Mon Sep 17 00:00:00 2001 From: Patrick O'Doherty Date: Thu, 22 May 2025 12:26:02 -0700 Subject: [PATCH 004/263] client/web: add Sec-Fetch-Site CSRF protection (#16046) RELNOTE=Fix CSRF errors in the client Web UI Replace gorilla/csrf with a Sec-Fetch-Site based CSRF protection middleware that falls back to comparing the Host & Origin headers if no SFS value is passed by the client. Add an -origin override to the web CLI that allows callers to specify the origin at which the web UI will be available if it is hosted behind a reverse proxy or within another application via CGI. Updates #14872 Updates #15065 Signed-off-by: Patrick O'Doherty --- client/web/src/api.ts | 10 --- client/web/web.go | 137 +++++++++++++++------------- client/web/web_test.go | 163 +++++++++++++++++++--------------- cmd/k8s-operator/depaware.txt | 9 +- cmd/tailscale/cli/web.go | 5 ++ cmd/tailscale/depaware.txt | 9 +- cmd/tailscaled/depaware.txt | 9 +- tsnet/depaware.txt | 9 +- 8 files changed, 183 insertions(+), 168 deletions(-) diff --git a/client/web/src/api.ts b/client/web/src/api.ts index 9414e2d5d7e16..e780c76459dfd 100644 --- a/client/web/src/api.ts +++ b/client/web/src/api.ts @@ -249,7 +249,6 @@ export function useAPI() { return api } -let csrfToken: string let synoToken: string | undefined // required for synology API requests let unraidCsrfToken: string | undefined // required for unraid POST requests (#8062) @@ -298,12 +297,10 @@ export function apiFetch( headers: { Accept: "application/json", "Content-Type": contentType, - "X-CSRF-Token": csrfToken, }, body: body, }) .then((r) => { - updateCsrfToken(r) if (!r.ok) { return r.text().then((err) => { throw new Error(err) @@ -322,13 +319,6 @@ export function apiFetch( }) } -function updateCsrfToken(r: Response) { - const tok = r.headers.get("X-CSRF-Token") - if (tok) { - csrfToken = tok - } -} - export function setSynoToken(token?: string) { synoToken = token } diff --git a/client/web/web.go b/client/web/web.go index 6eccdadcfdb65..f3158cd1f6ff5 100644 --- a/client/web/web.go +++ b/client/web/web.go @@ -6,7 +6,6 @@ package web import ( "context" - "crypto/rand" "encoding/json" "errors" "fmt" @@ -14,14 +13,14 @@ import ( "log" "net/http" "net/netip" + "net/url" "os" "path" - "path/filepath" + "slices" "strings" "sync" "time" - "github.com/gorilla/csrf" "tailscale.com/client/local" "tailscale.com/client/tailscale/apitype" "tailscale.com/clientupdate" @@ -60,6 +59,12 @@ type Server struct { cgiMode bool pathPrefix string + // originOverride is the origin that the web UI is accessible from. + // This value is used in the fallback CSRF checks when Sec-Fetch-Site is not + // available. In this case the application will compare Host and Origin + // header values to determine if the request is from the same origin. + originOverride string + apiHandler http.Handler // serves api endpoints; csrf-protected assetsHandler http.Handler // serves frontend assets assetsCleanup func() // called from Server.Shutdown @@ -150,6 +155,9 @@ type ServerOpts struct { // as completed. // This field is required for ManageServerMode mode. WaitAuthURL func(ctx context.Context, id string, src tailcfg.NodeID) (*tailcfg.WebClientAuthResponse, error) + + // OriginOverride specifies the origin that the web UI will be accessible from if hosted behind a reverse proxy or CGI. + OriginOverride string } // NewServer constructs a new Tailscale web client server. @@ -169,15 +177,16 @@ func NewServer(opts ServerOpts) (s *Server, err error) { opts.LocalClient = &local.Client{} } s = &Server{ - mode: opts.Mode, - logf: opts.Logf, - devMode: envknob.Bool("TS_DEBUG_WEB_CLIENT_DEV"), - lc: opts.LocalClient, - cgiMode: opts.CGIMode, - pathPrefix: opts.PathPrefix, - timeNow: opts.TimeNow, - newAuthURL: opts.NewAuthURL, - waitAuthURL: opts.WaitAuthURL, + mode: opts.Mode, + logf: opts.Logf, + devMode: envknob.Bool("TS_DEBUG_WEB_CLIENT_DEV"), + lc: opts.LocalClient, + cgiMode: opts.CGIMode, + pathPrefix: opts.PathPrefix, + timeNow: opts.TimeNow, + newAuthURL: opts.NewAuthURL, + waitAuthURL: opts.WaitAuthURL, + originOverride: opts.OriginOverride, } if opts.PathPrefix != "" { // Enforce that path prefix always has a single leading '/' @@ -205,7 +214,7 @@ func NewServer(opts ServerOpts) (s *Server, err error) { var metric string s.apiHandler, metric = s.modeAPIHandler(s.mode) - s.apiHandler = s.withCSRF(s.apiHandler) + s.apiHandler = s.csrfProtect(s.apiHandler) // Don't block startup on reporting metric. // Report in separate go routine with 5 second timeout. @@ -218,23 +227,64 @@ func NewServer(opts ServerOpts) (s *Server, err error) { return s, nil } -func (s *Server) withCSRF(h http.Handler) http.Handler { - csrfProtect := csrf.Protect(s.csrfKey(), csrf.Secure(false)) +func (s *Server) csrfProtect(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // CSRF is not required for GET, HEAD, or OPTIONS requests. + if slices.Contains([]string{"GET", "HEAD", "OPTIONS"}, r.Method) { + h.ServeHTTP(w, r) + return + } - // ref https://github.com/tailscale/tailscale/pull/14822 - // signal to the CSRF middleware that the request is being served over - // plaintext HTTP to skip TLS-only header checks. - withSetPlaintext := func(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - r = csrf.PlaintextHTTPRequest(r) + // first attempt to use Sec-Fetch-Site header (sent by all modern + // browsers to "potentially trustworthy" origins i.e. localhost or those + // served over HTTPS) + secFetchSite := r.Header.Get("Sec-Fetch-Site") + if secFetchSite == "same-origin" { h.ServeHTTP(w, r) - }) - } + return + } else if secFetchSite != "" { + http.Error(w, fmt.Sprintf("CSRF request denied with Sec-Fetch-Site %q", secFetchSite), http.StatusForbidden) + return + } + + // if Sec-Fetch-Site is not available we presume we are operating over HTTP. + // We fall back to comparing the Origin & Host headers. + + // use the Host header to determine the expected origin + // (use the override if set to allow for reverse proxying) + host := r.Host + if host == "" { + http.Error(w, "CSRF request denied with no Host header", http.StatusForbidden) + return + } + if s.originOverride != "" { + host = s.originOverride + } + + originHeader := r.Header.Get("Origin") + if originHeader == "" { + http.Error(w, "CSRF request denied with no Origin header", http.StatusForbidden) + return + } + parsedOrigin, err := url.Parse(originHeader) + if err != nil { + http.Error(w, fmt.Sprintf("CSRF request denied with invalid Origin %q", r.Header.Get("Origin")), http.StatusForbidden) + return + } + origin := parsedOrigin.Host + if origin == "" { + http.Error(w, "CSRF request denied with no host in the Origin header", http.StatusForbidden) + return + } + + if origin != host { + http.Error(w, fmt.Sprintf("CSRF request denied with mismatched Origin %q and Host %q", origin, host), http.StatusForbidden) + return + } + + h.ServeHTTP(w, r) - // NB: the order of the withSetPlaintext and csrfProtect calls is important - // to ensure that we signal to the CSRF middleware that the request is being - // served over plaintext HTTP and not over TLS as it presumes by default. - return withSetPlaintext(csrfProtect(h)) + }) } func (s *Server) modeAPIHandler(mode ServerMode) (http.Handler, string) { @@ -452,7 +502,6 @@ func (s *Server) authorizeRequest(w http.ResponseWriter, r *http.Request) (ok bo // It should only be called by Server.ServeHTTP, via Server.apiHandler, // which protects the handler using gorilla csrf. func (s *Server) serveLoginAPI(w http.ResponseWriter, r *http.Request) { - w.Header().Set("X-CSRF-Token", csrf.Token(r)) switch { case r.URL.Path == "/api/data" && r.Method == httpm.GET: s.serveGetNodeData(w, r) @@ -575,7 +624,6 @@ func (s *Server) serveAPI(w http.ResponseWriter, r *http.Request) { } } - w.Header().Set("X-CSRF-Token", csrf.Token(r)) path := strings.TrimPrefix(r.URL.Path, "/api") switch { case path == "/data" && r.Method == httpm.GET: @@ -1276,37 +1324,6 @@ func (s *Server) proxyRequestToLocalAPI(w http.ResponseWriter, r *http.Request) } } -// csrfKey returns a key that can be used for CSRF protection. -// If an error occurs during key creation, the error is logged and the active process terminated. -// If the server is running in CGI mode, the key is cached to disk and reused between requests. -// If an error occurs during key storage, the error is logged and the active process terminated. -func (s *Server) csrfKey() []byte { - csrfFile := filepath.Join(os.TempDir(), "tailscale-web-csrf.key") - - // if running in CGI mode, try to read from disk, but ignore errors - if s.cgiMode { - key, _ := os.ReadFile(csrfFile) - if len(key) == 32 { - return key - } - } - - // create a new key - key := make([]byte, 32) - if _, err := rand.Read(key); err != nil { - log.Fatalf("error generating CSRF key: %v", err) - } - - // if running in CGI mode, try to write the newly created key to disk, and exit if it fails. - if s.cgiMode { - if err := os.WriteFile(csrfFile, key, 0600); err != nil { - log.Fatalf("unable to store CSRF key: %v", err) - } - } - - return key -} - // enforcePrefix returns a HandlerFunc that enforces a given path prefix is used in requests, // then strips it before invoking h. // Unlike http.StripPrefix, it does not return a 404 if the prefix is not present. diff --git a/client/web/web_test.go b/client/web/web_test.go index 2a6bc787ac396..12dbb5c79b13a 100644 --- a/client/web/web_test.go +++ b/client/web/web_test.go @@ -11,7 +11,6 @@ import ( "fmt" "io" "net/http" - "net/http/cookiejar" "net/http/httptest" "net/netip" "net/url" @@ -21,14 +20,12 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/gorilla/csrf" "tailscale.com/client/local" "tailscale.com/client/tailscale/apitype" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/net/memnet" "tailscale.com/tailcfg" - "tailscale.com/tstest/nettest" "tailscale.com/types/views" "tailscale.com/util/httpm" ) @@ -1492,81 +1489,99 @@ func mockWaitAuthURL(_ context.Context, id string, src tailcfg.NodeID) (*tailcfg } func TestCSRFProtect(t *testing.T) { - s := &Server{} - - mux := http.NewServeMux() - mux.HandleFunc("GET /test/csrf-token", func(w http.ResponseWriter, r *http.Request) { - token := csrf.Token(r) - _, err := io.WriteString(w, token) - if err != nil { - t.Fatal(err) - } - }) - mux.HandleFunc("POST /test/csrf-protected", func(w http.ResponseWriter, r *http.Request) { - _, err := io.WriteString(w, "ok") - if err != nil { - t.Fatal(err) - } - }) - h := s.withCSRF(mux) - ser := nettest.NewHTTPServer(nettest.GetNetwork(t), h) - defer ser.Close() - - jar, err := cookiejar.New(nil) - if err != nil { - t.Fatalf("unable to construct cookie jar: %v", err) + tests := []struct { + name string + method string + secFetchSite string + host string + origin string + originOverride string + wantError bool + }{ + { + name: "GET requests with no header are allowed", + method: "GET", + }, + { + name: "POST requests with same-origin are allowed", + method: "POST", + secFetchSite: "same-origin", + }, + { + name: "POST requests with cross-site are not allowed", + method: "POST", + secFetchSite: "cross-site", + wantError: true, + }, + { + name: "POST requests with unknown sec-fetch-site values are not allowed", + method: "POST", + secFetchSite: "new-unknown-value", + wantError: true, + }, + { + name: "POST requests with none are not allowed", + method: "POST", + secFetchSite: "none", + wantError: true, + }, + { + name: "POST requests with no sec-fetch-site header but matching host and origin are allowed", + method: "POST", + host: "example.com", + origin: "https://example.com", + }, + { + name: "POST requests with no sec-fetch-site and non-matching host and origin are not allowed", + method: "POST", + host: "example.com", + origin: "https://example.net", + wantError: true, + }, + { + name: "POST requests with no sec-fetch-site and and origin that matches the override are allowed", + method: "POST", + originOverride: "example.net", + host: "internal.example.foo", // Host can be changed by reverse proxies + origin: "http://example.net", + }, } - client := ser.Client() - client.Jar = jar - - // make GET request to populate cookie jar - resp, err := client.Get(ser.URL + "/test/csrf-token") - if err != nil { - t.Fatalf("unable to make request: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status: %v", resp.Status) - } - tokenBytes, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatalf("unable to read body: %v", err) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "OK") + }) - csrfToken := strings.TrimSpace(string(tokenBytes)) - if csrfToken == "" { - t.Fatal("empty csrf token") - } + s := &Server{ + originOverride: tt.originOverride, + } + withCSRF := s.csrfProtect(handler) - // make a POST request without the CSRF header; ensure it fails - resp, err = client.Post(ser.URL+"/test/csrf-protected", "text/plain", nil) - if err != nil { - t.Fatalf("unable to make request: %v", err) - } - if resp.StatusCode != http.StatusForbidden { - t.Fatalf("unexpected status: %v", resp.Status) - } + r := httptest.NewRequest(tt.method, "http://example.com/", nil) + if tt.secFetchSite != "" { + r.Header.Set("Sec-Fetch-Site", tt.secFetchSite) + } + if tt.host != "" { + r.Host = tt.host + } + if tt.origin != "" { + r.Header.Set("Origin", tt.origin) + } - // make a POST request with the CSRF header; ensure it succeeds - req, err := http.NewRequest("POST", ser.URL+"/test/csrf-protected", nil) - if err != nil { - t.Fatalf("error building request: %v", err) - } - req.Header.Set("X-CSRF-Token", csrfToken) - resp, err = client.Do(req) - if err != nil { - t.Fatalf("unable to make request: %v", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status: %v", resp.Status) - } - defer resp.Body.Close() - out, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatalf("unable to read body: %v", err) - } - if string(out) != "ok" { - t.Fatalf("unexpected body: %q", out) + w := httptest.NewRecorder() + withCSRF.ServeHTTP(w, r) + res := w.Result() + defer res.Body.Close() + if tt.wantError { + if res.StatusCode != http.StatusForbidden { + t.Errorf("expected status forbidden, got %v", res.StatusCode) + } + return + } + if res.StatusCode != http.StatusOK { + t.Errorf("expected status ok, got %v", res.StatusCode) + } + }) } } diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 12fb5cf2e5a65..782603df09709 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -144,8 +144,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ github.com/google/uuid from github.com/prometheus-community/pro-bing+ - github.com/gorilla/csrf from tailscale.com/client/web - github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L 💣 github.com/illarion/gonotify/v3 from tailscale.com/net/dns L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 @@ -1112,13 +1110,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/tailscale/web-client-prebuilt+ - encoding from encoding/gob+ + encoding from encoding/json+ encoding/asn1 from crypto/x509+ encoding/base32 from github.com/fxamacker/cbor/v2+ encoding/base64 from encoding/json+ encoding/binary from compress/gzip+ encoding/csv from github.com/spf13/pflag - encoding/gob from github.com/gorilla/securecookie encoding/hex from crypto/x509+ encoding/json from expvar+ encoding/pem from crypto/tls+ @@ -1140,7 +1137,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ hash/fnv from google.golang.org/protobuf/internal/detrand hash/maphash from go4.org/mem html from html/template+ - html/template from github.com/gorilla/csrf+ + html/template from tailscale.com/util/eventbus internal/abi from crypto/x509/internal/macos+ internal/asan from internal/runtime/maps+ internal/bisect from internal/godebug @@ -1172,7 +1169,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ internal/runtime/math from internal/runtime/maps+ internal/runtime/sys from crypto/subtle+ L internal/runtime/syscall from runtime+ - internal/saferio from debug/pe+ + W internal/saferio from debug/pe internal/singleflight from net internal/stringslite from embed+ internal/sync from sync+ diff --git a/cmd/tailscale/cli/web.go b/cmd/tailscale/cli/web.go index e209d388eb123..5e1821dd011eb 100644 --- a/cmd/tailscale/cli/web.go +++ b/cmd/tailscale/cli/web.go @@ -43,6 +43,7 @@ Tailscale, as opposed to a CLI or a native app. webf.BoolVar(&webArgs.cgi, "cgi", false, "run as CGI script") webf.StringVar(&webArgs.prefix, "prefix", "", "URL prefix added to requests (for cgi or reverse proxies)") webf.BoolVar(&webArgs.readonly, "readonly", false, "run web UI in read-only mode") + webf.StringVar(&webArgs.origin, "origin", "", "origin at which the web UI is served (if behind a reverse proxy or used with cgi)") return webf })(), Exec: runWeb, @@ -53,6 +54,7 @@ var webArgs struct { cgi bool prefix string readonly bool + origin string } func tlsConfigFromEnvironment() *tls.Config { @@ -115,6 +117,9 @@ func runWeb(ctx context.Context, args []string) error { if webArgs.readonly { opts.Mode = web.ReadOnlyServerMode } + if webArgs.origin != "" { + opts.OriginOverride = webArgs.origin + } webServer, err := web.NewServer(opts) if err != nil { log.Printf("tailscale.web: %v", err) diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 03bf2f94ca4df..8c3b404b1fadc 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -27,8 +27,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ DW github.com/google/uuid from tailscale.com/clientupdate+ - github.com/gorilla/csrf from tailscale.com/client/web - github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink @@ -319,12 +317,11 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/peterbourgon/ff/v3+ - encoding from encoding/gob+ + encoding from encoding/json+ encoding/asn1 from crypto/x509+ encoding/base32 from github.com/fxamacker/cbor/v2+ encoding/base64 from encoding/json+ encoding/binary from compress/gzip+ - encoding/gob from github.com/gorilla/securecookie encoding/hex from crypto/x509+ encoding/json from expvar+ encoding/pem from crypto/tls+ @@ -338,7 +335,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep hash/crc32 from compress/gzip+ hash/maphash from go4.org/mem html from html/template+ - html/template from github.com/gorilla/csrf+ + html/template from tailscale.com/util/eventbus image from github.com/skip2/go-qrcode+ image/color from github.com/skip2/go-qrcode+ image/png from github.com/skip2/go-qrcode @@ -372,7 +369,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep internal/runtime/math from internal/runtime/maps+ internal/runtime/sys from crypto/subtle+ L internal/runtime/syscall from runtime+ - internal/saferio from debug/pe+ + W internal/saferio from debug/pe internal/singleflight from net internal/stringslite from embed+ internal/sync from sync+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 6de0ddc391794..d9a9cac65e5f3 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -123,8 +123,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ DW github.com/google/uuid from tailscale.com/clientupdate+ - github.com/gorilla/csrf from tailscale.com/client/web - github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L 💣 github.com/illarion/gonotify/v3 from tailscale.com/net/dns L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 @@ -590,12 +588,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/tailscale/web-client-prebuilt+ - encoding from encoding/gob+ + encoding from encoding/json+ encoding/asn1 from crypto/x509+ encoding/base32 from github.com/fxamacker/cbor/v2+ encoding/base64 from encoding/json+ encoding/binary from compress/gzip+ - encoding/gob from github.com/gorilla/securecookie encoding/hex from crypto/x509+ encoding/json from expvar+ encoding/pem from crypto/tls+ @@ -609,7 +606,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de hash/crc32 from compress/gzip+ hash/maphash from go4.org/mem html from html/template+ - html/template from github.com/gorilla/csrf+ + html/template from tailscale.com/util/eventbus internal/abi from crypto/x509/internal/macos+ internal/asan from internal/runtime/maps+ internal/bisect from internal/godebug @@ -640,7 +637,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de internal/runtime/math from internal/runtime/maps+ internal/runtime/sys from crypto/subtle+ L internal/runtime/syscall from runtime+ - internal/saferio from debug/pe+ + W internal/saferio from debug/pe internal/singleflight from net internal/stringslite from embed+ internal/sync from sync+ diff --git a/tsnet/depaware.txt b/tsnet/depaware.txt index 662752554d02e..3b705f68023ee 100644 --- a/tsnet/depaware.txt +++ b/tsnet/depaware.txt @@ -113,8 +113,6 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ DWI github.com/google/uuid from github.com/prometheus-community/pro-bing+ - LDW github.com/gorilla/csrf from tailscale.com/client/web - LDW github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L 💣 github.com/illarion/gonotify/v3 from tailscale.com/net/dns L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 @@ -534,12 +532,11 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/tailscale/web-client-prebuilt+ - encoding from encoding/gob+ + encoding from encoding/json+ encoding/asn1 from crypto/x509+ encoding/base32 from github.com/fxamacker/cbor/v2+ encoding/base64 from encoding/json+ encoding/binary from compress/gzip+ - LDW encoding/gob from github.com/gorilla/securecookie encoding/hex from crypto/x509+ encoding/json from expvar+ encoding/pem from crypto/tls+ @@ -553,7 +550,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) hash/crc32 from compress/gzip+ hash/maphash from go4.org/mem html from html/template+ - LDW html/template from github.com/gorilla/csrf+ + LDW html/template from tailscale.com/util/eventbus internal/abi from crypto/x509/internal/macos+ internal/asan from internal/runtime/maps+ internal/bisect from internal/godebug @@ -584,7 +581,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) internal/runtime/math from internal/runtime/maps+ internal/runtime/sys from crypto/subtle+ LA internal/runtime/syscall from runtime+ - LDW internal/saferio from debug/pe+ + W internal/saferio from debug/pe internal/singleflight from net internal/stringslite from embed+ internal/sync from sync+ From 7a5af6e6e7d4938923378fd93418615934bad8d8 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 21 May 2025 20:30:55 -0700 Subject: [PATCH 005/263] ssh/tailssh: exclude Android from Linux build tags As noted in #16048, the ./ssh/tailssh package failed to build on Android, because GOOS=android also matches the "linux" build tag. Exclude Android like iOS is excluded from macOS (darwin). This now works: $ GOOS=android go install ./ipn/ipnlocal ./ssh/tailssh The original PR at #16048 is also fine, but this stops the problem earlier. Updates #16048 Change-Id: Ie4a6f6966a012e510c9cb11dd0d1fa88c48fac37 Signed-off-by: Brad Fitzpatrick --- .github/workflows/test.yml | 4 ++-- ssh/tailssh/incubator.go | 2 +- ssh/tailssh/incubator_linux.go | 2 +- ssh/tailssh/tailssh.go | 2 +- ssh/tailssh/user.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fcd39e391caef..8cbb6f351c66b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -305,7 +305,7 @@ jobs: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: build some - run: ./tool/go build ./ipn/... ./wgengine/ ./types/... ./control/controlclient + run: ./tool/go build ./ipn/... ./ssh/tailssh ./wgengine/ ./types/... ./control/controlclient env: GOOS: ios GOARCH: arm64 @@ -375,7 +375,7 @@ jobs: # some Android breakages early. # TODO(bradfitz): better; see https://github.com/tailscale/tailscale/issues/4482 - name: build some - run: ./tool/go install ./net/netns ./ipn/ipnlocal ./wgengine/magicsock/ ./wgengine/ ./wgengine/router/ ./wgengine/netstack ./util/dnsname/ ./ipn/ ./net/netmon ./wgengine/router/ ./tailcfg/ ./types/logger/ ./net/dns ./hostinfo ./version + run: ./tool/go install ./net/netns ./ipn/ipnlocal ./wgengine/magicsock/ ./wgengine/ ./wgengine/router/ ./wgengine/netstack ./util/dnsname/ ./ipn/ ./net/netmon ./wgengine/router/ ./tailcfg/ ./types/logger/ ./net/dns ./hostinfo ./version ./ssh/tailssh env: GOOS: android GOARCH: arm64 diff --git a/ssh/tailssh/incubator.go b/ssh/tailssh/incubator.go index 442fedcf22242..9e1a9ea94e424 100644 --- a/ssh/tailssh/incubator.go +++ b/ssh/tailssh/incubator.go @@ -7,7 +7,7 @@ // and groups to the specified `--uid`, `--gid` and `--groups`, and // then launches the requested `--cmd`. -//go:build linux || (darwin && !ios) || freebsd || openbsd +//go:build (linux && !android) || (darwin && !ios) || freebsd || openbsd package tailssh diff --git a/ssh/tailssh/incubator_linux.go b/ssh/tailssh/incubator_linux.go index bcbe0e240a24d..4dfb9f27cc097 100644 --- a/ssh/tailssh/incubator_linux.go +++ b/ssh/tailssh/incubator_linux.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux +//go:build linux && !android package tailssh diff --git a/ssh/tailssh/tailssh.go b/ssh/tailssh/tailssh.go index e42f09bdfb4e4..19a2b11fd3800 100644 --- a/ssh/tailssh/tailssh.go +++ b/ssh/tailssh/tailssh.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux || (darwin && !ios) || freebsd || openbsd || plan9 +//go:build (linux && !android) || (darwin && !ios) || freebsd || openbsd || plan9 // Package tailssh is an SSH server integrated into Tailscale. package tailssh diff --git a/ssh/tailssh/user.go b/ssh/tailssh/user.go index 097f0d296e92c..ac92c762a875e 100644 --- a/ssh/tailssh/user.go +++ b/ssh/tailssh/user.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux || (darwin && !ios) || freebsd || openbsd || plan9 +//go:build (linux && !android) || (darwin && !ios) || freebsd || openbsd || plan9 package tailssh From 00a7dd180a7582502773c71a7ea52e051dbc67cd Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 23 May 2025 12:23:58 +0100 Subject: [PATCH 006/263] cmd/k8s-operator: validate Service tags, catch duplicate Tailscale Services (#16058) Validate that any tags that users have specified via tailscale.com/tags annotation are valid Tailscale ACL tags. Validate that no more than one HA Tailscale Kubernetes Services in a single cluster refer to the same Tailscale Service. Updates tailscale/tailscale#16054 Updates tailscale/tailscale#16035 Signed-off-by: Irbe Krumina --- cmd/k8s-operator/ingress-for-pg.go | 34 ++++++++---- cmd/k8s-operator/ingress-for-pg_test.go | 5 +- cmd/k8s-operator/operator_test.go | 2 +- cmd/k8s-operator/svc-for-pg.go | 32 +++++++++-- cmd/k8s-operator/svc-for-pg_test.go | 73 +++++++++++++++++++++++-- cmd/k8s-operator/svc.go | 1 + 6 files changed, 122 insertions(+), 25 deletions(-) diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 9cdd9cba96fca..4779014f37bc7 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -660,14 +660,9 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki var errs []error // Validate tags if present - if tstr, ok := ing.Annotations[AnnotationTags]; ok { - tags := strings.Split(tstr, ",") - for _, tag := range tags { - tag = strings.TrimSpace(tag) - if err := tailcfg.CheckTag(tag); err != nil { - errs = append(errs, fmt.Errorf("tailscale.com/tags annotation contains invalid tag %q: %w", tag, err)) - } - } + violations := tagViolations(ing) + if len(violations) > 0 { + errs = append(errs, fmt.Errorf("Ingress contains invalid tags: %v", strings.Join(violations, ","))) } // Validate TLS configuration @@ -699,8 +694,8 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki return errors.Join(errs...) } for _, i := range ingList.Items { - if r.shouldExpose(&i) && hostnameForIngress(&i) == hostname && i.Name != ing.Name { - errs = append(errs, fmt.Errorf("found duplicate Ingress %q for hostname %q - multiple Ingresses for the same hostname in the same cluster are not allowed", i.Name, hostname)) + if r.shouldExpose(&i) && hostnameForIngress(&i) == hostname && i.UID != ing.UID { + errs = append(errs, fmt.Errorf("found duplicate Ingress %q for hostname %q - multiple Ingresses for the same hostname in the same cluster are not allowed", client.ObjectKeyFromObject(&i), hostname)) } } return errors.Join(errs...) @@ -1113,3 +1108,22 @@ func isErrorTailscaleServiceNotFound(err error) bool { ok := errors.As(err, &errResp) return ok && errResp.Status == http.StatusNotFound } + +func tagViolations(obj client.Object) []string { + var violations []string + if obj == nil { + return nil + } + tags, ok := obj.GetAnnotations()[AnnotationTags] + if !ok { + return nil + } + + for _, tag := range strings.Split(tags, ",") { + tag = strings.TrimSpace(tag) + if err := tailcfg.CheckTag(tag); err != nil { + violations = append(violations, fmt.Sprintf("invalid tag %q: %v", tag, err)) + } + } + return violations +} diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 3330da8d001b0..9ce90f7716068 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -272,6 +272,7 @@ func TestValidateIngress(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test-ingress", Namespace: "default", + UID: types.UID("1234-UID"), Annotations: map[string]string{ AnnotationProxyGroup: "test-pg", }, @@ -339,7 +340,7 @@ func TestValidateIngress(t *testing.T) { }, }, pg: readyProxyGroup, - wantErr: "tailscale.com/tags annotation contains invalid tag \"tag:invalid!\": tag names can only contain numbers, letters, or dashes", + wantErr: "Ingress contains invalid tags: invalid tag \"tag:invalid!\": tag names can only contain numbers, letters, or dashes", }, { name: "multiple_TLS_entries", @@ -417,7 +418,7 @@ func TestValidateIngress(t *testing.T) { }, }, }}, - wantErr: `found duplicate Ingress "existing-ingress" for hostname "test" - multiple Ingresses for the same hostname in the same cluster are not allowed`, + wantErr: `found duplicate Ingress "default/existing-ingress" for hostname "test" - multiple Ingresses for the same hostname in the same cluster are not allowed`, }, } diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index f4b0db01cf108..33bf23e844d9a 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -1804,7 +1804,7 @@ func Test_metricsResourceCreation(t *testing.T) { func TestIgnorePGService(t *testing.T) { // NOTE: creating proxygroup stuff just to be sure that it's all ignored - _, _, fc, _ := setupServiceTest(t) + _, _, fc, _, _ := setupServiceTest(t) ft := &fakeTSClient{} zl, err := zap.NewDevelopment() diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index 779f2714e72f5..c9b5b8ae69a18 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -169,12 +169,9 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin return false, nil } - // Validate Service configuration - if violations := validateService(svc); len(violations) > 0 { - msg := fmt.Sprintf("unable to provision proxy resources: invalid Service: %s", strings.Join(violations, ", ")) - r.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg) - r.logger.Error(msg) - tsoperator.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, msg, r.clock, logger) + if err := r.validateService(ctx, svc, pg); err != nil { + r.recorder.Event(svc, corev1.EventTypeWarning, reasonIngressSvcInvalid, err.Error()) + tsoperator.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, err.Error(), r.clock, logger) return false, nil } @@ -857,3 +854,26 @@ func (r *HAServiceReconciler) checkEndpointsReady(ctx context.Context, svc *core logger.Debugf("could not find any ready Endpoints in EndpointSlice") return false, nil } + +func (r *HAServiceReconciler) validateService(ctx context.Context, svc *corev1.Service, pg *tsapi.ProxyGroup) error { + var errs []error + if pg.Spec.Type != tsapi.ProxyGroupTypeIngress { + errs = append(errs, fmt.Errorf("ProxyGroup %q is of type %q but must be of type %q", + pg.Name, pg.Spec.Type, tsapi.ProxyGroupTypeIngress)) + } + if violations := validateService(svc); len(violations) > 0 { + errs = append(errs, fmt.Errorf("invalid Service: %s", strings.Join(violations, ", "))) + } + svcList := &corev1.ServiceList{} + if err := r.List(ctx, svcList); err != nil { + errs = append(errs, fmt.Errorf("[unexpected] error listing Services: %w", err)) + return errors.Join(errs...) + } + svcName := nameForService(svc) + for _, s := range svcList.Items { + if r.shouldExpose(&s) && nameForService(&s) == svcName && s.UID != svc.UID { + errs = append(errs, fmt.Errorf("found duplicate Service %q for hostname %q - multiple HA Services for the same hostname in the same cluster are not allowed", client.ObjectKeyFromObject(&s), svcName)) + } + } + return errors.Join(errs...) +} diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go index 4bb633cb8ff66..ecd60af50f220 100644 --- a/cmd/k8s-operator/svc-for-pg_test.go +++ b/cmd/k8s-operator/svc-for-pg_test.go @@ -12,6 +12,7 @@ import ( "math/rand/v2" "net/netip" "testing" + "time" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -33,7 +34,7 @@ import ( ) func TestServicePGReconciler(t *testing.T) { - svcPGR, stateSecret, fc, ft := setupServiceTest(t) + svcPGR, stateSecret, fc, ft, _ := setupServiceTest(t) svcs := []*corev1.Service{} config := []string{} for i := range 4 { @@ -79,7 +80,7 @@ func TestServicePGReconciler(t *testing.T) { } func TestServicePGReconciler_UpdateHostname(t *testing.T) { - svcPGR, stateSecret, fc, ft := setupServiceTest(t) + svcPGR, stateSecret, fc, ft, _ := setupServiceTest(t) cip := "4.1.6.7" svc, _ := setupTestService(t, "test-service", "", cip, fc, stateSecret) @@ -110,7 +111,7 @@ func TestServicePGReconciler_UpdateHostname(t *testing.T) { } } -func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, client.Client, *fakeTSClient) { +func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, client.Client, *fakeTSClient, *tstest.Clock) { // Pre-create the ProxyGroup pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ @@ -215,14 +216,74 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien lc: lc, } - return svcPGR, pgStateSecret, fc, ft + return svcPGR, pgStateSecret, fc, ft, cl +} + +func TestValidateService(t *testing.T) { + // Test that no more than one Kubernetes Service in a cluster refers to the same Tailscale Service. + pgr, _, lc, _, cl := setupServiceTest(t) + svc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app", + Namespace: "ns-1", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + "tailscale.com/hostname": "my-app", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: ptr.To("tailscale"), + }, + } + svc2 := &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app2", + Namespace: "ns-2", + UID: types.UID("1235-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + "tailscale.com/hostname": "my-app", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.5", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: ptr.To("tailscale"), + }, + } + wantSvc := &corev1.Service{ + ObjectMeta: svc.ObjectMeta, + TypeMeta: svc.TypeMeta, + Spec: svc.Spec, + Status: corev1.ServiceStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.IngressSvcValid), + Status: metav1.ConditionFalse, + Reason: reasonIngressSvcInvalid, + LastTransitionTime: metav1.NewTime(cl.Now().Truncate(time.Second)), + Message: `found duplicate Service "ns-2/my-app2" for hostname "my-app" - multiple HA Services for the same hostname in the same cluster are not allowed`, + }, + }, + }, + } + + mustCreate(t, lc, svc) + mustCreate(t, lc, svc2) + expectReconciled(t, pgr, svc.Namespace, svc.Name) + expectEqual(t, lc, wantSvc) } func TestServicePGReconciler_MultiCluster(t *testing.T) { var ft *fakeTSClient var lc localClient for i := 0; i <= 10; i++ { - pgr, stateSecret, fc, fti := setupServiceTest(t) + pgr, stateSecret, fc, fti, _ := setupServiceTest(t) if i == 0 { ft = fti lc = pgr.lc @@ -250,7 +311,7 @@ func TestServicePGReconciler_MultiCluster(t *testing.T) { } func TestIgnoreRegularService(t *testing.T) { - pgr, _, fc, ft := setupServiceTest(t) + pgr, _, fc, ft, _ := setupServiceTest(t) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index d6a6f440feda9..c880f59f5012a 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -392,6 +392,7 @@ func validateService(svc *corev1.Service) []string { violations = append(violations, fmt.Sprintf("invalid Tailscale hostname %q, use %q annotation to override: %s", svcName, AnnotationHostname, err)) } } + violations = append(violations, tagViolations(svc)...) return violations } From 4a11514db5fc4fe77225cd032f9f76cc8610b9ce Mon Sep 17 00:00:00 2001 From: Zach Buchheit Date: Fri, 23 May 2025 14:17:28 -0700 Subject: [PATCH 007/263] ipn/ipnlocal: improve dohQuery error to suggest `?dns=` and `?q=` (#16056) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, a missing or invalid `dns` parameter on GET `/dns-query` returned only “missing ‘dns’ parameter”. Now the error message guides users to use `?dns=` or `?q=`. Updates: #16055 Signed-off-by: Zach Buchheit --- ipn/ipnlocal/peerapi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipn/ipnlocal/peerapi.go b/ipn/ipnlocal/peerapi.go index 84aaecf7efb78..60dd4102413f0 100644 --- a/ipn/ipnlocal/peerapi.go +++ b/ipn/ipnlocal/peerapi.go @@ -859,7 +859,7 @@ func dohQuery(r *http.Request) (dnsQuery []byte, publicErr string) { case "GET": q64 := r.FormValue("dns") if q64 == "" { - return nil, "missing 'dns' parameter" + return nil, "missing ‘dns’ parameter; try '?dns=' (DoH standard) or use '?q=' for JSON debug mode" } if base64.RawURLEncoding.DecodedLen(len(q64)) > maxQueryLen { return nil, "query too large" From 4980869977302612c77518adbd6351f568c264a4 Mon Sep 17 00:00:00 2001 From: Tim Klocke Date: Sat, 24 May 2025 18:05:57 +0200 Subject: [PATCH 008/263] cmd/tsidp: Fix sending string for refresh_token In accordance with the OIDC/OAuth 2.0 protocol, do not send an empty refresh_token and instead omit the field when empty. Fixes https://github.com/tailscale/tailscale/issues/16073 Signed-off-by: Tim Klocke --- cmd/tsidp/tsidp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tsidp/tsidp.go b/cmd/tsidp/tsidp.go index e2b777fa1b68e..2d9450e9675b4 100644 --- a/cmd/tsidp/tsidp.go +++ b/cmd/tsidp/tsidp.go @@ -795,7 +795,7 @@ type oidcTokenResponse struct { IDToken string `json:"id_token"` TokenType string `json:"token_type"` AccessToken string `json:"access_token"` - RefreshToken string `json:"refresh_token"` + RefreshToken string `json:"refresh_token,omitempty"` ExpiresIn int `json:"expires_in"` } From 09582bdc009fc6faeb5a17b657570fd2d7b9dd3c Mon Sep 17 00:00:00 2001 From: Raj Singh Date: Sat, 24 May 2025 18:16:29 -0400 Subject: [PATCH 009/263] cmd/tsidp: add web UI for managing OIDC clients (#16068) Add comprehensive web interface at ui for managing OIDC clients, similar to tsrecorder's design. Features include list view, create/edit forms with validation, client secret management, delete functionality with confirmation dialogs, responsive design, and restricted tailnet access only. Fixes #16067 Signed-off-by: Raj Singh --- cmd/tsidp/tsidp.go | 8 +- cmd/tsidp/ui-edit.html | 199 +++++++++++++++++ cmd/tsidp/ui-header.html | 53 +++++ cmd/tsidp/ui-list.html | 73 +++++++ cmd/tsidp/ui-style.css | 446 +++++++++++++++++++++++++++++++++++++++ cmd/tsidp/ui.go | 325 ++++++++++++++++++++++++++++ 6 files changed, 1097 insertions(+), 7 deletions(-) create mode 100644 cmd/tsidp/ui-edit.html create mode 100644 cmd/tsidp/ui-header.html create mode 100644 cmd/tsidp/ui-list.html create mode 100644 cmd/tsidp/ui-style.css create mode 100644 cmd/tsidp/ui.go diff --git a/cmd/tsidp/tsidp.go b/cmd/tsidp/tsidp.go index 2d9450e9675b4..5df99e1b82232 100644 --- a/cmd/tsidp/tsidp.go +++ b/cmd/tsidp/tsidp.go @@ -452,13 +452,7 @@ func (s *idpServer) newMux() *http.ServeMux { mux.HandleFunc("/userinfo", s.serveUserInfo) mux.HandleFunc("/token", s.serveToken) mux.HandleFunc("/clients/", s.serveClients) - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/" { - io.WriteString(w, "

Tailscale OIDC IdP

") - return - } - http.Error(w, "tsidp: not found", http.StatusNotFound) - }) + mux.HandleFunc("/", s.handleUI) return mux } diff --git a/cmd/tsidp/ui-edit.html b/cmd/tsidp/ui-edit.html new file mode 100644 index 0000000000000..d463981aa34df --- /dev/null +++ b/cmd/tsidp/ui-edit.html @@ -0,0 +1,199 @@ + + + + + {{if .IsNew}}Add New Client{{else}}Edit Client{{end}} - Tailscale OIDC Identity Provider + + + + + + + {{template "header"}} + +
+
+
+

+ {{if .IsNew}}Add New OIDC Client{{else}}Edit OIDC Client{{end}} +

+ ← Back to Clients +
+ + {{if .Success}} +
+ {{.Success}} +
+ {{end}} + + {{if .Error}} +
+ {{.Error}} +
+ {{end}} + + {{if and .Secret .IsNew}} +
+

Client Created Successfully!

+

⚠️ Save both the Client ID and Secret now! The secret will not be shown again.

+ +
+ +
+ + +
+
+ +
+ +
+ + +
+
+
+ {{end}} + + {{if and .Secret .IsEdit}} +
+

New Client Secret

+

⚠️ Save this secret now! It will not be shown again.

+
+ + +
+
+ {{end}} + +
+
+ + +
+ A descriptive name for this OIDC client (optional). +
+
+ +
+ + +
+ The URL where users will be redirected after authentication. +
+
+ + {{if .IsEdit}} +
+ + +
+ The client ID cannot be changed. +
+
+ {{end}} + +
+ + + {{if .IsEdit}} + + + + {{end}} +
+
+ + {{if .IsEdit}} +
+

Client Information

+
+
Client ID
+
{{.ID}}
+
Secret Status
+
+ {{if .HasSecret}} + Secret configured + {{else}} + No secret + {{end}} +
+
+
+ {{end}} +
+
+ + + + \ No newline at end of file diff --git a/cmd/tsidp/ui-header.html b/cmd/tsidp/ui-header.html new file mode 100644 index 0000000000000..68e9bc0df4d7e --- /dev/null +++ b/cmd/tsidp/ui-header.html @@ -0,0 +1,53 @@ +
+ +
\ No newline at end of file diff --git a/cmd/tsidp/ui-list.html b/cmd/tsidp/ui-list.html new file mode 100644 index 0000000000000..d45b883494403 --- /dev/null +++ b/cmd/tsidp/ui-list.html @@ -0,0 +1,73 @@ + + + + Tailscale OIDC Identity Provider + + + + + + {{template "header"}} + +
+
+
+

OIDC Clients

+ {{if .}} +

{{len .}} client{{if ne (len .) 1}}s{{end}} configured

+ {{end}} +
+ Add New Client +
+ + {{if .}} + + + + + + + + + + + + {{range .}} + + + + + + + + {{end}} + +
NameClient IDRedirect URIStatusActions
+ {{if .Name}} + {{.Name}} + {{else}} + Unnamed Client + {{end}} + + {{.ID}} + + {{.RedirectURI}} + + {{if .HasSecret}} + Active + {{else}} + No Secret + {{end}} + + Edit +
+ {{else}} +
+

No OIDC clients configured

+

Create your first OIDC client to get started with authentication.

+ Add New Client +
+ {{end}} +
+ + \ No newline at end of file diff --git a/cmd/tsidp/ui-style.css b/cmd/tsidp/ui-style.css new file mode 100644 index 0000000000000..148ec3030d0b3 --- /dev/null +++ b/cmd/tsidp/ui-style.css @@ -0,0 +1,446 @@ +:root { + --tw-text-opacity: 1; + --color-gray-100: 247 245 244; + --color-gray-200: 238 235 234; + --color-gray-500: 112 110 109; + --color-gray-700: 46 45 45; + --color-gray-800: 35 34 34; + --color-gray-900: 31 30 30; + --color-bg-app: rgb(var(--color-gray-900) / 1); + --color-border-base: rgb(var(--color-gray-200) / 1); + --color-primary: 59 130 246; + --color-primary-hover: 37 99 235; + --color-secondary: 107 114 128; + --color-secondary-hover: 75 85 99; + --color-success: 34 197 94; + --color-warning: 245 158 11; + --color-danger: 239 68 68; + --color-danger-hover: 220 38 38; +} + +* { + box-sizing: border-box; + padding: 0; + margin: 0; +} + +body { + font-family: Inter, -apple-system, BlinkMacSystemFont, Helvetica, Arial, + sans-serif; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + font-size: 16px; + line-height: 1.4; + margin: 0; + background-color: var(--color-bg-app); + color: rgb(var(--color-gray-200)); +} + +a { + text-decoration: none; + color: inherit; +} + +header { + margin-top: 40px; +} +header nav { + margin: 0 auto; + max-width: 1120px; + display: flex; + align-items: center; + justify-content: center; +} +header nav h1 { + display: inline; + font-weight: 600; + font-size: 1.125rem; + line-height: 1.75rem; + margin-left: 0.75rem; +} + +main { + margin: 40px auto 60px auto; + max-width: 1120px; + padding: 0 20px; +} + +/* Header actions */ +.header-actions { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 2rem; +} + +.header-actions h2 { + font-size: 1.5rem; + font-weight: 600; + margin: 0 0 0.25rem 0; +} + +.client-count { + font-size: 0.875rem; + color: rgb(var(--color-gray-500)); + margin: 0; +} + +/* Buttons */ +.btn { + display: inline-flex; + align-items: center; + padding: 8px 16px; + border-radius: 6px; + font-size: 14px; + font-weight: 500; + text-decoration: none; + border: none; + cursor: pointer; + transition: all 0.2s ease; +} + +.btn-small { + padding: 4px 8px; + font-size: 12px; +} + +.btn-primary { + background-color: rgb(var(--color-primary)); + color: white; +} + +.btn-primary:hover { + background-color: rgb(var(--color-primary-hover)); +} + +.btn-secondary { + background-color: rgb(var(--color-secondary)); + color: white; +} + +.btn-secondary:hover { + background-color: rgb(var(--color-secondary-hover)); +} + +.btn-success { + background-color: rgb(var(--color-success)); + color: white; +} + +.btn-warning { + background-color: rgb(var(--color-warning)); + color: white; +} + +.btn-danger { + background-color: rgb(var(--color-danger)); + color: white; +} + +.btn-danger:hover { + background-color: rgb(var(--color-danger-hover)); +} + +/* Tables */ +table { + width: 100%; + border-spacing: 0; + border: 1px solid rgb(var(--color-gray-700)); + border-bottom-width: 0; + border-radius: 8px; + overflow: hidden; +} + +td { + border: 0 solid rgb(var(--color-gray-700)); + border-bottom-width: 1px; + padding: 12px 16px; +} + +thead td { + text-transform: uppercase; + color: rgb(var(--color-gray-500) / var(--tw-text-opacity)); + font-size: 12px; + letter-spacing: 0.08em; + font-weight: 600; + background-color: rgb(var(--color-gray-800)); +} + +tbody tr:hover { + background-color: rgb(var(--color-gray-800)); +} + +/* Client display elements */ +.client-id { + font-family: "SF Mono", SFMono-Regular, ui-monospace, "DejaVu Sans Mono", + Menlo, Consolas, monospace; + font-size: 12px; + background-color: rgb(var(--color-gray-800)); + padding: 2px 6px; + border-radius: 4px; + color: rgb(var(--color-gray-200)); +} + +.redirect-uri { + font-size: 14px; + color: rgb(var(--color-gray-200)); + word-break: break-all; +} + +.status-active { + color: rgb(var(--color-success)); + font-weight: 500; +} + +.status-inactive { + color: rgb(var(--color-gray-500)); + font-weight: 500; +} + +.text-muted { + color: rgb(var(--color-gray-500)); +} + +/* Empty state */ +.empty-state { + text-align: center; + padding: 60px 20px; + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 8px; + background-color: rgb(var(--color-gray-800) / 0.5); +} + +.empty-state h3 { + font-size: 1.25rem; + font-weight: 600; + margin-bottom: 0.5rem; + color: rgb(var(--color-gray-200)); +} + +.empty-state p { + color: rgb(var(--color-gray-500)); + margin-bottom: 1.5rem; +} + +/* Forms */ +.form-container { + max-width: 600px; + margin: 0 auto; +} + +.form-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 2rem; +} + +.form-header h2 { + font-size: 1.5rem; + font-weight: 600; + margin: 0; +} + +.client-form { + background-color: rgb(var(--color-gray-800) / 0.5); + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 8px; + padding: 24px; + margin-bottom: 2rem; +} + +.form-group { + margin-bottom: 1.5rem; +} + +.form-group:last-child { + margin-bottom: 0; +} + +.form-group label { + display: block; + font-weight: 500; + margin-bottom: 0.5rem; + color: rgb(var(--color-gray-200)); +} + +.required { + color: rgb(var(--color-danger)); +} + +.form-input { + width: 100%; + padding: 10px 12px; + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 6px; + background-color: rgb(var(--color-gray-900)); + color: rgb(var(--color-gray-200)); + font-size: 14px; +} + +.form-input:focus { + outline: none; + border-color: rgb(var(--color-primary)); + box-shadow: 0 0 0 3px rgb(var(--color-primary) / 0.1); +} + +.form-input-readonly { + background-color: rgb(var(--color-gray-800)); + color: rgb(var(--color-gray-500)); +} + +.form-help { + font-size: 12px; + color: rgb(var(--color-gray-500)); + margin-top: 0.25rem; +} + +.form-actions { + display: flex; + gap: 1rem; + margin-top: 2rem; + padding-top: 1.5rem; + border-top: 1px solid rgb(var(--color-gray-700)); +} + +/* Alerts */ +.alert { + padding: 12px 16px; + border-radius: 6px; + margin-bottom: 1.5rem; + font-size: 14px; +} + +.alert-success { + background-color: rgb(var(--color-success) / 0.1); + border: 1px solid rgb(var(--color-success) / 0.3); + color: rgb(var(--color-success)); +} + +.alert-error { + background-color: rgb(var(--color-danger) / 0.1); + border: 1px solid rgb(var(--color-danger) / 0.3); + color: rgb(var(--color-danger)); +} + +/* Secret display */ +.secret-display { + background-color: rgb(var(--color-gray-800) / 0.5); + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 8px; + padding: 20px; + margin-bottom: 2rem; +} + +.secret-display h3 { + font-size: 1.125rem; + font-weight: 600; + margin-bottom: 0.5rem; + color: rgb(var(--color-gray-200)); +} + +.warning { + color: rgb(var(--color-warning)); + font-weight: 500; + margin-bottom: 1rem; +} + +.secret-field { + display: flex; + gap: 0.5rem; +} + +.secret-input { + flex: 1; + padding: 10px 12px; + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 6px; + background-color: rgb(var(--color-gray-900)); + color: rgb(var(--color-gray-200)); + font-family: "SF Mono", SFMono-Regular, ui-monospace, "DejaVu Sans Mono", + Menlo, Consolas, monospace; + font-size: 12px; +} + +/* Client info */ +.client-info { + background-color: rgb(var(--color-gray-800) / 0.5); + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 8px; + padding: 20px; +} + +.client-info h3 { + font-size: 1.125rem; + font-weight: 600; + margin-bottom: 1rem; + color: rgb(var(--color-gray-200)); +} + +.client-info dl { + display: grid; + grid-template-columns: auto 1fr; + gap: 0.5rem 1rem; + border: none; + border-radius: 0; + padding: 0; +} + +.client-info dt { + font-weight: 600; + color: rgb(var(--color-gray-400)); + border: none; + padding: 0; +} + +.client-info dd { + color: rgb(var(--color-gray-200)); + border: none; + padding: 0; +} + +.client-info code { + font-family: "SF Mono", SFMono-Regular, ui-monospace, "DejaVu Sans Mono", + Menlo, Consolas, monospace; + font-size: 12px; + background-color: rgb(var(--color-gray-800)); + padding: 2px 6px; + border-radius: 4px; + color: rgb(var(--color-gray-200)); +} + +/* Responsive design */ +@media (max-width: 768px) { + .header-actions { + flex-direction: column; + align-items: stretch; + gap: 1rem; + } + + .form-header { + flex-direction: column; + align-items: stretch; + gap: 1rem; + } + + .form-actions { + flex-direction: column; + } + + .secret-field { + flex-direction: column; + } + + table { + font-size: 14px; + } + + td { + padding: 8px 12px; + } + + .client-id { + font-size: 10px; + } +} \ No newline at end of file diff --git a/cmd/tsidp/ui.go b/cmd/tsidp/ui.go new file mode 100644 index 0000000000000..d37b64990cac8 --- /dev/null +++ b/cmd/tsidp/ui.go @@ -0,0 +1,325 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "bytes" + _ "embed" + "html/template" + "log" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "tailscale.com/util/rands" +) + +//go:embed ui-header.html +var headerHTML string + +//go:embed ui-list.html +var listHTML string + +//go:embed ui-edit.html +var editHTML string + +//go:embed ui-style.css +var styleCSS string + +var headerTmpl = template.Must(template.New("header").Parse(headerHTML)) +var listTmpl = template.Must(headerTmpl.New("list").Parse(listHTML)) +var editTmpl = template.Must(headerTmpl.New("edit").Parse(editHTML)) + +var processStart = time.Now() + +func (s *idpServer) handleUI(w http.ResponseWriter, r *http.Request) { + if isFunnelRequest(r) { + http.Error(w, "tsidp: UI not available over Funnel", http.StatusNotFound) + return + } + + switch r.URL.Path { + case "/": + s.handleClientsList(w, r) + return + case "/new": + s.handleNewClient(w, r) + return + case "/style.css": + http.ServeContent(w, r, "ui-style.css", processStart, strings.NewReader(styleCSS)) + return + } + + if strings.HasPrefix(r.URL.Path, "/edit/") { + s.handleEditClient(w, r) + return + } + + http.Error(w, "tsidp: not found", http.StatusNotFound) +} + +func (s *idpServer) handleClientsList(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + clients := make([]clientDisplayData, 0, len(s.funnelClients)) + for _, c := range s.funnelClients { + clients = append(clients, clientDisplayData{ + ID: c.ID, + Name: c.Name, + RedirectURI: c.RedirectURI, + HasSecret: c.Secret != "", + }) + } + s.mu.Unlock() + + sort.Slice(clients, func(i, j int) bool { + if clients[i].Name != clients[j].Name { + return clients[i].Name < clients[j].Name + } + return clients[i].ID < clients[j].ID + }) + + var buf bytes.Buffer + if err := listTmpl.Execute(&buf, clients); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + buf.WriteTo(w) +} + +func (s *idpServer) handleNewClient(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + if err := s.renderClientForm(w, clientDisplayData{IsNew: true}); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + if r.Method == "POST" { + if err := r.ParseForm(); err != nil { + http.Error(w, "Failed to parse form", http.StatusBadRequest) + return + } + + name := strings.TrimSpace(r.FormValue("name")) + redirectURI := strings.TrimSpace(r.FormValue("redirect_uri")) + + baseData := clientDisplayData{ + IsNew: true, + Name: name, + RedirectURI: redirectURI, + } + + if errMsg := validateRedirectURI(redirectURI); errMsg != "" { + s.renderFormError(w, baseData, errMsg) + return + } + + clientID := rands.HexString(32) + clientSecret := rands.HexString(64) + newClient := funnelClient{ + ID: clientID, + Secret: clientSecret, + Name: name, + RedirectURI: redirectURI, + } + + s.mu.Lock() + if s.funnelClients == nil { + s.funnelClients = make(map[string]*funnelClient) + } + s.funnelClients[clientID] = &newClient + err := s.storeFunnelClientsLocked() + s.mu.Unlock() + + if err != nil { + log.Printf("could not write funnel clients db: %v", err) + s.renderFormError(w, baseData, "Failed to save client") + return + } + + successData := clientDisplayData{ + ID: clientID, + Name: name, + RedirectURI: redirectURI, + Secret: clientSecret, + IsNew: true, + } + s.renderFormSuccess(w, successData, "Client created successfully! Save the client secret - it won't be shown again.") + return + } + + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) +} + +func (s *idpServer) handleEditClient(w http.ResponseWriter, r *http.Request) { + clientID := strings.TrimPrefix(r.URL.Path, "/edit/") + if clientID == "" { + http.Error(w, "Client ID required", http.StatusBadRequest) + return + } + + s.mu.Lock() + client, exists := s.funnelClients[clientID] + s.mu.Unlock() + + if !exists { + http.Error(w, "Client not found", http.StatusNotFound) + return + } + + if r.Method == "GET" { + data := createEditBaseData(client, client.Name, client.RedirectURI) + if err := s.renderClientForm(w, data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + if r.Method == "POST" { + action := r.FormValue("action") + + if action == "delete" { + s.mu.Lock() + delete(s.funnelClients, clientID) + err := s.storeFunnelClientsLocked() + s.mu.Unlock() + + if err != nil { + log.Printf("could not write funnel clients db: %v", err) + s.mu.Lock() + s.funnelClients[clientID] = client + s.mu.Unlock() + + baseData := createEditBaseData(client, client.Name, client.RedirectURI) + s.renderFormError(w, baseData, "Failed to delete client. Please try again.") + return + } + + http.Redirect(w, r, "/", http.StatusSeeOther) + return + } + + if action == "regenerate_secret" { + newSecret := rands.HexString(64) + s.mu.Lock() + s.funnelClients[clientID].Secret = newSecret + err := s.storeFunnelClientsLocked() + s.mu.Unlock() + + baseData := createEditBaseData(client, client.Name, client.RedirectURI) + baseData.HasSecret = true + + if err != nil { + log.Printf("could not write funnel clients db: %v", err) + s.renderFormError(w, baseData, "Failed to regenerate secret") + return + } + + baseData.Secret = newSecret + s.renderFormSuccess(w, baseData, "New client secret generated! Save it - it won't be shown again.") + return + } + + if err := r.ParseForm(); err != nil { + http.Error(w, "Failed to parse form", http.StatusBadRequest) + return + } + + name := strings.TrimSpace(r.FormValue("name")) + redirectURI := strings.TrimSpace(r.FormValue("redirect_uri")) + baseData := createEditBaseData(client, name, redirectURI) + + if errMsg := validateRedirectURI(redirectURI); errMsg != "" { + s.renderFormError(w, baseData, errMsg) + return + } + + s.mu.Lock() + s.funnelClients[clientID].Name = name + s.funnelClients[clientID].RedirectURI = redirectURI + err := s.storeFunnelClientsLocked() + s.mu.Unlock() + + if err != nil { + log.Printf("could not write funnel clients db: %v", err) + s.renderFormError(w, baseData, "Failed to update client") + return + } + + s.renderFormSuccess(w, baseData, "Client updated successfully!") + return + } + + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) +} + +type clientDisplayData struct { + ID string + Name string + RedirectURI string + Secret string + HasSecret bool + IsNew bool + IsEdit bool + Success string + Error string +} + +func (s *idpServer) renderClientForm(w http.ResponseWriter, data clientDisplayData) error { + var buf bytes.Buffer + if err := editTmpl.Execute(&buf, data); err != nil { + return err + } + if _, err := buf.WriteTo(w); err != nil { + return err + } + return nil +} + +func (s *idpServer) renderFormError(w http.ResponseWriter, data clientDisplayData, errorMsg string) { + data.Error = errorMsg + if err := s.renderClientForm(w, data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func (s *idpServer) renderFormSuccess(w http.ResponseWriter, data clientDisplayData, successMsg string) { + data.Success = successMsg + if err := s.renderClientForm(w, data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func createEditBaseData(client *funnelClient, name, redirectURI string) clientDisplayData { + return clientDisplayData{ + ID: client.ID, + Name: name, + RedirectURI: redirectURI, + HasSecret: client.Secret != "", + IsEdit: true, + } +} + +func validateRedirectURI(redirectURI string) string { + if redirectURI == "" { + return "Redirect URI is required" + } + + u, err := url.Parse(redirectURI) + if err != nil { + return "Invalid URL format" + } + + if u.Scheme != "http" && u.Scheme != "https" { + return "Redirect URI must be a valid HTTP or HTTPS URL" + } + + if u.Host == "" { + return "Redirect URI must include a valid host" + } + + return "" +} From cd49faa123131414cf990e4ad6ff7e2157f2e82a Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Mon, 26 May 2025 10:23:30 -0400 Subject: [PATCH 010/263] feature/capture: fix wireshark decoding and add new disco frame types (#16089) Fix the wireshark lua dissector to support 0 bit position and not throw modulo div by 0 errors. Add new disco frame types to the decoder. Updates tailscale/corp#29036 Signed-off-by: Mike O'Driscoll --- feature/capture/dissector/ts-dissector.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/feature/capture/dissector/ts-dissector.lua b/feature/capture/dissector/ts-dissector.lua index ad553d7674193..c2ee2b755f266 100644 --- a/feature/capture/dissector/ts-dissector.lua +++ b/feature/capture/dissector/ts-dissector.lua @@ -1,5 +1,5 @@ function hasbit(x, p) - return x % (p + p) >= p + return bit.band(x, p) ~= 0 end tsdebug_ll = Proto("tsdebug", "Tailscale debug") @@ -128,6 +128,10 @@ function tsdisco_frame.dissector(buffer, pinfo, tree) if message_type == 1 then subtree:add(DISCO_TYPE, "Ping") elseif message_type == 2 then subtree:add(DISCO_TYPE, "Pong") elseif message_type == 3 then subtree:add(DISCO_TYPE, "Call me maybe") + elseif message_type == 4 then subtree:add(DISCO_TYPE, "Bind UDP Relay Endpoint") + elseif message_type == 5 then subtree:add(DISCO_TYPE, "Bind UDP Relay Endpoint Challenge") + elseif message_type == 6 then subtree:add(DISCO_TYPE, "Bind UDP Relay Endpoint Answer") + elseif message_type == 7 then subtree:add(DISCO_TYPE, "Call me maybe via") end -- Message version From 4b59f1dfe6f0e1566f21573565204d07b49a3013 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Tue, 27 May 2025 16:03:45 +0100 Subject: [PATCH 011/263] .github/workflows: use Ubuntu 24.04 images (#16097) Bumps Ubuntu version for test container images 22.04 -> 24.04. Updates#cleanup Signed-off-by: Irbe Krumina --- .github/workflows/test.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8cbb6f351c66b..2aad005ae6ee0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ concurrency: jobs: race-root-integration: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false # don't abort the entire matrix if one element fails matrix: @@ -74,7 +74,7 @@ jobs: buildflags: "-race" shard: '3/3' - goarch: "386" # thanks yaml - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -187,7 +187,7 @@ jobs: find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete privileged: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: image: golang:latest options: --privileged @@ -214,7 +214,7 @@ jobs: XDG_CACHE_HOME: "/var/lib/ghrunner/cache" race-build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -258,7 +258,7 @@ jobs: - goos: openbsd goarch: amd64 - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -300,7 +300,7 @@ jobs: ios: # similar to cross above, but iOS can't build most of the repo. So, just #make it build a few smoke packages. - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -328,7 +328,7 @@ jobs: - goos: illumos goarch: amd64 - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -366,7 +366,7 @@ jobs: # similar to cross above, but android fails to build a few pieces of the # repo. We should fix those pieces, they're small, but as a stepping stone, # only test the subset of android that our past smoke test checked. - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -381,7 +381,7 @@ jobs: GOARCH: arm64 wasm: # builds tsconnect, which is the only wasm build we support - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -420,7 +420,7 @@ jobs: find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete tailscale_go: # Subset of tests that depend on our custom Go toolchain. - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -440,7 +440,7 @@ jobs: # explicit 'if' condition, because the default condition for steps is # 'success()', meaning "only run this if no previous steps failed". if: github.event_name == 'pull_request' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: build fuzzers id: build @@ -492,7 +492,7 @@ jobs: path: ${{ env.artifacts_path }}/out/artifacts depaware: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -501,7 +501,7 @@ jobs: make depaware go_generate: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -514,7 +514,7 @@ jobs: git diff --name-only --exit-code || (echo "The files above need updating. Please run 'go generate'."; exit 1) go_mod_tidy: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -526,7 +526,7 @@ jobs: git diff --name-only --exit-code || (echo "Please run 'go mod tidy'."; exit 1) licenses: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -534,7 +534,7 @@ jobs: run: ./scripts/check_license_headers.sh . staticcheck: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false # don't abort the entire matrix if one element fails matrix: @@ -575,7 +575,7 @@ jobs: - go_mod_tidy - licenses - staticcheck - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: notify # Only notify slack for merged commits, not PR failures. @@ -604,7 +604,7 @@ jobs: check_mergeability: if: always() - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - android - test From 842df378037579f35f996c9f3bb89dc53ba8e720 Mon Sep 17 00:00:00 2001 From: Jonathan Nobels Date: Wed, 28 May 2025 10:08:06 -0400 Subject: [PATCH 012/263] ipn: set RouteAll=true by default for new accounts on iOS and Android (#16110) fixes tailscale/tailscale#16082 RouteAll should be true by default on iOS and Android. Signed-off-by: Jonathan Nobels --- ipn/prefs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipn/prefs.go b/ipn/prefs.go index caf9ccfc3af09..01275a7e25bdc 100644 --- a/ipn/prefs.go +++ b/ipn/prefs.go @@ -721,9 +721,10 @@ func (p *Prefs) ControlURLOrDefault() string { // of the platform it's running on. func (p *Prefs) DefaultRouteAll(goos string) bool { switch goos { - case "windows": + case "windows", "android", "ios": return true case "darwin": + // Only true for macAppStore and macsys, false for darwin tailscaled. return version.IsSandboxedMacOS() default: return false From ffc8ec289b7ebd001963d45ce11efc140030deb7 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 28 May 2025 10:45:59 -0700 Subject: [PATCH 013/263] wgengine/magicsock: implement relayManager endpoint probing (#16029) relayManager is responsible for disco ping/pong probing of relay endpoints once a handshake is complete. Future work will enable relayManager to set a relay endpoint as the best UDP path on an endpoint if appropriate. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 10 +- wgengine/magicsock/magicsock.go | 32 ++- wgengine/magicsock/relaymanager.go | 297 +++++++++++++++--------- wgengine/magicsock/relaymanager_test.go | 2 +- 4 files changed, 215 insertions(+), 126 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 3788708a8260f..c2d18d707128d 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1562,10 +1562,18 @@ func pktLenToPingSize(mtu tstun.WireMTU, is6 bool) int { // It should be called with the Conn.mu held. // // It reports whether m.TxID corresponds to a ping that this endpoint sent. -func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip.AddrPort) (knownTxID bool) { +func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip.AddrPort, vni virtualNetworkID) (knownTxID bool) { de.mu.Lock() defer de.mu.Unlock() + if vni.isSet() { + // TODO(jwhited): check for matching [endpoint.bestAddr] once that data + // structure is VNI-aware and [relayManager] can mutate it. We do not + // need to reference any [endpointState] for Geneve-encapsulated disco, + // we store nothing about them there. + return false + } + isDerp := src.Addr() == tailcfg.DerpMagicIPAddr sp, ok := de.sentPing[m.TxID] diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 61cdf49543493..5b0f28a33c051 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -1802,6 +1802,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke return } var geneve packet.GeneveHeader + var vni virtualNetworkID if isGeneveEncap { err := geneve.Decode(msg) if err != nil { @@ -1810,6 +1811,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke c.logf("[unexpected] geneve header decoding error: %v", err) return } + vni.set(geneve.VNI) msg = msg[packet.GeneveFixedHeaderLength:] } // The control bit should only be set for relay handshake messages @@ -1923,33 +1925,30 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke c.logf("[unexpected] %T packets should not come from a relay server with Geneve control bit set", dm) return } - c.relayManager.handleBindUDPRelayEndpointChallenge(challenge, di, src, geneve.VNI) + c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(challenge, di, src, geneve.VNI) return } switch dm := dm.(type) { case *disco.Ping: metricRecvDiscoPing.Add(1) - if isGeneveEncap { - // TODO(jwhited): handle Geneve-encapsulated disco ping. - return - } - c.handlePingLocked(dm, src, di, derpNodeSrc) + c.handlePingLocked(dm, src, vni, di, derpNodeSrc) case *disco.Pong: metricRecvDiscoPong.Add(1) - if isGeneveEncap { - // TODO(jwhited): handle Geneve-encapsulated disco pong. - return - } // There might be multiple nodes for the sender's DiscoKey. // Ask each to handle it, stopping once one reports that // the Pong's TxID was theirs. + knownTxID := false c.peerMap.forEachEndpointWithDiscoKey(sender, func(ep *endpoint) (keepGoing bool) { - if ep.handlePongConnLocked(dm, di, src) { + if ep.handlePongConnLocked(dm, di, src, vni) { + knownTxID = true return false } return true }) + if !knownTxID && vni.isSet() { + c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(dm, di, src, vni.get()) + } case *disco.CallMeMaybe, *disco.CallMeMaybeVia: var via *disco.CallMeMaybeVia isVia := false @@ -2048,12 +2047,21 @@ func (c *Conn) unambiguousNodeKeyOfPingLocked(dm *disco.Ping, dk key.DiscoPublic // di is the discoInfo of the source of the ping. // derpNodeSrc is non-zero if the ping arrived via DERP. -func (c *Conn) handlePingLocked(dm *disco.Ping, src netip.AddrPort, di *discoInfo, derpNodeSrc key.NodePublic) { +func (c *Conn) handlePingLocked(dm *disco.Ping, src netip.AddrPort, vni virtualNetworkID, di *discoInfo, derpNodeSrc key.NodePublic) { likelyHeartBeat := src == di.lastPingFrom && time.Since(di.lastPingTime) < 5*time.Second di.lastPingFrom = src di.lastPingTime = time.Now() isDerp := src.Addr() == tailcfg.DerpMagicIPAddr + if vni.isSet() { + // TODO(jwhited): check for matching [endpoint.bestAddr] once that data + // structure is VNI-aware and [relayManager] can mutate it. We do not + // need to reference any [endpointState] for Geneve-encapsulated disco, + // we store nothing about them there. + c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(dm, di, src, vni.get()) + return + } + // If we can figure out with certainty which node key this disco // message is for, eagerly update our IP:port<>node and disco<>node // mappings to make p2p path discovery faster in simple diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 0b19bb83fcc1a..d9fd1fa24ba2f 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -14,15 +14,16 @@ import ( "time" "tailscale.com/disco" + "tailscale.com/net/stun" udprelay "tailscale.com/net/udprelay/endpoint" "tailscale.com/types/key" "tailscale.com/util/httpm" "tailscale.com/util/set" ) -// relayManager manages allocation and handshaking of -// [tailscale.com/net/udprelay.Server] endpoints. The zero value is ready for -// use. +// relayManager manages allocation, handshaking, and initial probing (disco +// ping/pong) of [tailscale.com/net/udprelay.Server] endpoints. The zero value +// is ready for use. type relayManager struct { initOnce sync.Once @@ -33,16 +34,18 @@ type relayManager struct { allocWorkByEndpoint map[*endpoint]*relayEndpointAllocWork handshakeWorkByEndpointByServerDisco map[*endpoint]map[key.DiscoPublic]*relayHandshakeWork handshakeWorkByServerDiscoVNI map[serverDiscoVNI]*relayHandshakeWork + handshakeWorkAwaitingPong map[*relayHandshakeWork]addrPortVNI + addrPortVNIToHandshakeWork map[addrPortVNI]*relayHandshakeWork // =================================================================== // The following chan fields serve event inputs to a single goroutine, // runLoop(). - allocateHandshakeCh chan *endpoint - allocateWorkDoneCh chan relayEndpointAllocWorkDoneEvent - handshakeWorkDoneCh chan relayEndpointHandshakeWorkDoneEvent - cancelWorkCh chan *endpoint - newServerEndpointCh chan newRelayServerEndpointEvent - rxChallengeCh chan relayHandshakeChallengeEvent + allocateHandshakeCh chan *endpoint + allocateWorkDoneCh chan relayEndpointAllocWorkDoneEvent + handshakeWorkDoneCh chan relayEndpointHandshakeWorkDoneEvent + cancelWorkCh chan *endpoint + newServerEndpointCh chan newRelayServerEndpointEvent + rxHandshakeDiscoMsgCh chan relayHandshakeDiscoMsgEvent discoInfoMu sync.Mutex // guards the following field discoInfoByServerDisco map[key.DiscoPublic]*relayHandshakeDiscoInfo @@ -66,16 +69,16 @@ type relayHandshakeWork struct { ep *endpoint se udprelay.ServerEndpoint - // In order to not deadlock, runLoop() must select{} read doneCh when - // attempting to write into rxChallengeCh, and the handshake work goroutine - // must close(doneCh) before attempting to write to - // relayManager.handshakeWorkDoneCh. - rxChallengeCh chan relayHandshakeChallengeEvent - doneCh chan struct{} + // handshakeServerEndpoint() always writes to doneCh (len 1) when it + // returns. It may end up writing the same event afterward to + // relayManager.handshakeWorkDoneCh if runLoop() can receive it. runLoop() + // must select{} read on doneCh to prevent deadlock when attempting to write + // to rxDiscoMsgCh. + rxDiscoMsgCh chan relayHandshakeDiscoMsgEvent + doneCh chan relayEndpointHandshakeWorkDoneEvent ctx context.Context cancel context.CancelFunc - wg *sync.WaitGroup } // newRelayServerEndpointEvent indicates a new [udprelay.ServerEndpoint] has @@ -99,8 +102,9 @@ type relayEndpointAllocWorkDoneEvent struct { // work for an [*endpoint] has completed. This structure is immutable once // initialized. type relayEndpointHandshakeWorkDoneEvent struct { - work *relayHandshakeWork - answerSentTo netip.AddrPort // zero value if answer was not transmitted + work *relayHandshakeWork + pongReceivedFrom netip.AddrPort // or zero value if handshake or ping/pong did not complete + latency time.Duration // only relevant if pongReceivedFrom.IsValid() } // activeWorkRunLoop returns true if there is outstanding allocation or @@ -150,8 +154,8 @@ func (r *relayManager) runLoop() { if !r.activeWorkRunLoop() { return } - case challenge := <-r.rxChallengeCh: - r.handleRxChallengeRunLoop(challenge) + case discoMsgEvent := <-r.rxHandshakeDiscoMsgCh: + r.handleRxHandshakeDiscoMsgRunLoop(discoMsgEvent) if !r.activeWorkRunLoop() { return } @@ -159,12 +163,12 @@ func (r *relayManager) runLoop() { } } -type relayHandshakeChallengeEvent struct { - challenge [32]byte - disco key.DiscoPublic - from netip.AddrPort - vni uint32 - at time.Time +type relayHandshakeDiscoMsgEvent struct { + msg disco.Message + disco key.DiscoPublic + from netip.AddrPort + vni uint32 + at time.Time } // relayEndpointAllocWork serves to track in-progress relay endpoint allocation @@ -187,12 +191,14 @@ func (r *relayManager) init() { r.allocWorkByEndpoint = make(map[*endpoint]*relayEndpointAllocWork) r.handshakeWorkByEndpointByServerDisco = make(map[*endpoint]map[key.DiscoPublic]*relayHandshakeWork) r.handshakeWorkByServerDiscoVNI = make(map[serverDiscoVNI]*relayHandshakeWork) + r.handshakeWorkAwaitingPong = make(map[*relayHandshakeWork]addrPortVNI) + r.addrPortVNIToHandshakeWork = make(map[addrPortVNI]*relayHandshakeWork) r.allocateHandshakeCh = make(chan *endpoint) r.allocateWorkDoneCh = make(chan relayEndpointAllocWorkDoneEvent) r.handshakeWorkDoneCh = make(chan relayEndpointHandshakeWorkDoneEvent) r.cancelWorkCh = make(chan *endpoint) r.newServerEndpointCh = make(chan newRelayServerEndpointEvent) - r.rxChallengeCh = make(chan relayHandshakeChallengeEvent) + r.rxHandshakeDiscoMsgCh = make(chan relayHandshakeDiscoMsgEvent) r.runLoopStoppedCh = make(chan struct{}, 1) go r.runLoop() }) @@ -270,8 +276,11 @@ func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, dm *disco.CallMeMaybeV }) } -func (r *relayManager) handleBindUDPRelayEndpointChallenge(dm *disco.BindUDPRelayEndpointChallenge, di *discoInfo, src netip.AddrPort, vni uint32) { - relayManagerInputEvent(r, nil, &r.rxChallengeCh, relayHandshakeChallengeEvent{challenge: dm.Challenge, disco: di.discoKey, from: src, vni: vni, at: time.Now()}) +// handleGeneveEncapDiscoMsgNotBestAddr handles reception of Geneve-encapsulated +// disco messages if they are not associated with any known +// [*endpoint.bestAddr]. +func (r *relayManager) handleGeneveEncapDiscoMsgNotBestAddr(dm disco.Message, di *discoInfo, src netip.AddrPort, vni uint32) { + relayManagerInputEvent(r, nil, &r.rxHandshakeDiscoMsgCh, relayHandshakeDiscoMsgEvent{msg: dm, disco: di.discoKey, from: src, vni: vni, at: time.Now()}) } // relayManagerInputEvent initializes [relayManager] if necessary, starts @@ -337,26 +346,68 @@ func (r *relayManager) stopWorkRunLoop(ep *endpoint, f stopHandshakeWorkFilter) _, knownServer := r.serversByDisco[disco] if knownServer || f == stopHandshakeWorkAllServers { handshakeWork.cancel() - handshakeWork.wg.Wait() - delete(byServerDisco, disco) - delete(r.handshakeWorkByServerDiscoVNI, serverDiscoVNI{handshakeWork.se.ServerDisco, handshakeWork.se.VNI}) + done := <-handshakeWork.doneCh + r.handleHandshakeWorkDoneRunLoop(done) } } - if len(byServerDisco) == 0 { - delete(r.handshakeWorkByEndpointByServerDisco, ep) - } } } -func (r *relayManager) handleRxChallengeRunLoop(challenge relayHandshakeChallengeEvent) { - work, ok := r.handshakeWorkByServerDiscoVNI[serverDiscoVNI{challenge.disco, challenge.vni}] - if !ok { +// addrPortVNI represents a combined netip.AddrPort and Geneve header virtual +// network identifier. +type addrPortVNI struct { + addrPort netip.AddrPort + vni uint32 +} + +func (r *relayManager) handleRxHandshakeDiscoMsgRunLoop(event relayHandshakeDiscoMsgEvent) { + var ( + work *relayHandshakeWork + ok bool + ) + apv := addrPortVNI{event.from, event.vni} + switch event.msg.(type) { + case *disco.BindUDPRelayEndpointChallenge: + work, ok = r.handshakeWorkByServerDiscoVNI[serverDiscoVNI{event.disco, event.vni}] + if !ok { + // No outstanding work tied to this challenge, discard. + return + } + _, ok = r.handshakeWorkAwaitingPong[work] + if ok { + // We've seen a challenge for this relay endpoint previously, + // discard. Servers only respond to the first src ip:port they see + // binds from. + return + } + _, ok = r.addrPortVNIToHandshakeWork[apv] + if ok { + // There is existing work for the same [addrPortVNI] that is not + // 'work'. If both instances happen to be on the same server we + // could attempt to resolve event order using LamportID. For now + // just leave both work instances alone and take no action other + // than to discard this challenge msg. + return + } + // Update state so that future ping/pong will route to 'work'. + r.handshakeWorkAwaitingPong[work] = apv + r.addrPortVNIToHandshakeWork[apv] = work + case *disco.Ping, *disco.Pong: + work, ok = r.addrPortVNIToHandshakeWork[apv] + if !ok { + // No outstanding work tied to this [addrPortVNI], discard. + return + } + default: + // Unexpected message type, discard. return } select { - case <-work.doneCh: + case done := <-work.doneCh: + // handshakeServerEndpoint() returned, clean up its state. + r.handleHandshakeWorkDoneRunLoop(done) return - case work.rxChallengeCh <- challenge: + case work.rxDiscoMsgCh <- event: return } } @@ -375,14 +426,17 @@ func (r *relayManager) handleHandshakeWorkDoneRunLoop(done relayEndpointHandshak delete(r.handshakeWorkByEndpointByServerDisco, done.work.ep) } delete(r.handshakeWorkByServerDiscoVNI, serverDiscoVNI{done.work.se.ServerDisco, done.work.se.VNI}) - if !done.answerSentTo.IsValid() { - // The handshake timed out. + apv, ok := r.handshakeWorkAwaitingPong[work] + if ok { + delete(r.handshakeWorkAwaitingPong, work) + delete(r.addrPortVNIToHandshakeWork, apv) + } + if !done.pongReceivedFrom.IsValid() { + // The handshake or ping/pong probing timed out. return } - // We received a challenge from and transmitted an answer towards the relay - // server. - // TODO(jwhited): Make the associated [*endpoint] aware of this - // [tailscale.com/net/udprelay.ServerEndpoint]. + // This relay endpoint is functional. + // TODO(jwhited): Set it on done.work.ep.bestAddr if it is a betterAddr(). } func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelayServerEndpointEvent) { @@ -398,19 +452,10 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay return } - // The existing work is no longer valid, clean it up. Be sure to lookup - // by the existing work's [*endpoint], not the incoming "new" work as - // they are not necessarily matching. + // The existing work is no longer valid, clean it up. existingWork.cancel() - existingWork.wg.Wait() - delete(r.handshakeWorkByServerDiscoVNI, sdv) - byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[existingWork.ep] - if ok { - delete(byServerDisco, sdv.serverDisco) - if len(byServerDisco) == 0 { - delete(r.handshakeWorkByEndpointByServerDisco, existingWork.ep) - } - } + done := <-existingWork.doneCh + r.handleHandshakeWorkDoneRunLoop(done) } // Check for duplicate work by [*endpoint] + server disco. @@ -425,12 +470,8 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay } // Cancel existing handshake that has a lower lamport ID. existingWork.cancel() - existingWork.wg.Wait() - delete(r.handshakeWorkByServerDiscoVNI, sdv) - delete(byServerDisco, sdv.serverDisco) - if len(byServerDisco) == 0 { - delete(r.handshakeWorkByEndpointByServerDisco, existingWork.ep) - } + done := <-existingWork.doneCh + r.handleHandshakeWorkDoneRunLoop(done) } } @@ -464,14 +505,12 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay // We're ready to start a new handshake. ctx, cancel := context.WithCancel(context.Background()) - wg := &sync.WaitGroup{} work := &relayHandshakeWork{ ep: newServerEndpoint.ep, se: newServerEndpoint.se, - doneCh: make(chan struct{}), + doneCh: make(chan relayEndpointHandshakeWorkDoneEvent, 1), ctx: ctx, cancel: cancel, - wg: wg, } if byServerDisco == nil { byServerDisco = make(map[key.DiscoPublic]*relayHandshakeWork) @@ -480,19 +519,16 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay byServerDisco[newServerEndpoint.se.ServerDisco] = work r.handshakeWorkByServerDiscoVNI[sdv] = work - wg.Add(1) go r.handshakeServerEndpoint(work) } func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { - defer work.wg.Done() - done := relayEndpointHandshakeWorkDoneEvent{work: work} r.ensureDiscoInfoFor(work) defer func() { r.derefDiscoInfoFor(work) - close(work.doneCh) + work.doneCh <- done relayManagerInputEvent(r, work.ctx, &r.handshakeWorkDoneCh, done) work.cancel() }() @@ -504,7 +540,7 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { for _, addrPort := range work.se.AddrPorts { if addrPort.IsValid() { sentBindAny = true - go work.ep.c.sendDiscoMessage(addrPort, vni, key.NodePublic{}, work.se.ServerDisco, bind, discoLog) + go work.ep.c.sendDiscoMessage(addrPort, vni, key.NodePublic{}, work.se.ServerDisco, bind, discoVerboseLog) } } if !sentBindAny { @@ -518,46 +554,83 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { timer := time.NewTimer(min(work.se.BindLifetime.Duration, maxHandshakeLifetime)) defer timer.Stop() - // Wait for cancellation, a challenge to be rx'd, or handshake lifetime to - // expire. Our initial implementation values simplicity over other aspects, - // e.g. it is not resilient to any packet loss. - // - // We may want to eventually consider [disc.BindUDPRelayEndpoint] - // retransmission lacking challenge rx, and - // [disco.BindUDPRelayEndpointAnswer] duplication in front of - // [disco.Ping] until [disco.Ping] or [disco.Pong] is received. - select { - case <-work.ctx.Done(): - return - case challenge := <-work.rxChallengeCh: - answer := &disco.BindUDPRelayEndpointAnswer{Answer: challenge.challenge} - done.answerSentTo = challenge.from - // Send answer back to relay server. Typically sendDiscoMessage() calls - // are invoked via a new goroutine in attempt to limit crypto+syscall - // time contributing to system backpressure, and to fire roundtrip - // latency-relevant messages as closely together as possible. We - // intentionally don't do that here, because: - // 1. The primary backpressure concern is around the work.rxChallengeCh - // writer on the [Conn] packet rx path, who is already unblocked - // since we read from the channel. Relay servers only ever tx one - // challenge per rx'd bind message for a given (the first seen) src. - // 2. runLoop() may be waiting for this 'work' to complete if - // explicitly canceled for some reason elsewhere, but this is - // typically only around [*endpoint] and/or [Conn] shutdown. - // 3. It complicates the defer()'d [*discoInfo] deref and 'work' - // completion event order. sendDiscoMessage() assumes the related - // [*discoInfo] is still available. We also don't want the - // [*endpoint] to send a [disco.Ping] before the - // [disco.BindUDPRelayEndpointAnswer] has gone out, otherwise the - // remote side will never see the ping, delaying/preventing the - // [udprelay.ServerEndpoint] from becoming fully operational. - // 4. This is a singular tx with no roundtrip latency measurements - // involved. - work.ep.c.sendDiscoMessage(challenge.from, vni, key.NodePublic{}, work.se.ServerDisco, answer, discoLog) - return - case <-timer.C: - // The handshake timed out. - return + // Limit the number of pings we will transmit. Inbound pings trigger + // outbound pings, so we want to be a little defensive. + const limitPings = 10 + + var ( + handshakeState disco.BindUDPRelayHandshakeState = disco.BindUDPRelayHandshakeStateBindSent + sentPingAt = make(map[stun.TxID]time.Time) + ) + + txPing := func(to netip.AddrPort, withAnswer *[32]byte) { + if len(sentPingAt) == limitPings { + return + } + epDisco := work.ep.disco.Load() + if epDisco == nil { + return + } + txid := stun.NewTxID() + sentPingAt[txid] = time.Now() + ping := &disco.Ping{ + TxID: txid, + NodeKey: work.ep.c.publicKeyAtomic.Load(), + } + go func() { + if withAnswer != nil { + answer := &disco.BindUDPRelayEndpointAnswer{Answer: *withAnswer} + work.ep.c.sendDiscoMessage(to, vni, key.NodePublic{}, work.se.ServerDisco, answer, discoVerboseLog) + } + work.ep.c.sendDiscoMessage(to, vni, key.NodePublic{}, epDisco.key, ping, discoVerboseLog) + }() + } + + // This for{select{}} is responsible for handshaking and tx'ing ping/pong + // when the handshake is complete. + for { + select { + case <-work.ctx.Done(): + return + case msgEvent := <-work.rxDiscoMsgCh: + switch msg := msgEvent.msg.(type) { + case *disco.BindUDPRelayEndpointChallenge: + if handshakeState >= disco.BindUDPRelayHandshakeStateAnswerSent { + continue + } + txPing(msgEvent.from, &msg.Challenge) + handshakeState = disco.BindUDPRelayHandshakeStateAnswerSent + case *disco.Ping: + if handshakeState < disco.BindUDPRelayHandshakeStateAnswerSent { + continue + } + // An inbound ping from the remote peer indicates we completed a + // handshake with the relay server (our answer msg was + // received). Chances are our ping was dropped before the remote + // handshake was complete. We need to rx a pong to determine + // latency, so send another ping. Since the handshake is + // complete we do not need to send an answer in front of this + // one. + txPing(msgEvent.from, nil) + case *disco.Pong: + at, ok := sentPingAt[msg.TxID] + if !ok { + continue + } + // The relay server endpoint is functional! Record the + // round-trip latency and return. + done.pongReceivedFrom = msgEvent.from + done.latency = time.Since(at) + return + default: + // unexpected message type, silently discard + continue + } + return + case <-timer.C: + // The handshake timed out. + return + } } } diff --git a/wgengine/magicsock/relaymanager_test.go b/wgengine/magicsock/relaymanager_test.go index 3b75db9f6e1f9..8276849aafd8b 100644 --- a/wgengine/magicsock/relaymanager_test.go +++ b/wgengine/magicsock/relaymanager_test.go @@ -25,6 +25,6 @@ func TestRelayManagerInitAndIdle(t *testing.T) { <-rm.runLoopStoppedCh rm = relayManager{} - rm.handleBindUDPRelayEndpointChallenge(&disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, netip.AddrPort{}, 0) + rm.handleGeneveEncapDiscoMsgNotBestAddr(&disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, netip.AddrPort{}, 0) <-rm.runLoopStoppedCh } From 5e54819ceecd789bce87c42b09b632932931d794 Mon Sep 17 00:00:00 2001 From: Jonathan Nobels Date: Wed, 28 May 2025 15:43:12 -0400 Subject: [PATCH 014/263] net/dns: cache dns.Config for reuse when compileConfig fails (#16059) fixes tailscale/corp#25612 We now keep track of any dns configurations which we could not compile. This gives RecompileDNSConfig a configuration to attempt to recompile and apply when the OS pokes us to indicate that the interface dns servers have changed/updated. The manager config will remain unset until we have the required information to compile it correctly which should eliminate the problematic SERVFAIL responses (especially on macOS 15). This also removes the missingUpstreamRecovery func in the forwarder which is no longer required now that we have proper error handling and recovery manager and the client. Signed-off-by: Jonathan Nobels --- net/dns/manager.go | 46 ++++++++++------------------ net/dns/manager_test.go | 56 +++++++++++++++++++++++++++++++++-- net/dns/resolver/forwarder.go | 26 ++++------------ net/dns/resolver/tsdns.go | 9 ------ 4 files changed, 76 insertions(+), 61 deletions(-) diff --git a/net/dns/manager.go b/net/dns/manager.go index 64bf12c6b1c26..5d6f225ce032f 100644 --- a/net/dns/manager.go +++ b/net/dns/manager.go @@ -25,7 +25,6 @@ import ( "tailscale.com/net/netmon" "tailscale.com/net/tsdial" "tailscale.com/syncs" - "tailscale.com/tstime/rate" "tailscale.com/types/dnstype" "tailscale.com/types/logger" "tailscale.com/util/clientmetric" @@ -63,10 +62,8 @@ type Manager struct { knobs *controlknobs.Knobs // or nil goos string // if empty, gets set to runtime.GOOS - mu sync.Mutex // guards following - // config is the last configuration we successfully compiled or nil if there - // was any failure applying the last configuration. - config *Config + mu sync.Mutex // guards following + config *Config // Tracks the last viable DNS configuration set by Set. nil on failures other than compilation failures or if set has never been called. } // NewManagers created a new manager from the given config. @@ -93,22 +90,6 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, health *health.Tracker, goos: goos, } - // Rate limit our attempts to correct our DNS configuration. - // This is done on incoming queries, we don't want to spam it. - limiter := rate.NewLimiter(1.0/5.0, 1) - - // This will recompile the DNS config, which in turn will requery the system - // DNS settings. The recovery func should triggered only when we are missing - // upstream nameservers and require them to forward a query. - m.resolver.SetMissingUpstreamRecovery(func() { - if limiter.Allow() { - m.logf("resolution failed due to missing upstream nameservers. Recompiling DNS configuration.") - if err := m.RecompileDNSConfig(); err != nil { - m.logf("config recompilation failed: %v", err) - } - } - }) - m.ctx, m.ctxCancel = context.WithCancel(context.Background()) m.logf("using %T", m.os) return m @@ -117,7 +98,7 @@ func NewManager(logf logger.Logf, oscfg OSConfigurator, health *health.Tracker, // Resolver returns the Manager's DNS Resolver. func (m *Manager) Resolver() *resolver.Resolver { return m.resolver } -// RecompileDNSConfig sets the DNS config to the current value, which has +// RecompileDNSConfig recompiles the last attempted DNS configuration, which has // the side effect of re-querying the OS's interface nameservers. This should be used // on platforms where the interface nameservers can change. Darwin, for example, // where the nameservers aren't always available when we process a major interface @@ -127,14 +108,14 @@ func (m *Manager) Resolver() *resolver.Resolver { return m.resolver } // give a better or different result than when [Manager.Set] was last called. The // logic for making that determination is up to the caller. // -// It returns [ErrNoDNSConfig] if the [Manager] has no existing DNS configuration. +// It returns [ErrNoDNSConfig] if [Manager.Set] has never been called. func (m *Manager) RecompileDNSConfig() error { m.mu.Lock() defer m.mu.Unlock() - if m.config == nil { - return ErrNoDNSConfig + if m.config != nil { + return m.setLocked(*m.config) } - return m.setLocked(*m.config) + return ErrNoDNSConfig } func (m *Manager) Set(cfg Config) error { @@ -154,15 +135,15 @@ func (m *Manager) GetBaseConfig() (OSConfig, error) { func (m *Manager) setLocked(cfg Config) error { syncs.AssertLocked(&m.mu) - // On errors, the 'set' config is cleared. - m.config = nil - m.logf("Set: %v", logger.ArgWriter(func(w *bufio.Writer) { cfg.WriteToBufioWriter(w) })) rcfg, ocfg, err := m.compileConfig(cfg) if err != nil { + // On a compilation failure, set m.config set for later reuse by + // [Manager.RecompileDNSConfig] and return the error. + m.config = &cfg return err } @@ -174,9 +155,11 @@ func (m *Manager) setLocked(cfg Config) error { })) if err := m.resolver.SetConfig(rcfg); err != nil { + m.config = nil return err } if err := m.os.SetDNS(ocfg); err != nil { + m.config = nil m.health.SetUnhealthy(osConfigurationSetWarnable, health.Args{health.ArgError: err.Error()}) return err } @@ -355,7 +338,10 @@ func (m *Manager) compileConfig(cfg Config) (rcfg resolver.Config, ocfg OSConfig // that as the forwarder for all DNS traffic that quad-100 doesn't handle. if isApple || !m.os.SupportsSplitDNS() { // If the OS can't do native split-dns, read out the underlying - // resolver config and blend it into our config. + // resolver config and blend it into our config. On apple platforms, [OSConfigurator.GetBaseConfig] + // has a tendency to temporarily fail if called immediately following + // an interface change. These failures should be retried if/when the OS + // indicates that the DNS configuration has changed via [RecompileDNSConfig]. cfg, err := m.os.GetBaseConfig() if err == nil { baseCfg = &cfg diff --git a/net/dns/manager_test.go b/net/dns/manager_test.go index 2bdbc72e26093..522f9636abefe 100644 --- a/net/dns/manager_test.go +++ b/net/dns/manager_test.go @@ -4,6 +4,7 @@ package dns import ( + "errors" "net/netip" "runtime" "strings" @@ -24,8 +25,9 @@ type fakeOSConfigurator struct { SplitDNS bool BaseConfig OSConfig - OSConfig OSConfig - ResolverConfig resolver.Config + OSConfig OSConfig + ResolverConfig resolver.Config + GetBaseConfigErr *error } func (c *fakeOSConfigurator) SetDNS(cfg OSConfig) error { @@ -45,6 +47,9 @@ func (c *fakeOSConfigurator) SupportsSplitDNS() bool { } func (c *fakeOSConfigurator) GetBaseConfig() (OSConfig, error) { + if c.GetBaseConfigErr != nil { + return OSConfig{}, *c.GetBaseConfigErr + } return c.BaseConfig, nil } @@ -1019,3 +1024,50 @@ func upstreams(strs ...string) (ret map[dnsname.FQDN][]*dnstype.Resolver) { } return ret } + +func TestConfigRecompilation(t *testing.T) { + fakeErr := errors.New("fake os configurator error") + f := &fakeOSConfigurator{} + f.GetBaseConfigErr = &fakeErr + f.BaseConfig = OSConfig{ + Nameservers: mustIPs("1.1.1.1"), + } + + config := Config{ + Routes: upstreams("ts.net", "69.4.2.0", "foo.ts.net", ""), + SearchDomains: fqdns("foo.ts.net"), + } + + m := NewManager(t.Logf, f, new(health.Tracker), tsdial.NewDialer(netmon.NewStatic()), nil, nil, "darwin") + + var managerConfig *resolver.Config + m.resolver.TestOnlySetHook(func(cfg resolver.Config) { + managerConfig = &cfg + }) + + // Initial set should error out and store the config + if err := m.Set(config); err == nil { + t.Fatalf("Want non-nil error. Got nil") + } + if m.config == nil { + t.Fatalf("Want persisted config. Got nil.") + } + if managerConfig != nil { + t.Fatalf("Want nil managerConfig. Got %v", managerConfig) + } + + // Clear the error. We should take the happy path now and + // set m.manager's Config. + f.GetBaseConfigErr = nil + + // Recompilation without an error should succeed and set m.config and m.manager's [resolver.Config] + if err := m.RecompileDNSConfig(); err != nil { + t.Fatalf("Want nil error. Got err %v", err) + } + if m.config == nil { + t.Fatalf("Want non-nil config. Got nil") + } + if managerConfig == nil { + t.Fatalf("Want non nil managerConfig. Got nil") + } +} diff --git a/net/dns/resolver/forwarder.go b/net/dns/resolver/forwarder.go index 321401a843e4e..c87fbd5041a93 100644 --- a/net/dns/resolver/forwarder.go +++ b/net/dns/resolver/forwarder.go @@ -245,12 +245,6 @@ type forwarder struct { // /etc/resolv.conf is missing/corrupt, and the peerapi ExitDNS stub // resolver lookup. cloudHostFallback []resolverAndDelay - - // missingUpstreamRecovery, if non-nil, is set called when a SERVFAIL is - // returned due to missing upstream resolvers. - // - // This should attempt to properly (re)set the upstream resolvers. - missingUpstreamRecovery func() } func newForwarder(logf logger.Logf, netMon *netmon.Monitor, linkSel ForwardLinkSelector, dialer *tsdial.Dialer, health *health.Tracker, knobs *controlknobs.Knobs) *forwarder { @@ -258,13 +252,12 @@ func newForwarder(logf logger.Logf, netMon *netmon.Monitor, linkSel ForwardLinkS panic("nil netMon") } f := &forwarder{ - logf: logger.WithPrefix(logf, "forward: "), - netMon: netMon, - linkSel: linkSel, - dialer: dialer, - health: health, - controlKnobs: knobs, - missingUpstreamRecovery: func() {}, + logf: logger.WithPrefix(logf, "forward: "), + netMon: netMon, + linkSel: linkSel, + dialer: dialer, + health: health, + controlKnobs: knobs, } f.ctx, f.ctxCancel = context.WithCancel(context.Background()) return f @@ -962,13 +955,6 @@ func (f *forwarder) forwardWithDestChan(ctx context.Context, query packet, respo f.health.SetUnhealthy(dnsForwarderFailing, health.Args{health.ArgDNSServers: ""}) f.logf("no upstream resolvers set, returning SERVFAIL") - // Attempt to recompile the DNS configuration - // If we are being asked to forward queries and we have no - // nameservers, the network is in a bad state. - if f.missingUpstreamRecovery != nil { - f.missingUpstreamRecovery() - } - res, err := servfailResponse(query) if err != nil { return err diff --git a/net/dns/resolver/tsdns.go b/net/dns/resolver/tsdns.go index 107740b136d54..33fa9c3c07d4c 100644 --- a/net/dns/resolver/tsdns.go +++ b/net/dns/resolver/tsdns.go @@ -251,15 +251,6 @@ func New(logf logger.Logf, linkSel ForwardLinkSelector, dialer *tsdial.Dialer, h return r } -// SetMissingUpstreamRecovery sets a callback to be called upon encountering -// a SERVFAIL due to missing upstream resolvers. -// -// This call should only happen before the resolver is used. It is not safe -// for concurrent use. -func (r *Resolver) SetMissingUpstreamRecovery(f func()) { - r.forwarder.missingUpstreamRecovery = f -} - func (r *Resolver) TestOnlySetHook(hook func(Config)) { r.saveConfigForTests = hook } func (r *Resolver) SetConfig(cfg Config) error { From 36df320e6a66546f4921d359c555b64059a0aded Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Wed, 28 May 2025 14:12:24 -0700 Subject: [PATCH 015/263] tsnet: remove an expired configuration-path migration step (#16120) As note in the comment, it now being more than six months since this was deprecated and there being no (further) uses of the old pattern in our internal services, let's drop the migrator. Updates #cleanup Change-Id: Ie4fb9518b2ca04a9b361e09c51cbbacf1e2633a8 Signed-off-by: M. J. Fromberger --- tsnet/tsnet.go | 48 +----------------------------------------------- 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index 4664a66a796d4..65367f235482f 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -536,10 +536,7 @@ func (s *Server) start() (reterr error) { if err != nil { return err } - s.rootPath, err = getTSNetDir(s.logf, confDir, prog) - if err != nil { - return err - } + s.rootPath = filepath.Join(confDir, "tsnet-"+prog) } if err := os.MkdirAll(s.rootPath, 0700); err != nil { return err @@ -897,49 +894,6 @@ func (s *Server) getUDPHandlerForFlow(src, dst netip.AddrPort) (handler func(net return func(c nettype.ConnPacketConn) { ln.handle(c) }, true } -// getTSNetDir usually just returns filepath.Join(confDir, "tsnet-"+prog) -// with no error. -// -// One special case is that it renames old "tslib-" directories to -// "tsnet-", and that rename might return an error. -// -// TODO(bradfitz): remove this maybe 6 months after 2022-03-17, -// once people (notably Tailscale corp services) have updated. -func getTSNetDir(logf logger.Logf, confDir, prog string) (string, error) { - oldPath := filepath.Join(confDir, "tslib-"+prog) - newPath := filepath.Join(confDir, "tsnet-"+prog) - - fi, err := os.Lstat(oldPath) - if os.IsNotExist(err) { - // Common path. - return newPath, nil - } - if err != nil { - return "", err - } - if !fi.IsDir() { - return "", fmt.Errorf("expected old tslib path %q to be a directory; got %v", oldPath, fi.Mode()) - } - - // At this point, oldPath exists and is a directory. But does - // the new path exist? - - fi, err = os.Lstat(newPath) - if err == nil && fi.IsDir() { - // New path already exists somehow. Ignore the old one and - // don't try to migrate it. - return newPath, nil - } - if err != nil && !os.IsNotExist(err) { - return "", err - } - if err := os.Rename(oldPath, newPath); err != nil { - return "", err - } - logf("renamed old tsnet state storage directory %q to %q", oldPath, newPath) - return newPath, nil -} - // APIClient returns a tailscale.Client that can be used to make authenticated // requests to the Tailscale control server. // It requires the user to set tailscale.I_Acknowledge_This_API_Is_Unstable. From b0d35975c0462a8499667ac7b52d685b5e90465a Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Wed, 28 May 2025 17:54:04 -0700 Subject: [PATCH 016/263] go.toolchain.rev: bump to 1.24.3 (#16060) Updates https://github.com/tailscale/corp/issues/28916 Signed-off-by: Andrew Lytvynov --- go.toolchain.rev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.toolchain.rev b/go.toolchain.rev index e8ede337c11cb..a5d73929c61b0 100644 --- a/go.toolchain.rev +++ b/go.toolchain.rev @@ -1 +1 @@ -982da8f24fa0504f2214f24b0d68b2febd5983f8 +98e8c99c256a5aeaa13725d2e43fdd7f465ba200 From dca4036a207b5f7edeb1d54cce30c7dfe1914499 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 27 May 2025 13:31:39 -0700 Subject: [PATCH 017/263] util/set: add SmallSet Updates tailscale/corp#29093 Change-Id: I0e07e83dee51b4915597a913b0583c99756d90e2 Signed-off-by: Brad Fitzpatrick --- util/set/smallset.go | 134 ++++++++++++++++++++++++++++++++++++++ util/set/smallset_test.go | 91 ++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 util/set/smallset.go create mode 100644 util/set/smallset_test.go diff --git a/util/set/smallset.go b/util/set/smallset.go new file mode 100644 index 0000000000000..51cad6a2560cc --- /dev/null +++ b/util/set/smallset.go @@ -0,0 +1,134 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package set + +import ( + "iter" + "maps" + + "tailscale.com/types/structs" +) + +// SmallSet is a set that is optimized for reducing memory overhead when the +// expected size of the set is 0 or 1 elements. +// +// The zero value of SmallSet is a usable empty set. +// +// When storing a SmallSet in a map as a value type, it is important to re-assign +// the map entry after calling Add or Delete, as the SmallSet's representation +// may change. +// +// Copying a SmallSet by value may alias the previous value. Use the Clone method +// to create a new SmallSet with the same contents. +type SmallSet[T comparable] struct { + _ structs.Incomparable // to prevent == mistakes + one T // if non-zero, then single item in set + m Set[T] // if non-nil, the set of items, which might be size 1 if it's the zero value of T +} + +// Values returns an iterator over the elements of the set. +// The iterator will yield the elements in no particular order. +func (s SmallSet[T]) Values() iter.Seq[T] { + if s.m != nil { + return maps.Keys(s.m) + } + var zero T + return func(yield func(T) bool) { + if s.one != zero { + yield(s.one) + } + } +} + +// Contains reports whether e is in the set. +func (s SmallSet[T]) Contains(e T) bool { + if s.m != nil { + return s.m.Contains(e) + } + var zero T + return e != zero && s.one == e +} + +// Add adds e to the set. +// +// When storing a SmallSet in a map as a value type, it is important to +// re-assign the map entry after calling Add or Delete, as the SmallSet's +// representation may change. +func (s *SmallSet[T]) Add(e T) { + var zero T + if s.m != nil { + s.m.Add(e) + return + } + // Size zero to one non-zero element. + if s.one == zero && e != zero { + s.one = e + return + } + // Need to make a multi map, either + // because we now have two items, or + // because e is the zero value. + s.m = Set[T]{} + if s.one != zero { + s.m.Add(s.one) // move single item to multi + } + s.m.Add(e) // add new item + s.one = zero +} + +// Len reports the number of elements in the set. +func (s SmallSet[T]) Len() int { + var zero T + if s.m != nil { + return s.m.Len() + } + if s.one != zero { + return 1 + } + return 0 +} + +// Delete removes e from the set. +// +// When storing a SmallSet in a map as a value type, it is important to +// re-assign the map entry after calling Add or Delete, as the SmallSet's +// representation may change. +func (s *SmallSet[T]) Delete(e T) { + var zero T + if s.m == nil { + if s.one == e { + s.one = zero + } + return + } + s.m.Delete(e) + + // If the map size drops to zero, that means + // it only contained the zero value of T. + if s.m.Len() == 0 { + s.m = nil + return + } + + // If the map size drops to one element and doesn't + // contain the zero value, we can switch back to the + // single-item representation. + if s.m.Len() == 1 { + for v := range s.m { + if v != zero { + s.one = v + s.m = nil + } + } + } + return +} + +// Clone returns a copy of s that doesn't alias the original. +func (s SmallSet[T]) Clone() SmallSet[T] { + return SmallSet[T]{ + one: s.one, + m: maps.Clone(s.m), // preserves nilness + } +} diff --git a/util/set/smallset_test.go b/util/set/smallset_test.go new file mode 100644 index 0000000000000..2635bc893678f --- /dev/null +++ b/util/set/smallset_test.go @@ -0,0 +1,91 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package set + +import ( + "fmt" + "iter" + "maps" + "reflect" + "slices" + "testing" +) + +func TestSmallSet(t *testing.T) { + t.Parallel() + + wantSize := reflect.TypeFor[int64]().Size() + reflect.TypeFor[map[int]struct{}]().Size() + if wantSize > 16 { + t.Errorf("wantSize should be no more than 16") // it might be smaller on 32-bit systems + } + if size := reflect.TypeFor[SmallSet[int64]]().Size(); size != wantSize { + t.Errorf("SmallSet[int64] size is %d, want %v", size, wantSize) + } + + type op struct { + add bool + v int + } + ops := iter.Seq[op](func(yield func(op) bool) { + for _, add := range []bool{false, true} { + for v := range 4 { + if !yield(op{add: add, v: v}) { + return + } + } + } + }) + type setLike interface { + Add(int) + Delete(int) + } + apply := func(s setLike, o op) { + if o.add { + s.Add(o.v) + } else { + s.Delete(o.v) + } + } + + // For all combinations of 4 operations, + // apply them to both a regular map and SmallSet + // and make sure all the invariants hold. + + for op1 := range ops { + for op2 := range ops { + for op3 := range ops { + for op4 := range ops { + + normal := Set[int]{} + small := &SmallSet[int]{} + for _, op := range []op{op1, op2, op3, op4} { + apply(normal, op) + apply(small, op) + } + + name := func() string { + return fmt.Sprintf("op1=%v, op2=%v, op3=%v, op4=%v", op1, op2, op3, op4) + } + if normal.Len() != small.Len() { + t.Errorf("len mismatch after ops %s: normal=%d, small=%d", name(), normal.Len(), small.Len()) + } + if got := small.Clone().Len(); normal.Len() != got { + t.Errorf("len mismatch after ops %s: normal=%d, clone=%d", name(), normal.Len(), got) + } + + normalEle := slices.Sorted(maps.Keys(normal)) + smallEle := slices.Sorted(small.Values()) + if !slices.Equal(normalEle, smallEle) { + t.Errorf("elements mismatch after ops %s: normal=%v, small=%v", name(), normalEle, smallEle) + } + for e := range 5 { + if normal.Contains(e) != small.Contains(e) { + t.Errorf("contains(%v) mismatch after ops %s: normal=%v, small=%v", e, name(), normal.Contains(e), small.Contains(e)) + } + } + } + } + } + } +} From 4cccd15eeb13d5d7f8f831e5406a567b8f18378b Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Thu, 29 May 2025 13:51:46 -0500 Subject: [PATCH 018/263] ipn/ipnlocal: fix data race when accessing b.appConnector The field must only be accessed while holding LocalBackend's mutex, but there are two places where it's accessed without the mutex: - (LocalBackend).MaybeClearAppConnector() - handleC2NAppConnectorDomainRoutesGet() Fixes #16123 Signed-off-by: Nick Khyl --- ipn/ipnlocal/c2n.go | 5 +++-- ipn/ipnlocal/local.go | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ipn/ipnlocal/c2n.go b/ipn/ipnlocal/c2n.go index b3379475164ae..876c130641b81 100644 --- a/ipn/ipnlocal/c2n.go +++ b/ipn/ipnlocal/c2n.go @@ -240,13 +240,14 @@ func handleC2NAppConnectorDomainRoutesGet(b *LocalBackend, w http.ResponseWriter b.logf("c2n: GET /appconnector/routes received") var res tailcfg.C2NAppConnectorDomainRoutesResponse - if b.appConnector == nil { + appConnector := b.AppConnector() + if appConnector == nil { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(res) return } - res.Domains = b.appConnector.DomainRoutes() + res.Domains = appConnector.DomainRoutes() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(res) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index d2f6c86f7f113..d69c07a9fc944 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -4150,8 +4150,8 @@ func (b *LocalBackend) SetUseExitNodeEnabled(v bool) (ipn.PrefsView, error) { // AdvertiseRoutes has been set in the MaskedPrefs. func (b *LocalBackend) MaybeClearAppConnector(mp *ipn.MaskedPrefs) error { var err error - if b.appConnector != nil && mp.AdvertiseRoutesSet { - err = b.appConnector.ClearRoutes() + if ac := b.AppConnector(); ac != nil && mp.AdvertiseRoutesSet { + err = ac.ClearRoutes() if err != nil { b.logf("appc: clear routes error: %v", err) } @@ -4755,9 +4755,7 @@ func (b *LocalBackend) readvertiseAppConnectorRoutes() { // // Grab a copy of the field, since b.mu only guards access to the // b.appConnector field itself. - b.mu.Lock() - appConnector := b.appConnector - b.mu.Unlock() + appConnector := b.AppConnector() if appConnector == nil { return @@ -6432,6 +6430,15 @@ func (b *LocalBackend) OfferingAppConnector() bool { return b.appConnector != nil } +// AppConnector returns the current AppConnector, or nil if not configured. +// +// TODO(nickkhyl): move app connectors to [nodeBackend], or perhaps a feature package? +func (b *LocalBackend) AppConnector() *appc.AppConnector { + b.mu.Lock() + defer b.mu.Unlock() + return b.appConnector +} + // allowExitNodeDNSProxyToServeName reports whether the Exit Node DNS // proxy is allowed to serve responses for the provided DNS name. func (b *LocalBackend) allowExitNodeDNSProxyToServeName(name string) bool { From 191afd3390f08354515af9d6b0c3f6b919b5a0fb Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Thu, 29 May 2025 10:41:23 -0500 Subject: [PATCH 019/263] net/tshttpproxy: fix WDAP/PAC proxy detection on Win10 1607 and earlier Using WINHTTP_AUTOPROXY_ALLOW_AUTOCONFIG on Windows versions older than Windows 10 1703 (build 15063) is not supported and causes WinHttpGetProxyForUrl to fail with ERROR_INVALID_PARAMETER. This results in failures reaching the control on environments where a proxy is required. We use wingoes version detection to conditionally set the WINHTTP_AUTOPROXY_ALLOW_AUTOCONFIG flag on Windows builds greater than 15063. While there, we also update proxy detection to use WINHTTP_AUTO_DETECT_TYPE_DNS_A, as DNS-based proxy discovery might be required with Active Directory and in certain other environments. Updates tailscale/corp#29168 Fixes #879 Signed-off-by: Nick Khyl --- cmd/derper/depaware.txt | 2 +- net/tshttpproxy/tshttpproxy_windows.go | 28 ++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index ca772353079dc..640e64d6cab21 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -12,7 +12,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa github.com/coder/websocket/internal/util from github.com/coder/websocket github.com/coder/websocket/internal/xsync from github.com/coder/websocket L github.com/coreos/go-iptables/iptables from tailscale.com/util/linuxfw - W 💣 github.com/dblohm7/wingoes from tailscale.com/util/winutil + W 💣 github.com/dblohm7/wingoes from tailscale.com/util/winutil+ github.com/fxamacker/cbor/v2 from tailscale.com/tka github.com/go-json-experiment/json from tailscale.com/types/opt+ github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json+ diff --git a/net/tshttpproxy/tshttpproxy_windows.go b/net/tshttpproxy/tshttpproxy_windows.go index 06a1f5ae445d0..7163c786307ac 100644 --- a/net/tshttpproxy/tshttpproxy_windows.go +++ b/net/tshttpproxy/tshttpproxy_windows.go @@ -18,6 +18,7 @@ import ( "unsafe" "github.com/alexbrainman/sspi/negotiate" + "github.com/dblohm7/wingoes" "golang.org/x/sys/windows" "tailscale.com/hostinfo" "tailscale.com/syncs" @@ -97,9 +98,7 @@ func proxyFromWinHTTPOrCache(req *http.Request) (*url.URL, error) { } if err == windows.ERROR_INVALID_PARAMETER { metricErrInvalidParameters.Add(1) - // Seen on Windows 8.1. (https://github.com/tailscale/tailscale/issues/879) - // TODO(bradfitz): figure this out. - setNoProxyUntil(time.Hour) + setNoProxyUntil(10 * time.Second) proxyErrorf("tshttpproxy: winhttp: GetProxyForURL(%q): ERROR_INVALID_PARAMETER [unexpected]", urlStr) return nil, nil } @@ -238,17 +237,30 @@ func (pi *winHTTPProxyInfo) free() { } } -var proxyForURLOpts = &winHTTPAutoProxyOptions{ - DwFlags: winHTTP_AUTOPROXY_ALLOW_AUTOCONFIG | winHTTP_AUTOPROXY_AUTO_DETECT, - DwAutoDetectFlags: winHTTP_AUTO_DETECT_TYPE_DHCP, // | winHTTP_AUTO_DETECT_TYPE_DNS_A, -} +var getProxyForURLOpts = sync.OnceValue(func() *winHTTPAutoProxyOptions { + opts := &winHTTPAutoProxyOptions{ + DwFlags: winHTTP_AUTOPROXY_AUTO_DETECT, + DwAutoDetectFlags: winHTTP_AUTO_DETECT_TYPE_DHCP | winHTTP_AUTO_DETECT_TYPE_DNS_A, + } + // Support for the WINHTTP_AUTOPROXY_ALLOW_AUTOCONFIG flag was added in Windows 10, version 1703. + // + // Using it on earlier versions causes GetProxyForURL to fail with ERROR_INVALID_PARAMETER, + // which prevents proxy detection and can lead to failures reaching the control server + // on environments where a proxy is required. + // + // https://web.archive.org/web/20250529044903/https://learn.microsoft.com/en-us/windows/win32/api/winhttp/ns-winhttp-winhttp_autoproxy_options + if wingoes.IsWin10BuildOrGreater(wingoes.Win10Build1703) { + opts.DwFlags |= winHTTP_AUTOPROXY_ALLOW_AUTOCONFIG + } + return opts +}) func (hi winHTTPInternet) GetProxyForURL(urlStr string) (string, error) { var out winHTTPProxyInfo err := winHTTPGetProxyForURL( hi, windows.StringToUTF16Ptr(urlStr), - proxyForURLOpts, + getProxyForURLOpts(), &out, ) if err != nil { From 401d6c0cfaae0a2caf50640b79287505838704a9 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 29 May 2025 12:05:41 -0700 Subject: [PATCH 020/263] go.mod: bump golang.org/x deps Updates #8043 Change-Id: I8702a17130559353ccdecbe8b64eeee461ff09c3 Signed-off-by: Brad Fitzpatrick --- cmd/k8s-operator/depaware.txt | 3 ++- cmd/tailscaled/depaware.txt | 3 ++- go.mod | 22 +++++++++--------- go.sum | 44 +++++++++++++++++------------------ tsnet/depaware.txt | 3 ++- 5 files changed, 39 insertions(+), 36 deletions(-) diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 782603df09709..2e467843ac07e 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -1072,7 +1072,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ crypto/internal/fips140/edwards25519/field from crypto/ecdh+ crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls + crypto/internal/fips140/mlkem from crypto/tls+ crypto/internal/fips140/nistec from crypto/elliptic+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa @@ -1092,6 +1092,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ crypto/internal/randutil from crypto/dsa+ crypto/internal/sysrand from crypto/internal/entropy+ crypto/md5 from crypto/tls+ + LD crypto/mlkem from golang.org/x/crypto/ssh crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls+ crypto/rsa from crypto/tls+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index d9a9cac65e5f3..c6011a12cc7ef 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -551,7 +551,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/internal/fips140/edwards25519/field from crypto/ecdh+ crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls + crypto/internal/fips140/mlkem from crypto/tls+ crypto/internal/fips140/nistec from crypto/elliptic+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa @@ -571,6 +571,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/internal/randutil from crypto/dsa+ crypto/internal/sysrand from crypto/internal/entropy+ crypto/md5 from crypto/tls+ + LD crypto/mlkem from golang.org/x/crypto/ssh crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls+ crypto/rsa from crypto/tls+ diff --git a/go.mod b/go.mod index f346b1e4095dd..d44a14aef6731 100644 --- a/go.mod +++ b/go.mod @@ -99,16 +99,16 @@ require ( go.uber.org/zap v1.27.0 go4.org/mem v0.0.0-20240501181205-ae6ca9944745 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba - golang.org/x/crypto v0.37.0 + golang.org/x/crypto v0.38.0 golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac - golang.org/x/mod v0.23.0 - golang.org/x/net v0.36.0 - golang.org/x/oauth2 v0.26.0 - golang.org/x/sync v0.13.0 - golang.org/x/sys v0.32.0 - golang.org/x/term v0.31.0 - golang.org/x/time v0.10.0 - golang.org/x/tools v0.30.0 + golang.org/x/mod v0.24.0 + golang.org/x/net v0.40.0 + golang.org/x/oauth2 v0.30.0 + golang.org/x/sync v0.14.0 + golang.org/x/sys v0.33.0 + golang.org/x/term v0.32.0 + golang.org/x/time v0.11.0 + golang.org/x/tools v0.33.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 golang.zx2c4.com/wireguard/windows v0.5.3 gopkg.in/square/go-jose.v2 v2.6.0 @@ -392,8 +392,8 @@ require ( gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect - golang.org/x/image v0.24.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/image v0.27.0 // indirect + golang.org/x/text v0.25.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index bdbae11bb2e3a..73d87fd664fb0 100644 --- a/go.sum +++ b/go.sum @@ -1091,8 +1091,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1111,8 +1111,8 @@ golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9 golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ= -golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8= +golang.org/x/image v0.27.0 h1:C8gA4oWU/tKkdCfYT6T2u4faJu3MeNS5O8UPWlPF61w= +golang.org/x/image v0.27.0/go.mod h1:xbdrClrAUway1MUTEZDq9mz/UpRwYAkFFNUslZtcB+g= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1140,8 +1140,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= -golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1181,16 +1181,16 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.26.0 h1:afQXWNNaeC4nvZ0Ed9XvCCzXM6UHJG7iCg0W4fPqSBE= -golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1204,8 +1204,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1268,16 +1268,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= -golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1288,13 +1288,13 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= -golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1359,8 +1359,8 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= -golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/tsnet/depaware.txt b/tsnet/depaware.txt index 3b705f68023ee..242cd8f1b36d5 100644 --- a/tsnet/depaware.txt +++ b/tsnet/depaware.txt @@ -495,7 +495,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) crypto/internal/fips140/edwards25519/field from crypto/ecdh+ crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls + crypto/internal/fips140/mlkem from crypto/tls+ crypto/internal/fips140/nistec from crypto/elliptic+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa @@ -515,6 +515,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) crypto/internal/randutil from crypto/dsa+ crypto/internal/sysrand from crypto/internal/entropy+ crypto/md5 from crypto/tls+ + LD crypto/mlkem from golang.org/x/crypto/ssh crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls+ crypto/rsa from crypto/tls+ From ef49e75b10a30b32c0c4e79c7e78392b95435eed Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 29 May 2025 12:40:29 -0700 Subject: [PATCH 021/263] util/set: add SmallSet.SoleElement, fix bug, add more tests This adds SmallSet.SoleElement, which I need in another repo for efficiency. I added tests, but those tests failed because Add(1) + Add(1) was promoting the first Add's sole element to a map of one item. So fix that, and add more tests. Updates tailscale/corp#29093 Change-Id: Iadd5ad08afe39721ee5449343095e389214d8389 Signed-off-by: Brad Fitzpatrick --- util/set/smallset.go | 24 +++++++++++++++++++----- util/set/smallset_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/util/set/smallset.go b/util/set/smallset.go index 51cad6a2560cc..1b77419d27dc9 100644 --- a/util/set/smallset.go +++ b/util/set/smallset.go @@ -50,6 +50,15 @@ func (s SmallSet[T]) Contains(e T) bool { return e != zero && s.one == e } +// SoleElement returns the single value in the set, if the set has exactly one +// element. +// +// If the set is empty or has more than one element, ok will be false and e will +// be the zero value of T. +func (s SmallSet[T]) SoleElement() (e T, ok bool) { + return s.one, s.Len() == 1 +} + // Add adds e to the set. // // When storing a SmallSet in a map as a value type, it is important to @@ -61,10 +70,15 @@ func (s *SmallSet[T]) Add(e T) { s.m.Add(e) return } - // Size zero to one non-zero element. - if s.one == zero && e != zero { - s.one = e - return + // Non-zero elements can go into s.one. + if e != zero { + if s.one == zero { + s.one = e // Len 0 to Len 1 + return + } + if s.one == e { + return // dup + } } // Need to make a multi map, either // because we now have two items, or @@ -73,7 +87,7 @@ func (s *SmallSet[T]) Add(e T) { if s.one != zero { s.m.Add(s.one) // move single item to multi } - s.m.Add(e) // add new item + s.m.Add(e) // add new item, possibly zero s.one = zero } diff --git a/util/set/smallset_test.go b/util/set/smallset_test.go index 2635bc893678f..d6f446df08e81 100644 --- a/util/set/smallset_test.go +++ b/util/set/smallset_test.go @@ -84,8 +84,43 @@ func TestSmallSet(t *testing.T) { t.Errorf("contains(%v) mismatch after ops %s: normal=%v, small=%v", e, name(), normal.Contains(e), small.Contains(e)) } } + + if err := small.checkInvariants(); err != nil { + t.Errorf("checkInvariants failed after ops %s: %v", name(), err) + } + + if !t.Failed() { + sole, ok := small.SoleElement() + if ok != (small.Len() == 1) { + t.Errorf("SoleElement ok mismatch after ops %s: SoleElement ok=%v, want=%v", name(), ok, !ok) + } + if ok && sole != smallEle[0] { + t.Errorf("SoleElement value mismatch after ops %s: SoleElement=%v, want=%v", name(), sole, smallEle[0]) + t.Errorf("Internals: %+v", small) + } + } + } + } + } + } +} + +func (s *SmallSet[T]) checkInvariants() error { + var zero T + if s.m != nil && s.one != zero { + return fmt.Errorf("both m and one are non-zero") + } + if s.m != nil { + switch len(s.m) { + case 0: + return fmt.Errorf("m is non-nil but empty") + case 1: + for k := range s.m { + if k != zero { + return fmt.Errorf("m contains exactly 1 non-zero element, %v", k) } } } } + return nil } From 5b670eb3a5f1749a655692d97a2e7086c78d1580 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 30 May 2025 11:30:03 +0100 Subject: [PATCH 022/263] cmd/containerboot: allow setting --accept-dns via TS_EXTRA_ARGS again (#16129) In 1.84 we made 'tailscale set'/'tailscale up' error out if duplicate command line flags are passed. This broke some container configurations as we have two env vars that can be used to set --accept-dns flag: - TS_ACCEPT_DNS- specifically for --accept-dns - TS_EXTRA_ARGS- accepts any arbitrary 'tailscale up'/'tailscale set' flag. We default TS_ACCEPT_DNS to false (to make the container behaviour more declarative), which with the new restrictive CLI behaviour resulted in failure for users who had set --accept-dns via TS_EXTRA_ARGS as the flag would be provided twice. This PR re-instates the previous behaviour by checking if TS_EXTRA_ARGS contains --accept-dns flag and if so using its value to override TS_ACCEPT_DNS. Updates tailscale/tailscale#16108 Signed-off-by: Irbe Krumina --- cmd/containerboot/main_test.go | 248 ++++++++++++++++++----------- cmd/containerboot/settings.go | 57 +++++++ cmd/containerboot/settings_test.go | 108 +++++++++++++ 3 files changed, 322 insertions(+), 91 deletions(-) create mode 100644 cmd/containerboot/settings_test.go diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index a0ccce3dd86a2..c7293c77a4afa 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -41,97 +41,6 @@ import ( "tailscale.com/types/ptr" ) -// testEnv represents the environment needed for a single sub-test so that tests -// can run in parallel. -type testEnv struct { - kube *kubeServer // Fake kube server. - lapi *localAPI // Local TS API server. - d string // Temp dir for the specific test. - argFile string // File with commands test_tailscale{,d}.sh were invoked with. - runningSockPath string // Path to the running tailscaled socket. - localAddrPort int // Port for the containerboot HTTP server. - healthAddrPort int // Port for the (deprecated) containerboot health server. -} - -func newTestEnv(t *testing.T) testEnv { - d := t.TempDir() - - lapi := localAPI{FSRoot: d} - if err := lapi.Start(); err != nil { - t.Fatal(err) - } - t.Cleanup(lapi.Close) - - kube := kubeServer{FSRoot: d} - kube.Start(t) - t.Cleanup(kube.Close) - - tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"} - serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}} - egressCfg := egressSvcConfig("foo", "foo.tailnetxyz.ts.net") - - dirs := []string{ - "var/lib", - "usr/bin", - "tmp", - "dev/net", - "proc/sys/net/ipv4", - "proc/sys/net/ipv6/conf/all", - "etc/tailscaled", - } - for _, path := range dirs { - if err := os.MkdirAll(filepath.Join(d, path), 0700); err != nil { - t.Fatal(err) - } - } - files := map[string][]byte{ - "usr/bin/tailscaled": fakeTailscaled, - "usr/bin/tailscale": fakeTailscale, - "usr/bin/iptables": fakeTailscale, - "usr/bin/ip6tables": fakeTailscale, - "dev/net/tun": []byte(""), - "proc/sys/net/ipv4/ip_forward": []byte("0"), - "proc/sys/net/ipv6/conf/all/forwarding": []byte("0"), - "etc/tailscaled/cap-95.hujson": mustJSON(t, tailscaledConf), - "etc/tailscaled/serve-config.json": mustJSON(t, serveConf), - filepath.Join("etc/tailscaled/", egressservices.KeyEgressServices): mustJSON(t, egressCfg), - filepath.Join("etc/tailscaled/", egressservices.KeyHEPPings): []byte("4"), - } - for path, content := range files { - // Making everything executable is a little weird, but the - // stuff that doesn't need to be executable doesn't care if we - // do make it executable. - if err := os.WriteFile(filepath.Join(d, path), content, 0700); err != nil { - t.Fatal(err) - } - } - - argFile := filepath.Join(d, "args") - runningSockPath := filepath.Join(d, "tmp/tailscaled.sock") - var localAddrPort, healthAddrPort int - for _, p := range []*int{&localAddrPort, &healthAddrPort} { - ln, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatalf("Failed to open listener: %v", err) - } - if err := ln.Close(); err != nil { - t.Fatalf("Failed to close listener: %v", err) - } - port := ln.Addr().(*net.TCPAddr).Port - *p = port - } - - return testEnv{ - kube: &kube, - lapi: &lapi, - d: d, - argFile: argFile, - runningSockPath: runningSockPath, - localAddrPort: localAddrPort, - healthAddrPort: healthAddrPort, - } -} - func TestContainerBoot(t *testing.T) { boot := filepath.Join(t.TempDir(), "containerboot") if err := exec.Command("go", "build", "-ldflags", "-X main.testSleepDuration=1ms", "-o", boot, "tailscale.com/cmd/containerboot").Run(); err != nil { @@ -515,6 +424,37 @@ func TestContainerBoot(t *testing.T) { }, } }, + "auth_key_once_extra_args_override_dns": func(env *testEnv) testCase { + return testCase{ + Env: map[string]string{ + "TS_AUTHKEY": "tskey-key", + "TS_AUTH_ONCE": "true", + "TS_ACCEPT_DNS": "false", + "TS_EXTRA_ARGS": "--accept-dns", + }, + Phases: []phase{ + { + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", + }, + }, + { + Notify: &ipn.Notify{ + State: ptr.To(ipn.NeedsLogin), + }, + WantCmds: []string{ + "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true --authkey=tskey-key", + }, + }, + { + Notify: runningNotify, + WantCmds: []string{ + "/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=true", + }, + }, + }, + } + }, "kube_storage": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ @@ -766,6 +706,41 @@ func TestContainerBoot(t *testing.T) { }, } }, + "extra_args_accept_dns": func(env *testEnv) testCase { + return testCase{ + Env: map[string]string{ + "TS_EXTRA_ARGS": "--accept-dns", + }, + Phases: []phase{ + { + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", + "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=true", + }, + }, { + Notify: runningNotify, + }, + }, + } + }, + "extra_args_accept_dns_overrides_env_var": func(env *testEnv) testCase { + return testCase{ + Env: map[string]string{ + "TS_ACCEPT_DNS": "true", // Overridden by TS_EXTRA_ARGS. + "TS_EXTRA_ARGS": "--accept-dns=false", + }, + Phases: []phase{ + { + WantCmds: []string{ + "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=mem: --statedir=/tmp --tun=userspace-networking", + "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false", + }, + }, { + Notify: runningNotify, + }, + }, + } + }, "hostname": func(env *testEnv) testCase { return testCase{ Env: map[string]string{ @@ -1604,3 +1579,94 @@ func egressSvcConfig(name, fqdn string) egressservices.Configs { }, } } + +// testEnv represents the environment needed for a single sub-test so that tests +// can run in parallel. +type testEnv struct { + kube *kubeServer // Fake kube server. + lapi *localAPI // Local TS API server. + d string // Temp dir for the specific test. + argFile string // File with commands test_tailscale{,d}.sh were invoked with. + runningSockPath string // Path to the running tailscaled socket. + localAddrPort int // Port for the containerboot HTTP server. + healthAddrPort int // Port for the (deprecated) containerboot health server. +} + +func newTestEnv(t *testing.T) testEnv { + d := t.TempDir() + + lapi := localAPI{FSRoot: d} + if err := lapi.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(lapi.Close) + + kube := kubeServer{FSRoot: d} + kube.Start(t) + t.Cleanup(kube.Close) + + tailscaledConf := &ipn.ConfigVAlpha{AuthKey: ptr.To("foo"), Version: "alpha0"} + serveConf := ipn.ServeConfig{TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}} + egressCfg := egressSvcConfig("foo", "foo.tailnetxyz.ts.net") + + dirs := []string{ + "var/lib", + "usr/bin", + "tmp", + "dev/net", + "proc/sys/net/ipv4", + "proc/sys/net/ipv6/conf/all", + "etc/tailscaled", + } + for _, path := range dirs { + if err := os.MkdirAll(filepath.Join(d, path), 0700); err != nil { + t.Fatal(err) + } + } + files := map[string][]byte{ + "usr/bin/tailscaled": fakeTailscaled, + "usr/bin/tailscale": fakeTailscale, + "usr/bin/iptables": fakeTailscale, + "usr/bin/ip6tables": fakeTailscale, + "dev/net/tun": []byte(""), + "proc/sys/net/ipv4/ip_forward": []byte("0"), + "proc/sys/net/ipv6/conf/all/forwarding": []byte("0"), + "etc/tailscaled/cap-95.hujson": mustJSON(t, tailscaledConf), + "etc/tailscaled/serve-config.json": mustJSON(t, serveConf), + filepath.Join("etc/tailscaled/", egressservices.KeyEgressServices): mustJSON(t, egressCfg), + filepath.Join("etc/tailscaled/", egressservices.KeyHEPPings): []byte("4"), + } + for path, content := range files { + // Making everything executable is a little weird, but the + // stuff that doesn't need to be executable doesn't care if we + // do make it executable. + if err := os.WriteFile(filepath.Join(d, path), content, 0700); err != nil { + t.Fatal(err) + } + } + + argFile := filepath.Join(d, "args") + runningSockPath := filepath.Join(d, "tmp/tailscaled.sock") + var localAddrPort, healthAddrPort int + for _, p := range []*int{&localAddrPort, &healthAddrPort} { + ln, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatalf("Failed to open listener: %v", err) + } + if err := ln.Close(); err != nil { + t.Fatalf("Failed to close listener: %v", err) + } + port := ln.Addr().(*net.TCPAddr).Port + *p = port + } + + return testEnv{ + kube: &kube, + lapi: &lapi, + d: d, + argFile: argFile, + runningSockPath: runningSockPath, + localAddrPort: localAddrPort, + healthAddrPort: healthAddrPort, + } +} diff --git a/cmd/containerboot/settings.go b/cmd/containerboot/settings.go index 0ac9c828e2f76..5a8be9036b3ca 100644 --- a/cmd/containerboot/settings.go +++ b/cmd/containerboot/settings.go @@ -147,12 +147,69 @@ func configFromEnv() (*settings, error) { } } + // See https://github.com/tailscale/tailscale/issues/16108 for context- we + // do this to preserve the previous behaviour where --accept-dns could be + // set either via TS_ACCEPT_DNS or TS_EXTRA_ARGS. + acceptDNS := cfg.AcceptDNS != nil && *cfg.AcceptDNS + tsExtraArgs, acceptDNSNew := parseAcceptDNS(cfg.ExtraArgs, acceptDNS) + cfg.ExtraArgs = tsExtraArgs + if acceptDNS != acceptDNSNew { + cfg.AcceptDNS = &acceptDNSNew + } + if err := cfg.validate(); err != nil { return nil, fmt.Errorf("invalid configuration: %v", err) } return cfg, nil } +// parseAcceptDNS parses any values for Tailscale --accept-dns flag set via +// TS_ACCEPT_DNS and TS_EXTRA_ARGS env vars. If TS_EXTRA_ARGS contains +// --accept-dns flag, override the acceptDNS value with the one from +// TS_EXTRA_ARGS. +// The value of extraArgs can be empty string or one or more whitespace-separate +// key value pairs for 'tailscale up' command. The value for boolean flags can +// be omitted (default to true). +func parseAcceptDNS(extraArgs string, acceptDNS bool) (string, bool) { + if !strings.Contains(extraArgs, "--accept-dns") { + return extraArgs, acceptDNS + } + // TODO(irbekrm): we should validate that TS_EXTRA_ARGS contains legit + // 'tailscale up' flag values separated by whitespace. + argsArr := strings.Fields(extraArgs) + i := -1 + for key, val := range argsArr { + if strings.HasPrefix(val, "--accept-dns") { + i = key + break + } + } + if i == -1 { + return extraArgs, acceptDNS + } + a := strings.TrimSpace(argsArr[i]) + var acceptDNSFromExtraArgsS string + keyval := strings.Split(a, "=") + if len(keyval) == 2 { + acceptDNSFromExtraArgsS = keyval[1] + } else if len(keyval) == 1 && keyval[0] == "--accept-dns" { + // If the arg is just --accept-dns, we assume it means true. + acceptDNSFromExtraArgsS = "true" + } else { + log.Printf("TS_EXTRA_ARGS contains --accept-dns, but it is not in the expected format --accept-dns=, ignoring it") + return extraArgs, acceptDNS + } + acceptDNSFromExtraArgs, err := strconv.ParseBool(acceptDNSFromExtraArgsS) + if err != nil { + log.Printf("TS_EXTRA_ARGS contains --accept-dns=%q, which is not a valid boolean value, ignoring it", acceptDNSFromExtraArgsS) + return extraArgs, acceptDNS + } + if acceptDNSFromExtraArgs != acceptDNS { + log.Printf("TS_EXTRA_ARGS contains --accept-dns=%v, which overrides TS_ACCEPT_DNS=%v", acceptDNSFromExtraArgs, acceptDNS) + } + return strings.Join(append(argsArr[:i], argsArr[i+1:]...), " "), acceptDNSFromExtraArgs +} + func (s *settings) validate() error { if s.TailscaledConfigFilePath != "" { dir, file := path.Split(s.TailscaledConfigFilePath) diff --git a/cmd/containerboot/settings_test.go b/cmd/containerboot/settings_test.go new file mode 100644 index 0000000000000..dbec066c9ab0d --- /dev/null +++ b/cmd/containerboot/settings_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux + +package main + +import "testing" + +func Test_parseAcceptDNS(t *testing.T) { + tests := []struct { + name string + extraArgs string + acceptDNS bool + wantExtraArgs string + wantAcceptDNS bool + }{ + { + name: "false_extra_args_unset", + extraArgs: "", + wantExtraArgs: "", + wantAcceptDNS: false, + }, + { + name: "false_unrelated_args_set", + extraArgs: "--accept-routes=true --advertise-routes=10.0.0.1/32", + wantExtraArgs: "--accept-routes=true --advertise-routes=10.0.0.1/32", + wantAcceptDNS: false, + }, + { + name: "true_extra_args_unset", + extraArgs: "", + acceptDNS: true, + wantExtraArgs: "", + wantAcceptDNS: true, + }, + { + name: "true_unrelated_args_set", + acceptDNS: true, + extraArgs: "--accept-routes=true --advertise-routes=10.0.0.1/32", + wantExtraArgs: "--accept-routes=true --advertise-routes=10.0.0.1/32", + wantAcceptDNS: true, + }, + { + name: "false_extra_args_set_to_false", + extraArgs: "--accept-dns=false", + wantExtraArgs: "", + wantAcceptDNS: false, + }, + { + name: "false_extra_args_set_to_true", + extraArgs: "--accept-dns=true", + wantExtraArgs: "", + wantAcceptDNS: true, + }, + { + name: "true_extra_args_set_to_false", + extraArgs: "--accept-dns=false", + acceptDNS: true, + wantExtraArgs: "", + wantAcceptDNS: false, + }, + { + name: "true_extra_args_set_to_true", + extraArgs: "--accept-dns=true", + acceptDNS: true, + wantExtraArgs: "", + wantAcceptDNS: true, + }, + { + name: "false_extra_args_set_to_true_implicitly", + extraArgs: "--accept-dns", + wantExtraArgs: "", + wantAcceptDNS: true, + }, + { + name: "false_extra_args_set_to_true_implicitly_with_unrelated_args", + extraArgs: "--accept-dns --accept-routes --advertise-routes=10.0.0.1/32", + wantExtraArgs: "--accept-routes --advertise-routes=10.0.0.1/32", + wantAcceptDNS: true, + }, + { + name: "false_extra_args_set_to_true_implicitly_surrounded_with_unrelated_args", + extraArgs: "--accept-routes --accept-dns --advertise-routes=10.0.0.1/32", + wantExtraArgs: "--accept-routes --advertise-routes=10.0.0.1/32", + wantAcceptDNS: true, + }, + { + name: "true_extra_args_set_to_false_with_unrelated_args", + extraArgs: "--accept-routes --accept-dns=false", + acceptDNS: true, + wantExtraArgs: "--accept-routes", + wantAcceptDNS: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotExtraArgs, gotAcceptDNS := parseAcceptDNS(tt.extraArgs, tt.acceptDNS) + if gotExtraArgs != tt.wantExtraArgs { + t.Errorf("parseAcceptDNS() gotExtraArgs = %v, want %v", gotExtraArgs, tt.wantExtraArgs) + } + if gotAcceptDNS != tt.wantAcceptDNS { + t.Errorf("parseAcceptDNS() gotAcceptDNS = %v, want %v", gotAcceptDNS, tt.wantAcceptDNS) + } + }) + } +} From 11e83f9da5eb4e11d50464ac6ab01bb663218b22 Mon Sep 17 00:00:00 2001 From: James Sanderson Date: Wed, 7 May 2025 17:01:40 +0100 Subject: [PATCH 023/263] controlclient,health,ipnlocal,tailcfg: add DisplayMessage support Updates tailscale/corp#27759 Signed-off-by: James Sanderson --- control/controlclient/map.go | 35 ++++- control/controlclient/map_test.go | 238 +++++++++++++++++++++++++++++- health/state.go | 31 +++- ipn/ipnlocal/local.go | 9 +- ipn/ipnlocal/local_test.go | 65 ++++++++ tailcfg/tailcfg.go | 60 +++++++- types/netmap/nodemut.go | 1 + 7 files changed, 417 insertions(+), 22 deletions(-) diff --git a/control/controlclient/map.go b/control/controlclient/map.go index abfc5eb170c44..f346e19d494f6 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -90,6 +90,7 @@ type mapSession struct { lastDomain string lastDomainAuditLogID string lastHealth []string + lastDisplayMessages map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage lastPopBrowserURL string lastTKAInfo *tailcfg.TKAInfo lastNetmapSummary string // from NetworkMap.VeryConcise @@ -412,6 +413,21 @@ func (ms *mapSession) updateStateFromResponse(resp *tailcfg.MapResponse) { if resp.Health != nil { ms.lastHealth = resp.Health } + if resp.DisplayMessages != nil { + if v, ok := resp.DisplayMessages["*"]; ok && v == nil { + ms.lastDisplayMessages = nil + } + for k, v := range resp.DisplayMessages { + if k == "*" { + continue + } + if v != nil { + mak.Set(&ms.lastDisplayMessages, k, *v) + } else { + delete(ms.lastDisplayMessages, k) + } + } + } if resp.TKAInfo != nil { ms.lastTKAInfo = resp.TKAInfo } @@ -831,14 +847,19 @@ func (ms *mapSession) sortedPeers() []tailcfg.NodeView { func (ms *mapSession) netmap() *netmap.NetworkMap { peerViews := ms.sortedPeers() - // Convert all ms.lastHealth to the new [netmap.NetworkMap.DisplayMessages]. var msgs map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage - for _, h := range ms.lastHealth { - mak.Set(&msgs, tailcfg.DisplayMessageID("control-health-"+strhash(h)), tailcfg.DisplayMessage{ - Title: "Coordination server reports an issue", - Severity: tailcfg.SeverityMedium, - Text: "The coordination server is reporting a health issue: " + h, - }) + if len(ms.lastDisplayMessages) != 0 { + msgs = ms.lastDisplayMessages + } else if len(ms.lastHealth) > 0 { + // Convert all ms.lastHealth to the new [netmap.NetworkMap.DisplayMessages] + for _, h := range ms.lastHealth { + id := "control-health-" + strhash(h) // Unique ID in case there is more than one health message + mak.Set(&msgs, tailcfg.DisplayMessageID(id), tailcfg.DisplayMessage{ + Title: "Coordination server reports an issue", + Severity: tailcfg.SeverityMedium, + Text: "The coordination server is reporting a health issue: " + h, + }) + } } nm := &netmap.NetworkMap{ diff --git a/control/controlclient/map_test.go b/control/controlclient/map_test.go index 9abaae9236b6f..013640f47b08a 100644 --- a/control/controlclient/map_test.go +++ b/control/controlclient/map_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "go4.org/mem" "tailscale.com/control/controlknobs" "tailscale.com/health" @@ -1139,8 +1140,190 @@ func BenchmarkMapSessionDelta(b *testing.B) { } } +// TestNetmapDisplayMessage checks that the various diff operations +// (add/update/delete/clear) for [tailcfg.DisplayMessage] in a +// [tailcfg.MapResponse] work as expected. +func TestNetmapDisplayMessage(t *testing.T) { + type test struct { + name string + initialState *tailcfg.MapResponse + mapResponse tailcfg.MapResponse + wantMessages map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage + } + + tests := []test{ + { + name: "basic-set", + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "test-message": { + Title: "Testing", + Text: "This is a test message", + Severity: tailcfg.SeverityHigh, + ImpactsConnectivity: true, + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "https://www.example.com", + Label: "Learn more", + }, + }, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-message": { + Title: "Testing", + Text: "This is a test message", + Severity: tailcfg.SeverityHigh, + ImpactsConnectivity: true, + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "https://www.example.com", + Label: "Learn more", + }, + }, + }, + }, + { + name: "delete-one", + initialState: &tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": nil, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "message-b": { + Title: "Message B", + }, + }, + }, + { + name: "update-one", + initialState: &tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A updated", + }, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A updated", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + { + name: "add-one", + initialState: &tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + }, + }, + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-b": { + Title: "Message B", + }, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + { + name: "delete-all", + initialState: &tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "*": nil, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{}, + }, + { + name: "delete-all-and-add", + initialState: &tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "*": nil, + "message-c": { + Title: "Message C", + }, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "message-c": { + Title: "Message C", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ms := newTestMapSession(t, nil) + + if test.initialState != nil { + ms.netmapForResponse(test.initialState) + } + + nm := ms.netmapForResponse(&test.mapResponse) + + if diff := cmp.Diff(test.wantMessages, nm.DisplayMessages, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("unexpected warnings (-want +got):\n%s", diff) + } + }) + } +} + // TestNetmapHealthIntegration checks that we get the expected health warnings -// from processing a map response and passing the NetworkMap to a health tracker +// from processing a [tailcfg.MapResponse] containing health messages and passing the +// [netmap.NetworkMap] to a [health.Tracker]. func TestNetmapHealthIntegration(t *testing.T) { ms := newTestMapSession(t, nil) ht := health.Tracker{} @@ -1182,3 +1365,56 @@ func TestNetmapHealthIntegration(t *testing.T) { t.Fatalf("CurrentStatus().Warnings[\"control-health*\"] different than expected (-want +got)\n%s", d) } } + +// TestNetmapDisplayMessageIntegration checks that we get the expected health +// warnings from processing a [tailcfg.MapResponse] that contains DisplayMessages and +// passing the [netmap.NetworkMap] to a [health.Tracker]. +func TestNetmapDisplayMessageIntegration(t *testing.T) { + ms := newTestMapSession(t, nil) + ht := health.Tracker{} + + ht.SetIPNState("NeedsLogin", true) + ht.GotStreamedMapResponse() + baseWarnings := ht.CurrentState().Warnings + + nm := ms.netmapForResponse(&tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "test-message": { + Title: "Testing", + Text: "This is a test message", + Severity: tailcfg.SeverityHigh, + ImpactsConnectivity: true, + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "https://www.example.com", + Label: "Learn more", + }, + }, + }, + }) + ht.SetControlHealth(nm.DisplayMessages) + + state := ht.CurrentState() + + // Ignore warnings that aren't from the netmap + for k := range baseWarnings { + delete(state.Warnings, k) + } + + want := map[health.WarnableCode]health.UnhealthyState{ + "test-message": { + WarnableCode: "test-message", + Title: "Testing", + Text: "This is a test message", + Severity: health.SeverityHigh, + ImpactsConnectivity: true, + PrimaryAction: &health.UnhealthyStateAction{ + URL: "https://www.example.com", + Label: "Learn more", + }, + }, + } + + if diff := cmp.Diff(want, state.Warnings); diff != "" { + t.Errorf("unexpected message contents (-want +got):\n%s", diff) + } +} diff --git a/health/state.go b/health/state.go index cf4f922d7563e..cec967931af92 100644 --- a/health/state.go +++ b/health/state.go @@ -30,10 +30,19 @@ type UnhealthyState struct { Severity Severity Title string Text string - BrokenSince *time.Time `json:",omitempty"` - Args Args `json:",omitempty"` - DependsOn []WarnableCode `json:",omitempty"` - ImpactsConnectivity bool `json:",omitempty"` + BrokenSince *time.Time `json:",omitempty"` + Args Args `json:",omitempty"` + DependsOn []WarnableCode `json:",omitempty"` + ImpactsConnectivity bool `json:",omitempty"` + PrimaryAction *UnhealthyStateAction `json:",omitempty"` +} + +// UnhealthyStateAction represents an action (URL and link) to be presented to +// the user associated with an [UnhealthyState]. Analogous to +// [tailcfg.DisplayMessageAction]. +type UnhealthyStateAction struct { + URL string + Label string } // unhealthyState returns a unhealthyState of the Warnable given its current warningState. @@ -102,15 +111,23 @@ func (t *Tracker) CurrentState() *State { } for id, msg := range t.lastNotifiedControlMessages { - code := WarnableCode(id) - wm[code] = UnhealthyState{ - WarnableCode: code, + state := UnhealthyState{ + WarnableCode: WarnableCode(id), Severity: severityFromTailcfg(msg.Severity), Title: msg.Title, Text: msg.Text, ImpactsConnectivity: msg.ImpactsConnectivity, // TODO(tailscale/corp#27759): DependsOn? } + + if msg.PrimaryAction != nil { + state.PrimaryAction = &UnhealthyStateAction{ + URL: msg.PrimaryAction.URL, + Label: msg.PrimaryAction.Label, + } + } + + wm[state.WarnableCode] = state } return &State{ diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index d69c07a9fc944..05f0266317338 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -5828,7 +5828,14 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { b.pauseOrResumeControlClientLocked() if nm != nil { - b.health.SetControlHealth(nm.DisplayMessages) + messages := make(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage) + for id, msg := range nm.DisplayMessages { + if msg.PrimaryAction != nil && !b.validPopBrowserURL(msg.PrimaryAction.URL) { + msg.PrimaryAction = nil + } + messages[id] = msg + } + b.health.SetControlHealth(messages) } else { b.health.SetControlHealth(nil) } diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 19cfd91953e4c..1ad3225a5e27d 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -5339,3 +5339,68 @@ func TestSrcCapPacketFilter(t *testing.T) { t.Error("IsDrop() for node without cap = false, want true") } } + +func TestDisplayMessages(t *testing.T) { + b := newTestLocalBackend(t) + + // Pretend we're in a map poll so health updates get processed + ht := b.HealthTracker() + ht.SetIPNState("NeedsLogin", true) + ht.GotStreamedMapResponse() + + b.setNetMapLocked(&netmap.NetworkMap{ + DisplayMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-message": { + Title: "Testing", + }, + }, + }) + + state := ht.CurrentState() + _, ok := state.Warnings["test-message"] + + if !ok { + t.Error("no warning found with id 'test-message'") + } +} + +// TestDisplayMessagesURLFilter tests that we filter out any URLs that are not +// valid as a pop browser URL (see [LocalBackend.validPopBrowserURL]). +func TestDisplayMessagesURLFilter(t *testing.T) { + b := newTestLocalBackend(t) + + // Pretend we're in a map poll so health updates get processed + ht := b.HealthTracker() + ht.SetIPNState("NeedsLogin", true) + ht.GotStreamedMapResponse() + + b.setNetMapLocked(&netmap.NetworkMap{ + DisplayMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-message": { + Title: "Testing", + Severity: tailcfg.SeverityHigh, + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "https://www.evil.com", + Label: "Phishing Link", + }, + }, + }, + }) + + state := ht.CurrentState() + got, ok := state.Warnings["test-message"] + + if !ok { + t.Fatal("no warning found with id 'test-message'") + } + + want := health.UnhealthyState{ + WarnableCode: "test-message", + Title: "Testing", + Severity: health.SeverityHigh, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Unexpected message content (-want/+got):\n%s", diff) + } +} diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 7e2fa3ffc5f8e..4679609f3e9d4 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -161,7 +161,8 @@ type CapabilityVersion int // - 114: 2025-01-30: NodeAttrMaxKeyDuration CapMap defined, clients might use it (no tailscaled code change) (#14829) // - 115: 2025-03-07: Client understands DERPRegion.NoMeasureNoHome. // - 116: 2025-05-05: Client serves MagicDNS "AAAA" if NodeAttrMagicDNSPeerAAAA set on self node -const CurrentCapabilityVersion CapabilityVersion = 116 +// - 117: 2025-05-28: Client understands DisplayMessages (structured health messages), but not necessarily PrimaryAction. +const CurrentCapabilityVersion CapabilityVersion = 117 // ID is an integer ID for a user, node, or login allocated by the // control plane. @@ -2030,11 +2031,29 @@ type MapResponse struct { // known problems). A non-zero length slice are the list of problems that // the control plane sees. // + // Either this will be set, or DisplayMessages will be set, but not both. + // // Note that this package's type, due its use of a slice and omitempty, is // unable to marshal a zero-length non-nil slice. The control server needs // to marshal this type using a separate type. See MapResponse docs. Health []string `json:",omitempty"` + // DisplayMessages sets the health state of the node from the control + // plane's perspective. + // + // Either this will be set, or Health will be set, but not both. + // + // The map keys are IDs that uniquely identify the type of health issue. The + // map values are the messages. If the server sends down a map with entries, + // the client treats it as a patch: new entries are added, keys with a value + // of nil are deleted, existing entries with new values are updated. A nil + // map and an empty map both mean no change has occurred since the last + // update. + // + // As a special case, the map key "*" with a value of nil means to clear all + // prior display messages before processing the other map entries. + DisplayMessages map[DisplayMessageID]*DisplayMessage `json:",omitempty"` + // SSHPolicy, if non-nil, updates the SSH policy for how incoming // SSH connections should be handled. SSHPolicy *SSHPolicy `json:",omitempty"` @@ -2079,24 +2098,53 @@ type MapResponse struct { } // DisplayMessage represents a health state of the node from the control plane's -// perspective. It is deliberately similar to health.Warnable as both get -// converted into health.UnhealthyState to be sent to the GUI. +// perspective. It is deliberately similar to [health.Warnable] as both get +// converted into [health.UnhealthyState] to be sent to the GUI. type DisplayMessage struct { // Title is a string that the GUI uses as title for this message. The title - // should be short and fit in a single line. + // should be short and fit in a single line. It should not end in a period. + // + // Example: "Network may be blocking Tailscale". + // + // See the various instantiations of [health.Warnable] for more examples. Title string - // Text is an extended string that the GUI will display to the user. + // Text is an extended string that the GUI will display to the user. This + // could be multiple sentences explaining the issue in more detail. + // + // Example: "macOS Screen Time seems to be blocking Tailscale. Try disabling + // Screen Time in System Settings > Screen Time > Content & Privacy > Access + // to Web Content." + // + // See the various instantiations of [health.Warnable] for more examples. Text string // Severity is the severity of the DisplayMessage, which the GUI can use to - // determine how to display it. Maps to health.Severity. + // determine how to display it. Maps to [health.Severity]. Severity DisplayMessageSeverity // ImpactsConnectivity is whether the health problem will impact the user's // ability to connect to the Internet or other nodes on the tailnet, which // the GUI can use to determine how to display it. ImpactsConnectivity bool `json:",omitempty"` + + // Primary action, if present, represents the action to allow the user to + // take when interacting with this message. For example, if the + // DisplayMessage is shown via a notification, the action label might be a + // button on that notification and clicking the button would open the URL. + PrimaryAction *DisplayMessageAction `json:",omitempty"` +} + +// DisplayMessageAction represents an action (URL and link) to be presented to +// the user associated with a [DisplayMessage]. +type DisplayMessageAction struct { + // URL is the URL to navigate to when the user interacts with this action + URL string + + // Label is the call to action for the UI to display on the UI element that + // will open the URL (such as a button or link). For example, "Sign in" or + // "Learn more". + Label string } // DisplayMessageID is a string that uniquely identifies the kind of health diff --git a/types/netmap/nodemut.go b/types/netmap/nodemut.go index e31c731becbf1..ccbdeae3f5ec7 100644 --- a/types/netmap/nodemut.go +++ b/types/netmap/nodemut.go @@ -163,6 +163,7 @@ func mapResponseContainsNonPatchFields(res *tailcfg.MapResponse) bool { res.PacketFilters != nil || res.UserProfiles != nil || res.Health != nil || + res.DisplayMessages != nil || res.SSHPolicy != nil || res.TKAInfo != nil || res.DomainDataPlaneAuditLogID != "" || From 84aa7ff3bbcabd1bbc272c99de0c898b799cb144 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Fri, 30 May 2025 08:06:16 -1000 Subject: [PATCH 024/263] syncs: fix AtomicValue.CompareAndSwap (#16137) Fix CompareAndSwap in the edge-case where the underlying sync.AtomicValue is uninitialized (i.e., Store was never called) and the oldV is the zero value, then perform CompareAndSwap with any(nil). Also, document that T must be comparable. This is a pre-existing restriction. Fixes #16135 Signed-off-by: Joe Tsai --- syncs/syncs.go | 10 ++++++++-- syncs/syncs_test.go | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/syncs/syncs.go b/syncs/syncs.go index 337fca7557f34..cf0be919b5b6b 100644 --- a/syncs/syncs.go +++ b/syncs/syncs.go @@ -67,12 +67,18 @@ func (v *AtomicValue[T]) Swap(x T) (old T) { if oldV != nil { return oldV.(wrappedValue[T]).v } - return old + return old // zero value of T } // CompareAndSwap executes the compare-and-swap operation for the Value. +// It panics if T is not comparable. func (v *AtomicValue[T]) CompareAndSwap(oldV, newV T) (swapped bool) { - return v.v.CompareAndSwap(wrappedValue[T]{oldV}, wrappedValue[T]{newV}) + var zero T + return v.v.CompareAndSwap(wrappedValue[T]{oldV}, wrappedValue[T]{newV}) || + // In the edge-case where [atomic.Value.Store] is uninitialized + // and trying to compare with the zero value of T, + // then compare-and-swap with the nil any value. + (any(oldV) == any(zero) && v.v.CompareAndSwap(any(nil), wrappedValue[T]{newV})) } // MutexValue is a value protected by a mutex. diff --git a/syncs/syncs_test.go b/syncs/syncs_test.go index 901d429486d13..2439b6068391b 100644 --- a/syncs/syncs_test.go +++ b/syncs/syncs_test.go @@ -64,6 +64,23 @@ func TestAtomicValue(t *testing.T) { t.Fatalf("LoadOk = (%v, %v), want (nil, true)", got, gotOk) } } + + { + c1, c2, c3 := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var v AtomicValue[chan struct{}] + if v.CompareAndSwap(c1, c2) != false { + t.Fatalf("CompareAndSwap = true, want false") + } + if v.CompareAndSwap(nil, c1) != true { + t.Fatalf("CompareAndSwap = false, want true") + } + if v.CompareAndSwap(c2, c3) != false { + t.Fatalf("CompareAndSwap = true, want false") + } + if v.CompareAndSwap(c1, c2) != true { + t.Fatalf("CompareAndSwap = false, want true") + } + } } func TestMutexValue(t *testing.T) { From c9a5d638e9d8895eda0cb175afcb8dc738e75a09 Mon Sep 17 00:00:00 2001 From: Fran Bull Date: Tue, 27 May 2025 08:06:45 -0700 Subject: [PATCH 025/263] tsconsensus: enable writing state to disk The comments in the raft code say to only use the InMemStore for tests. Updates #16027 Signed-off-by: Fran Bull --- go.mod | 3 +++ go.sum | 10 +++++++++ tsconsensus/bolt_store.go | 19 ++++++++++++++++ tsconsensus/bolt_store_no_bolt.go | 18 +++++++++++++++ tsconsensus/tsconsensus.go | 37 +++++++++++++++++++++++++------ 5 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 tsconsensus/bolt_store.go create mode 100644 tsconsensus/bolt_store_no_bolt.go diff --git a/go.mod b/go.mod index d44a14aef6731..9ea25446bb57f 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/goreleaser/nfpm/v2 v2.33.1 github.com/hashicorp/go-hclog v1.6.2 github.com/hashicorp/raft v1.7.2 + github.com/hashicorp/raft-boltdb/v2 v2.3.1 github.com/hdevalence/ed25519consensus v0.2.0 github.com/illarion/gonotify/v3 v3.0.2 github.com/inetaf/tcpproxy v0.0.0-20250203165043-ded522cbd03f @@ -135,6 +136,7 @@ require ( github.com/alecthomas/go-check-sumtype v0.1.4 // indirect github.com/alexkohler/nakedret/v2 v2.0.4 // indirect github.com/armon/go-metrics v0.4.1 // indirect + github.com/boltdb/bolt v1.3.1 // indirect github.com/bombsimon/wsl/v4 v4.2.1 // indirect github.com/butuzov/mirror v1.1.0 // indirect github.com/catenacyber/perfsprint v0.7.1 // indirect @@ -166,6 +168,7 @@ require ( github.com/ykadowak/zerologlint v0.1.5 // indirect go-simpler.org/musttag v0.9.0 // indirect go-simpler.org/sloglint v0.5.0 // indirect + go.etcd.io/bbolt v1.3.11 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect go.opentelemetry.io/otel v1.33.0 // indirect diff --git a/go.sum b/go.sum index 73d87fd664fb0..318eae1ea886d 100644 --- a/go.sum +++ b/go.sum @@ -180,6 +180,8 @@ github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bombsimon/wsl/v4 v4.2.1 h1:Cxg6u+XDWff75SIFFmNsqnIOgob+Q9hG6y/ioKbRFiM= github.com/bombsimon/wsl/v4 v4.2.1/go.mod h1:Xu/kDxGZTofQcDGCtQe9KCzhHphIe0fDuyWTxER9Feo= github.com/bramvdbogaerde/go-scp v1.4.0 h1:jKMwpwCbcX1KyvDbm/PDJuXcMuNVlLGi0Q0reuzjyKY= @@ -555,6 +557,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY= github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0= github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= @@ -571,6 +575,10 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/raft v1.7.2 h1:pyvxhfJ4R8VIAlHKvLoKQWElZspsCVT6YWuxVxsPAgc= github.com/hashicorp/raft v1.7.2/go.mod h1:DfvCGFxpAUPE0L4Uc8JLlTPtc3GzSbdH0MTJCLgnmJQ= +github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKcZ+qp2VlaaZsYZfLOmIiuJNpEi48Rl8u9cQ= +github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0= +github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc= +github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE= github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU= github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -1046,6 +1054,8 @@ go-simpler.org/musttag v0.9.0 h1:Dzt6/tyP9ONr5g9h9P3cnYWCxeBFRkd0uJL/w+1Mxos= go-simpler.org/musttag v0.9.0/go.mod h1:gA9nThnalvNSKpEoyp3Ko4/vCX2xTpqKoUtNqXOnVR4= go-simpler.org/sloglint v0.5.0 h1:2YCcd+YMuYpuqthCgubcF5lBSjb6berc5VMOYUHKrpY= go-simpler.org/sloglint v0.5.0/go.mod h1:EUknX5s8iXqf18KQxKnaBHUPVriiPnOrPjjJcsaTcSQ= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/tsconsensus/bolt_store.go b/tsconsensus/bolt_store.go new file mode 100644 index 0000000000000..ca347cfc049b2 --- /dev/null +++ b/tsconsensus/bolt_store.go @@ -0,0 +1,19 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !loong64 + +package tsconsensus + +import ( + "github.com/hashicorp/raft" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" +) + +func boltStore(path string) (raft.StableStore, raft.LogStore, error) { + store, err := raftboltdb.NewBoltStore(path) + if err != nil { + return nil, nil, err + } + return store, store, nil +} diff --git a/tsconsensus/bolt_store_no_bolt.go b/tsconsensus/bolt_store_no_bolt.go new file mode 100644 index 0000000000000..33b3bd6c7a29f --- /dev/null +++ b/tsconsensus/bolt_store_no_bolt.go @@ -0,0 +1,18 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build loong64 + +package tsconsensus + +import ( + "errors" + + "github.com/hashicorp/raft" +) + +func boltStore(path string) (raft.StableStore, raft.LogStore, error) { + // "github.com/hashicorp/raft-boltdb/v2" doesn't build on loong64 + // see https://github.com/hashicorp/raft-boltdb/issues/27 + return nil, nil, errors.New("not implemented") +} diff --git a/tsconsensus/tsconsensus.go b/tsconsensus/tsconsensus.go index 74094782f4383..b6bf373102aa6 100644 --- a/tsconsensus/tsconsensus.go +++ b/tsconsensus/tsconsensus.go @@ -32,6 +32,7 @@ import ( "net" "net/http" "net/netip" + "path/filepath" "time" "github.com/hashicorp/go-hclog" @@ -71,6 +72,7 @@ type Config struct { MaxConnPool int ConnTimeout time.Duration ServeDebugMonitor bool + StateDirPath string } // DefaultConfig returns a Config populated with default values ready for use. @@ -223,10 +225,31 @@ func Start(ctx context.Context, ts *tsnet.Server, fsm raft.FSM, clusterTag strin func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, self selfRaftNode, auth *authorization, cfg Config) (*raft.Raft, error) { cfg.Raft.LocalID = raft.ServerID(self.id) - // no persistence (for now?) - logStore := raft.NewInmemStore() - stableStore := raft.NewInmemStore() - snapshots := raft.NewInmemSnapshotStore() + var logStore raft.LogStore + var stableStore raft.StableStore + var snapStore raft.SnapshotStore + + if cfg.StateDirPath == "" { + // comments in raft code say to only use for tests + logStore = raft.NewInmemStore() + stableStore = raft.NewInmemStore() + snapStore = raft.NewInmemSnapshotStore() + } else { + var err error + stableStore, logStore, err = boltStore(filepath.Join(cfg.StateDirPath, "store")) + if err != nil { + return nil, err + } + snaplogger := hclog.New(&hclog.LoggerOptions{ + Name: "raft-snap", + Output: cfg.Raft.LogOutput, + Level: hclog.LevelFromString(cfg.Raft.LogLevel), + }) + snapStore, err = raft.NewFileSnapshotStoreWithLogger(filepath.Join(cfg.StateDirPath, "snapstore"), 2, snaplogger) + if err != nil { + return nil, err + } + } // opens the listener on the raft port, raft will close it when it thinks it's appropriate ln, err := ts.Listen("tcp", raftAddr(self.hostAddr, cfg)) @@ -234,7 +257,7 @@ func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, sel return nil, err } - logger := hclog.New(&hclog.LoggerOptions{ + transportLogger := hclog.New(&hclog.LoggerOptions{ Name: "raft-net", Output: cfg.Raft.LogOutput, Level: hclog.LevelFromString(cfg.Raft.LogLevel), @@ -248,9 +271,9 @@ func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, sel }, cfg.MaxConnPool, cfg.ConnTimeout, - logger) + transportLogger) - return raft.NewRaft(cfg.Raft, *fsm, logStore, stableStore, snapshots, transport) + return raft.NewRaft(cfg.Raft, *fsm, logStore, stableStore, snapStore, transport) } // A Consensus is the consensus algorithm for a tsnet.Server From 5f35143d83016520b6870fedd116ada0a84e856a Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 2 Jun 2025 13:22:28 -0700 Subject: [PATCH 026/263] go.mod,wgengine/magicsock: update wireguard-go (#16148) Our conn.Bind implementation is updated to make Send() offset-aware for future VXLAN/Geneve encapsulation support. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- go.mod | 2 +- go.sum | 4 +-- wgengine/magicsock/batching_conn.go | 2 +- wgengine/magicsock/batching_conn_linux.go | 9 +++--- .../magicsock/batching_conn_linux_test.go | 29 +++++++++++-------- wgengine/magicsock/endpoint.go | 7 +++-- wgengine/magicsock/magicsock.go | 12 ++++---- wgengine/magicsock/magicsock_test.go | 2 +- wgengine/magicsock/rebinding_conn.go | 5 ++-- wgengine/wgcfg/device_test.go | 6 ++-- 10 files changed, 43 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index 9ea25446bb57f..ec98275e599f0 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/tailscale/setec v0.0.0-20250205144240-8898a29c3fbb github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 - github.com/tailscale/wireguard-go v0.0.0-20250304000100-91a0587fb251 + github.com/tailscale/wireguard-go v0.0.0-20250530210235-65cd6eed7d7f github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e github.com/tc-hib/winres v0.2.1 github.com/tcnksm/go-httpstat v0.2.0 diff --git a/go.sum b/go.sum index 318eae1ea886d..0b521da8ca4c4 100644 --- a/go.sum +++ b/go.sum @@ -975,8 +975,8 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 h1:U github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y= -github.com/tailscale/wireguard-go v0.0.0-20250304000100-91a0587fb251 h1:h/41LFTrwMxB9Xvvug0kRdQCU5TlV1+pAMQw0ZtDE3U= -github.com/tailscale/wireguard-go v0.0.0-20250304000100-91a0587fb251/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= +github.com/tailscale/wireguard-go v0.0.0-20250530210235-65cd6eed7d7f h1:vg3PmQdq1BbB2V81iC1VBICQtfwbVGZ/4A/p7QKXTK0= +github.com/tailscale/wireguard-go v0.0.0-20250530210235-65cd6eed7d7f/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg= github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA= diff --git a/wgengine/magicsock/batching_conn.go b/wgengine/magicsock/batching_conn.go index 5320d1cafa59a..58cfe28aa940d 100644 --- a/wgengine/magicsock/batching_conn.go +++ b/wgengine/magicsock/batching_conn.go @@ -21,5 +21,5 @@ var ( type batchingConn interface { nettype.PacketConn ReadBatch(msgs []ipv6.Message, flags int) (n int, err error) - WriteBatchTo(buffs [][]byte, addr netip.AddrPort) error + WriteBatchTo(buffs [][]byte, addr netip.AddrPort, offset int) error } diff --git a/wgengine/magicsock/batching_conn_linux.go b/wgengine/magicsock/batching_conn_linux.go index 25bf974b022ba..9ad5e44749d27 100644 --- a/wgengine/magicsock/batching_conn_linux.go +++ b/wgengine/magicsock/batching_conn_linux.go @@ -94,7 +94,7 @@ const ( // coalesceMessages iterates msgs, coalescing them where possible while // maintaining datagram order. All msgs have their Addr field set to addr. -func (c *linuxBatchingConn) coalesceMessages(addr *net.UDPAddr, buffs [][]byte, msgs []ipv6.Message) int { +func (c *linuxBatchingConn) coalesceMessages(addr *net.UDPAddr, buffs [][]byte, msgs []ipv6.Message, offset int) int { var ( base = -1 // index of msg we are currently coalescing into gsoSize int // segmentation size of msgs[base] @@ -106,6 +106,7 @@ func (c *linuxBatchingConn) coalesceMessages(addr *net.UDPAddr, buffs [][]byte, maxPayloadLen = maxIPv6PayloadLen } for i, buff := range buffs { + buff = buff[offset:] if i > 0 { msgLen := len(buff) baseLenBefore := len(msgs[base].Buffers[0]) @@ -162,7 +163,7 @@ func (c *linuxBatchingConn) putSendBatch(batch *sendBatch) { c.sendBatchPool.Put(batch) } -func (c *linuxBatchingConn) WriteBatchTo(buffs [][]byte, addr netip.AddrPort) error { +func (c *linuxBatchingConn) WriteBatchTo(buffs [][]byte, addr netip.AddrPort, offset int) error { batch := c.getSendBatch() defer c.putSendBatch(batch) if addr.Addr().Is6() { @@ -181,10 +182,10 @@ func (c *linuxBatchingConn) WriteBatchTo(buffs [][]byte, addr netip.AddrPort) er ) retry: if c.txOffload.Load() { - n = c.coalesceMessages(batch.ua, buffs, batch.msgs) + n = c.coalesceMessages(batch.ua, buffs, batch.msgs, offset) } else { for i := range buffs { - batch.msgs[i].Buffers[0] = buffs[i] + batch.msgs[i].Buffers[0] = buffs[i][offset:] batch.msgs[i].Addr = batch.ua batch.msgs[i].OOB = batch.msgs[i].OOB[:0] } diff --git a/wgengine/magicsock/batching_conn_linux_test.go b/wgengine/magicsock/batching_conn_linux_test.go index 5c22bf1c73cf4..effd5a2cc5001 100644 --- a/wgengine/magicsock/batching_conn_linux_test.go +++ b/wgengine/magicsock/batching_conn_linux_test.go @@ -9,6 +9,7 @@ import ( "testing" "golang.org/x/net/ipv6" + "tailscale.com/net/packet" ) func setGSOSize(control *[]byte, gsoSize uint16) { @@ -154,6 +155,10 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { getGSOSizeFromControl: getGSOSize, } + withGeneveSpace := func(len, cap int) []byte { + return make([]byte, len+packet.GeneveFixedHeaderLength, cap+packet.GeneveFixedHeaderLength) + } + cases := []struct { name string buffs [][]byte @@ -163,7 +168,7 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { { name: "one message no coalesce", buffs: [][]byte{ - make([]byte, 1, 1), + withGeneveSpace(1, 1), }, wantLens: []int{1}, wantGSO: []int{0}, @@ -171,8 +176,8 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { { name: "two messages equal len coalesce", buffs: [][]byte{ - make([]byte, 1, 2), - make([]byte, 1, 1), + withGeneveSpace(1, 2), + withGeneveSpace(1, 1), }, wantLens: []int{2}, wantGSO: []int{1}, @@ -180,8 +185,8 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { { name: "two messages unequal len coalesce", buffs: [][]byte{ - make([]byte, 2, 3), - make([]byte, 1, 1), + withGeneveSpace(2, 3), + withGeneveSpace(1, 1), }, wantLens: []int{3}, wantGSO: []int{2}, @@ -189,9 +194,9 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { { name: "three messages second unequal len coalesce", buffs: [][]byte{ - make([]byte, 2, 3), - make([]byte, 1, 1), - make([]byte, 2, 2), + withGeneveSpace(2, 3), + withGeneveSpace(1, 1), + withGeneveSpace(2, 2), }, wantLens: []int{3, 2}, wantGSO: []int{2, 0}, @@ -199,9 +204,9 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { { name: "three messages limited cap coalesce", buffs: [][]byte{ - make([]byte, 2, 4), - make([]byte, 2, 2), - make([]byte, 2, 2), + withGeneveSpace(2, 4), + withGeneveSpace(2, 2), + withGeneveSpace(2, 2), }, wantLens: []int{4, 2}, wantGSO: []int{2, 0}, @@ -219,7 +224,7 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { msgs[i].Buffers = make([][]byte, 1) msgs[i].OOB = make([]byte, 0, 2) } - got := c.coalesceMessages(addr, tt.buffs, msgs) + got := c.coalesceMessages(addr, tt.buffs, msgs, packet.GeneveFixedHeaderLength) if got != len(tt.wantLens) { t.Fatalf("got len %d want: %d", got, len(tt.wantLens)) } diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index c2d18d707128d..243d0f4de26a8 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -927,7 +927,7 @@ var ( errPingTooBig = errors.New("ping size too big") ) -func (de *endpoint) send(buffs [][]byte) error { +func (de *endpoint) send(buffs [][]byte, offset int) error { de.mu.Lock() if de.expired { de.mu.Unlock() @@ -961,7 +961,7 @@ func (de *endpoint) send(buffs [][]byte) error { } var err error if udpAddr.IsValid() { - _, err = de.c.sendUDPBatch(udpAddr, buffs) + _, err = de.c.sendUDPBatch(udpAddr, buffs, offset) // If the error is known to indicate that the endpoint is no longer // usable, clear the endpoint statistics so that the next send will @@ -972,7 +972,7 @@ func (de *endpoint) send(buffs [][]byte) error { var txBytes int for _, b := range buffs { - txBytes += len(b) + txBytes += len(b[offset:]) } switch { @@ -993,6 +993,7 @@ func (de *endpoint) send(buffs [][]byte) error { allOk := true var txBytes int for _, buff := range buffs { + buff = buff[offset:] const isDisco = false ok, _ := de.c.sendAddr(derpAddr, de.publicKey, buff, isDisco) txBytes += len(buff) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 5b0f28a33c051..3a4fdf8a248f9 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -1264,8 +1264,8 @@ func (c *Conn) networkDown() bool { return !c.networkUp.Load() } // Send implements conn.Bind. // -// See https://pkg.go.dev/golang.zx2c4.com/wireguard/conn#Bind.Send -func (c *Conn) Send(buffs [][]byte, ep conn.Endpoint) (err error) { +// See https://pkg.go.dev/github.com/tailscale/wireguard-go/conn#Bind.Send +func (c *Conn) Send(buffs [][]byte, ep conn.Endpoint, offset int) (err error) { n := int64(len(buffs)) defer func() { if err != nil { @@ -1278,7 +1278,7 @@ func (c *Conn) Send(buffs [][]byte, ep conn.Endpoint) (err error) { return errNetworkDown } if ep, ok := ep.(*endpoint); ok { - return ep.send(buffs) + return ep.send(buffs, offset) } // If it's not of type *endpoint, it's probably *lazyEndpoint, which means // we don't actually know who the peer is and we're waiting for wireguard-go @@ -1294,7 +1294,7 @@ var errNoUDP = errors.New("no UDP available on platform") var errUnsupportedConnType = errors.New("unsupported connection type") -func (c *Conn) sendUDPBatch(addr netip.AddrPort, buffs [][]byte) (sent bool, err error) { +func (c *Conn) sendUDPBatch(addr netip.AddrPort, buffs [][]byte, offset int) (sent bool, err error) { isIPv6 := false switch { case addr.Addr().Is4(): @@ -1304,9 +1304,9 @@ func (c *Conn) sendUDPBatch(addr netip.AddrPort, buffs [][]byte) (sent bool, err panic("bogus sendUDPBatch addr type") } if isIPv6 { - err = c.pconn6.WriteBatchTo(buffs, addr) + err = c.pconn6.WriteBatchTo(buffs, addr, offset) } else { - err = c.pconn4.WriteBatchTo(buffs, addr) + err = c.pconn4.WriteBatchTo(buffs, addr, offset) } if err != nil { var errGSO neterror.ErrUDPGSODisabled diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index ddbf3e3940efe..e1801187352d5 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -3147,7 +3147,7 @@ func TestNetworkDownSendErrors(t *testing.T) { defer conn.Close() conn.SetNetworkUp(false) - if err := conn.Send([][]byte{{00}}, &lazyEndpoint{}); err == nil { + if err := conn.Send([][]byte{{00}}, &lazyEndpoint{}, 0); err == nil { t.Error("expected error, got nil") } resp := httptest.NewRecorder() diff --git a/wgengine/magicsock/rebinding_conn.go b/wgengine/magicsock/rebinding_conn.go index c27abbadc9ced..7a9dd1821f306 100644 --- a/wgengine/magicsock/rebinding_conn.go +++ b/wgengine/magicsock/rebinding_conn.go @@ -71,12 +71,13 @@ func (c *RebindingUDPConn) ReadFromUDPAddrPort(b []byte) (int, netip.AddrPort, e } // WriteBatchTo writes buffs to addr. -func (c *RebindingUDPConn) WriteBatchTo(buffs [][]byte, addr netip.AddrPort) error { +func (c *RebindingUDPConn) WriteBatchTo(buffs [][]byte, addr netip.AddrPort, offset int) error { for { pconn := *c.pconnAtomic.Load() b, ok := pconn.(batchingConn) if !ok { for _, buf := range buffs { + buf = buf[offset:] _, err := c.writeToUDPAddrPortWithInitPconn(pconn, buf, addr) if err != nil { return err @@ -84,7 +85,7 @@ func (c *RebindingUDPConn) WriteBatchTo(buffs [][]byte, addr netip.AddrPort) err } return nil } - err := b.WriteBatchTo(buffs, addr) + err := b.WriteBatchTo(buffs, addr, offset) if err != nil { if pconn != c.currentConn() { continue diff --git a/wgengine/wgcfg/device_test.go b/wgengine/wgcfg/device_test.go index d54282e4bdf04..9138d6e5a0f47 100644 --- a/wgengine/wgcfg/device_test.go +++ b/wgengine/wgcfg/device_test.go @@ -242,9 +242,9 @@ type noopBind struct{} func (noopBind) Open(port uint16) (fns []conn.ReceiveFunc, actualPort uint16, err error) { return nil, 1, nil } -func (noopBind) Close() error { return nil } -func (noopBind) SetMark(mark uint32) error { return nil } -func (noopBind) Send(b [][]byte, ep conn.Endpoint) error { return nil } +func (noopBind) Close() error { return nil } +func (noopBind) SetMark(mark uint32) error { return nil } +func (noopBind) Send(b [][]byte, ep conn.Endpoint, offset int) error { return nil } func (noopBind) ParseEndpoint(s string) (conn.Endpoint, error) { return dummyEndpoint(s), nil } From 8a3afa5963f425e42a02016b7059434c3962f2c3 Mon Sep 17 00:00:00 2001 From: James Sanderson Date: Mon, 2 Jun 2025 15:52:16 +0100 Subject: [PATCH 027/263] ipn/ipnlocal: fix deadlock when filtering DisplayMessage URLs Updates tailscale/corp#27759 Signed-off-by: James Sanderson --- ipn/ipnlocal/local.go | 14 ++++++++++++-- ipn/ipnlocal/local_test.go | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 05f0266317338..e494920b1f5b0 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -3289,6 +3289,16 @@ func (b *LocalBackend) popBrowserAuthNow(url string, keyExpired bool, recipient // // b.mu must *not* be held. func (b *LocalBackend) validPopBrowserURL(urlStr string) bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.validPopBrowserURLLocked(urlStr) +} + +// validPopBrowserURLLocked reports whether urlStr is a valid value for a +// control server to send in a *URL field. +// +// b.mu must be held. +func (b *LocalBackend) validPopBrowserURLLocked(urlStr string) bool { if urlStr == "" { return false } @@ -3296,7 +3306,7 @@ func (b *LocalBackend) validPopBrowserURL(urlStr string) bool { if err != nil { return false } - serverURL := b.Prefs().ControlURLOrDefault() + serverURL := b.sanitizedPrefsLocked().ControlURLOrDefault() if ipn.IsLoginServerSynonym(serverURL) { // When connected to the official Tailscale control plane, only allow // URLs from tailscale.com or its subdomains. @@ -5830,7 +5840,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { if nm != nil { messages := make(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage) for id, msg := range nm.DisplayMessages { - if msg.PrimaryAction != nil && !b.validPopBrowserURL(msg.PrimaryAction.URL) { + if msg.PrimaryAction != nil && !b.validPopBrowserURLLocked(msg.PrimaryAction.URL) { msg.PrimaryAction = nil } messages[id] = msg diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 1ad3225a5e27d..d23bd1e262dd8 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -5374,6 +5374,7 @@ func TestDisplayMessagesURLFilter(t *testing.T) { ht.SetIPNState("NeedsLogin", true) ht.GotStreamedMapResponse() + defer b.lockAndGetUnlock()() b.setNetMapLocked(&netmap.NetworkMap{ DisplayMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ "test-message": { From cc988596a214e7bb429dc85e4413586c13ea99d3 Mon Sep 17 00:00:00 2001 From: Anton Tolchanov Date: Fri, 30 May 2025 13:03:46 +0100 Subject: [PATCH 028/263] posture: propagate serial number from MDM on Android Updates #16010 Signed-off-by: Anton Tolchanov --- posture/serialnumber_stub.go | 3 +-- posture/{serialnumber_ios.go => serialnumber_syspolicy.go} | 6 ++++-- util/syspolicy/policy_keys.go | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) rename posture/{serialnumber_ios.go => serialnumber_syspolicy.go} (75%) diff --git a/posture/serialnumber_stub.go b/posture/serialnumber_stub.go index cdabf03e5a417..4cc84fa133489 100644 --- a/posture/serialnumber_stub.go +++ b/posture/serialnumber_stub.go @@ -1,13 +1,12 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -// android: not implemented // js: not implemented // plan9: not implemented // solaris: currently unsupported by go-smbios: // https://github.com/digitalocean/go-smbios/pull/21 -//go:build android || solaris || plan9 || js || wasm || tamago || aix || (darwin && !cgo && !ios) +//go:build solaris || plan9 || js || wasm || tamago || aix || (darwin && !cgo && !ios) package posture diff --git a/posture/serialnumber_ios.go b/posture/serialnumber_syspolicy.go similarity index 75% rename from posture/serialnumber_ios.go rename to posture/serialnumber_syspolicy.go index 55d0e438b54d5..d6491ff214b95 100644 --- a/posture/serialnumber_ios.go +++ b/posture/serialnumber_syspolicy.go @@ -1,6 +1,8 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause +//go:build android || ios + package posture import ( @@ -10,9 +12,9 @@ import ( "tailscale.com/util/syspolicy" ) -// GetSerialNumbers returns the serial number of the iOS/tvOS device as reported by an +// GetSerialNumbers returns the serial number of the device as reported by an // MDM solution. It requires configuration via the DeviceSerialNumber system policy. -// This is the only way to gather serial numbers on iOS and tvOS. +// This is the only way to gather serial numbers on iOS, tvOS and Android. func GetSerialNumbers(_ logger.Logf) ([]string, error) { s, err := syspolicy.GetString(syspolicy.DeviceSerialNumber, "") if err != nil { diff --git a/util/syspolicy/policy_keys.go b/util/syspolicy/policy_keys.go index 29b2dfd281c4a..ed00d0004abb1 100644 --- a/util/syspolicy/policy_keys.go +++ b/util/syspolicy/policy_keys.go @@ -126,8 +126,8 @@ const ( // The default is "user-decides" unless otherwise stated. PostureChecking Key = "PostureChecking" // DeviceSerialNumber is the serial number of the device that is running Tailscale. - // This is used on iOS/tvOS to allow IT administrators to manually give us a serial number via MDM. - // We are unable to programmatically get the serial number from IOKit due to sandboxing restrictions. + // This is used on Android, iOS and tvOS to allow IT administrators to manually give us a serial number via MDM. + // We are unable to programmatically get the serial number on mobile due to sandboxing restrictions. DeviceSerialNumber Key = "DeviceSerialNumber" // ManagedByOrganizationName indicates the name of the organization managing the Tailscale From 5f0e1390123db8231244107706145a48a9d60938 Mon Sep 17 00:00:00 2001 From: Raj Singh Date: Tue, 3 Jun 2025 12:52:00 -0400 Subject: [PATCH 029/263] cmd/tsidp: add Docker image building support (#16078) - Add tsidp target to build_docker.sh for standard Tailscale image builds - Add publishdevtsidp Makefile target for development image publishing - Remove Dockerfile, using standard build process - Include tsidp in depaware dependency tracking - Update README with comprehensive Docker usage examples This enables tsidp to be built and published like other Tailscale components (tailscale/tailscale, tailscale/k8s-operator, tailscale/k8s-nameserver). Fixes #16077 Signed-off-by: Raj Singh --- Makefile | 14 +- build_docker.sh | 18 ++ cmd/tsidp/Dockerfile | 41 --- cmd/tsidp/README.md | 69 ++--- cmd/tsidp/depaware.txt | 657 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 719 insertions(+), 80 deletions(-) delete mode 100644 cmd/tsidp/Dockerfile create mode 100644 cmd/tsidp/depaware.txt diff --git a/Makefile b/Makefile index c30818c965b77..1978af90d259a 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,8 @@ updatedeps: ## Update depaware deps tailscale.com/cmd/tailscale \ tailscale.com/cmd/derper \ tailscale.com/cmd/k8s-operator \ - tailscale.com/cmd/stund + tailscale.com/cmd/stund \ + tailscale.com/cmd/tsidp PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update -goos=linux,darwin,windows,android,ios --internal \ tailscale.com/tsnet @@ -34,7 +35,8 @@ depaware: ## Run depaware checks tailscale.com/cmd/tailscale \ tailscale.com/cmd/derper \ tailscale.com/cmd/k8s-operator \ - tailscale.com/cmd/stund + tailscale.com/cmd/stund \ + tailscale.com/cmd/tsidp PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --goos=linux,darwin,windows,android,ios --internal \ tailscale.com/tsnet @@ -114,6 +116,14 @@ publishdevnameserver: ## Build and publish k8s-nameserver image to location spec @test "${REPO}" != "ghcr.io/tailscale/k8s-nameserver" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-nameserver" && exit 1) TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-nameserver ./build_docker.sh +publishdevtsidp: ## Build and publish tsidp image to location specified by ${REPO} + @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) + @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) + @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) + @test "${REPO}" != "tailscale/tsidp" || (echo "REPO=... must not be tailscale/tsidp" && exit 1) + @test "${REPO}" != "ghcr.io/tailscale/tsidp" || (echo "REPO=... must not be ghcr.io/tailscale/tsidp" && exit 1) + TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=tsidp ./build_docker.sh + .PHONY: sshintegrationtest sshintegrationtest: ## Run the SSH integration tests in various Docker containers @GOOS=linux GOARCH=amd64 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ diff --git a/build_docker.sh b/build_docker.sh index 15105c2ef5541..bdc9dc08609fa 100755 --- a/build_docker.sh +++ b/build_docker.sh @@ -90,6 +90,24 @@ case "$TARGET" in --annotations="${ANNOTATIONS}" \ /usr/local/bin/k8s-nameserver ;; + tsidp) + DEFAULT_REPOS="tailscale/tsidp" + REPOS="${REPOS:-${DEFAULT_REPOS}}" + go run github.com/tailscale/mkctr \ + --gopaths="tailscale.com/cmd/tsidp:/usr/local/bin/tsidp" \ + --ldflags=" \ + -X tailscale.com/version.longStamp=${VERSION_LONG} \ + -X tailscale.com/version.shortStamp=${VERSION_SHORT} \ + -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \ + --base="${BASE}" \ + --tags="${TAGS}" \ + --gotags="ts_package_container" \ + --repos="${REPOS}" \ + --push="${PUSH}" \ + --target="${PLATFORM}" \ + --annotations="${ANNOTATIONS}" \ + /usr/local/bin/tsidp + ;; *) echo "unknown target: $TARGET" exit 1 diff --git a/cmd/tsidp/Dockerfile b/cmd/tsidp/Dockerfile deleted file mode 100644 index c4f352ed01839..0000000000000 --- a/cmd/tsidp/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Build stage -FROM golang:alpine AS builder - -# Install build dependencies -RUN apk add --no-cache git - -# Set working directory -WORKDIR /src - -# Copy only go.mod and go.sum first to leverage Docker caching -COPY go.mod go.sum ./ -RUN go mod download - -# Copy the entire repository -COPY . . - -# Build the tsidp binary -RUN go build -o /bin/tsidp ./cmd/tsidp - -# Final stage -FROM alpine:latest - -# Create necessary directories -RUN mkdir -p /var/lib/tsidp - -# Copy binary from builder stage -COPY --from=builder /bin/tsidp /app/tsidp - -# Set working directory -WORKDIR /app - -# Environment variables -ENV TAILSCALE_USE_WIP_CODE=1 \ - TS_HOSTNAME=idp \ - TS_STATE_DIR=/var/lib/tsidp - -# Expose the default port -EXPOSE 443 - -# Run the application -ENTRYPOINT ["/bin/sh", "-c", "/app/tsidp --hostname=${TS_HOSTNAME} --dir=${TS_STATE_DIR}"] diff --git a/cmd/tsidp/README.md b/cmd/tsidp/README.md index 61a81e8aee7ae..fce844e0b309c 100644 --- a/cmd/tsidp/README.md +++ b/cmd/tsidp/README.md @@ -12,43 +12,38 @@ ## Installation using Docker -1. **Build the Docker Image** - - The Dockerfile uses a multi-stage build process to: - - Build the `tsidp` binary from source - - Create a minimal Alpine-based image with just the necessary components - - ```bash - # Clone the Tailscale repository - git clone https://github.com/tailscale/tailscale.git - cd tailscale - ``` - - ```bash - # Build the Docker image - docker build -t tsidp:latest -f cmd/tsidp/Dockerfile . - ``` - -2. **Run the Container** - - Replace `YOUR_TAILSCALE_AUTHKEY` with your Tailscale authentication key. - - ```bash - docker run -d \ - --name tsidp \ - -p 443:443 \ - -e TS_AUTHKEY=YOUR_TAILSCALE_AUTHKEY \ - -e TS_HOSTNAME=idp \ - -v tsidp-data:/var/lib/tsidp \ - tsidp:latest - ``` - -3. **Verify Installation** - ```bash - docker logs tsidp - ``` - - Visit `https://idp.tailnet.ts.net` to confirm the service is running. +### Building from Source + +```bash +# Clone the Tailscale repository +git clone https://github.com/tailscale/tailscale.git +cd tailscale + +# Build and publish to your own registry +make publishdevtsidp REPO=ghcr.io/yourusername/tsidp TAGS=v0.0.1 PUSH=true +``` + +### Running the Container + +Replace `YOUR_TAILSCALE_AUTHKEY` with your Tailscale authentication key: + +```bash +docker run -d \ + --name tsidp \ + -p 443:443 \ + -e TS_AUTHKEY=YOUR_TAILSCALE_AUTHKEY \ + -e TAILSCALE_USE_WIP_CODE=1 \ + -v tsidp-data:/var/lib/tsidp \ + ghcr.io/yourusername/tsidp:v0.0.1 \ + tsidp --hostname=idp --dir=/var/lib/tsidp +``` + +### Verify Installation +```bash +docker logs tsidp +``` + +Visit `https://idp.tailnet.ts.net` to confirm the service is running. ## Usage Example: Proxmox Integration diff --git a/cmd/tsidp/depaware.txt b/cmd/tsidp/depaware.txt new file mode 100644 index 0000000000000..1ea4b3d885987 --- /dev/null +++ b/cmd/tsidp/depaware.txt @@ -0,0 +1,657 @@ +tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depaware) + + filippo.io/edwards25519 from github.com/hdevalence/ed25519consensus + filippo.io/edwards25519/field from filippo.io/edwards25519 + W 💣 github.com/alexbrainman/sspi from github.com/alexbrainman/sspi/internal/common+ + W github.com/alexbrainman/sspi/internal/common from github.com/alexbrainman/sspi/negotiate + W 💣 github.com/alexbrainman/sspi/negotiate from tailscale.com/net/tshttpproxy + L github.com/aws/aws-sdk-go-v2/aws from github.com/aws/aws-sdk-go-v2/aws/defaults+ + L github.com/aws/aws-sdk-go-v2/aws/arn from tailscale.com/ipn/store/awsstore + L github.com/aws/aws-sdk-go-v2/aws/defaults from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/aws/middleware from github.com/aws/aws-sdk-go-v2/aws/retry+ + L github.com/aws/aws-sdk-go-v2/aws/protocol/query from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/aws/protocol/restjson from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/aws/protocol/xml from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/aws/ratelimit from github.com/aws/aws-sdk-go-v2/aws/retry + L github.com/aws/aws-sdk-go-v2/aws/retry from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client+ + L github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 from github.com/aws/aws-sdk-go-v2/aws/signer/v4 + L github.com/aws/aws-sdk-go-v2/aws/signer/v4 from github.com/aws/aws-sdk-go-v2/internal/auth/smithy+ + L github.com/aws/aws-sdk-go-v2/aws/transport/http from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/config from tailscale.com/ipn/store/awsstore + L github.com/aws/aws-sdk-go-v2/credentials from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/credentials/endpointcreds from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds + L github.com/aws/aws-sdk-go-v2/credentials/processcreds from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/credentials/ssocreds from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/credentials/stscreds from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/feature/ec2/imds from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config from github.com/aws/aws-sdk-go-v2/feature/ec2/imds + L github.com/aws/aws-sdk-go-v2/internal/auth from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ + L github.com/aws/aws-sdk-go-v2/internal/auth/smithy from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/internal/configsources from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/internal/context from github.com/aws/aws-sdk-go-v2/aws/retry+ + L github.com/aws/aws-sdk-go-v2/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+ + L github.com/aws/aws-sdk-go-v2/internal/ini from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/internal/middleware from github.com/aws/aws-sdk-go-v2/service/sso+ + L github.com/aws/aws-sdk-go-v2/internal/rand from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/aws-sdk-go-v2/internal/sdk from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/aws-sdk-go-v2/internal/sdkio from github.com/aws/aws-sdk-go-v2/credentials/processcreds + L github.com/aws/aws-sdk-go-v2/internal/shareddefaults from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 + L github.com/aws/aws-sdk-go-v2/internal/sync/singleflight from github.com/aws/aws-sdk-go-v2/aws + L github.com/aws/aws-sdk-go-v2/internal/timeconv from github.com/aws/aws-sdk-go-v2/aws/retry + L github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/service/internal/presigned-url from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/service/ssm from tailscale.com/ipn/store/awsstore + L github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm + L github.com/aws/aws-sdk-go-v2/service/ssm/types from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/service/sso from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sso + L github.com/aws/aws-sdk-go-v2/service/sso/types from github.com/aws/aws-sdk-go-v2/service/sso + L github.com/aws/aws-sdk-go-v2/service/ssooidc from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/service/ssooidc/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssooidc + L github.com/aws/aws-sdk-go-v2/service/ssooidc/types from github.com/aws/aws-sdk-go-v2/service/ssooidc + L github.com/aws/aws-sdk-go-v2/service/sts from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/service/sts/types from github.com/aws/aws-sdk-go-v2/credentials/stscreds+ + L github.com/aws/smithy-go from github.com/aws/aws-sdk-go-v2/aws/protocol/restjson+ + L github.com/aws/smithy-go/auth from github.com/aws/aws-sdk-go-v2/internal/auth+ + L github.com/aws/smithy-go/auth/bearer from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/smithy-go/context from github.com/aws/smithy-go/auth/bearer + L github.com/aws/smithy-go/document from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/smithy-go/encoding from github.com/aws/smithy-go/encoding/json+ + L github.com/aws/smithy-go/encoding/httpbinding from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ + L github.com/aws/smithy-go/encoding/json from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/smithy-go/encoding/xml from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/smithy-go/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/smithy-go/internal/sync/singleflight from github.com/aws/smithy-go/auth/bearer + L github.com/aws/smithy-go/io from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+ + L github.com/aws/smithy-go/logging from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/smithy-go/metrics from github.com/aws/aws-sdk-go-v2/aws/retry+ + L github.com/aws/smithy-go/middleware from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/smithy-go/private/requestcompression from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/smithy-go/ptr from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/smithy-go/rand from github.com/aws/aws-sdk-go-v2/aws/middleware+ + L github.com/aws/smithy-go/time from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/smithy-go/tracing from github.com/aws/aws-sdk-go-v2/aws/middleware+ + L github.com/aws/smithy-go/transport/http from github.com/aws/aws-sdk-go-v2/aws/middleware+ + L github.com/aws/smithy-go/transport/http/internal/io from github.com/aws/smithy-go/transport/http + L github.com/aws/smithy-go/waiter from github.com/aws/aws-sdk-go-v2/service/ssm + github.com/coder/websocket from tailscale.com/util/eventbus + github.com/coder/websocket/internal/errd from github.com/coder/websocket + github.com/coder/websocket/internal/util from github.com/coder/websocket + github.com/coder/websocket/internal/xsync from github.com/coder/websocket + L github.com/coreos/go-iptables/iptables from tailscale.com/util/linuxfw + W 💣 github.com/dblohm7/wingoes from github.com/dblohm7/wingoes/com+ + W 💣 github.com/dblohm7/wingoes/com from tailscale.com/util/osdiag+ + W 💣 github.com/dblohm7/wingoes/com/automation from tailscale.com/util/osdiag/internal/wsc + W github.com/dblohm7/wingoes/internal from github.com/dblohm7/wingoes/com + W 💣 github.com/dblohm7/wingoes/pe from tailscale.com/util/osdiag+ + LW 💣 github.com/digitalocean/go-smbios/smbios from tailscale.com/posture + github.com/fxamacker/cbor/v2 from tailscale.com/tka + github.com/gaissmai/bart from tailscale.com/net/ipset+ + github.com/gaissmai/bart/internal/bitset from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/sparse from github.com/gaissmai/bart + github.com/go-json-experiment/json from tailscale.com/types/opt+ + github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonflags from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + W 💣 github.com/go-ole/go-ole from github.com/go-ole/go-ole/oleutil+ + W 💣 github.com/go-ole/go-ole/oleutil from tailscale.com/wgengine/winnet + L 💣 github.com/godbus/dbus/v5 from tailscale.com/net/dns + github.com/golang/groupcache/lru from tailscale.com/net/dnscache + github.com/google/btree from gvisor.dev/gvisor/pkg/tcpip/header+ + L github.com/google/nftables from tailscale.com/util/linuxfw + L 💣 github.com/google/nftables/alignedbuff from github.com/google/nftables/xt + L 💣 github.com/google/nftables/binaryutil from github.com/google/nftables+ + L github.com/google/nftables/expr from github.com/google/nftables+ + L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ + L github.com/google/nftables/xt from github.com/google/nftables/expr+ + DW github.com/google/uuid from github.com/prometheus-community/pro-bing+ + github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ + L 💣 github.com/illarion/gonotify/v3 from tailscale.com/net/dns + L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 + L github.com/jmespath/go-jmespath from github.com/aws/aws-sdk-go-v2/service/ssm + L 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon + L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink + github.com/klauspost/compress from github.com/klauspost/compress/zstd + github.com/klauspost/compress/fse from github.com/klauspost/compress/huff0 + github.com/klauspost/compress/huff0 from github.com/klauspost/compress/zstd + github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+ + github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd + github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe + github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd + L github.com/mdlayher/genetlink from tailscale.com/net/tstun + L 💣 github.com/mdlayher/netlink from github.com/google/nftables+ + L 💣 github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+ + L github.com/mdlayher/netlink/nltest from github.com/google/nftables + L github.com/mdlayher/sdnotify from tailscale.com/util/systemd + L 💣 github.com/mdlayher/socket from github.com/mdlayher/netlink+ + github.com/miekg/dns from tailscale.com/net/dns/recursive + 💣 github.com/mitchellh/go-ps from tailscale.com/safesocket + D github.com/prometheus-community/pro-bing from tailscale.com/wgengine/netstack + L 💣 github.com/safchain/ethtool from tailscale.com/doctor/ethtool+ + W 💣 github.com/tailscale/certstore from tailscale.com/control/controlclient + W 💣 github.com/tailscale/go-winio from tailscale.com/safesocket + W 💣 github.com/tailscale/go-winio/internal/fs from github.com/tailscale/go-winio + W 💣 github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio + W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs + W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ + github.com/tailscale/goupnp from github.com/tailscale/goupnp/dcps/internetgateway2+ + github.com/tailscale/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper + github.com/tailscale/goupnp/httpu from github.com/tailscale/goupnp+ + github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp + github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+ + github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp + github.com/tailscale/hujson from tailscale.com/ipn/conffile + L 💣 github.com/tailscale/netlink from tailscale.com/net/routetable+ + L 💣 github.com/tailscale/netlink/nl from github.com/tailscale/netlink + github.com/tailscale/peercred from tailscale.com/ipn/ipnauth + github.com/tailscale/web-client-prebuilt from tailscale.com/client/web + 💣 github.com/tailscale/wireguard-go/conn from github.com/tailscale/wireguard-go/device+ + W 💣 github.com/tailscale/wireguard-go/conn/winrio from github.com/tailscale/wireguard-go/conn + 💣 github.com/tailscale/wireguard-go/device from tailscale.com/net/tstun+ + 💣 github.com/tailscale/wireguard-go/ipc from github.com/tailscale/wireguard-go/device + W 💣 github.com/tailscale/wireguard-go/ipc/namedpipe from github.com/tailscale/wireguard-go/ipc + github.com/tailscale/wireguard-go/ratelimiter from github.com/tailscale/wireguard-go/device + github.com/tailscale/wireguard-go/replay from github.com/tailscale/wireguard-go/device + github.com/tailscale/wireguard-go/rwcancel from github.com/tailscale/wireguard-go/device+ + github.com/tailscale/wireguard-go/tai64n from github.com/tailscale/wireguard-go/device + 💣 github.com/tailscale/wireguard-go/tun from github.com/tailscale/wireguard-go/device+ + L github.com/vishvananda/netns from github.com/tailscale/netlink+ + github.com/x448/float16 from github.com/fxamacker/cbor/v2 + 💣 go4.org/mem from tailscale.com/client/local+ + go4.org/netipx from tailscale.com/ipn/ipnlocal+ + W 💣 golang.zx2c4.com/wintun from github.com/tailscale/wireguard-go/tun + W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg from tailscale.com/net/dns+ + gopkg.in/square/go-jose.v2 from gopkg.in/square/go-jose.v2/jwt+ + gopkg.in/square/go-jose.v2/cipher from gopkg.in/square/go-jose.v2 + gopkg.in/square/go-jose.v2/json from gopkg.in/square/go-jose.v2+ + gopkg.in/square/go-jose.v2/jwt from tailscale.com/cmd/tsidp + gvisor.dev/gvisor/pkg/atomicbitops from gvisor.dev/gvisor/pkg/buffer+ + gvisor.dev/gvisor/pkg/bits from gvisor.dev/gvisor/pkg/buffer + 💣 gvisor.dev/gvisor/pkg/buffer from gvisor.dev/gvisor/pkg/tcpip+ + gvisor.dev/gvisor/pkg/context from gvisor.dev/gvisor/pkg/refs + 💣 gvisor.dev/gvisor/pkg/gohacks from gvisor.dev/gvisor/pkg/state/wire+ + gvisor.dev/gvisor/pkg/linewriter from gvisor.dev/gvisor/pkg/log + gvisor.dev/gvisor/pkg/log from gvisor.dev/gvisor/pkg/context+ + gvisor.dev/gvisor/pkg/rand from gvisor.dev/gvisor/pkg/tcpip+ + gvisor.dev/gvisor/pkg/refs from gvisor.dev/gvisor/pkg/buffer+ + 💣 gvisor.dev/gvisor/pkg/sleep from gvisor.dev/gvisor/pkg/tcpip/transport/tcp + 💣 gvisor.dev/gvisor/pkg/state from gvisor.dev/gvisor/pkg/atomicbitops+ + gvisor.dev/gvisor/pkg/state/wire from gvisor.dev/gvisor/pkg/state + 💣 gvisor.dev/gvisor/pkg/sync from gvisor.dev/gvisor/pkg/atomicbitops+ + 💣 gvisor.dev/gvisor/pkg/sync/locking from gvisor.dev/gvisor/pkg/tcpip/stack + gvisor.dev/gvisor/pkg/tcpip from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ + gvisor.dev/gvisor/pkg/tcpip/adapters/gonet from tailscale.com/wgengine/netstack + 💣 gvisor.dev/gvisor/pkg/tcpip/checksum from gvisor.dev/gvisor/pkg/buffer+ + gvisor.dev/gvisor/pkg/tcpip/hash/jenkins from gvisor.dev/gvisor/pkg/tcpip/stack+ + gvisor.dev/gvisor/pkg/tcpip/header from gvisor.dev/gvisor/pkg/tcpip/header/parse+ + gvisor.dev/gvisor/pkg/tcpip/header/parse from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ + gvisor.dev/gvisor/pkg/tcpip/internal/tcp from gvisor.dev/gvisor/pkg/tcpip/transport/tcp + gvisor.dev/gvisor/pkg/tcpip/network/hash from gvisor.dev/gvisor/pkg/tcpip/network/ipv4 + gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ + gvisor.dev/gvisor/pkg/tcpip/network/internal/ip from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ + gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ + gvisor.dev/gvisor/pkg/tcpip/network/ipv4 from tailscale.com/wgengine/netstack + gvisor.dev/gvisor/pkg/tcpip/network/ipv6 from tailscale.com/wgengine/netstack + gvisor.dev/gvisor/pkg/tcpip/ports from gvisor.dev/gvisor/pkg/tcpip/stack+ + gvisor.dev/gvisor/pkg/tcpip/seqnum from gvisor.dev/gvisor/pkg/tcpip/header+ + 💣 gvisor.dev/gvisor/pkg/tcpip/stack from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ + gvisor.dev/gvisor/pkg/tcpip/stack/gro from tailscale.com/wgengine/netstack/gro + gvisor.dev/gvisor/pkg/tcpip/transport from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+ + gvisor.dev/gvisor/pkg/tcpip/transport/icmp from tailscale.com/wgengine/netstack + gvisor.dev/gvisor/pkg/tcpip/transport/internal/network from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+ + gvisor.dev/gvisor/pkg/tcpip/transport/internal/noop from gvisor.dev/gvisor/pkg/tcpip/transport/raw + gvisor.dev/gvisor/pkg/tcpip/transport/packet from gvisor.dev/gvisor/pkg/tcpip/transport/raw + gvisor.dev/gvisor/pkg/tcpip/transport/raw from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+ + 💣 gvisor.dev/gvisor/pkg/tcpip/transport/tcp from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ + gvisor.dev/gvisor/pkg/tcpip/transport/tcpconntrack from gvisor.dev/gvisor/pkg/tcpip/stack + gvisor.dev/gvisor/pkg/tcpip/transport/udp from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ + gvisor.dev/gvisor/pkg/waiter from gvisor.dev/gvisor/pkg/context+ + tailscale.com from tailscale.com/version + tailscale.com/appc from tailscale.com/ipn/ipnlocal + 💣 tailscale.com/atomicfile from tailscale.com/ipn+ + tailscale.com/client/local from tailscale.com/client/tailscale+ + tailscale.com/client/tailscale from tailscale.com/derp+ + tailscale.com/client/tailscale/apitype from tailscale.com/client/local+ + tailscale.com/client/web from tailscale.com/ipn/ipnlocal + tailscale.com/clientupdate from tailscale.com/client/web+ + LW tailscale.com/clientupdate/distsign from tailscale.com/clientupdate + tailscale.com/control/controlbase from tailscale.com/control/controlhttp+ + tailscale.com/control/controlclient from tailscale.com/ipn/ipnext+ + tailscale.com/control/controlhttp from tailscale.com/control/controlclient + tailscale.com/control/controlhttp/controlhttpcommon from tailscale.com/control/controlhttp + tailscale.com/control/controlknobs from tailscale.com/control/controlclient+ + tailscale.com/derp from tailscale.com/derp/derphttp+ + tailscale.com/derp/derpconst from tailscale.com/derp+ + tailscale.com/derp/derphttp from tailscale.com/ipn/localapi+ + tailscale.com/disco from tailscale.com/derp+ + tailscale.com/doctor from tailscale.com/ipn/ipnlocal + tailscale.com/doctor/ethtool from tailscale.com/ipn/ipnlocal + 💣 tailscale.com/doctor/permissions from tailscale.com/ipn/ipnlocal + tailscale.com/doctor/routetable from tailscale.com/ipn/ipnlocal + tailscale.com/drive from tailscale.com/client/local+ + tailscale.com/envknob from tailscale.com/client/local+ + tailscale.com/envknob/featureknob from tailscale.com/client/web+ + tailscale.com/feature from tailscale.com/ipn/ipnext+ + tailscale.com/health from tailscale.com/control/controlclient+ + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal + tailscale.com/hostinfo from tailscale.com/client/web+ + tailscale.com/internal/noiseconn from tailscale.com/control/controlclient + tailscale.com/ipn from tailscale.com/client/local+ + tailscale.com/ipn/conffile from tailscale.com/ipn/ipnlocal+ + 💣 tailscale.com/ipn/ipnauth from tailscale.com/ipn/ipnext+ + tailscale.com/ipn/ipnext from tailscale.com/ipn/ipnlocal + tailscale.com/ipn/ipnlocal from tailscale.com/ipn/localapi+ + tailscale.com/ipn/ipnstate from tailscale.com/client/local+ + tailscale.com/ipn/localapi from tailscale.com/tsnet + tailscale.com/ipn/policy from tailscale.com/ipn/ipnlocal + tailscale.com/ipn/store from tailscale.com/ipn/ipnlocal+ + L tailscale.com/ipn/store/awsstore from tailscale.com/ipn/store + L tailscale.com/ipn/store/kubestore from tailscale.com/ipn/store + tailscale.com/ipn/store/mem from tailscale.com/ipn/ipnlocal+ + L tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+ + L tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore + tailscale.com/kube/kubetypes from tailscale.com/envknob+ + tailscale.com/licenses from tailscale.com/client/web + tailscale.com/log/filelogger from tailscale.com/logpolicy + tailscale.com/log/sockstatlog from tailscale.com/ipn/ipnlocal + tailscale.com/logpolicy from tailscale.com/ipn/ipnlocal+ + tailscale.com/logtail from tailscale.com/control/controlclient+ + tailscale.com/logtail/backoff from tailscale.com/control/controlclient+ + tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+ + tailscale.com/metrics from tailscale.com/derp+ + tailscale.com/net/bakedroots from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/connstats from tailscale.com/net/tstun+ + tailscale.com/net/dns from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/dns/publicdns from tailscale.com/net/dns+ + tailscale.com/net/dns/recursive from tailscale.com/net/dnsfallback + tailscale.com/net/dns/resolvconffile from tailscale.com/net/dns+ + tailscale.com/net/dns/resolver from tailscale.com/net/dns+ + tailscale.com/net/dnscache from tailscale.com/control/controlclient+ + tailscale.com/net/dnsfallback from tailscale.com/control/controlclient+ + tailscale.com/net/flowtrack from tailscale.com/net/packet+ + tailscale.com/net/ipset from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/memnet from tailscale.com/tsnet + tailscale.com/net/netaddr from tailscale.com/ipn+ + tailscale.com/net/netcheck from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/neterror from tailscale.com/net/dns/resolver+ + tailscale.com/net/netkernelconf from tailscale.com/ipn/ipnlocal + tailscale.com/net/netknob from tailscale.com/logpolicy+ + 💣 tailscale.com/net/netmon from tailscale.com/control/controlclient+ + 💣 tailscale.com/net/netns from tailscale.com/derp/derphttp+ + W 💣 tailscale.com/net/netstat from tailscale.com/portlist + tailscale.com/net/netutil from tailscale.com/client/local+ + tailscale.com/net/netx from tailscale.com/control/controlclient+ + tailscale.com/net/packet from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/packet/checksum from tailscale.com/net/tstun + tailscale.com/net/ping from tailscale.com/net/netcheck+ + tailscale.com/net/portmapper from tailscale.com/ipn/localapi+ + tailscale.com/net/proxymux from tailscale.com/tsnet + tailscale.com/net/routetable from tailscale.com/doctor/routetable + tailscale.com/net/socks5 from tailscale.com/tsnet + tailscale.com/net/sockstats from tailscale.com/control/controlclient+ + tailscale.com/net/stun from tailscale.com/ipn/localapi+ + L tailscale.com/net/tcpinfo from tailscale.com/derp + tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ + tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/tsaddr from tailscale.com/client/web+ + tailscale.com/net/tsdial from tailscale.com/control/controlclient+ + 💣 tailscale.com/net/tshttpproxy from tailscale.com/clientupdate/distsign+ + tailscale.com/net/tstun from tailscale.com/tsd+ + tailscale.com/net/udprelay/endpoint from tailscale.com/wgengine/magicsock + tailscale.com/omit from tailscale.com/ipn/conffile + tailscale.com/paths from tailscale.com/client/local+ + 💣 tailscale.com/portlist from tailscale.com/ipn/ipnlocal + tailscale.com/posture from tailscale.com/ipn/ipnlocal + tailscale.com/proxymap from tailscale.com/tsd+ + 💣 tailscale.com/safesocket from tailscale.com/client/local+ + tailscale.com/syncs from tailscale.com/control/controlhttp+ + tailscale.com/tailcfg from tailscale.com/client/local+ + tailscale.com/tempfork/acme from tailscale.com/ipn/ipnlocal + tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock + tailscale.com/tempfork/httprec from tailscale.com/control/controlclient + tailscale.com/tka from tailscale.com/client/local+ + tailscale.com/tsconst from tailscale.com/ipn/ipnlocal+ + tailscale.com/tsd from tailscale.com/ipn/ipnext+ + tailscale.com/tsnet from tailscale.com/cmd/tsidp + tailscale.com/tstime from tailscale.com/control/controlclient+ + tailscale.com/tstime/mono from tailscale.com/net/tstun+ + tailscale.com/tstime/rate from tailscale.com/derp+ + tailscale.com/tsweb from tailscale.com/util/eventbus + tailscale.com/tsweb/varz from tailscale.com/tsweb+ + tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal + tailscale.com/types/bools from tailscale.com/tsnet + tailscale.com/types/dnstype from tailscale.com/client/local+ + tailscale.com/types/empty from tailscale.com/ipn+ + tailscale.com/types/ipproto from tailscale.com/ipn+ + tailscale.com/types/key from tailscale.com/client/local+ + tailscale.com/types/lazy from tailscale.com/clientupdate+ + tailscale.com/types/logger from tailscale.com/appc+ + tailscale.com/types/logid from tailscale.com/ipn/ipnlocal+ + tailscale.com/types/mapx from tailscale.com/ipn/ipnext + tailscale.com/types/netlogtype from tailscale.com/net/connstats+ + tailscale.com/types/netmap from tailscale.com/control/controlclient+ + tailscale.com/types/nettype from tailscale.com/ipn/localapi+ + tailscale.com/types/opt from tailscale.com/client/tailscale+ + tailscale.com/types/persist from tailscale.com/control/controlclient+ + tailscale.com/types/preftype from tailscale.com/ipn+ + tailscale.com/types/ptr from tailscale.com/control/controlclient+ + tailscale.com/types/result from tailscale.com/util/lineiter + tailscale.com/types/structs from tailscale.com/control/controlclient+ + tailscale.com/types/tkatype from tailscale.com/client/local+ + tailscale.com/types/views from tailscale.com/appc+ + tailscale.com/util/cibuild from tailscale.com/health + tailscale.com/util/clientmetric from tailscale.com/appc+ + tailscale.com/util/cloudenv from tailscale.com/hostinfo+ + tailscale.com/util/cmpver from tailscale.com/clientupdate+ + tailscale.com/util/ctxkey from tailscale.com/client/tailscale/apitype+ + 💣 tailscale.com/util/deephash from tailscale.com/ipn/ipnlocal+ + L 💣 tailscale.com/util/dirwalk from tailscale.com/metrics+ + tailscale.com/util/dnsname from tailscale.com/appc+ + tailscale.com/util/eventbus from tailscale.com/ipn/localapi+ + tailscale.com/util/execqueue from tailscale.com/appc+ + tailscale.com/util/goroutines from tailscale.com/ipn/ipnlocal + tailscale.com/util/groupmember from tailscale.com/client/web+ + 💣 tailscale.com/util/hashx from tailscale.com/util/deephash + tailscale.com/util/httpm from tailscale.com/client/tailscale+ + tailscale.com/util/lineiter from tailscale.com/hostinfo+ + L tailscale.com/util/linuxfw from tailscale.com/net/netns+ + tailscale.com/util/mak from tailscale.com/appc+ + tailscale.com/util/multierr from tailscale.com/control/controlclient+ + tailscale.com/util/must from tailscale.com/clientupdate/distsign+ + tailscale.com/util/nocasemaps from tailscale.com/types/ipproto + 💣 tailscale.com/util/osdiag from tailscale.com/ipn/localapi + W 💣 tailscale.com/util/osdiag/internal/wsc from tailscale.com/util/osdiag + tailscale.com/util/osuser from tailscale.com/ipn/ipnlocal + tailscale.com/util/race from tailscale.com/net/dns/resolver + tailscale.com/util/racebuild from tailscale.com/logpolicy + tailscale.com/util/rands from tailscale.com/cmd/tsidp+ + tailscale.com/util/ringbuffer from tailscale.com/wgengine/magicsock + tailscale.com/util/set from tailscale.com/control/controlclient+ + tailscale.com/util/singleflight from tailscale.com/control/controlclient+ + tailscale.com/util/slicesx from tailscale.com/appc+ + tailscale.com/util/syspolicy from tailscale.com/control/controlclient+ + tailscale.com/util/syspolicy/internal from tailscale.com/util/syspolicy+ + tailscale.com/util/syspolicy/internal/loggerx from tailscale.com/util/syspolicy+ + tailscale.com/util/syspolicy/internal/metrics from tailscale.com/util/syspolicy/source + tailscale.com/util/syspolicy/rsop from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/syspolicy/setting from tailscale.com/client/local+ + tailscale.com/util/syspolicy/source from tailscale.com/util/syspolicy+ + tailscale.com/util/sysresources from tailscale.com/wgengine/magicsock + tailscale.com/util/systemd from tailscale.com/control/controlclient+ + tailscale.com/util/testenv from tailscale.com/control/controlclient+ + tailscale.com/util/truncate from tailscale.com/logtail + tailscale.com/util/usermetric from tailscale.com/health+ + tailscale.com/util/vizerror from tailscale.com/tailcfg+ + 💣 tailscale.com/util/winutil from tailscale.com/clientupdate+ + W 💣 tailscale.com/util/winutil/authenticode from tailscale.com/clientupdate+ + W 💣 tailscale.com/util/winutil/gp from tailscale.com/net/dns+ + W tailscale.com/util/winutil/policy from tailscale.com/ipn/ipnlocal + W 💣 tailscale.com/util/winutil/winenv from tailscale.com/hostinfo+ + tailscale.com/util/zstdframe from tailscale.com/control/controlclient+ + tailscale.com/version from tailscale.com/client/web+ + tailscale.com/version/distro from tailscale.com/client/web+ + tailscale.com/wgengine from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ + tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap+ + 💣 tailscale.com/wgengine/magicsock from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/netlog from tailscale.com/wgengine + tailscale.com/wgengine/netstack from tailscale.com/tsnet + tailscale.com/wgengine/netstack/gro from tailscale.com/net/tstun+ + tailscale.com/wgengine/router from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/wgcfg from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/wgcfg/nmcfg from tailscale.com/ipn/ipnlocal + 💣 tailscale.com/wgengine/wgint from tailscale.com/wgengine+ + tailscale.com/wgengine/wglog from tailscale.com/wgengine + W 💣 tailscale.com/wgengine/winnet from tailscale.com/wgengine/router + golang.org/x/crypto/argon2 from tailscale.com/tka + golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ + golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ + LD golang.org/x/crypto/blowfish from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf + golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305+ + golang.org/x/crypto/chacha20poly1305 from crypto/internal/hpke+ + golang.org/x/crypto/cryptobyte from crypto/ecdsa+ + golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + golang.org/x/crypto/curve25519 from github.com/tailscale/wireguard-go/device+ + golang.org/x/crypto/ed25519 from gopkg.in/square/go-jose.v2 + golang.org/x/crypto/hkdf from tailscale.com/control/controlbase + golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ + golang.org/x/crypto/nacl/box from tailscale.com/types/key + golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box + golang.org/x/crypto/pbkdf2 from gopkg.in/square/go-jose.v2 + golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device + golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ + LD golang.org/x/crypto/ssh from tailscale.com/ipn/ipnlocal + LD golang.org/x/crypto/ssh/internal/bcrypt_pbkdf from golang.org/x/crypto/ssh + golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+ + golang.org/x/exp/maps from tailscale.com/ipn/store/mem+ + golang.org/x/net/bpf from github.com/mdlayher/genetlink+ + golang.org/x/net/dns/dnsmessage from net+ + golang.org/x/net/http/httpguts from golang.org/x/net/http2+ + golang.org/x/net/http/httpproxy from net/http+ + golang.org/x/net/http2 from golang.org/x/net/http2/h2c+ + golang.org/x/net/http2/h2c from tailscale.com/ipn/ipnlocal + golang.org/x/net/http2/hpack from golang.org/x/net/http2+ + golang.org/x/net/icmp from github.com/prometheus-community/pro-bing+ + golang.org/x/net/idna from golang.org/x/net/http/httpguts+ + golang.org/x/net/internal/httpcommon from golang.org/x/net/http2 + golang.org/x/net/internal/iana from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socks from golang.org/x/net/proxy + golang.org/x/net/ipv4 from github.com/miekg/dns+ + golang.org/x/net/ipv6 from github.com/miekg/dns+ + golang.org/x/net/proxy from tailscale.com/net/netns + D golang.org/x/net/route from net+ + golang.org/x/sync/errgroup from github.com/mdlayher/socket+ + golang.org/x/sys/cpu from github.com/tailscale/certstore+ + LD golang.org/x/sys/unix from github.com/google/nftables+ + W golang.org/x/sys/windows from github.com/dblohm7/wingoes+ + W golang.org/x/sys/windows/registry from github.com/dblohm7/wingoes+ + W golang.org/x/sys/windows/svc from golang.org/x/sys/windows/svc/mgr+ + W golang.org/x/sys/windows/svc/mgr from tailscale.com/util/winutil + golang.org/x/term from tailscale.com/logpolicy + golang.org/x/text/secure/bidirule from golang.org/x/net/idna + golang.org/x/text/transform from golang.org/x/text/secure/bidirule+ + golang.org/x/text/unicode/bidi from golang.org/x/net/idna+ + golang.org/x/text/unicode/norm from golang.org/x/net/idna + golang.org/x/time/rate from gvisor.dev/gvisor/pkg/log+ + archive/tar from tailscale.com/clientupdate + bufio from compress/flate+ + bytes from archive/tar+ + cmp from encoding/json+ + compress/flate from compress/gzip+ + compress/gzip from github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding+ + W compress/zlib from debug/pe + container/heap from gvisor.dev/gvisor/pkg/tcpip/transport/tcp + container/list from crypto/tls+ + context from crypto/tls+ + crypto from crypto/ecdh+ + crypto/aes from crypto/internal/hpke+ + crypto/cipher from crypto/aes+ + crypto/des from crypto/tls+ + crypto/dsa from crypto/x509+ + crypto/ecdh from crypto/ecdsa+ + crypto/ecdsa from crypto/tls+ + crypto/ed25519 from crypto/tls+ + crypto/elliptic from crypto/ecdsa+ + crypto/hmac from crypto/tls+ + crypto/internal/boring from crypto/aes+ + crypto/internal/boring/bbig from crypto/ecdsa+ + crypto/internal/boring/sig from crypto/internal/boring + crypto/internal/entropy from crypto/internal/fips140/drbg + crypto/internal/fips140 from crypto/internal/fips140/aes+ + crypto/internal/fips140/aes from crypto/aes+ + crypto/internal/fips140/aes/gcm from crypto/cipher+ + crypto/internal/fips140/alias from crypto/cipher+ + crypto/internal/fips140/bigmod from crypto/internal/fips140/ecdsa+ + crypto/internal/fips140/check from crypto/internal/fips140/aes+ + crypto/internal/fips140/drbg from crypto/internal/fips140/aes/gcm+ + crypto/internal/fips140/ecdh from crypto/ecdh + crypto/internal/fips140/ecdsa from crypto/ecdsa + crypto/internal/fips140/ed25519 from crypto/ed25519 + crypto/internal/fips140/edwards25519 from crypto/internal/fips140/ed25519 + crypto/internal/fips140/edwards25519/field from crypto/ecdh+ + crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ + crypto/internal/fips140/hmac from crypto/hmac+ + crypto/internal/fips140/mlkem from crypto/tls+ + crypto/internal/fips140/nistec from crypto/elliptic+ + crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec + crypto/internal/fips140/rsa from crypto/rsa + crypto/internal/fips140/sha256 from crypto/internal/fips140/check+ + crypto/internal/fips140/sha3 from crypto/internal/fips140/hmac+ + crypto/internal/fips140/sha512 from crypto/internal/fips140/ecdsa+ + crypto/internal/fips140/subtle from crypto/internal/fips140/aes+ + crypto/internal/fips140/tls12 from crypto/tls + crypto/internal/fips140/tls13 from crypto/tls + crypto/internal/fips140deps/byteorder from crypto/internal/fips140/aes+ + crypto/internal/fips140deps/cpu from crypto/internal/fips140/aes+ + crypto/internal/fips140deps/godebug from crypto/internal/fips140+ + crypto/internal/fips140hash from crypto/ecdsa+ + crypto/internal/fips140only from crypto/cipher+ + crypto/internal/hpke from crypto/tls + crypto/internal/impl from crypto/internal/fips140/aes+ + crypto/internal/randutil from crypto/dsa+ + crypto/internal/sysrand from crypto/internal/entropy+ + crypto/md5 from crypto/tls+ + LD crypto/mlkem from golang.org/x/crypto/ssh + crypto/rand from crypto/ed25519+ + crypto/rc4 from crypto/tls+ + crypto/rsa from crypto/tls+ + crypto/sha1 from crypto/tls+ + crypto/sha256 from crypto/tls+ + crypto/sha3 from crypto/internal/fips140hash + crypto/sha512 from crypto/ecdsa+ + crypto/subtle from crypto/cipher+ + crypto/tls from github.com/aws/aws-sdk-go-v2/aws/transport/http+ + crypto/tls/internal/fips140tls from crypto/tls + crypto/x509 from crypto/tls+ + D crypto/x509/internal/macos from crypto/x509 + crypto/x509/pkix from crypto/x509+ + DW database/sql/driver from github.com/google/uuid + W debug/dwarf from debug/pe + W debug/pe from github.com/dblohm7/wingoes/pe + embed from github.com/tailscale/web-client-prebuilt+ + encoding from encoding/json+ + encoding/asn1 from crypto/x509+ + encoding/base32 from github.com/fxamacker/cbor/v2+ + encoding/base64 from encoding/json+ + encoding/binary from compress/gzip+ + encoding/hex from crypto/x509+ + encoding/json from expvar+ + encoding/pem from crypto/tls+ + encoding/xml from github.com/aws/aws-sdk-go-v2/aws/protocol/xml+ + errors from archive/tar+ + expvar from tailscale.com/derp+ + flag from tailscale.com/cmd/tsidp+ + fmt from archive/tar+ + hash from compress/zlib+ + W hash/adler32 from compress/zlib + hash/crc32 from compress/gzip+ + hash/maphash from go4.org/mem + html from html/template+ + html/template from tailscale.com/util/eventbus+ + internal/abi from crypto/x509/internal/macos+ + internal/asan from internal/runtime/maps+ + internal/bisect from internal/godebug + internal/bytealg from bytes+ + internal/byteorder from crypto/cipher+ + internal/chacha8rand from math/rand/v2+ + internal/coverage/rtcov from runtime + internal/cpu from crypto/internal/fips140deps/cpu+ + internal/filepathlite from os+ + internal/fmtsort from fmt+ + internal/goarch from crypto/internal/fips140deps/cpu+ + internal/godebug from archive/tar+ + internal/godebugs from internal/godebug+ + internal/goexperiment from hash/maphash+ + internal/goos from crypto/x509+ + internal/itoa from internal/poll+ + internal/msan from internal/runtime/maps+ + internal/nettrace from net+ + internal/oserror from io/fs+ + internal/poll from net+ + internal/profile from net/http/pprof + internal/profilerecord from runtime+ + internal/race from internal/poll+ + internal/reflectlite from context+ + internal/runtime/atomic from internal/runtime/exithook+ + internal/runtime/exithook from runtime + internal/runtime/maps from reflect+ + internal/runtime/math from internal/runtime/maps+ + internal/runtime/sys from crypto/subtle+ + L internal/runtime/syscall from runtime+ + W internal/saferio from debug/pe + internal/singleflight from net + internal/stringslite from embed+ + internal/sync from sync+ + internal/syscall/execenv from os+ + LD internal/syscall/unix from crypto/internal/sysrand+ + W internal/syscall/windows from crypto/internal/sysrand+ + W internal/syscall/windows/registry from mime+ + W internal/syscall/windows/sysdll from internal/syscall/windows+ + internal/testlog from os + internal/unsafeheader from internal/reflectlite+ + io from archive/tar+ + io/fs from archive/tar+ + io/ioutil from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ + iter from bytes+ + log from expvar+ + log/internal from log + maps from archive/tar+ + math from archive/tar+ + math/big from crypto/dsa+ + math/bits from bytes+ + math/rand from github.com/fxamacker/cbor/v2+ + math/rand/v2 from crypto/ecdsa+ + mime from mime/multipart+ + mime/multipart from net/http + mime/quotedprintable from mime/multipart + net from crypto/tls+ + net/http from expvar+ + net/http/httptrace from github.com/aws/smithy-go/transport/http+ + net/http/httputil from github.com/aws/smithy-go/transport/http+ + net/http/internal from net/http+ + net/http/internal/ascii from net/http+ + net/http/pprof from tailscale.com/ipn/localapi+ + net/netip from crypto/x509+ + net/textproto from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ + net/url from crypto/x509+ + os from crypto/internal/sysrand+ + os/exec from github.com/aws/aws-sdk-go-v2/credentials/processcreds+ + os/signal from tailscale.com/cmd/tsidp + os/user from archive/tar+ + path from archive/tar+ + path/filepath from archive/tar+ + reflect from archive/tar+ + regexp from github.com/aws/aws-sdk-go-v2/internal/endpoints+ + regexp/syntax from regexp + runtime from archive/tar+ + runtime/debug from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+ + runtime/pprof from net/http/pprof+ + runtime/trace from net/http/pprof + slices from archive/tar+ + sort from compress/flate+ + strconv from archive/tar+ + strings from archive/tar+ + sync from archive/tar+ + sync/atomic from context+ + syscall from archive/tar+ + text/tabwriter from runtime/pprof + text/template from html/template + text/template/parse from html/template+ + time from archive/tar+ + unicode from bytes+ + unicode/utf16 from crypto/x509+ + unicode/utf8 from bufio+ + unique from net/netip + unsafe from bytes+ + weak from unique From 1635ccca275fe3223f96f35f9ec5393f5613685e Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Thu, 29 May 2025 09:11:31 -0500 Subject: [PATCH 030/263] ssh/tailssh: display more useful error messages when authentication fails Also add a trailing newline to error banners so that SSH client messages don't print on the same line. Updates tailscale/corp#29138 Signed-off-by: Percy Wegmann --- ssh/tailssh/tailssh.go | 59 ++++++++++++++++++++++++------------- ssh/tailssh/tailssh_test.go | 44 ++++++++++++++++++++------- 2 files changed, 73 insertions(+), 30 deletions(-) diff --git a/ssh/tailssh/tailssh.go b/ssh/tailssh/tailssh.go index 19a2b11fd3800..b249a10639c30 100644 --- a/ssh/tailssh/tailssh.go +++ b/ssh/tailssh/tailssh.go @@ -281,7 +281,7 @@ func (c *conn) errBanner(message string, err error) error { if err != nil { c.logf("%s: %s", message, err) } - if err := c.spac.SendAuthBanner("tailscale: " + message); err != nil { + if err := c.spac.SendAuthBanner("tailscale: " + message + "\n"); err != nil { c.logf("failed to send auth banner: %s", err) } return errTerminal @@ -324,9 +324,16 @@ func (c *conn) clientAuth(cm gossh.ConnMetadata) (perms *gossh.Permissions, retE return nil, c.errBanner("failed to get connection info", err) } - action, localUser, acceptEnv, err := c.evaluatePolicy() - if err != nil { - return nil, c.errBanner("failed to evaluate SSH policy", err) + action, localUser, acceptEnv, result := c.evaluatePolicy() + switch result { + case accepted: + // do nothing + case rejectedUser: + return nil, c.errBanner(fmt.Sprintf("tailnet policy does not permit you to SSH as user %q", c.info.sshUser), nil) + case rejected, noPolicy: + return nil, c.errBanner("tailnet policy does not permit you to SSH to this node", fmt.Errorf("failed to evaluate policy, result: %s", result)) + default: + return nil, c.errBanner("failed to evaluate tailnet policy", fmt.Errorf("failed to evaluate policy, result: %s", result)) } c.action0 = action @@ -597,18 +604,23 @@ func (c *conn) setInfo(cm gossh.ConnMetadata) error { return nil } +type evalResult string + +const ( + noPolicy evalResult = "no policy" + rejected evalResult = "rejected" + rejectedUser evalResult = "rejected user" + accepted evalResult = "accept" +) + // evaluatePolicy returns the SSHAction and localUser after evaluating // the SSHPolicy for this conn. -func (c *conn) evaluatePolicy() (_ *tailcfg.SSHAction, localUser string, acceptEnv []string, _ error) { +func (c *conn) evaluatePolicy() (_ *tailcfg.SSHAction, localUser string, acceptEnv []string, result evalResult) { pol, ok := c.sshPolicy() if !ok { - return nil, "", nil, fmt.Errorf("tailssh: rejecting connection; no SSH policy") - } - a, localUser, acceptEnv, ok := c.evalSSHPolicy(pol) - if !ok { - return nil, "", nil, fmt.Errorf("tailssh: rejecting connection; no matching policy") + return nil, "", nil, noPolicy } - return a, localUser, acceptEnv, nil + return c.evalSSHPolicy(pol) } // handleSessionPostSSHAuth runs an SSH session after the SSH-level authentication, @@ -706,9 +718,9 @@ func (c *conn) newSSHSession(s ssh.Session) *sshSession { // isStillValid reports whether the conn is still valid. func (c *conn) isStillValid() bool { - a, localUser, _, err := c.evaluatePolicy() - c.vlogf("stillValid: %+v %v %v", a, localUser, err) - if err != nil { + a, localUser, _, result := c.evaluatePolicy() + c.vlogf("stillValid: %+v %v %v", a, localUser, result) + if result != accepted { return false } if !a.Accept && a.HoldAndDelegate == "" { @@ -1089,13 +1101,20 @@ func (c *conn) ruleExpired(r *tailcfg.SSHRule) bool { return r.RuleExpires.Before(c.srv.now()) } -func (c *conn) evalSSHPolicy(pol *tailcfg.SSHPolicy) (a *tailcfg.SSHAction, localUser string, acceptEnv []string, ok bool) { +func (c *conn) evalSSHPolicy(pol *tailcfg.SSHPolicy) (a *tailcfg.SSHAction, localUser string, acceptEnv []string, result evalResult) { + failedOnUser := false for _, r := range pol.Rules { if a, localUser, acceptEnv, err := c.matchRule(r); err == nil { - return a, localUser, acceptEnv, true + return a, localUser, acceptEnv, accepted + } else if errors.Is(err, errUserMatch) { + failedOnUser = true } } - return nil, "", nil, false + result = rejected + if failedOnUser { + result = rejectedUser + } + return nil, "", nil, result } // internal errors for testing; they don't escape to callers or logs. @@ -1129,6 +1148,9 @@ func (c *conn) matchRule(r *tailcfg.SSHRule) (a *tailcfg.SSHAction, localUser st if c.ruleExpired(r) { return nil, "", nil, errRuleExpired } + if !c.anyPrincipalMatches(r.Principals) { + return nil, "", nil, errPrincipalMatch + } if !r.Action.Reject { // For all but Reject rules, SSHUsers is required. // If SSHUsers is nil or empty, mapLocalUser will return an @@ -1138,9 +1160,6 @@ func (c *conn) matchRule(r *tailcfg.SSHRule) (a *tailcfg.SSHAction, localUser st return nil, "", nil, errUserMatch } } - if !c.anyPrincipalMatches(r.Principals) { - return nil, "", nil, errPrincipalMatch - } return r.Action, localUser, r.AcceptEnv, nil } diff --git a/ssh/tailssh/tailssh_test.go b/ssh/tailssh/tailssh_test.go index 79479d7fbf5c7..96fb87f4903c0 100644 --- a/ssh/tailssh/tailssh_test.go +++ b/ssh/tailssh/tailssh_test.go @@ -253,7 +253,7 @@ func TestEvalSSHPolicy(t *testing.T) { name string policy *tailcfg.SSHPolicy ci *sshConnInfo - wantMatch bool + wantResult evalResult wantUser string wantAcceptEnv []string }{ @@ -299,10 +299,20 @@ func TestEvalSSHPolicy(t *testing.T) { ci: &sshConnInfo{sshUser: "alice"}, wantUser: "thealice", wantAcceptEnv: []string{"EXAMPLE", "?_?", "TEST_*"}, - wantMatch: true, + wantResult: accepted, }, { - name: "no-matches-returns-failure", + name: "no-matches-returns-rejected", + policy: &tailcfg.SSHPolicy{ + Rules: []*tailcfg.SSHRule{}, + }, + ci: &sshConnInfo{sshUser: "alice"}, + wantUser: "", + wantAcceptEnv: nil, + wantResult: rejected, + }, + { + name: "no-user-matches-returns-rejected-user", policy: &tailcfg.SSHPolicy{ Rules: []*tailcfg.SSHRule{ { @@ -340,7 +350,7 @@ func TestEvalSSHPolicy(t *testing.T) { ci: &sshConnInfo{sshUser: "alice"}, wantUser: "", wantAcceptEnv: nil, - wantMatch: false, + wantResult: rejectedUser, }, } for _, tt := range tests { @@ -349,14 +359,14 @@ func TestEvalSSHPolicy(t *testing.T) { info: tt.ci, srv: &server{logf: tstest.WhileTestRunningLogger(t)}, } - got, gotUser, gotAcceptEnv, match := c.evalSSHPolicy(tt.policy) - if match != tt.wantMatch { - t.Errorf("match = %v; want %v", match, tt.wantMatch) + got, gotUser, gotAcceptEnv, result := c.evalSSHPolicy(tt.policy) + if result != tt.wantResult { + t.Errorf("result = %v; want %v", result, tt.wantResult) } if gotUser != tt.wantUser { t.Errorf("user = %q; want %q", gotUser, tt.wantUser) } - if tt.wantMatch == true && got == nil { + if tt.wantResult == accepted && got == nil { t.Errorf("expected non-nil action on success") } if !slices.Equal(gotAcceptEnv, tt.wantAcceptEnv) { @@ -467,7 +477,7 @@ func (ts *localState) NodeKey() key.NodePublic { func newSSHRule(action *tailcfg.SSHAction) *tailcfg.SSHRule { return &tailcfg.SSHRule{ SSHUsers: map[string]string{ - "*": currentUser, + "alice": currentUser, }, Action: action, Principals: []*tailcfg.SSHPrincipal{ @@ -789,6 +799,11 @@ func TestSSHAuthFlow(t *testing.T) { Accept: true, Message: "Welcome to Tailscale SSH!", }) + bobRule := newSSHRule(&tailcfg.SSHAction{ + Accept: true, + Message: "Welcome to Tailscale SSH!", + }) + bobRule.SSHUsers = map[string]string{"bob": "bob"} rejectRule := newSSHRule(&tailcfg.SSHAction{ Reject: true, Message: "Go Away!", @@ -808,7 +823,16 @@ func TestSSHAuthFlow(t *testing.T) { sshEnabled: true, }, authErr: true, - wantBanners: []string{"tailscale: failed to evaluate SSH policy"}, + wantBanners: []string{"tailscale: tailnet policy does not permit you to SSH to this node\n"}, + }, + { + name: "user-mismatch", + state: &localState{ + sshEnabled: true, + matchingRule: bobRule, + }, + authErr: true, + wantBanners: []string{`tailscale: tailnet policy does not permit you to SSH as user "alice"` + "\n"}, }, { name: "accept", From 5fde183754c565c7a4f9c8cf956218d31ee30ba4 Mon Sep 17 00:00:00 2001 From: James Sanderson Date: Tue, 3 Jun 2025 15:09:34 +0100 Subject: [PATCH 031/263] ipn: add watch opt to include actions in health messages Updates tailscale/corp#27759 Signed-off-by: James Sanderson --- ipn/backend.go | 2 + ipn/ipnlocal/local.go | 70 ++++++++++++++++++------ ipn/ipnlocal/local_test.go | 106 ++++++++++++++++++++++++++++++++++++- 3 files changed, 161 insertions(+), 17 deletions(-) diff --git a/ipn/backend.go b/ipn/backend.go index 3e956f4734f5f..ab01d2fdef57a 100644 --- a/ipn/backend.go +++ b/ipn/backend.go @@ -81,6 +81,8 @@ const ( NotifyInitialHealthState NotifyWatchOpt = 1 << 7 // if set, the first Notify message (sent immediately) will contain the current health.State of the client NotifyRateLimit NotifyWatchOpt = 1 << 8 // if set, rate limit spammy netmap updates to every few seconds + + NotifyHealthActions NotifyWatchOpt = 1 << 9 // if set, include PrimaryActions in health.State. Otherwise append the action URL to the text ) // Notify is a communication from a backend (e.g. tailscaled) to a frontend diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index e494920b1f5b0..0efec6b9fff29 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -2948,28 +2948,19 @@ func (b *LocalBackend) WatchNotifications(ctx context.Context, mask ipn.NotifyWa b.WatchNotificationsAs(ctx, nil, mask, onWatchAdded, fn) } -// WatchNotificationsAs is like WatchNotifications but takes an [ipnauth.Actor] +// WatchNotificationsAs is like [LocalBackend.WatchNotifications] but takes an [ipnauth.Actor] // as an additional parameter. If non-nil, the specified callback is invoked // only for notifications relevant to this actor. func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.Actor, mask ipn.NotifyWatchOpt, onWatchAdded func(), fn func(roNotify *ipn.Notify) (keepGoing bool)) { ch := make(chan *ipn.Notify, 128) sessionID := rands.HexString(16) - origFn := fn if mask&ipn.NotifyNoPrivateKeys != 0 { - fn = func(n *ipn.Notify) bool { - if n.NetMap == nil || n.NetMap.PrivateKey.IsZero() { - return origFn(n) - } - - // The netmap in n is shared across all watchers, so to mutate it for a - // single watcher we have to clone the notify and the netmap. We can - // make shallow clones, at least. - nm2 := *n.NetMap - n2 := *n - n2.NetMap = &nm2 - n2.NetMap.PrivateKey = key.NodePrivate{} - return origFn(&n2) - } + fn = filterPrivateKeys(fn) + } + if mask&ipn.NotifyHealthActions == 0 { + // if UI does not support PrimaryAction in health warnings, append + // action URLs to the warning text instead. + fn = appendHealthActions(fn) } var ini *ipn.Notify @@ -3060,6 +3051,53 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A sender.Run(ctx, ch) } +// filterPrivateKeys returns an IPN listener func that wraps the supplied IPN +// listener and zeroes out the PrivateKey in the NetMap passed to the wrapped +// listener. +func filterPrivateKeys(fn func(roNotify *ipn.Notify) (keepGoing bool)) func(*ipn.Notify) bool { + return func(n *ipn.Notify) bool { + if n.NetMap == nil || n.NetMap.PrivateKey.IsZero() { + return fn(n) + } + + // The netmap in n is shared across all watchers, so to mutate it for a + // single watcher we have to clone the notify and the netmap. We can + // make shallow clones, at least. + nm2 := *n.NetMap + n2 := *n + n2.NetMap = &nm2 + n2.NetMap.PrivateKey = key.NodePrivate{} + return fn(&n2) + } +} + +// appendHealthActions returns an IPN listener func that wraps the supplied IPN +// listener func and transforms health messages passed to the wrapped listener. +// If health messages with PrimaryActions are present, it appends the label & +// url in the PrimaryAction to the text of the message. For use for clients that +// do not process the PrimaryAction. +func appendHealthActions(fn func(roNotify *ipn.Notify) (keepGoing bool)) func(*ipn.Notify) bool { + return func(n *ipn.Notify) bool { + if n.Health == nil || len(n.Health.Warnings) == 0 { + return fn(n) + } + + // Shallow clone the notify and health so we can mutate them + h2 := *n.Health + n2 := *n + n2.Health = &h2 + n2.Health.Warnings = make(map[health.WarnableCode]health.UnhealthyState, len(n.Health.Warnings)) + for k, v := range n.Health.Warnings { + if v.PrimaryAction != nil { + v.Text = fmt.Sprintf("%s %s: %s", v.Text, v.PrimaryAction.Label, v.PrimaryAction.URL) + v.PrimaryAction = nil + } + n2.Health.Warnings[k] = v + } + return fn(&n2) + } +} + // pollRequestEngineStatus calls b.e.RequestStatus every 2 seconds until ctx // is done. func (b *LocalBackend) pollRequestEngineStatus(ctx context.Context) { diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index d23bd1e262dd8..8f9b6ee68f62b 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -5348,6 +5348,8 @@ func TestDisplayMessages(t *testing.T) { ht.SetIPNState("NeedsLogin", true) ht.GotStreamedMapResponse() + b.mu.Lock() + defer b.mu.Unlock() b.setNetMapLocked(&netmap.NetworkMap{ DisplayMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ "test-message": { @@ -5374,7 +5376,8 @@ func TestDisplayMessagesURLFilter(t *testing.T) { ht.SetIPNState("NeedsLogin", true) ht.GotStreamedMapResponse() - defer b.lockAndGetUnlock()() + b.mu.Lock() + defer b.mu.Unlock() b.setNetMapLocked(&netmap.NetworkMap{ DisplayMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ "test-message": { @@ -5405,3 +5408,104 @@ func TestDisplayMessagesURLFilter(t *testing.T) { t.Errorf("Unexpected message content (-want/+got):\n%s", diff) } } + +// TestDisplayMessageIPNBus checks that we send health messages appropriately +// based on whether the watcher has sent the [ipn.NotifyHealthActions] watch +// option or not. +func TestDisplayMessageIPNBus(t *testing.T) { + type test struct { + name string + mask ipn.NotifyWatchOpt + wantWarning health.UnhealthyState + } + + msgs := map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-message": { + Title: "Message title", + Text: "Message text.", + Severity: tailcfg.SeverityMedium, + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "https://example.com", + Label: "Learn more", + }, + }, + } + + for _, tt := range []test{ + { + name: "older-client-no-actions", + mask: 0, + wantWarning: health.UnhealthyState{ + WarnableCode: "test-message", + Severity: health.SeverityMedium, + Title: "Message title", + Text: "Message text. Learn more: https://example.com", // PrimaryAction appended to text + PrimaryAction: nil, // PrimaryAction not included + }, + }, + { + name: "new-client-with-actions", + mask: ipn.NotifyHealthActions, + wantWarning: health.UnhealthyState{ + WarnableCode: "test-message", + Severity: health.SeverityMedium, + Title: "Message title", + Text: "Message text.", + PrimaryAction: &health.UnhealthyStateAction{ + URL: "https://example.com", + Label: "Learn more", + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + lb := newLocalBackendWithTestControl(t, false, func(tb testing.TB, opts controlclient.Options) controlclient.Client { + return newClient(tb, opts) + }) + + ipnWatcher := newNotificationWatcher(t, lb, nil) + ipnWatcher.watch(tt.mask, []wantedNotification{{ + name: "test", + cond: func(_ testing.TB, _ ipnauth.Actor, n *ipn.Notify) bool { + if n.Health == nil { + return false + } + got, ok := n.Health.Warnings["test-message"] + if ok { + if diff := cmp.Diff(tt.wantWarning, got); diff != "" { + t.Errorf("unexpected warning details (-want/+got):\n%s", diff) + return true // we failed the test so tell the watcher we've seen what we need to to stop it waiting + } + } + return ok + }, + }}) + + lb.SetPrefsForTest(&ipn.Prefs{ + ControlURL: "https://localhost:1/", + WantRunning: true, + LoggedOut: false, + }) + if err := lb.Start(ipn.Options{}); err != nil { + t.Fatalf("(*LocalBackend).Start(): %v", err) + } + + cc := lb.cc.(*mockControl) + + // Assert that we are logged in and authorized, and also send our DisplayMessages + cc.send(nil, "", true, &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{MachineAuthorized: true}).View(), + DisplayMessages: msgs, + }) + + // Tell the health tracker that we are in a map poll because + // mockControl doesn't tell it + lb.HealthTracker().GotStreamedMapResponse() + + // Assert that we got the expected notification + ipnWatcher.check() + }) + } +} From 13ee2856752525a0ec4462f04eeb97f68b4f3830 Mon Sep 17 00:00:00 2001 From: James Sanderson Date: Wed, 4 Jun 2025 12:10:15 +0100 Subject: [PATCH 032/263] health: show DisplayMessage actions in 'tailscale status' Updates tailscale/corp#27759 Signed-off-by: James Sanderson --- health/health.go | 11 ++++++++--- health/health_test.go | 35 +++++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/health/health.go b/health/health.go index 6dbbf782cabce..05887043814ea 100644 --- a/health/health.go +++ b/health/health.go @@ -1054,13 +1054,18 @@ func (t *Tracker) stringsLocked() []string { warnLen := len(result) for _, c := range t.controlMessages { + var msg string if c.Title != "" && c.Text != "" { - result = append(result, c.Title+": "+c.Text) + msg = c.Title + ": " + c.Text } else if c.Title != "" { - result = append(result, c.Title) + msg = c.Title + "." } else if c.Text != "" { - result = append(result, c.Text) + msg = c.Text } + if c.PrimaryAction != nil { + msg = msg + " " + c.PrimaryAction.Label + ": " + c.PrimaryAction.URL + } + result = append(result, msg) } sort.Strings(result[warnLen:]) diff --git a/health/health_test.go b/health/health_test.go index f609cfb1613b1..aa519e92c081e 100644 --- a/health/health_test.go +++ b/health/health_test.go @@ -467,15 +467,24 @@ func TestControlHealth(t *testing.T) { baseWarns := ht.CurrentState().Warnings baseStrs := ht.Strings() - ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + msgs := map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ "control-health-test": { Title: "Control health message", - Text: "Extra help", + Text: "Extra help.", }, "control-health-title": { Title: "Control health title only", }, - }) + "control-health-with-action": { + Title: "Control health message", + Text: "Extra help.", + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "http://www.example.com", + Label: "Learn more", + }, + }, + } + ht.SetControlHealth(msgs) t.Run("Warnings", func(t *testing.T) { wantWarns := map[WarnableCode]UnhealthyState{ @@ -483,13 +492,23 @@ func TestControlHealth(t *testing.T) { WarnableCode: "control-health-test", Severity: SeverityMedium, Title: "Control health message", - Text: "Extra help", + Text: "Extra help.", }, "control-health-title": { WarnableCode: "control-health-title", Severity: SeverityMedium, Title: "Control health title only", }, + "control-health-with-action": { + WarnableCode: "control-health-with-action", + Severity: SeverityMedium, + Title: "Control health message", + Text: "Extra help.", + PrimaryAction: &UnhealthyStateAction{ + URL: "http://www.example.com", + Label: "Learn more", + }, + }, } state := ht.CurrentState() gotWarns := maps.Clone(state.Warnings) @@ -505,8 +524,9 @@ func TestControlHealth(t *testing.T) { t.Run("Strings()", func(t *testing.T) { wantStrs := []string{ - "Control health message: Extra help", - "Control health title only", + "Control health message: Extra help.", + "Control health message: Extra help. Learn more: http://www.example.com", + "Control health title only.", } var gotStrs []string for _, s := range ht.Strings() { @@ -527,8 +547,7 @@ func TestControlHealth(t *testing.T) { Type: MetricLabelWarning, }).String() want := strconv.Itoa( - 2 + // from SetControlHealth - len(baseStrs), + len(msgs) + len(baseStrs), ) if got != want { t.Errorf("metricsHealthMessage.Get(warning) = %q, want %q", got, want) From 486a55f0a9bffc45eb34350f24fea5a76be5169a Mon Sep 17 00:00:00 2001 From: Fran Bull Date: Wed, 16 Apr 2025 10:21:50 -0700 Subject: [PATCH 033/263] cmd/natc: add optional consensus backend Enable nat connector to be run on a cluster of machines for high availability. Updates #14667 Signed-off-by: Fran Bull --- cmd/natc/ippool/consensusippool.go | 434 ++++++++++++++++++++ cmd/natc/ippool/consensusippool_test.go | 383 +++++++++++++++++ cmd/natc/ippool/consensusippoolserialize.go | 164 ++++++++ cmd/natc/ippool/ippool.go | 21 +- cmd/natc/ippool/ippool_test.go | 7 +- cmd/natc/natc.go | 28 +- cmd/natc/natc_test.go | 2 +- 7 files changed, 1029 insertions(+), 10 deletions(-) create mode 100644 cmd/natc/ippool/consensusippool.go create mode 100644 cmd/natc/ippool/consensusippool_test.go create mode 100644 cmd/natc/ippool/consensusippoolserialize.go diff --git a/cmd/natc/ippool/consensusippool.go b/cmd/natc/ippool/consensusippool.go new file mode 100644 index 0000000000000..4783209b252e2 --- /dev/null +++ b/cmd/natc/ippool/consensusippool.go @@ -0,0 +1,434 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ippool + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net/netip" + "time" + + "github.com/hashicorp/raft" + "go4.org/netipx" + "tailscale.com/syncs" + "tailscale.com/tailcfg" + "tailscale.com/tsconsensus" + "tailscale.com/tsnet" + "tailscale.com/util/mak" +) + +// ConsensusIPPool implements an [IPPool] that is distributed among members of a cluster for high availability. +// Writes are directed to a leader among the cluster and are slower than reads, reads are performed locally +// using information replicated from the leader. +// The cluster maintains consistency, reads can be stale and writes can be unavailable if sufficient cluster +// peers are unavailable. +type ConsensusIPPool struct { + IPSet *netipx.IPSet + perPeerMap *syncs.Map[tailcfg.NodeID, *consensusPerPeerState] + consensus commandExecutor + unusedAddressLifetime time.Duration +} + +func NewConsensusIPPool(ipSet *netipx.IPSet) *ConsensusIPPool { + return &ConsensusIPPool{ + unusedAddressLifetime: 48 * time.Hour, // TODO (fran) is this appropriate? should it be configurable? + IPSet: ipSet, + perPeerMap: &syncs.Map[tailcfg.NodeID, *consensusPerPeerState]{}, + } +} + +// IPForDomain looks up or creates an IP address allocation for the tailcfg.NodeID and domain pair. +// If no address association is found, one is allocated from the range of free addresses for this tailcfg.NodeID. +// If no more address are available, an error is returned. +func (ipp *ConsensusIPPool) IPForDomain(nid tailcfg.NodeID, domain string) (netip.Addr, error) { + now := time.Now() + // Check local state; local state may be stale. If we have an IP for this domain, and we are not + // close to the expiry time for the domain, it's safe to return what we have. + ps, psFound := ipp.perPeerMap.Load(nid) + if psFound { + if addr, addrFound := ps.domainToAddr[domain]; addrFound { + if ww, wwFound := ps.addrToDomain.Load(addr); wwFound { + if !isCloseToExpiry(ww.LastUsed, now, ipp.unusedAddressLifetime) { + ipp.fireAndForgetMarkLastUsed(nid, addr, ww, now) + return addr, nil + } + } + } + } + + // go via consensus + args := checkoutAddrArgs{ + NodeID: nid, + Domain: domain, + ReuseDeadline: now.Add(-1 * ipp.unusedAddressLifetime), + UpdatedAt: now, + } + bs, err := json.Marshal(args) + if err != nil { + return netip.Addr{}, err + } + c := tsconsensus.Command{ + Name: "checkoutAddr", + Args: bs, + } + result, err := ipp.consensus.ExecuteCommand(c) + if err != nil { + log.Printf("IPForDomain: raft error executing command: %v", err) + return netip.Addr{}, err + } + if result.Err != nil { + log.Printf("IPForDomain: error returned from state machine: %v", err) + return netip.Addr{}, result.Err + } + var addr netip.Addr + err = json.Unmarshal(result.Result, &addr) + return addr, err +} + +// DomainForIP looks up the domain associated with a tailcfg.NodeID and netip.Addr pair. +// If there is no association, the result is empty and ok is false. +func (ipp *ConsensusIPPool) DomainForIP(from tailcfg.NodeID, addr netip.Addr, updatedAt time.Time) (string, bool) { + // Look in local state, to save a consensus round trip; local state may be stale. + // + // The only time we expect ordering of commands to matter to clients is on first + // connection to a domain. In that case it may be that although we don't find the + // domain in our local state, it is in fact in the state of the state machine (ie + // the client did a DNS lookup, and we responded with an IP and _should_ know that + // domain when the TCP connection for that IP arrives.) + // + // So it's ok to return local state, unless local state doesn't recognize the domain, + // in which case we should check the consensus state machine to know for sure. + var domain string + ww, ok := ipp.domainLookup(from, addr) + if ok { + domain = ww.Domain + } else { + d, err := ipp.readDomainForIP(from, addr) + if err != nil { + log.Printf("error reading domain from consensus: %v", err) + return "", false + } + domain = d + } + if domain == "" { + log.Printf("did not find domain for node: %v, addr: %s", from, addr) + return "", false + } + ipp.fireAndForgetMarkLastUsed(from, addr, ww, updatedAt) + return domain, true +} + +func (ipp *ConsensusIPPool) fireAndForgetMarkLastUsed(from tailcfg.NodeID, addr netip.Addr, ww whereWhen, updatedAt time.Time) { + window := 5 * time.Minute + if updatedAt.Sub(ww.LastUsed).Abs() < window { + return + } + go func() { + err := ipp.markLastUsed(from, addr, ww.Domain, updatedAt) + if err != nil { + log.Printf("error marking last used: %v", err) + } + }() +} + +func (ipp *ConsensusIPPool) domainLookup(from tailcfg.NodeID, addr netip.Addr) (whereWhen, bool) { + ps, ok := ipp.perPeerMap.Load(from) + if !ok { + log.Printf("domainLookup: peer state absent for: %d", from) + return whereWhen{}, false + } + ww, ok := ps.addrToDomain.Load(addr) + if !ok { + log.Printf("domainLookup: peer state doesn't recognize addr: %s", addr) + return whereWhen{}, false + } + return ww, true +} + +// StartConsensus is part of the IPPool interface. It starts the raft background routines that handle consensus. +func (ipp *ConsensusIPPool) StartConsensus(ctx context.Context, ts *tsnet.Server, clusterTag string) error { + cfg := tsconsensus.DefaultConfig() + cfg.ServeDebugMonitor = true + cns, err := tsconsensus.Start(ctx, ts, ipp, clusterTag, cfg) + if err != nil { + return err + } + ipp.consensus = cns + return nil +} + +type whereWhen struct { + Domain string + LastUsed time.Time +} + +type consensusPerPeerState struct { + domainToAddr map[string]netip.Addr + addrToDomain *syncs.Map[netip.Addr, whereWhen] +} + +// StopConsensus is part of the IPPool interface. It stops the raft background routines that handle consensus. +func (ipp *ConsensusIPPool) StopConsensus(ctx context.Context) error { + return (ipp.consensus).(*tsconsensus.Consensus).Stop(ctx) +} + +// unusedIPV4 finds the next unused or expired IP address in the pool. +// IP addresses in the pool should be reused if they haven't been used for some period of time. +// reuseDeadline is the time before which addresses are considered to be expired. +// So if addresses are being reused after they haven't been used for 24 hours say, reuseDeadline +// would be 24 hours ago. +func (ps *consensusPerPeerState) unusedIPV4(ipset *netipx.IPSet, reuseDeadline time.Time) (netip.Addr, bool, string, error) { + // If we want to have a random IP choice behavior we could make that work with the state machine by doing something like + // passing the randomly chosen IP into the state machine call (so replaying logs would still be deterministic). + for _, r := range ipset.Ranges() { + ip := r.From() + toIP := r.To() + if !ip.IsValid() || !toIP.IsValid() { + continue + } + for toIP.Compare(ip) != -1 { + ww, ok := ps.addrToDomain.Load(ip) + if !ok { + return ip, false, "", nil + } + if ww.LastUsed.Before(reuseDeadline) { + return ip, true, ww.Domain, nil + } + ip = ip.Next() + } + } + return netip.Addr{}, false, "", errors.New("ip pool exhausted") +} + +// isCloseToExpiry returns true if the lastUsed and now times are more than +// half the lifetime apart +func isCloseToExpiry(lastUsed, now time.Time, lifetime time.Duration) bool { + return now.Sub(lastUsed).Abs() > (lifetime / 2) +} + +type readDomainForIPArgs struct { + NodeID tailcfg.NodeID + Addr netip.Addr +} + +// executeReadDomainForIP parses a readDomainForIP log entry and applies it. +func (ipp *ConsensusIPPool) executeReadDomainForIP(bs []byte) tsconsensus.CommandResult { + var args readDomainForIPArgs + err := json.Unmarshal(bs, &args) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + return ipp.applyReadDomainForIP(args.NodeID, args.Addr) +} + +func (ipp *ConsensusIPPool) applyReadDomainForIP(from tailcfg.NodeID, addr netip.Addr) tsconsensus.CommandResult { + domain := func() string { + ps, ok := ipp.perPeerMap.Load(from) + if !ok { + return "" + } + ww, ok := ps.addrToDomain.Load(addr) + if !ok { + return "" + } + return ww.Domain + }() + resultBs, err := json.Marshal(domain) + return tsconsensus.CommandResult{Result: resultBs, Err: err} +} + +// readDomainForIP executes a readDomainForIP command on the leader with raft. +func (ipp *ConsensusIPPool) readDomainForIP(nid tailcfg.NodeID, addr netip.Addr) (string, error) { + args := readDomainForIPArgs{ + NodeID: nid, + Addr: addr, + } + bs, err := json.Marshal(args) + if err != nil { + return "", err + } + c := tsconsensus.Command{ + Name: "readDomainForIP", + Args: bs, + } + result, err := ipp.consensus.ExecuteCommand(c) + if err != nil { + log.Printf("readDomainForIP: raft error executing command: %v", err) + return "", err + } + if result.Err != nil { + log.Printf("readDomainForIP: error returned from state machine: %v", err) + return "", result.Err + } + var domain string + err = json.Unmarshal(result.Result, &domain) + return domain, err +} + +type markLastUsedArgs struct { + NodeID tailcfg.NodeID + Addr netip.Addr + Domain string + UpdatedAt time.Time +} + +// executeMarkLastUsed parses a markLastUsed log entry and applies it. +func (ipp *ConsensusIPPool) executeMarkLastUsed(bs []byte) tsconsensus.CommandResult { + var args markLastUsedArgs + err := json.Unmarshal(bs, &args) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + err = ipp.applyMarkLastUsed(args.NodeID, args.Addr, args.Domain, args.UpdatedAt) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + return tsconsensus.CommandResult{} +} + +// applyMarkLastUsed applies the arguments from the log entry to the state. It updates an entry in the AddrToDomain +// map with a new LastUsed timestamp. +// applyMarkLastUsed is not safe for concurrent access. It's only called from raft which will +// not call it concurrently. +func (ipp *ConsensusIPPool) applyMarkLastUsed(from tailcfg.NodeID, addr netip.Addr, domain string, updatedAt time.Time) error { + ps, ok := ipp.perPeerMap.Load(from) + if !ok { + // There's nothing to mark. But this is unexpected, because we mark last used after we do things with peer state. + log.Printf("applyMarkLastUsed: could not find peer state, nodeID: %s", from) + return nil + } + ww, ok := ps.addrToDomain.Load(addr) + if !ok { + // The peer state didn't have an entry for the IP address (possibly it expired), so there's nothing to mark. + return nil + } + if ww.Domain != domain { + // The IP address expired and was reused for a new domain. Don't mark. + return nil + } + if ww.LastUsed.After(updatedAt) { + // This has been marked more recently. Don't mark. + return nil + } + ww.LastUsed = updatedAt + ps.addrToDomain.Store(addr, ww) + return nil +} + +// markLastUsed executes a markLastUsed command on the leader with raft. +func (ipp *ConsensusIPPool) markLastUsed(nid tailcfg.NodeID, addr netip.Addr, domain string, lastUsed time.Time) error { + args := markLastUsedArgs{ + NodeID: nid, + Addr: addr, + Domain: domain, + UpdatedAt: lastUsed, + } + bs, err := json.Marshal(args) + if err != nil { + return err + } + c := tsconsensus.Command{ + Name: "markLastUsed", + Args: bs, + } + result, err := ipp.consensus.ExecuteCommand(c) + if err != nil { + log.Printf("markLastUsed: raft error executing command: %v", err) + return err + } + if result.Err != nil { + log.Printf("markLastUsed: error returned from state machine: %v", err) + return result.Err + } + return nil +} + +type checkoutAddrArgs struct { + NodeID tailcfg.NodeID + Domain string + ReuseDeadline time.Time + UpdatedAt time.Time +} + +// executeCheckoutAddr parses a checkoutAddr raft log entry and applies it. +func (ipp *ConsensusIPPool) executeCheckoutAddr(bs []byte) tsconsensus.CommandResult { + var args checkoutAddrArgs + err := json.Unmarshal(bs, &args) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + addr, err := ipp.applyCheckoutAddr(args.NodeID, args.Domain, args.ReuseDeadline, args.UpdatedAt) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + resultBs, err := json.Marshal(addr) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + return tsconsensus.CommandResult{Result: resultBs} +} + +// applyCheckoutAddr finds the IP address for a nid+domain +// Each nid can use all of the addresses in the pool. +// updatedAt is the current time, the time at which we are wanting to get a new IP address. +// reuseDeadline is the time before which addresses are considered to be expired. +// So if addresses are being reused after they haven't been used for 24 hours say updatedAt would be now +// and reuseDeadline would be 24 hours ago. +// It is not safe for concurrent access (it's only called from raft, which will not call concurrently +// so that's fine). +func (ipp *ConsensusIPPool) applyCheckoutAddr(nid tailcfg.NodeID, domain string, reuseDeadline, updatedAt time.Time) (netip.Addr, error) { + ps, ok := ipp.perPeerMap.Load(nid) + if !ok { + ps = &consensusPerPeerState{ + addrToDomain: &syncs.Map[netip.Addr, whereWhen]{}, + } + ipp.perPeerMap.Store(nid, ps) + } + if existing, ok := ps.domainToAddr[domain]; ok { + ww, ok := ps.addrToDomain.Load(existing) + if ok { + ww.LastUsed = updatedAt + ps.addrToDomain.Store(existing, ww) + return existing, nil + } + log.Printf("applyCheckoutAddr: data out of sync, allocating new IP") + } + addr, wasInUse, previousDomain, err := ps.unusedIPV4(ipp.IPSet, reuseDeadline) + if err != nil { + return netip.Addr{}, err + } + mak.Set(&ps.domainToAddr, domain, addr) + if wasInUse { + delete(ps.domainToAddr, previousDomain) + } + ps.addrToDomain.Store(addr, whereWhen{Domain: domain, LastUsed: updatedAt}) + return addr, nil +} + +// Apply is part of the raft.FSM interface. It takes an incoming log entry and applies it to the state. +func (ipp *ConsensusIPPool) Apply(l *raft.Log) any { + var c tsconsensus.Command + if err := json.Unmarshal(l.Data, &c); err != nil { + panic(fmt.Sprintf("failed to unmarshal command: %s", err.Error())) + } + switch c.Name { + case "checkoutAddr": + return ipp.executeCheckoutAddr(c.Args) + case "markLastUsed": + return ipp.executeMarkLastUsed(c.Args) + case "readDomainForIP": + return ipp.executeReadDomainForIP(c.Args) + default: + panic(fmt.Sprintf("unrecognized command: %s", c.Name)) + } +} + +// commandExecutor is an interface covering the routing parts of consensus +// used to allow a fake in the tests +type commandExecutor interface { + ExecuteCommand(tsconsensus.Command) (tsconsensus.CommandResult, error) +} diff --git a/cmd/natc/ippool/consensusippool_test.go b/cmd/natc/ippool/consensusippool_test.go new file mode 100644 index 0000000000000..242cdffaf26d3 --- /dev/null +++ b/cmd/natc/ippool/consensusippool_test.go @@ -0,0 +1,383 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ippool + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/netip" + "testing" + "time" + + "github.com/hashicorp/raft" + "go4.org/netipx" + "tailscale.com/tailcfg" + "tailscale.com/tsconsensus" + "tailscale.com/util/must" +) + +func makeSetFromPrefix(pfx netip.Prefix) *netipx.IPSet { + var ipsb netipx.IPSetBuilder + ipsb.AddPrefix(pfx) + return must.Get(ipsb.IPSet()) +} + +type FakeConsensus struct { + ipp *ConsensusIPPool +} + +func (c *FakeConsensus) ExecuteCommand(cmd tsconsensus.Command) (tsconsensus.CommandResult, error) { + b, err := json.Marshal(cmd) + if err != nil { + return tsconsensus.CommandResult{}, err + } + result := c.ipp.Apply(&raft.Log{Data: b}) + return result.(tsconsensus.CommandResult), nil +} + +func makePool(pfx netip.Prefix) *ConsensusIPPool { + ipp := NewConsensusIPPool(makeSetFromPrefix(pfx)) + ipp.consensus = &FakeConsensus{ipp: ipp} + return ipp +} + +func TestConsensusIPForDomain(t *testing.T) { + pfx := netip.MustParsePrefix("100.64.0.0/16") + ipp := makePool(pfx) + from := tailcfg.NodeID(1) + + a, err := ipp.IPForDomain(from, "example.com") + if err != nil { + t.Fatal(err) + } + if !pfx.Contains(a) { + t.Fatalf("expected %v to be in the prefix %v", a, pfx) + } + + b, err := ipp.IPForDomain(from, "a.example.com") + if err != nil { + t.Fatal(err) + } + if !pfx.Contains(b) { + t.Fatalf("expected %v to be in the prefix %v", b, pfx) + } + if b == a { + t.Fatalf("same address issued twice %v, %v", a, b) + } + + c, err := ipp.IPForDomain(from, "example.com") + if err != nil { + t.Fatal(err) + } + if c != a { + t.Fatalf("expected %v to be remembered as the addr for example.com, but got %v", a, c) + } +} + +func TestConsensusPoolExhaustion(t *testing.T) { + ipp := makePool(netip.MustParsePrefix("100.64.0.0/31")) + from := tailcfg.NodeID(1) + + subdomains := []string{"a", "b", "c"} + for i, sd := range subdomains { + _, err := ipp.IPForDomain(from, fmt.Sprintf("%s.example.com", sd)) + if i < 2 && err != nil { + t.Fatal(err) + } + expected := "ip pool exhausted" + if i == 2 && err.Error() != expected { + t.Fatalf("expected error to be '%s', got '%s'", expected, err.Error()) + } + } +} + +func TestConsensusPoolExpiry(t *testing.T) { + ipp := makePool(netip.MustParsePrefix("100.64.0.0/31")) + firstIP := netip.MustParseAddr("100.64.0.0") + secondIP := netip.MustParseAddr("100.64.0.1") + timeOfUse := time.Now() + beforeTimeOfUse := timeOfUse.Add(-1 * time.Hour) + afterTimeOfUse := timeOfUse.Add(1 * time.Hour) + from := tailcfg.NodeID(1) + + // the pool is unused, we get an address, and it's marked as being used at timeOfUse + aAddr, err := ipp.applyCheckoutAddr(from, "a.example.com", time.Time{}, timeOfUse) + if err != nil { + t.Fatal(err) + } + if aAddr.Compare(firstIP) != 0 { + t.Fatalf("expected %s, got %s", firstIP, aAddr) + } + ww, ok := ipp.domainLookup(from, firstIP) + if !ok { + t.Fatal("expected wherewhen to be found") + } + if ww.Domain != "a.example.com" { + t.Fatalf("expected aAddr to look up to a.example.com, got: %s", ww.Domain) + } + + // the time before which we will reuse addresses is prior to timeOfUse, so no reuse + bAddr, err := ipp.applyCheckoutAddr(from, "b.example.com", beforeTimeOfUse, timeOfUse) + if err != nil { + t.Fatal(err) + } + if bAddr.Compare(secondIP) != 0 { + t.Fatalf("expected %s, got %s", secondIP, bAddr) + } + + // the time before which we will reuse addresses is after timeOfUse, so reuse addresses that were marked as used at timeOfUse. + cAddr, err := ipp.applyCheckoutAddr(from, "c.example.com", afterTimeOfUse, timeOfUse) + if err != nil { + t.Fatal(err) + } + if cAddr.Compare(firstIP) != 0 { + t.Fatalf("expected %s, got %s", firstIP, cAddr) + } + ww, ok = ipp.domainLookup(from, firstIP) + if !ok { + t.Fatal("expected wherewhen to be found") + } + if ww.Domain != "c.example.com" { + t.Fatalf("expected firstIP to look up to c.example.com, got: %s", ww.Domain) + } + + // the addr remains associated with c.example.com + cAddrAgain, err := ipp.applyCheckoutAddr(from, "c.example.com", afterTimeOfUse, timeOfUse) + if err != nil { + t.Fatal(err) + } + if cAddrAgain.Compare(cAddr) != 0 { + t.Fatalf("expected cAddrAgain to be cAddr, but they are different. cAddrAgain=%s cAddr=%s", cAddrAgain, cAddr) + } + ww, ok = ipp.domainLookup(from, firstIP) + if !ok { + t.Fatal("expected wherewhen to be found") + } + if ww.Domain != "c.example.com" { + t.Fatalf("expected firstIP to look up to c.example.com, got: %s", ww.Domain) + } +} + +func TestConsensusPoolApplyMarkLastUsed(t *testing.T) { + ipp := makePool(netip.MustParsePrefix("100.64.0.0/31")) + firstIP := netip.MustParseAddr("100.64.0.0") + time1 := time.Now() + time2 := time1.Add(1 * time.Hour) + from := tailcfg.NodeID(1) + domain := "example.com" + + aAddr, err := ipp.applyCheckoutAddr(from, domain, time.Time{}, time1) + if err != nil { + t.Fatal(err) + } + if aAddr.Compare(firstIP) != 0 { + t.Fatalf("expected %s, got %s", firstIP, aAddr) + } + // example.com LastUsed is now time1 + ww, ok := ipp.domainLookup(from, firstIP) + if !ok { + t.Fatal("expected wherewhen to be found") + } + if ww.LastUsed != time1 { + t.Fatalf("expected %s, got %s", time1, ww.LastUsed) + } + if ww.Domain != domain { + t.Fatalf("expected %s, got %s", domain, ww.Domain) + } + + err = ipp.applyMarkLastUsed(from, firstIP, domain, time2) + if err != nil { + t.Fatal(err) + } + + // example.com LastUsed is now time2 + ww, ok = ipp.domainLookup(from, firstIP) + if !ok { + t.Fatal("expected wherewhen to be found") + } + if ww.LastUsed != time2 { + t.Fatalf("expected %s, got %s", time2, ww.LastUsed) + } + if ww.Domain != domain { + t.Fatalf("expected %s, got %s", domain, ww.Domain) + } +} + +func TestConsensusDomainForIP(t *testing.T) { + ipp := makePool(netip.MustParsePrefix("100.64.0.0/16")) + from := tailcfg.NodeID(1) + domain := "example.com" + now := time.Now() + + d, ok := ipp.DomainForIP(from, netip.MustParseAddr("100.64.0.1"), now) + if d != "" { + t.Fatalf("expected an empty string if the addr is not found but got %s", d) + } + if ok { + t.Fatalf("expected domain to not be found for IP, as it has never been looked up") + } + a, err := ipp.IPForDomain(from, domain) + if err != nil { + t.Fatal(err) + } + d2, ok := ipp.DomainForIP(from, a, now) + if d2 != domain { + t.Fatalf("expected %s but got %s", domain, d2) + } + if !ok { + t.Fatalf("expected domain to be found for IP that was handed out for it") + } +} + +func TestConsensusReadDomainForIP(t *testing.T) { + ipp := makePool(netip.MustParsePrefix("100.64.0.0/16")) + from := tailcfg.NodeID(1) + domain := "example.com" + + d, err := ipp.readDomainForIP(from, netip.MustParseAddr("100.64.0.1")) + if err != nil { + t.Fatal(err) + } + if d != "" { + t.Fatalf("expected an empty string if the addr is not found but got %s", d) + } + a, err := ipp.IPForDomain(from, domain) + if err != nil { + t.Fatal(err) + } + d2, err := ipp.readDomainForIP(from, a) + if err != nil { + t.Fatal(err) + } + if d2 != domain { + t.Fatalf("expected %s but got %s", domain, d2) + } +} + +func TestConsensusSnapshot(t *testing.T) { + pfx := netip.MustParsePrefix("100.64.0.0/16") + ipp := makePool(pfx) + domain := "example.com" + expectedAddr := netip.MustParseAddr("100.64.0.0") + expectedFrom := expectedAddr + expectedTo := netip.MustParseAddr("100.64.255.255") + from := tailcfg.NodeID(1) + + // pool allocates first addr for from + if _, err := ipp.IPForDomain(from, domain); err != nil { + t.Fatal(err) + } + // take a snapshot + fsmSnap, err := ipp.Snapshot() + if err != nil { + t.Fatal(err) + } + snap := fsmSnap.(fsmSnapshot) + + // verify snapshot state matches the state we know ipp will have + // ipset matches ipp.IPSet + if len(snap.IPSet.Ranges) != 1 { + t.Fatalf("expected 1, got %d", len(snap.IPSet.Ranges)) + } + if snap.IPSet.Ranges[0].From != expectedFrom { + t.Fatalf("want %s, got %s", expectedFrom, snap.IPSet.Ranges[0].From) + } + if snap.IPSet.Ranges[0].To != expectedTo { + t.Fatalf("want %s, got %s", expectedTo, snap.IPSet.Ranges[0].To) + } + + // perPeerMap has one entry, for from + if len(snap.PerPeerMap) != 1 { + t.Fatalf("expected 1, got %d", len(snap.PerPeerMap)) + } + ps := snap.PerPeerMap[from] + + // the one peer state has allocated one address, the first in the prefix + if len(ps.DomainToAddr) != 1 { + t.Fatalf("expected 1, got %d", len(ps.DomainToAddr)) + } + addr := ps.DomainToAddr[domain] + if addr != expectedAddr { + t.Fatalf("want %s, got %s", expectedAddr.String(), addr.String()) + } + if len(ps.AddrToDomain) != 1 { + t.Fatalf("expected 1, got %d", len(ps.AddrToDomain)) + } + ww := ps.AddrToDomain[addr] + if ww.Domain != domain { + t.Fatalf("want %s, got %s", domain, ww.Domain) + } +} + +func TestConsensusRestore(t *testing.T) { + pfx := netip.MustParsePrefix("100.64.0.0/16") + ipp := makePool(pfx) + domain := "example.com" + expectedAddr := netip.MustParseAddr("100.64.0.0") + from := tailcfg.NodeID(1) + + if _, err := ipp.IPForDomain(from, domain); err != nil { + t.Fatal(err) + } + // take the snapshot after only 1 addr allocated + fsmSnap, err := ipp.Snapshot() + if err != nil { + t.Fatal(err) + } + snap := fsmSnap.(fsmSnapshot) + + if _, err := ipp.IPForDomain(from, "b.example.com"); err != nil { + t.Fatal(err) + } + if _, err := ipp.IPForDomain(from, "c.example.com"); err != nil { + t.Fatal(err) + } + if _, err := ipp.IPForDomain(from, "d.example.com"); err != nil { + t.Fatal(err) + } + // ipp now has 4 entries in domainToAddr + ps, _ := ipp.perPeerMap.Load(from) + if len(ps.domainToAddr) != 4 { + t.Fatalf("want 4, got %d", len(ps.domainToAddr)) + } + + // restore the snapshot + bs, err := json.Marshal(snap) + if err != nil { + t.Fatal(err) + } + err = ipp.Restore(io.NopCloser(bytes.NewBuffer(bs))) + if err != nil { + t.Fatal(err) + } + + // everything should be as it was when the snapshot was taken + if ipp.perPeerMap.Len() != 1 { + t.Fatalf("want 1, got %d", ipp.perPeerMap.Len()) + } + psAfter, _ := ipp.perPeerMap.Load(from) + if len(psAfter.domainToAddr) != 1 { + t.Fatalf("want 1, got %d", len(psAfter.domainToAddr)) + } + if psAfter.domainToAddr[domain] != expectedAddr { + t.Fatalf("want %s, got %s", expectedAddr, psAfter.domainToAddr[domain]) + } + ww, _ := psAfter.addrToDomain.Load(expectedAddr) + if ww.Domain != domain { + t.Fatalf("want %s, got %s", domain, ww.Domain) + } +} + +func TestConsensusIsCloseToExpiry(t *testing.T) { + a := time.Now() + b := a.Add(5 * time.Second) + if !isCloseToExpiry(a, b, 8*time.Second) { + t.Fatal("times are not within half the lifetime, expected true") + } + if isCloseToExpiry(a, b, 12*time.Second) { + t.Fatal("times are within half the lifetime, expected false") + } +} diff --git a/cmd/natc/ippool/consensusippoolserialize.go b/cmd/natc/ippool/consensusippoolserialize.go new file mode 100644 index 0000000000000..97dc02f2c7d7c --- /dev/null +++ b/cmd/natc/ippool/consensusippoolserialize.go @@ -0,0 +1,164 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ippool + +import ( + "encoding/json" + "io" + "log" + "maps" + "net/netip" + + "github.com/hashicorp/raft" + "go4.org/netipx" + "tailscale.com/syncs" + "tailscale.com/tailcfg" +) + +// Snapshot and Restore enable the raft lib to do log compaction. +// https://pkg.go.dev/github.com/hashicorp/raft#FSM + +// Snapshot is part of the raft.FSM interface. +// According to the docs it: +// - should return quickly +// - will not be called concurrently with Apply +// - the snapshot returned will have Persist called on it concurrently with Apply +// (so it should not contain pointers to the original data that's being mutated) +func (ipp *ConsensusIPPool) Snapshot() (raft.FSMSnapshot, error) { + // everything is safe for concurrent reads and this is not called concurrently with Apply which is + // the only thing that writes, so we do not need to lock + return ipp.getPersistable(), nil +} + +type persistableIPSet struct { + Ranges []persistableIPRange +} + +func getPersistableIPSet(i *netipx.IPSet) persistableIPSet { + rs := []persistableIPRange{} + for _, r := range i.Ranges() { + rs = append(rs, getPersistableIPRange(r)) + } + return persistableIPSet{Ranges: rs} +} + +func (mips *persistableIPSet) toIPSet() (*netipx.IPSet, error) { + b := netipx.IPSetBuilder{} + for _, r := range mips.Ranges { + b.AddRange(r.toIPRange()) + } + return b.IPSet() +} + +type persistableIPRange struct { + From netip.Addr + To netip.Addr +} + +func getPersistableIPRange(r netipx.IPRange) persistableIPRange { + return persistableIPRange{ + From: r.From(), + To: r.To(), + } +} + +func (mipr *persistableIPRange) toIPRange() netipx.IPRange { + return netipx.IPRangeFrom(mipr.From, mipr.To) +} + +// Restore is part of the raft.FSM interface. +// According to the docs it: +// - will not be called concurrently with any other command +// - the FSM must discard all previous state before restoring +func (ipp *ConsensusIPPool) Restore(rc io.ReadCloser) error { + var snap fsmSnapshot + if err := json.NewDecoder(rc).Decode(&snap); err != nil { + return err + } + ipset, ppm, err := snap.getData() + if err != nil { + return err + } + ipp.IPSet = ipset + ipp.perPeerMap = ppm + return nil +} + +type fsmSnapshot struct { + IPSet persistableIPSet + PerPeerMap map[tailcfg.NodeID]persistablePPS +} + +// Persist is part of the raft.FSMSnapshot interface +// According to the docs Persist may be called concurrently with Apply +func (f fsmSnapshot) Persist(sink raft.SnapshotSink) error { + if err := json.NewEncoder(sink).Encode(f); err != nil { + log.Printf("Error encoding snapshot as JSON: %v", err) + return sink.Cancel() + } + return sink.Close() +} + +// Release is part of the raft.FSMSnapshot interface +func (f fsmSnapshot) Release() {} + +// getPersistable returns an object that: +// - contains all the data in ConsensusIPPool +// - doesn't share any pointers with it +// - can be marshalled to JSON +// +// part of the raft snapshotting, getPersistable will be called during Snapshot +// and the results used during persist (concurrently with Apply) +func (ipp *ConsensusIPPool) getPersistable() fsmSnapshot { + ppm := map[tailcfg.NodeID]persistablePPS{} + for k, v := range ipp.perPeerMap.All() { + ppm[k] = v.getPersistable() + } + return fsmSnapshot{ + IPSet: getPersistableIPSet(ipp.IPSet), + PerPeerMap: ppm, + } +} + +func (f fsmSnapshot) getData() (*netipx.IPSet, *syncs.Map[tailcfg.NodeID, *consensusPerPeerState], error) { + ppm := syncs.Map[tailcfg.NodeID, *consensusPerPeerState]{} + for k, v := range f.PerPeerMap { + ppm.Store(k, v.toPerPeerState()) + } + ipset, err := f.IPSet.toIPSet() + if err != nil { + return nil, nil, err + } + return ipset, &ppm, nil +} + +// getPersistable returns an object that: +// - contains all the data in consensusPerPeerState +// - doesn't share any pointers with it +// - can be marshalled to JSON +// +// part of the raft snapshotting, getPersistable will be called during Snapshot +// and the results used during persist (concurrently with Apply) +func (ps *consensusPerPeerState) getPersistable() persistablePPS { + return persistablePPS{ + AddrToDomain: maps.Collect(ps.addrToDomain.All()), + DomainToAddr: maps.Clone(ps.domainToAddr), + } +} + +type persistablePPS struct { + DomainToAddr map[string]netip.Addr + AddrToDomain map[netip.Addr]whereWhen +} + +func (p persistablePPS) toPerPeerState() *consensusPerPeerState { + atd := &syncs.Map[netip.Addr, whereWhen]{} + for k, v := range p.AddrToDomain { + atd.Store(k, v) + } + return &consensusPerPeerState{ + domainToAddr: p.DomainToAddr, + addrToDomain: atd, + } +} diff --git a/cmd/natc/ippool/ippool.go b/cmd/natc/ippool/ippool.go index 3a46a6e7ad186..5a2dcbec911e0 100644 --- a/cmd/natc/ippool/ippool.go +++ b/cmd/natc/ippool/ippool.go @@ -10,6 +10,7 @@ import ( "math/big" "net/netip" "sync" + "time" "github.com/gaissmai/bart" "go4.org/netipx" @@ -21,12 +22,26 @@ import ( var ErrNoIPsAvailable = errors.New("no IPs available") -type IPPool struct { +// IPPool allocates IPv4 addresses from a pool to DNS domains, on a per tailcfg.NodeID basis. +// For each tailcfg.NodeID, IPv4 addresses are associated with at most one DNS domain. +// Addresses may be reused across other tailcfg.NodeID's for the same or other domains. +type IPPool interface { + // DomainForIP looks up the domain associated with a tailcfg.NodeID and netip.Addr pair. + // If there is no association, the result is empty and ok is false. + DomainForIP(tailcfg.NodeID, netip.Addr, time.Time) (string, bool) + + // IPForDomain looks up or creates an IP address allocation for the tailcfg.NodeID and domain pair. + // If no address association is found, one is allocated from the range of free addresses for this tailcfg.NodeID. + // If no more address are available, an error is returned. + IPForDomain(tailcfg.NodeID, string) (netip.Addr, error) +} + +type SingleMachineIPPool struct { perPeerMap syncs.Map[tailcfg.NodeID, *perPeerState] IPSet *netipx.IPSet } -func (ipp *IPPool) DomainForIP(from tailcfg.NodeID, addr netip.Addr) (string, bool) { +func (ipp *SingleMachineIPPool) DomainForIP(from tailcfg.NodeID, addr netip.Addr, _ time.Time) (string, bool) { ps, ok := ipp.perPeerMap.Load(from) if !ok { log.Printf("handleTCPFlow: no perPeerState for %v", from) @@ -40,7 +55,7 @@ func (ipp *IPPool) DomainForIP(from tailcfg.NodeID, addr netip.Addr) (string, bo return domain, ok } -func (ipp *IPPool) IPForDomain(from tailcfg.NodeID, domain string) (netip.Addr, error) { +func (ipp *SingleMachineIPPool) IPForDomain(from tailcfg.NodeID, domain string) (netip.Addr, error) { npps := &perPeerState{ ipset: ipp.IPSet, } diff --git a/cmd/natc/ippool/ippool_test.go b/cmd/natc/ippool/ippool_test.go index 2919d7757af8c..8d474f86a97ed 100644 --- a/cmd/natc/ippool/ippool_test.go +++ b/cmd/natc/ippool/ippool_test.go @@ -8,6 +8,7 @@ import ( "fmt" "net/netip" "testing" + "time" "go4.org/netipx" "tailscale.com/tailcfg" @@ -19,7 +20,7 @@ func TestIPPoolExhaustion(t *testing.T) { var ipsb netipx.IPSetBuilder ipsb.AddPrefix(smallPrefix) addrPool := must.Get(ipsb.IPSet()) - pool := IPPool{IPSet: addrPool} + pool := SingleMachineIPPool{IPSet: addrPool} assignedIPs := make(map[netip.Addr]string) @@ -68,7 +69,7 @@ func TestIPPool(t *testing.T) { var ipsb netipx.IPSetBuilder ipsb.AddPrefix(netip.MustParsePrefix("100.64.1.0/24")) addrPool := must.Get(ipsb.IPSet()) - pool := IPPool{ + pool := SingleMachineIPPool{ IPSet: addrPool, } from := tailcfg.NodeID(12345) @@ -89,7 +90,7 @@ func TestIPPool(t *testing.T) { t.Errorf("IPv4 address %s not in range %s", addr, addrPool) } - domain, ok := pool.DomainForIP(from, addr) + domain, ok := pool.DomainForIP(from, addr, time.Now()) if !ok { t.Errorf("domainForIP(%s) not found", addr) } else if domain != "example.com" { diff --git a/cmd/natc/natc.go b/cmd/natc/natc.go index b327f55bdc3ea..2dcdc551f23cc 100644 --- a/cmd/natc/natc.go +++ b/cmd/natc/natc.go @@ -57,6 +57,8 @@ func main() { printULA = fs.Bool("print-ula", false, "print the ULA prefix and exit") ignoreDstPfxStr = fs.String("ignore-destinations", "", "comma-separated list of prefixes to ignore") wgPort = fs.Uint("wg-port", 0, "udp port for wireguard and peer to peer traffic") + clusterTag = fs.String("cluster-tag", "", "optionally run in a consensus cluster with other nodes with this tag") + server = fs.String("login-server", ipn.DefaultControlURL, "the base URL of control server") ) ff.Parse(fs, os.Args[1:], ff.WithEnvVarPrefix("TS_NATC")) @@ -94,6 +96,7 @@ func main() { ts := &tsnet.Server{ Hostname: *hostname, } + ts.ControlURL = *server if *wgPort != 0 { if *wgPort >= 1<<16 { log.Fatalf("wg-port must be in the range [0, 65535]") @@ -148,12 +151,31 @@ func main() { routes, dnsAddr, addrPool := calculateAddresses(prefixes) v6ULA := ula(uint16(*siteID)) + + var ipp ippool.IPPool + if *clusterTag != "" { + cipp := ippool.NewConsensusIPPool(addrPool) + err = cipp.StartConsensus(ctx, ts, *clusterTag) + if err != nil { + log.Fatalf("StartConsensus: %v", err) + } + defer func() { + err := cipp.StopConsensus(ctx) + if err != nil { + log.Printf("Error stopping consensus: %v", err) + } + }() + ipp = cipp + } else { + ipp = &ippool.SingleMachineIPPool{IPSet: addrPool} + } + c := &connector{ ts: ts, whois: lc, v6ULA: v6ULA, ignoreDsts: ignoreDstTable, - ipPool: &ippool.IPPool{IPSet: addrPool}, + ipPool: ipp, routes: routes, dnsAddr: dnsAddr, resolver: net.DefaultResolver, @@ -209,7 +231,7 @@ type connector struct { ignoreDsts *bart.Table[bool] // ipPool contains the per-peer IPv4 address assignments. - ipPool *ippool.IPPool + ipPool ippool.IPPool // resolver is used to lookup IP addresses for DNS queries. resolver lookupNetIPer @@ -453,7 +475,7 @@ func (c *connector) handleTCPFlow(src, dst netip.AddrPort) (handler func(net.Con if dstAddr.Is6() { dstAddr = v4ForV6(dstAddr) } - domain, ok := c.ipPool.DomainForIP(who.Node.ID, dstAddr) + domain, ok := c.ipPool.DomainForIP(who.Node.ID, dstAddr, time.Now()) if !ok { return nil, false } diff --git a/cmd/natc/natc_test.go b/cmd/natc/natc_test.go index 0320db8a4ea59..78dec86fdf02f 100644 --- a/cmd/natc/natc_test.go +++ b/cmd/natc/natc_test.go @@ -270,7 +270,7 @@ func TestDNSResponse(t *testing.T) { ignoreDsts: &bart.Table[bool]{}, routes: routes, v6ULA: v6ULA, - ipPool: &ippool.IPPool{IPSet: addrPool}, + ipPool: &ippool.SingleMachineIPPool{IPSet: addrPool}, dnsAddr: dnsAddr, } c.ignoreDsts.Insert(netip.MustParsePrefix("8.8.4.4/32"), true) From 75a7d28b079d1b16448e75c014eb689583a4d175 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 5 Jun 2025 10:33:16 -0700 Subject: [PATCH 034/263] net/packet: fix Parsed docs (#16200) Updates #cleanup Signed-off-by: Jordan Whited --- net/packet/packet.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/net/packet/packet.go b/net/packet/packet.go index b683b22126948..876a653ed4c79 100644 --- a/net/packet/packet.go +++ b/net/packet/packet.go @@ -51,10 +51,11 @@ type Parsed struct { IPVersion uint8 // IPProto is the IP subprotocol (UDP, TCP, etc.). Valid iff IPVersion != 0. IPProto ipproto.Proto - // SrcIP4 is the source address. Family matches IPVersion. Port is - // valid iff IPProto == TCP || IPProto == UDP. + // Src is the source address. Family matches IPVersion. Port is + // valid iff IPProto == TCP || IPProto == UDP || IPProto == SCTP. Src netip.AddrPort - // DstIP4 is the destination address. Family matches IPVersion. + // Dst is the destination address. Family matches IPVersion. Port is + // valid iff IPProto == TCP || IPProto == UDP || IPProto == SCTP. Dst netip.AddrPort // TCPFlags is the packet's TCP flag bits. Valid iff IPProto == TCP. TCPFlags TCPFlag From 3e08eab21e204bc3568762c2b49e0e1ab9ebf4b4 Mon Sep 17 00:00:00 2001 From: Fran Bull Date: Thu, 5 Jun 2025 08:51:10 -0700 Subject: [PATCH 035/263] cmd/natc: use new on disk state store for consensus Fixes #16027 Signed-off-by: Fran Bull --- cmd/natc/ippool/consensusippool.go | 33 +++++++++++++++++++++++++++++- cmd/natc/natc.go | 3 ++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/cmd/natc/ippool/consensusippool.go b/cmd/natc/ippool/consensusippool.go index 4783209b252e2..adf2090d1b76a 100644 --- a/cmd/natc/ippool/consensusippool.go +++ b/cmd/natc/ippool/consensusippool.go @@ -10,6 +10,8 @@ import ( "fmt" "log" "net/netip" + "os" + "path/filepath" "time" "github.com/hashicorp/raft" @@ -150,9 +152,14 @@ func (ipp *ConsensusIPPool) domainLookup(from tailcfg.NodeID, addr netip.Addr) ( } // StartConsensus is part of the IPPool interface. It starts the raft background routines that handle consensus. -func (ipp *ConsensusIPPool) StartConsensus(ctx context.Context, ts *tsnet.Server, clusterTag string) error { +func (ipp *ConsensusIPPool) StartConsensus(ctx context.Context, ts *tsnet.Server, clusterTag string, clusterStateDir string) error { cfg := tsconsensus.DefaultConfig() cfg.ServeDebugMonitor = true + var err error + cfg.StateDirPath, err = getStatePath(clusterStateDir) + if err != nil { + return err + } cns, err := tsconsensus.Start(ctx, ts, ipp, clusterTag, cfg) if err != nil { return err @@ -204,6 +211,30 @@ func (ps *consensusPerPeerState) unusedIPV4(ipset *netipx.IPSet, reuseDeadline t return netip.Addr{}, false, "", errors.New("ip pool exhausted") } +func getStatePath(pathFromFlag string) (string, error) { + var dirPath string + if pathFromFlag != "" { + dirPath = pathFromFlag + } else { + confDir, err := os.UserConfigDir() + if err != nil { + return "", err + } + dirPath = filepath.Join(confDir, "nat-connector-cluster-state") + } + + if err := os.MkdirAll(dirPath, 0700); err != nil { + return "", err + } + if fi, err := os.Stat(dirPath); err != nil { + return "", err + } else if !fi.IsDir() { + return "", fmt.Errorf("%v is not a directory", dirPath) + } + + return dirPath, nil +} + // isCloseToExpiry returns true if the lastUsed and now times are more than // half the lifetime apart func isCloseToExpiry(lastUsed, now time.Time, lifetime time.Duration) bool { diff --git a/cmd/natc/natc.go b/cmd/natc/natc.go index 2dcdc551f23cc..719d5d20d6b38 100644 --- a/cmd/natc/natc.go +++ b/cmd/natc/natc.go @@ -59,6 +59,7 @@ func main() { wgPort = fs.Uint("wg-port", 0, "udp port for wireguard and peer to peer traffic") clusterTag = fs.String("cluster-tag", "", "optionally run in a consensus cluster with other nodes with this tag") server = fs.String("login-server", ipn.DefaultControlURL, "the base URL of control server") + clusterStateDir = fs.String("cluster-state-dir", "", "path to directory in which to store raft state") ) ff.Parse(fs, os.Args[1:], ff.WithEnvVarPrefix("TS_NATC")) @@ -155,7 +156,7 @@ func main() { var ipp ippool.IPPool if *clusterTag != "" { cipp := ippool.NewConsensusIPPool(addrPool) - err = cipp.StartConsensus(ctx, ts, *clusterTag) + err = cipp.StartConsensus(ctx, ts, *clusterTag, *clusterStateDir) if err != nil { log.Fatalf("StartConsensus: %v", err) } From 3f7a9f82e39813ca42701aca18cf23a97b5652dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus=20Lensb=C3=B8l?= Date: Fri, 6 Jun 2025 11:42:33 -0400 Subject: [PATCH 036/263] wgengine/magicsock: fix bpf fragmentation jump offsets (#16204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fragmented datagrams would be processed instead of being dumped right away. In reality, thse datagrams would be dropped anyway later so there should functionally not be any change. Additionally, the feature is off by default. Closes #16203 Signed-off-by: Claus Lensbøl --- wgengine/magicsock/magicsock_linux.go | 4 +- wgengine/magicsock/magicsock_linux_test.go | 76 ++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/wgengine/magicsock/magicsock_linux.go b/wgengine/magicsock/magicsock_linux.go index c5df555cd4ecd..34c39fe62b773 100644 --- a/wgengine/magicsock/magicsock_linux.go +++ b/wgengine/magicsock/magicsock_linux.go @@ -66,10 +66,10 @@ var ( // fragmented, and we don't want to handle reassembly. bpf.LoadAbsolute{Off: 6, Size: 2}, // More Fragments bit set means this is part of a fragmented packet. - bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x2000, SkipTrue: 7, SkipFalse: 0}, + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x2000, SkipTrue: 8, SkipFalse: 0}, // Non-zero fragment offset with MF=0 means this is the last // fragment of packet. - bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x1fff, SkipTrue: 6, SkipFalse: 0}, + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x1fff, SkipTrue: 7, SkipFalse: 0}, // Load IP header length into X register. bpf.LoadMemShift{Off: 0}, diff --git a/wgengine/magicsock/magicsock_linux_test.go b/wgengine/magicsock/magicsock_linux_test.go index 6b86b04f2c8d4..28ccd220ee784 100644 --- a/wgengine/magicsock/magicsock_linux_test.go +++ b/wgengine/magicsock/magicsock_linux_test.go @@ -9,6 +9,7 @@ import ( "net/netip" "testing" + "golang.org/x/net/bpf" "golang.org/x/sys/cpu" "golang.org/x/sys/unix" "tailscale.com/disco" @@ -146,3 +147,78 @@ func TestEthernetProto(t *testing.T) { } } } + +func TestBpfDiscardV4(t *testing.T) { + // Good packet as a reference for what should not be rejected + udp4Packet := []byte{ + // IPv4 header + 0x45, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x11, 0x00, 0x00, + 0x7f, 0x00, 0x00, 0x01, // source ip + 0x7f, 0x00, 0x00, 0x02, // dest ip + + // UDP header + 0x30, 0x39, // src port + 0xd4, 0x31, // dest port + 0x00, 0x12, // length; 8 bytes header + 10 bytes payload = 18 bytes + 0x00, 0x00, // checksum; unused + + // Payload: disco magic plus 32 bytes for key and 24 bytes for nonce + 0x54, 0x53, 0xf0, 0x9f, 0x92, 0xac, 0x00, 0x01, 0x02, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, + } + + vm, err := bpf.NewVM(magicsockFilterV4) + if err != nil { + t.Fatalf("failed creating BPF VM: %v", err) + } + + tests := []struct { + name string + replace map[int]byte + accept bool + }{ + { + name: "base accepted datagram", + replace: map[int]byte{}, + accept: true, + }, + { + name: "more fragments", + replace: map[int]byte{ + 6: 0x20, + }, + accept: false, + }, + { + name: "some fragment", + replace: map[int]byte{ + 7: 0x01, + }, + accept: false, + }, + } + + udp4PacketChanged := make([]byte, len(udp4Packet)) + copy(udp4PacketChanged, udp4Packet) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.replace { + udp4PacketChanged[k] = v + } + ret, err := vm.Run(udp4PacketChanged) + if err != nil { + t.Fatalf("BPF VM error: %v", err) + } + + if (ret != 0) != tt.accept { + t.Errorf("expected accept=%v, got ret=%v", tt.accept, ret) + } + }) + } +} From 66ae8737f40bf5aebcff96824bf0d4f8439db9c7 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Fri, 6 Jun 2025 09:46:29 -0700 Subject: [PATCH 037/263] wgengine/magicsock: make endpoint.bestAddr Geneve-aware (#16195) This commit adds a new type to magicsock, epAddr, which largely ends up replacing netip.AddrPort in packet I/O paths throughout, enabling Geneve encapsulation over UDP awareness. The conn.ReceiveFunc for UDP has been revamped to fix and more clearly distinguish the different classes of packets we expect to receive: naked STUN binding messages, naked disco, naked WireGuard, Geneve-encapsulated disco, and Geneve-encapsulated WireGuard. Prior to this commit, STUN matching logic in the RX path could swallow a naked WireGuard packet if the keypair index, which is randomly generated, happened to overlap with a subset of the STUN magic cookie. Updates tailscale/corp#27502 Updates tailscale/corp#29326 Signed-off-by: Jordan Whited --- wgengine/magicsock/batching_conn.go | 4 +- wgengine/magicsock/batching_conn_linux.go | 44 ++- .../magicsock/batching_conn_linux_test.go | 57 ++- wgengine/magicsock/debughttp.go | 28 +- wgengine/magicsock/derp.go | 11 +- wgengine/magicsock/endpoint.go | 206 ++++++----- wgengine/magicsock/endpoint_test.go | 3 +- wgengine/magicsock/magicsock.go | 333 +++++++++++------- wgengine/magicsock/magicsock_linux.go | 8 +- wgengine/magicsock/magicsock_test.go | 221 ++++++------ wgengine/magicsock/peermap.go | 50 ++- wgengine/magicsock/rebinding_conn.go | 22 +- wgengine/magicsock/relaymanager.go | 12 +- wgengine/magicsock/relaymanager_test.go | 3 +- 14 files changed, 610 insertions(+), 392 deletions(-) diff --git a/wgengine/magicsock/batching_conn.go b/wgengine/magicsock/batching_conn.go index 58cfe28aa940d..b769907dbe88f 100644 --- a/wgengine/magicsock/batching_conn.go +++ b/wgengine/magicsock/batching_conn.go @@ -4,8 +4,6 @@ package magicsock import ( - "net/netip" - "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" "tailscale.com/types/nettype" @@ -21,5 +19,5 @@ var ( type batchingConn interface { nettype.PacketConn ReadBatch(msgs []ipv6.Message, flags int) (n int, err error) - WriteBatchTo(buffs [][]byte, addr netip.AddrPort, offset int) error + WriteBatchTo(buffs [][]byte, addr epAddr, offset int) error } diff --git a/wgengine/magicsock/batching_conn_linux.go b/wgengine/magicsock/batching_conn_linux.go index 9ad5e44749d27..c9aaff168b5b6 100644 --- a/wgengine/magicsock/batching_conn_linux.go +++ b/wgengine/magicsock/batching_conn_linux.go @@ -22,6 +22,7 @@ import ( "golang.org/x/sys/unix" "tailscale.com/hostinfo" "tailscale.com/net/neterror" + "tailscale.com/net/packet" "tailscale.com/types/nettype" ) @@ -92,9 +93,14 @@ const ( maxIPv6PayloadLen = 1<<16 - 1 - 8 ) -// coalesceMessages iterates msgs, coalescing them where possible while -// maintaining datagram order. All msgs have their Addr field set to addr. -func (c *linuxBatchingConn) coalesceMessages(addr *net.UDPAddr, buffs [][]byte, msgs []ipv6.Message, offset int) int { +// coalesceMessages iterates 'buffs', setting and coalescing them in 'msgs' +// where possible while maintaining datagram order. +// +// All msgs have their Addr field set to addr. +// +// All msgs[i].Buffers[0] are preceded by a Geneve header with vni.get() if +// vni.isSet(). +func (c *linuxBatchingConn) coalesceMessages(addr *net.UDPAddr, vni virtualNetworkID, buffs [][]byte, msgs []ipv6.Message, offset int) int { var ( base = -1 // index of msg we are currently coalescing into gsoSize int // segmentation size of msgs[base] @@ -105,8 +111,17 @@ func (c *linuxBatchingConn) coalesceMessages(addr *net.UDPAddr, buffs [][]byte, if addr.IP.To4() == nil { maxPayloadLen = maxIPv6PayloadLen } + vniIsSet := vni.isSet() + var gh packet.GeneveHeader + if vniIsSet { + gh.VNI = vni.get() + } for i, buff := range buffs { - buff = buff[offset:] + if vniIsSet { + gh.Encode(buffs[i]) + } else { + buff = buff[offset:] + } if i > 0 { msgLen := len(buff) baseLenBefore := len(msgs[base].Buffers[0]) @@ -163,28 +178,37 @@ func (c *linuxBatchingConn) putSendBatch(batch *sendBatch) { c.sendBatchPool.Put(batch) } -func (c *linuxBatchingConn) WriteBatchTo(buffs [][]byte, addr netip.AddrPort, offset int) error { +func (c *linuxBatchingConn) WriteBatchTo(buffs [][]byte, addr epAddr, offset int) error { batch := c.getSendBatch() defer c.putSendBatch(batch) - if addr.Addr().Is6() { - as16 := addr.Addr().As16() + if addr.ap.Addr().Is6() { + as16 := addr.ap.Addr().As16() copy(batch.ua.IP, as16[:]) batch.ua.IP = batch.ua.IP[:16] } else { - as4 := addr.Addr().As4() + as4 := addr.ap.Addr().As4() copy(batch.ua.IP, as4[:]) batch.ua.IP = batch.ua.IP[:4] } - batch.ua.Port = int(addr.Port()) + batch.ua.Port = int(addr.ap.Port()) var ( n int retried bool ) retry: if c.txOffload.Load() { - n = c.coalesceMessages(batch.ua, buffs, batch.msgs, offset) + n = c.coalesceMessages(batch.ua, addr.vni, buffs, batch.msgs, offset) } else { + vniIsSet := addr.vni.isSet() + var gh packet.GeneveHeader + if vniIsSet { + gh.VNI = addr.vni.get() + offset -= packet.GeneveFixedHeaderLength + } for i := range buffs { + if vniIsSet { + gh.Encode(buffs[i]) + } batch.msgs[i].Buffers[0] = buffs[i][offset:] batch.msgs[i].Addr = batch.ua batch.msgs[i].OOB = batch.msgs[i].OOB[:0] diff --git a/wgengine/magicsock/batching_conn_linux_test.go b/wgengine/magicsock/batching_conn_linux_test.go index effd5a2cc5001..7e0ab8fc485eb 100644 --- a/wgengine/magicsock/batching_conn_linux_test.go +++ b/wgengine/magicsock/batching_conn_linux_test.go @@ -159,9 +159,13 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { return make([]byte, len+packet.GeneveFixedHeaderLength, cap+packet.GeneveFixedHeaderLength) } + vni1 := virtualNetworkID{} + vni1.set(1) + cases := []struct { name string buffs [][]byte + vni virtualNetworkID wantLens []int wantGSO []int }{ @@ -173,6 +177,15 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { wantLens: []int{1}, wantGSO: []int{0}, }, + { + name: "one message no coalesce vni.isSet", + buffs: [][]byte{ + withGeneveSpace(1, 1), + }, + vni: vni1, + wantLens: []int{1 + packet.GeneveFixedHeaderLength}, + wantGSO: []int{0}, + }, { name: "two messages equal len coalesce", buffs: [][]byte{ @@ -182,6 +195,16 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { wantLens: []int{2}, wantGSO: []int{1}, }, + { + name: "two messages equal len coalesce vni.isSet", + buffs: [][]byte{ + withGeneveSpace(1, 2+packet.GeneveFixedHeaderLength), + withGeneveSpace(1, 1), + }, + vni: vni1, + wantLens: []int{2 + (2 * packet.GeneveFixedHeaderLength)}, + wantGSO: []int{1 + packet.GeneveFixedHeaderLength}, + }, { name: "two messages unequal len coalesce", buffs: [][]byte{ @@ -191,6 +214,16 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { wantLens: []int{3}, wantGSO: []int{2}, }, + { + name: "two messages unequal len coalesce vni.isSet", + buffs: [][]byte{ + withGeneveSpace(2, 3+packet.GeneveFixedHeaderLength), + withGeneveSpace(1, 1), + }, + vni: vni1, + wantLens: []int{3 + (2 * packet.GeneveFixedHeaderLength)}, + wantGSO: []int{2 + packet.GeneveFixedHeaderLength}, + }, { name: "three messages second unequal len coalesce", buffs: [][]byte{ @@ -201,6 +234,17 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { wantLens: []int{3, 2}, wantGSO: []int{2, 0}, }, + { + name: "three messages second unequal len coalesce vni.isSet", + buffs: [][]byte{ + withGeneveSpace(2, 3+(2*packet.GeneveFixedHeaderLength)), + withGeneveSpace(1, 1), + withGeneveSpace(2, 2), + }, + vni: vni1, + wantLens: []int{3 + (2 * packet.GeneveFixedHeaderLength), 2 + packet.GeneveFixedHeaderLength}, + wantGSO: []int{2 + packet.GeneveFixedHeaderLength, 0}, + }, { name: "three messages limited cap coalesce", buffs: [][]byte{ @@ -211,6 +255,17 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { wantLens: []int{4, 2}, wantGSO: []int{2, 0}, }, + { + name: "three messages limited cap coalesce vni.isSet", + buffs: [][]byte{ + withGeneveSpace(2, 4+packet.GeneveFixedHeaderLength), + withGeneveSpace(2, 2), + withGeneveSpace(2, 2), + }, + vni: vni1, + wantLens: []int{4 + (2 * packet.GeneveFixedHeaderLength), 2 + packet.GeneveFixedHeaderLength}, + wantGSO: []int{2 + packet.GeneveFixedHeaderLength, 0}, + }, } for _, tt := range cases { @@ -224,7 +279,7 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { msgs[i].Buffers = make([][]byte, 1) msgs[i].OOB = make([]byte, 0, 2) } - got := c.coalesceMessages(addr, tt.buffs, msgs, packet.GeneveFixedHeaderLength) + got := c.coalesceMessages(addr, tt.vni, tt.buffs, msgs, packet.GeneveFixedHeaderLength) if got != len(tt.wantLens) { t.Fatalf("got len %d want: %d", got, len(tt.wantLens)) } diff --git a/wgengine/magicsock/debughttp.go b/wgengine/magicsock/debughttp.go index aa109c242e27c..cfdf8c1e12d78 100644 --- a/wgengine/magicsock/debughttp.go +++ b/wgengine/magicsock/debughttp.go @@ -72,18 +72,18 @@ func (c *Conn) ServeHTTPDebug(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "

# ip:port to endpoint

    ") { type kv struct { - ipp netip.AddrPort - pi *peerInfo + addr epAddr + pi *peerInfo } - ent := make([]kv, 0, len(c.peerMap.byIPPort)) - for k, v := range c.peerMap.byIPPort { + ent := make([]kv, 0, len(c.peerMap.byEpAddr)) + for k, v := range c.peerMap.byEpAddr { ent = append(ent, kv{k, v}) } - sort.Slice(ent, func(i, j int) bool { return ipPortLess(ent[i].ipp, ent[j].ipp) }) + sort.Slice(ent, func(i, j int) bool { return epAddrLess(ent[i].addr, ent[j].addr) }) for _, e := range ent { ep := e.pi.ep shortStr := ep.publicKey.ShortString() - fmt.Fprintf(w, "
  • %v: %v
    • \n", e.ipp, strings.Trim(shortStr, "[]"), shortStr) + fmt.Fprintf(w, "
  • %v: %v
    • \n", e.addr, strings.Trim(shortStr, "[]"), shortStr) } } @@ -148,11 +148,11 @@ func printEndpointHTML(w io.Writer, ep *endpoint) { for ipp := range ep.endpointState { eps = append(eps, ipp) } - sort.Slice(eps, func(i, j int) bool { return ipPortLess(eps[i], eps[j]) }) + sort.Slice(eps, func(i, j int) bool { return addrPortLess(eps[i], eps[j]) }) io.WriteString(w, "

    Endpoints:

      ") for _, ipp := range eps { s := ep.endpointState[ipp] - if ipp == ep.bestAddr.AddrPort { + if ipp == ep.bestAddr.ap && !ep.bestAddr.vni.isSet() { fmt.Fprintf(w, "
    • %s: (best)
        ", ipp) } else { fmt.Fprintf(w, "
      • %s: ...
          ", ipp) @@ -196,9 +196,19 @@ func peerDebugName(p tailcfg.NodeView) string { return p.Hostinfo().Hostname() } -func ipPortLess(a, b netip.AddrPort) bool { +func addrPortLess(a, b netip.AddrPort) bool { if v := a.Addr().Compare(b.Addr()); v != 0 { return v < 0 } return a.Port() < b.Port() } + +func epAddrLess(a, b epAddr) bool { + if v := a.ap.Addr().Compare(b.ap.Addr()); v != 0 { + return v < 0 + } + if a.ap.Port() == b.ap.Port() { + return a.vni.get() < b.vni.get() + } + return a.ap.Port() < b.ap.Port() +} diff --git a/wgengine/magicsock/derp.go b/wgengine/magicsock/derp.go index ffdff14a11d47..5afdbc6d8718b 100644 --- a/wgengine/magicsock/derp.go +++ b/wgengine/magicsock/derp.go @@ -740,8 +740,11 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep *en return 0, nil } - ipp := netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, uint16(regionID)) - if c.handleDiscoMessage(b[:n], ipp, dm.src, discoRXPathDERP) { + srcAddr := epAddr{ap: netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, uint16(regionID))} + pt, isGeneveEncap := packetLooksLike(b[:n]) + if pt == packetLooksLikeDisco && + !isGeneveEncap { // We should never receive Geneve-encapsulated disco over DERP. + c.handleDiscoMessage(b[:n], srcAddr, false, dm.src, discoRXPathDERP) return 0, nil } @@ -755,9 +758,9 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep *en return 0, nil } - ep.noteRecvActivity(ipp, mono.Now()) + ep.noteRecvActivity(srcAddr, mono.Now()) if stats := c.stats.Load(); stats != nil { - stats.UpdateRxPhysical(ep.nodeAddr, ipp, 1, dm.n) + stats.UpdateRxPhysical(ep.nodeAddr, srcAddr.ap, 1, dm.n) } c.metrics.inboundPacketsDERPTotal.Add(1) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 243d0f4de26a8..faae49a97c83c 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -25,6 +25,7 @@ import ( "golang.org/x/net/ipv6" "tailscale.com/disco" "tailscale.com/ipn/ipnstate" + "tailscale.com/net/packet" "tailscale.com/net/stun" "tailscale.com/net/tstun" "tailscale.com/tailcfg" @@ -84,7 +85,7 @@ type endpoint struct { bestAddrAt mono.Time // time best address re-confirmed trustBestAddrUntil mono.Time // time when bestAddr expires sentPing map[stun.TxID]sentPing - endpointState map[netip.AddrPort]*endpointState + endpointState map[netip.AddrPort]*endpointState // netip.AddrPort type for key (instead of [epAddr]) as [endpointState] is irrelevant for Geneve-encapsulated paths isCallMeMaybeEP map[netip.AddrPort]bool // The following fields are related to the new "silent disco" @@ -99,7 +100,7 @@ type endpoint struct { } func (de *endpoint) setBestAddrLocked(v addrQuality) { - if v.AddrPort != de.bestAddr.AddrPort { + if v.epAddr != de.bestAddr.epAddr { de.probeUDPLifetime.resetCycleEndpointLocked() } de.bestAddr = v @@ -135,11 +136,11 @@ type probeUDPLifetime struct { // timeout cliff in the future. timer *time.Timer - // bestAddr contains the endpoint.bestAddr.AddrPort at the time a cycle was + // bestAddr contains the endpoint.bestAddr.epAddr at the time a cycle was // scheduled to start. A probing cycle is 1:1 with the current - // endpoint.bestAddr.AddrPort in the interest of simplicity. When - // endpoint.bestAddr.AddrPort changes, any active probing cycle will reset. - bestAddr netip.AddrPort + // endpoint.bestAddr.epAddr in the interest of simplicity. When + // endpoint.bestAddr.epAddr changes, any active probing cycle will reset. + bestAddr epAddr // cycleStartedAt contains the time at which the first cliff // (ProbeUDPLifetimeConfig.Cliffs[0]) was pinged for the current/last cycle. cycleStartedAt time.Time @@ -191,7 +192,7 @@ func (p *probeUDPLifetime) resetCycleEndpointLocked() { } p.cycleActive = false p.currentCliff = 0 - p.bestAddr = netip.AddrPort{} + p.bestAddr = epAddr{} } // ProbeUDPLifetimeConfig represents the configuration for probing UDP path @@ -334,7 +335,7 @@ type endpointDisco struct { } type sentPing struct { - to netip.AddrPort + to epAddr at mono.Time timer *time.Timer // timeout timer purpose discoPingPurpose @@ -446,7 +447,8 @@ func (de *endpoint) deleteEndpointLocked(why string, ep netip.AddrPort) { From: ep, }) delete(de.endpointState, ep) - if de.bestAddr.AddrPort == ep { + asEpAddr := epAddr{ap: ep} + if de.bestAddr.epAddr == asEpAddr { de.debugUpdates.Add(EndpointChange{ When: time.Now(), What: "deleteEndpointLocked-bestAddr-" + why, @@ -469,10 +471,10 @@ func (de *endpoint) initFakeUDPAddr() { // noteRecvActivity records receive activity on de, and invokes // Conn.noteRecvActivity no more than once every 10s. -func (de *endpoint) noteRecvActivity(ipp netip.AddrPort, now mono.Time) { +func (de *endpoint) noteRecvActivity(src epAddr, now mono.Time) { if de.isWireguardOnly { de.mu.Lock() - de.bestAddr.AddrPort = ipp + de.bestAddr.ap = src.ap de.bestAddrAt = now de.trustBestAddrUntil = now.Add(5 * time.Second) de.mu.Unlock() @@ -482,7 +484,7 @@ func (de *endpoint) noteRecvActivity(ipp netip.AddrPort, now mono.Time) { // kick off discovery disco pings every trustUDPAddrDuration and mirror // to DERP. de.mu.Lock() - if de.heartbeatDisabled && de.bestAddr.AddrPort == ipp { + if de.heartbeatDisabled && de.bestAddr.epAddr == src { de.trustBestAddrUntil = now.Add(trustUDPAddrDuration) } de.mu.Unlock() @@ -530,10 +532,10 @@ func (de *endpoint) DstToBytes() []byte { return packIPPort(de.fakeWGAddr) } // de.mu must be held. // // TODO(val): Rewrite the addrFor*Locked() variations to share code. -func (de *endpoint) addrForSendLocked(now mono.Time) (udpAddr, derpAddr netip.AddrPort, sendWGPing bool) { - udpAddr = de.bestAddr.AddrPort +func (de *endpoint) addrForSendLocked(now mono.Time) (udpAddr epAddr, derpAddr netip.AddrPort, sendWGPing bool) { + udpAddr = de.bestAddr.epAddr - if udpAddr.IsValid() && !now.After(de.trustBestAddrUntil) { + if udpAddr.ap.IsValid() && !now.After(de.trustBestAddrUntil) { return udpAddr, netip.AddrPort{}, false } @@ -557,7 +559,7 @@ func (de *endpoint) addrForSendLocked(now mono.Time) (udpAddr, derpAddr netip.Ad // best latency is used. // // de.mu must be held. -func (de *endpoint) addrForWireGuardSendLocked(now mono.Time) (udpAddr netip.AddrPort, shouldPing bool) { +func (de *endpoint) addrForWireGuardSendLocked(now mono.Time) (udpAddr epAddr, shouldPing bool) { if len(de.endpointState) == 0 { de.c.logf("magicsock: addrForSendWireguardLocked: [unexpected] no candidates available for endpoint") return udpAddr, false @@ -581,22 +583,22 @@ func (de *endpoint) addrForWireGuardSendLocked(now mono.Time) (udpAddr netip.Add // TODO(catzkorn): Consider a small increase in latency to use // IPv6 in comparison to IPv4, when possible. lowestLatency = latency - udpAddr = ipp + udpAddr.ap = ipp } } } needPing := len(de.endpointState) > 1 && now.Sub(oldestPing) > wireguardPingInterval - if !udpAddr.IsValid() { + if !udpAddr.ap.IsValid() { candidates := slicesx.MapKeys(de.endpointState) // Randomly select an address to use until we retrieve latency information // and give it a short trustBestAddrUntil time so we avoid flapping between // addresses while waiting on latency information to be populated. - udpAddr = candidates[rand.IntN(len(candidates))] + udpAddr.ap = candidates[rand.IntN(len(candidates))] } - de.bestAddr.AddrPort = udpAddr + de.bestAddr.epAddr = epAddr{ap: udpAddr.ap} // Only extend trustBestAddrUntil by one second to avoid packet // reordering and/or CPU usage from random selection during the first // second. We should receive a response due to a WireGuard handshake in @@ -614,18 +616,18 @@ func (de *endpoint) addrForWireGuardSendLocked(now mono.Time) (udpAddr netip.Add // both of the returned UDP address and DERP address may be non-zero. // // de.mu must be held. -func (de *endpoint) addrForPingSizeLocked(now mono.Time, size int) (udpAddr, derpAddr netip.AddrPort) { +func (de *endpoint) addrForPingSizeLocked(now mono.Time, size int) (udpAddr epAddr, derpAddr netip.AddrPort) { if size == 0 { udpAddr, derpAddr, _ = de.addrForSendLocked(now) return } - udpAddr = de.bestAddr.AddrPort + udpAddr = de.bestAddr.epAddr pathMTU := de.bestAddr.wireMTU - requestedMTU := pingSizeToPktLen(size, udpAddr.Addr().Is6()) + requestedMTU := pingSizeToPktLen(size, udpAddr) mtuOk := requestedMTU <= pathMTU - if udpAddr.IsValid() && mtuOk { + if udpAddr.ap.IsValid() && mtuOk { if !now.After(de.trustBestAddrUntil) { return udpAddr, netip.AddrPort{} } @@ -638,7 +640,7 @@ func (de *endpoint) addrForPingSizeLocked(now mono.Time, size int) (udpAddr, der // for the packet. Return a zero-value udpAddr to signal that we should // keep probing the path MTU to all addresses for this endpoint, and a // valid DERP addr to signal that we should also send via DERP. - return netip.AddrPort{}, de.derpAddr + return epAddr{}, de.derpAddr } // maybeProbeUDPLifetimeLocked returns an afterInactivityFor duration and true @@ -649,7 +651,7 @@ func (de *endpoint) maybeProbeUDPLifetimeLocked() (afterInactivityFor time.Durat if p == nil { return afterInactivityFor, false } - if !de.bestAddr.IsValid() { + if !de.bestAddr.ap.IsValid() { return afterInactivityFor, false } epDisco := de.disco.Load() @@ -701,7 +703,7 @@ func (de *endpoint) scheduleHeartbeatForLifetimeLocked(after time.Duration, via } de.c.dlogf("[v1] magicsock: disco: scheduling UDP lifetime probe for cliff=%v via=%v to %v (%v)", p.currentCliffDurationEndpointLocked(), via, de.publicKey.ShortString(), de.discoShort()) - p.bestAddr = de.bestAddr.AddrPort + p.bestAddr = de.bestAddr.epAddr p.timer = time.AfterFunc(after, de.heartbeatForLifetime) if via == heartbeatForLifetimeViaSelf { metricUDPLifetimeCliffsRescheduled.Add(1) @@ -729,7 +731,7 @@ func (de *endpoint) heartbeatForLifetime() { return } p.timer = nil - if !p.bestAddr.IsValid() || de.bestAddr.AddrPort != p.bestAddr { + if !p.bestAddr.ap.IsValid() || de.bestAddr.epAddr != p.bestAddr { // best path changed p.resetCycleEndpointLocked() return @@ -761,7 +763,7 @@ func (de *endpoint) heartbeatForLifetime() { } de.c.dlogf("[v1] magicsock: disco: sending disco ping for UDP lifetime probe cliff=%v to %v (%v)", p.currentCliffDurationEndpointLocked(), de.publicKey.ShortString(), de.discoShort()) - de.startDiscoPingLocked(de.bestAddr.AddrPort, mono.Now(), pingHeartbeatForUDPLifetime, 0, nil) + de.startDiscoPingLocked(de.bestAddr.epAddr, mono.Now(), pingHeartbeatForUDPLifetime, 0, nil) } // heartbeat is called every heartbeatInterval to keep the best UDP path alive, @@ -819,7 +821,7 @@ func (de *endpoint) heartbeat() { } udpAddr, _, _ := de.addrForSendLocked(now) - if udpAddr.IsValid() { + if udpAddr.ap.IsValid() { // We have a preferred path. Ping that every 'heartbeatInterval'. de.startDiscoPingLocked(udpAddr, now, pingHeartbeat, 0, nil) } @@ -846,7 +848,7 @@ func (de *endpoint) wantFullPingLocked(now mono.Time) bool { if runtime.GOOS == "js" { return false } - if !de.bestAddr.IsValid() || de.lastFullPing.IsZero() { + if !de.bestAddr.ap.IsValid() || de.lastFullPing.IsZero() { return true } if now.After(de.trustBestAddrUntil) { @@ -906,9 +908,9 @@ func (de *endpoint) discoPing(res *ipnstate.PingResult, size int, cb func(*ipnst udpAddr, derpAddr := de.addrForPingSizeLocked(now, size) if derpAddr.IsValid() { - de.startDiscoPingLocked(derpAddr, now, pingCLI, size, resCB) + de.startDiscoPingLocked(epAddr{ap: derpAddr}, now, pingCLI, size, resCB) } - if udpAddr.IsValid() && now.Before(de.trustBestAddrUntil) { + if udpAddr.ap.IsValid() && now.Before(de.trustBestAddrUntil) { // Already have an active session, so just ping the address we're using. // Otherwise "tailscale ping" results to a node on the local network // can look like they're bouncing between, say 10.0.0.0/9 and the peer's @@ -916,7 +918,7 @@ func (de *endpoint) discoPing(res *ipnstate.PingResult, size int, cb func(*ipnst de.startDiscoPingLocked(udpAddr, now, pingCLI, size, resCB) } else { for ep := range de.endpointState { - de.startDiscoPingLocked(ep, now, pingCLI, size, resCB) + de.startDiscoPingLocked(epAddr{ap: ep}, now, pingCLI, size, resCB) } } } @@ -941,14 +943,14 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { if startWGPing { de.sendWireGuardOnlyPingsLocked(now) } - } else if !udpAddr.IsValid() || now.After(de.trustBestAddrUntil) { + } else if !udpAddr.ap.IsValid() || now.After(de.trustBestAddrUntil) { de.sendDiscoPingsLocked(now, true) } de.noteTxActivityExtTriggerLocked(now) de.lastSendAny = now de.mu.Unlock() - if !udpAddr.IsValid() && !derpAddr.IsValid() { + if !udpAddr.ap.IsValid() && !derpAddr.IsValid() { // Make a last ditch effort to see if we have a DERP route for them. If // they contacted us over DERP and we don't know their UDP endpoints or // their DERP home, we can at least assume they're reachable over the @@ -960,7 +962,7 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { } } var err error - if udpAddr.IsValid() { + if udpAddr.ap.IsValid() { _, err = de.c.sendUDPBatch(udpAddr, buffs, offset) // If the error is known to indicate that the endpoint is no longer @@ -976,17 +978,17 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { } switch { - case udpAddr.Addr().Is4(): + case udpAddr.ap.Addr().Is4(): de.c.metrics.outboundPacketsIPv4Total.Add(int64(len(buffs))) de.c.metrics.outboundBytesIPv4Total.Add(int64(txBytes)) - case udpAddr.Addr().Is6(): + case udpAddr.ap.Addr().Is6(): de.c.metrics.outboundPacketsIPv6Total.Add(int64(len(buffs))) de.c.metrics.outboundBytesIPv6Total.Add(int64(txBytes)) } // TODO(raggi): needs updating for accuracy, as in error conditions we may have partial sends. if stats := de.c.stats.Load(); err == nil && stats != nil { - stats.UpdateTxPhysical(de.nodeAddr, udpAddr, len(buffs), txBytes) + stats.UpdateTxPhysical(de.nodeAddr, udpAddr.ap, len(buffs), txBytes) } } if derpAddr.IsValid() { @@ -1055,7 +1057,7 @@ func (de *endpoint) discoPingTimeout(txid stun.TxID) { if !ok { return } - if debugDisco() || !de.bestAddr.IsValid() || mono.Now().After(de.trustBestAddrUntil) { + if debugDisco() || !de.bestAddr.ap.IsValid() || mono.Now().After(de.trustBestAddrUntil) { de.c.dlogf("[v1] magicsock: disco: timeout waiting for pong %x from %v (%v, %v)", txid[:6], sp.to, de.publicKey.ShortString(), de.discoShort()) } de.removeSentDiscoPingLocked(txid, sp, discoPingTimedOut) @@ -1109,11 +1111,11 @@ const discoPingSize = len(disco.Magic) + key.DiscoPublicRawLen + disco.NonceLen // // The caller should use de.discoKey as the discoKey argument. // It is passed in so that sendDiscoPing doesn't need to lock de.mu. -func (de *endpoint) sendDiscoPing(ep netip.AddrPort, discoKey key.DiscoPublic, txid stun.TxID, size int, logLevel discoLogLevel) { +func (de *endpoint) sendDiscoPing(ep epAddr, discoKey key.DiscoPublic, txid stun.TxID, size int, logLevel discoLogLevel) { size = min(size, MaxDiscoPingSize) padding := max(size-discoPingSize, 0) - sent, _ := de.c.sendDiscoMessage(ep, virtualNetworkID{}, de.publicKey, discoKey, &disco.Ping{ + sent, _ := de.c.sendDiscoMessage(ep, de.publicKey, discoKey, &disco.Ping{ TxID: [12]byte(txid), NodeKey: de.c.publicKeyAtomic.Load(), Padding: padding, @@ -1125,7 +1127,7 @@ func (de *endpoint) sendDiscoPing(ep netip.AddrPort, discoKey key.DiscoPublic, t if size != 0 { metricSentDiscoPeerMTUProbes.Add(1) - metricSentDiscoPeerMTUProbeBytes.Add(int64(pingSizeToPktLen(size, ep.Addr().Is6()))) + metricSentDiscoPeerMTUProbeBytes.Add(int64(pingSizeToPktLen(size, ep))) } } @@ -1156,7 +1158,7 @@ const ( // if non-nil, means that a caller external to the magicsock package internals // is interested in the result (such as a CLI "tailscale ping" or a c2n ping // request, etc) -func (de *endpoint) startDiscoPingLocked(ep netip.AddrPort, now mono.Time, purpose discoPingPurpose, size int, resCB *pingResultAndCallback) { +func (de *endpoint) startDiscoPingLocked(ep epAddr, now mono.Time, purpose discoPingPurpose, size int, resCB *pingResultAndCallback) { if runtime.GOOS == "js" { return } @@ -1164,8 +1166,9 @@ func (de *endpoint) startDiscoPingLocked(ep netip.AddrPort, now mono.Time, purpo if epDisco == nil { return } - if purpose != pingCLI { - st, ok := de.endpointState[ep] + if purpose != pingCLI && + !ep.vni.isSet() { // de.endpointState is only relevant for direct/non-vni epAddr's + st, ok := de.endpointState[ep.ap] if !ok { // Shouldn't happen. But don't ping an endpoint that's // not active for us. @@ -1182,11 +1185,11 @@ func (de *endpoint) startDiscoPingLocked(ep netip.AddrPort, now mono.Time, purpo // Default to sending a single ping of the specified size sizes := []int{size} if de.c.PeerMTUEnabled() { - isDerp := ep.Addr() == tailcfg.DerpMagicIPAddr + isDerp := ep.ap.Addr() == tailcfg.DerpMagicIPAddr if !isDerp && ((purpose == pingDiscovery) || (purpose == pingCLI && size == 0)) { de.c.dlogf("[v1] magicsock: starting MTU probe") sizes = mtuProbePingSizesV4 - if ep.Addr().Is6() { + if ep.ap.Addr().Is6() { sizes = mtuProbePingSizesV6 } } @@ -1241,7 +1244,7 @@ func (de *endpoint) sendDiscoPingsLocked(now mono.Time, sendCallMeMaybe bool) { de.c.dlogf("[v1] magicsock: disco: send, starting discovery for %v (%v)", de.publicKey.ShortString(), de.discoShort()) } - de.startDiscoPingLocked(ep, now, pingDiscovery, 0, nil) + de.startDiscoPingLocked(epAddr{ap: ep}, now, pingDiscovery, 0, nil) } derpAddr := de.derpAddr if sentAny && sendCallMeMaybe && derpAddr.IsValid() { @@ -1496,17 +1499,19 @@ func (de *endpoint) clearBestAddrLocked() { de.trustBestAddrUntil = 0 } -// noteBadEndpoint marks ipp as a bad endpoint that would need to be +// noteBadEndpoint marks udpAddr as a bad endpoint that would need to be // re-evaluated before future use, this should be called for example if a send -// to ipp fails due to a host unreachable error or similar. -func (de *endpoint) noteBadEndpoint(ipp netip.AddrPort) { +// to udpAddr fails due to a host unreachable error or similar. +func (de *endpoint) noteBadEndpoint(udpAddr epAddr) { de.mu.Lock() defer de.mu.Unlock() de.clearBestAddrLocked() - if st, ok := de.endpointState[ipp]; ok { - st.clear() + if !udpAddr.vni.isSet() { + if st, ok := de.endpointState[udpAddr.ap]; ok { + st.clear() + } } } @@ -1526,17 +1531,20 @@ func (de *endpoint) noteConnectivityChange() { // pingSizeToPktLen calculates the minimum path MTU that would permit // a disco ping message of length size to reach its target at -// addr. size is the length of the entire disco message including +// udpAddr. size is the length of the entire disco message including // disco headers. If size is zero, assume it is the safe wire MTU. -func pingSizeToPktLen(size int, is6 bool) tstun.WireMTU { +func pingSizeToPktLen(size int, udpAddr epAddr) tstun.WireMTU { if size == 0 { return tstun.SafeWireMTU() } headerLen := ipv4.HeaderLen - if is6 { + if udpAddr.ap.Addr().Is6() { headerLen = ipv6.HeaderLen } headerLen += 8 // UDP header length + if udpAddr.vni.isSet() { + headerLen += packet.GeneveFixedHeaderLength + } return tstun.WireMTU(size + headerLen) } @@ -1563,19 +1571,19 @@ func pktLenToPingSize(mtu tstun.WireMTU, is6 bool) int { // It should be called with the Conn.mu held. // // It reports whether m.TxID corresponds to a ping that this endpoint sent. -func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip.AddrPort, vni virtualNetworkID) (knownTxID bool) { +func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAddr) (knownTxID bool) { de.mu.Lock() defer de.mu.Unlock() - if vni.isSet() { - // TODO(jwhited): check for matching [endpoint.bestAddr] once that data - // structure is VNI-aware and [relayManager] can mutate it. We do not - // need to reference any [endpointState] for Geneve-encapsulated disco, - // we store nothing about them there. + if src.vni.isSet() { + // TODO(jwhited): fall through once [relayManager] is able to set an + // [epAddr] as de.bestAddr. We do not need to reference any + // [endpointState] for Geneve-encapsulated disco, we store nothing + // about them there. return false } - isDerp := src.Addr() == tailcfg.DerpMagicIPAddr + isDerp := src.ap.Addr() == tailcfg.DerpMagicIPAddr sp, ok := de.sentPing[m.TxID] if !ok { @@ -1585,7 +1593,7 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip knownTxID = true // for naked returns below de.removeSentDiscoPingLocked(m.TxID, sp, discoPongReceived) - pktLen := int(pingSizeToPktLen(sp.size, sp.to.Addr().Is6())) + pktLen := int(pingSizeToPktLen(sp.size, src)) if sp.size != 0 { m := getPeerMTUsProbedMetric(tstun.WireMTU(pktLen)) m.Add(1) @@ -1598,18 +1606,18 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip latency := now.Sub(sp.at) if !isDerp { - st, ok := de.endpointState[sp.to] + st, ok := de.endpointState[sp.to.ap] if !ok { // This is no longer an endpoint we care about. return } - de.c.peerMap.setNodeKeyForIPPort(src, de.publicKey) + de.c.peerMap.setNodeKeyForEpAddr(src, de.publicKey) st.addPongReplyLocked(pongReply{ latency: latency, pongAt: now, - from: src, + from: src.ap, pongSrc: m.Src, }) } @@ -1633,7 +1641,7 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip // Promote this pong response to our current best address if it's lower latency. // TODO(bradfitz): decide how latency vs. preference order affects decision if !isDerp { - thisPong := addrQuality{sp.to, latency, tstun.WireMTU(pingSizeToPktLen(sp.size, sp.to.Addr().Is6()))} + thisPong := addrQuality{sp.to, latency, tstun.WireMTU(pingSizeToPktLen(sp.size, sp.to))} if betterAddr(thisPong, de.bestAddr) { de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v tx=%x", de.publicKey.ShortString(), de.discoShort(), sp.to, thisPong.wireMTU, m.TxID[:6]) de.debugUpdates.Add(EndpointChange{ @@ -1644,7 +1652,7 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip }) de.setBestAddrLocked(thisPong) } - if de.bestAddr.AddrPort == thisPong.AddrPort { + if de.bestAddr.epAddr == thisPong.epAddr { de.debugUpdates.Add(EndpointChange{ When: time.Now(), What: "handlePongConnLocked-bestAddr-latency", @@ -1659,20 +1667,34 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip return } -// addrQuality is an IPPort with an associated latency and path mtu. +// epAddr is a [netip.AddrPort] with an optional Geneve header (RFC8926) +// [virtualNetworkID]. +type epAddr struct { + ap netip.AddrPort // if ap == tailcfg.DerpMagicIPAddr then vni is never set + vni virtualNetworkID // vni.isSet() indicates if this [epAddr] involves a Geneve header +} + +func (e epAddr) String() string { + if !e.vni.isSet() { + return e.ap.String() + } + return fmt.Sprintf("%v:vni:%d", e.ap.String(), e.vni.get()) +} + +// addrQuality is an [epAddr] with an associated latency and path mtu. type addrQuality struct { - netip.AddrPort + epAddr latency time.Duration wireMTU tstun.WireMTU } func (a addrQuality) String() string { - return fmt.Sprintf("%v@%v+%v", a.AddrPort, a.latency, a.wireMTU) + return fmt.Sprintf("%v@%v+%v", a.epAddr, a.latency, a.wireMTU) } // betterAddr reports whether a is a better addr to use than b. func betterAddr(a, b addrQuality) bool { - if a.AddrPort == b.AddrPort { + if a.epAddr == b.epAddr { if a.wireMTU > b.wireMTU { // TODO(val): Think harder about the case of lower // latency and smaller or unknown MTU, and higher @@ -1683,10 +1705,19 @@ func betterAddr(a, b addrQuality) bool { } return false } - if !b.IsValid() { + if !b.ap.IsValid() { + return true + } + if !a.ap.IsValid() { + return false + } + + // Geneve-encapsulated paths (UDP relay servers) are lower preference in + // relation to non. + if !a.vni.isSet() && b.vni.isSet() { return true } - if !a.IsValid() { + if a.vni.isSet() && !b.vni.isSet() { return false } @@ -1710,27 +1741,27 @@ func betterAddr(a, b addrQuality) bool { // addresses, and prefer link-local unicast addresses over other types // of private IP addresses since it's definitionally more likely that // they'll be on the same network segment than a general private IP. - if a.Addr().IsLoopback() { + if a.ap.Addr().IsLoopback() { aPoints += 50 - } else if a.Addr().IsLinkLocalUnicast() { + } else if a.ap.Addr().IsLinkLocalUnicast() { aPoints += 30 - } else if a.Addr().IsPrivate() { + } else if a.ap.Addr().IsPrivate() { aPoints += 20 } - if b.Addr().IsLoopback() { + if b.ap.Addr().IsLoopback() { bPoints += 50 - } else if b.Addr().IsLinkLocalUnicast() { + } else if b.ap.Addr().IsLinkLocalUnicast() { bPoints += 30 - } else if b.Addr().IsPrivate() { + } else if b.ap.Addr().IsPrivate() { bPoints += 20 } // Prefer IPv6 for being a bit more robust, as long as // the latencies are roughly equivalent. - if a.Addr().Is6() { + if a.ap.Addr().Is6() { aPoints += 10 } - if b.Addr().Is6() { + if b.ap.Addr().Is6() { bPoints += 10 } @@ -1831,7 +1862,10 @@ func (de *endpoint) populatePeerStatus(ps *ipnstate.PeerStatus) { ps.LastWrite = de.lastSendExt.WallTime() ps.Active = now.Sub(de.lastSendExt) < sessionActiveTimeout - if udpAddr, derpAddr, _ := de.addrForSendLocked(now); udpAddr.IsValid() && !derpAddr.IsValid() { + if udpAddr, derpAddr, _ := de.addrForSendLocked(now); udpAddr.ap.IsValid() && !derpAddr.IsValid() { + // TODO(jwhited): if udpAddr.vni.isSet() we are using a Tailscale client + // as a UDP relay; update PeerStatus and its interpretation by + // "tailscale status" to make this clear. ps.CurAddr = udpAddr.String() } } diff --git a/wgengine/magicsock/endpoint_test.go b/wgengine/magicsock/endpoint_test.go index 1e2de8967511c..b1e8cab91bcd1 100644 --- a/wgengine/magicsock/endpoint_test.go +++ b/wgengine/magicsock/endpoint_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/dsnet/try" "tailscale.com/types/key" ) @@ -154,7 +153,7 @@ func Test_endpoint_maybeProbeUDPLifetimeLocked(t *testing.T) { lower = b higher = a } - addr := addrQuality{AddrPort: try.E1[netip.AddrPort](netip.ParseAddrPort("1.1.1.1:1"))} + addr := addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort("1.1.1.1:1")}} newProbeUDPLifetime := func() *probeUDPLifetime { return &probeUDPLifetime{ config: *defaultProbeUDPLifetimeConfig, diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 3a4fdf8a248f9..c446cff2cfd14 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -950,7 +950,7 @@ func (c *Conn) callNetInfoCallbackLocked(ni *tailcfg.NetInfo) { func (c *Conn) addValidDiscoPathForTest(nodeKey key.NodePublic, addr netip.AddrPort) { c.mu.Lock() defer c.mu.Unlock() - c.peerMap.setNodeKeyForIPPort(addr, nodeKey) + c.peerMap.setNodeKeyForEpAddr(epAddr{ap: addr}, nodeKey) } // SetNetInfoCallback sets the func to be called whenever the network conditions @@ -1019,13 +1019,16 @@ func (c *Conn) Ping(peer tailcfg.NodeView, res *ipnstate.PingResult, size int, c } // c.mu must be held -func (c *Conn) populateCLIPingResponseLocked(res *ipnstate.PingResult, latency time.Duration, ep netip.AddrPort) { +func (c *Conn) populateCLIPingResponseLocked(res *ipnstate.PingResult, latency time.Duration, ep epAddr) { res.LatencySeconds = latency.Seconds() - if ep.Addr() != tailcfg.DerpMagicIPAddr { + if ep.ap.Addr() != tailcfg.DerpMagicIPAddr { + // TODO(jwhited): if ep.vni.isSet() we are using a Tailscale client + // as a UDP relay; update PingResult and its interpretation by + // "tailscale ping" to make this clear. res.Endpoint = ep.String() return } - regionID := int(ep.Port()) + regionID := int(ep.ap.Port()) res.DERPRegionID = regionID res.DERPRegionCode = c.derpRegionCodeLocked(regionID) } @@ -1294,11 +1297,11 @@ var errNoUDP = errors.New("no UDP available on platform") var errUnsupportedConnType = errors.New("unsupported connection type") -func (c *Conn) sendUDPBatch(addr netip.AddrPort, buffs [][]byte, offset int) (sent bool, err error) { +func (c *Conn) sendUDPBatch(addr epAddr, buffs [][]byte, offset int) (sent bool, err error) { isIPv6 := false switch { - case addr.Addr().Is4(): - case addr.Addr().Is6(): + case addr.ap.Addr().Is4(): + case addr.ap.Addr().Is6(): isIPv6 = true default: panic("bogus sendUDPBatch addr type") @@ -1484,8 +1487,8 @@ func (c *Conn) receiveIPv6() conn.ReceiveFunc { // mkReceiveFunc creates a ReceiveFunc reading from ruc. // The provided healthItem and metrics are updated if non-nil. func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFuncStats, packetMetric, bytesMetric *expvar.Int) conn.ReceiveFunc { - // epCache caches an IPPort->endpoint for hot flows. - var epCache ippEndpointCache + // epCache caches an epAddr->endpoint for hot flows. + var epCache epAddrEndpointCache return func(buffs [][]byte, sizes []int, eps []conn.Endpoint) (_ int, retErr error) { if healthItem != nil { @@ -1519,7 +1522,7 @@ func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFu continue } ipp := msg.Addr.(*net.UDPAddr).AddrPort() - if ep, ok := c.receiveIP(msg.Buffers[0][:msg.N], ipp, &epCache); ok { + if ep, size, ok := c.receiveIP(msg.Buffers[0][:msg.N], ipp, &epCache); ok { if packetMetric != nil { packetMetric.Add(1) } @@ -1527,7 +1530,7 @@ func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFu bytesMetric.Add(int64(msg.N)) } eps[i] = ep - sizes[i] = msg.N + sizes[i] = size reportToCaller = true } else { sizes[i] = 0 @@ -1542,47 +1545,89 @@ func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFu // receiveIP is the shared bits of ReceiveIPv4 and ReceiveIPv6. // +// size is the length of 'b' to report up to wireguard-go (only relevant if +// 'ok' is true) +// // ok is whether this read should be reported up to wireguard-go (our // caller). -func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *ippEndpointCache) (_ conn.Endpoint, ok bool) { +func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCache) (_ conn.Endpoint, size int, ok bool) { var ep *endpoint - if stun.Is(b) { + size = len(b) + + var geneve packet.GeneveHeader + pt, isGeneveEncap := packetLooksLike(b) + src := epAddr{ap: ipp} + if isGeneveEncap { + err := geneve.Decode(b) + if err != nil { + // Decode only returns an error when 'b' is too short, and + // 'isGeneveEncap' indicates it's a sufficient length. + c.logf("[unexpected] geneve header decoding error: %v", err) + return nil, 0, false + } + src.vni.set(geneve.VNI) + } + switch pt { + case packetLooksLikeDisco: + if isGeneveEncap { + b = b[packet.GeneveFixedHeaderLength:] + } + // The Geneve header control bit should only be set for relay handshake + // messages terminating on or originating from a UDP relay server. We + // have yet to open the encrypted disco payload to determine the + // [disco.MessageType], but we assert it should be handshake-related. + shouldByRelayHandshakeMsg := geneve.Control == true + c.handleDiscoMessage(b, src, shouldByRelayHandshakeMsg, key.NodePublic{}, discoRXPathUDP) + return nil, 0, false + case packetLooksLikeSTUNBinding: c.netChecker.ReceiveSTUNPacket(b, ipp) - return nil, false - } - if c.handleDiscoMessage(b, ipp, key.NodePublic{}, discoRXPathUDP) { - return nil, false + return nil, 0, false + default: + // Fall through for all other packet types as they are assumed to + // be potentially WireGuard. } + if !c.havePrivateKey.Load() { // If we have no private key, we're logged out or // stopped. Don't try to pass these wireguard packets // up to wireguard-go; it'll just complain (issue 1167). - return nil, false + return nil, 0, false } - if cache.ipp == ipp && cache.de != nil && cache.gen == cache.de.numStopAndReset() { + + if src.vni.isSet() { + // Strip away the Geneve header before returning the packet to + // wireguard-go. + // + // TODO(jwhited): update [github.com/tailscale/wireguard-go/conn.ReceiveFunc] + // to support returning start offset in order to get rid of this memmove perf + // penalty. + size = copy(b, b[packet.GeneveFixedHeaderLength:]) + } + + if cache.epAddr == src && cache.de != nil && cache.gen == cache.de.numStopAndReset() { ep = cache.de } else { c.mu.Lock() - de, ok := c.peerMap.endpointForIPPort(ipp) + de, ok := c.peerMap.endpointForEpAddr(src) c.mu.Unlock() if !ok { if c.controlKnobs != nil && c.controlKnobs.DisableCryptorouting.Load() { - return nil, false + return nil, 0, false } - return &lazyEndpoint{c: c, src: ipp}, true + return &lazyEndpoint{c: c, src: src}, size, true } - cache.ipp = ipp + cache.epAddr = src cache.de = de cache.gen = de.numStopAndReset() ep = de } now := mono.Now() ep.lastRecvUDPAny.StoreAtomic(now) - ep.noteRecvActivity(ipp, now) + ep.noteRecvActivity(src, now) if stats := c.stats.Load(); stats != nil { stats.UpdateRxPhysical(ep.nodeAddr, ipp, 1, len(b)) } - return ep, true + return ep, size, true } // discoLogLevel controls the verbosity of discovery log messages. @@ -1632,16 +1677,16 @@ func (v *virtualNetworkID) get() uint32 { // sendDiscoMessage sends discovery message m to dstDisco at dst. // -// If dst is a DERP IP:port, then dstKey must be non-zero. +// If dst.ap is a DERP IP:port, then dstKey must be non-zero. // -// If vni.isSet(), the [disco.Message] will be preceded by a Geneve header with -// the VNI field set to the value returned by vni.get(). +// If dst.vni.isSet(), the [disco.Message] will be preceded by a Geneve header +// with the VNI field set to the value returned by vni.get(). // // The dstKey should only be non-zero if the dstDisco key // unambiguously maps to exactly one peer. -func (c *Conn) sendDiscoMessage(dst netip.AddrPort, vni virtualNetworkID, dstKey key.NodePublic, dstDisco key.DiscoPublic, m disco.Message, logLevel discoLogLevel) (sent bool, err error) { - isDERP := dst.Addr() == tailcfg.DerpMagicIPAddr - if _, isPong := m.(*disco.Pong); isPong && !isDERP && dst.Addr().Is4() { +func (c *Conn) sendDiscoMessage(dst epAddr, dstKey key.NodePublic, dstDisco key.DiscoPublic, m disco.Message, logLevel discoLogLevel) (sent bool, err error) { + isDERP := dst.ap.Addr() == tailcfg.DerpMagicIPAddr + if _, isPong := m.(*disco.Pong); isPong && !isDERP && dst.ap.Addr().Is4() { time.Sleep(debugIPv4DiscoPingPenalty()) } @@ -1678,11 +1723,11 @@ func (c *Conn) sendDiscoMessage(dst netip.AddrPort, vni virtualNetworkID, dstKey c.mu.Unlock() pkt := make([]byte, 0, 512) // TODO: size it correctly? pool? if it matters. - if vni.isSet() { + if dst.vni.isSet() { gh := packet.GeneveHeader{ Version: 0, Protocol: packet.GeneveProtocolDisco, - VNI: vni.get(), + VNI: dst.vni.get(), Control: isRelayHandshakeMsg, } pkt = append(pkt, make([]byte, packet.GeneveFixedHeaderLength)...) @@ -1703,7 +1748,7 @@ func (c *Conn) sendDiscoMessage(dst netip.AddrPort, vni virtualNetworkID, dstKey box := di.sharedKey.Seal(m.AppendMarshal(nil)) pkt = append(pkt, box...) const isDisco = true - sent, err = c.sendAddr(dst, dstKey, pkt, isDisco) + sent, err = c.sendAddr(dst.ap, dstKey, pkt, isDisco) if sent { if logLevel == discoLog || (logLevel == discoVerboseLog && debugDisco()) { node := "?" @@ -1745,45 +1790,96 @@ const ( const discoHeaderLen = len(disco.Magic) + key.DiscoPublicRawLen -// isDiscoMaybeGeneve reports whether msg is a Tailscale Disco protocol -// message, and if true, whether it is encapsulated by a Geneve header. +type packetLooksLikeType int + +const ( + packetLooksLikeWireGuard packetLooksLikeType = iota + packetLooksLikeSTUNBinding + packetLooksLikeDisco +) + +// packetLooksLike reports a [packetsLooksLikeType] for 'msg', and whether +// 'msg' is encapsulated by a Geneve header (or naked). +// +// [packetLooksLikeSTUNBinding] is never Geneve-encapsulated. // -// isGeneveEncap is only relevant when isDiscoMsg is true. +// Naked STUN binding, Naked Disco, Geneve followed by Disco, naked WireGuard, +// and Geneve followed by WireGuard can be confidently distinguished based on +// the following: // -// Naked Disco, Geneve followed by Disco, and naked WireGuard can be confidently -// distinguished based on the following: -// 1. [disco.Magic] is sufficiently non-overlapping with a Geneve protocol -// field value of [packet.GeneveProtocolDisco]. -// 2. [disco.Magic] is sufficiently non-overlapping with the first 4 bytes of -// a WireGuard packet. -// 3. [packet.GeneveHeader] with a Geneve protocol field value of -// [packet.GeneveProtocolDisco] is sufficiently non-overlapping with the -// first 4 bytes of a WireGuard packet. -func isDiscoMaybeGeneve(msg []byte) (isDiscoMsg bool, isGeneveEncap bool) { - if len(msg) < discoHeaderLen { - return false, false - } - if string(msg[:len(disco.Magic)]) == disco.Magic { - return true, false - } - if len(msg) < packet.GeneveFixedHeaderLength+discoHeaderLen { - return false, false - } - if msg[0]&0xC0 != 0 || // version bits that we always transmit as 0s - msg[1]&0x3F != 0 || // reserved bits that we always transmit as 0s - binary.BigEndian.Uint16(msg[2:4]) != packet.GeneveProtocolDisco || - msg[7] != 0 { // reserved byte that we always transmit as 0 - return false, false - } - msg = msg[packet.GeneveFixedHeaderLength:] - if string(msg[:len(disco.Magic)]) == disco.Magic { - return true, true - } - return false, false -} - -// handleDiscoMessage handles a discovery message and reports whether -// msg was a Tailscale inter-node discovery message. +// 1. STUN binding @ msg[1] (0x01) is sufficiently non-overlapping with the +// Geneve header where the LSB is always 0 (part of 6 "reserved" bits). +// +// 2. STUN binding @ msg[1] (0x01) is sufficiently non-overlapping with naked +// WireGuard, which is always a 0 byte value (WireGuard message type +// occupies msg[0:4], and msg[1:4] are always 0). +// +// 3. STUN binding @ msg[1] (0x01) is sufficiently non-overlapping with the +// second byte of [disco.Magic] (0x53). +// +// 4. [disco.Magic] @ msg[2:4] (0xf09f) is sufficiently non-overlapping with a +// Geneve protocol field value of [packet.GeneveProtocolDisco] or +// [packet.GeneveProtocolWireGuard] . +// +// 5. [disco.Magic] @ msg[0] (0x54) is sufficiently non-overlapping with the +// first byte of a WireGuard packet (0x01-0x04). +// +// 6. [packet.GeneveHeader] with a Geneve protocol field value of +// [packet.GeneveProtocolDisco] or [packet.GeneveProtocolWireGuard] +// (msg[2:4]) is sufficiently non-overlapping with the second 2 bytes of a +// WireGuard packet which are always 0x0000. +func packetLooksLike(msg []byte) (t packetLooksLikeType, isGeneveEncap bool) { + if stun.Is(msg) && + msg[1] == 0x01 { // method binding + return packetLooksLikeSTUNBinding, false + } + + // TODO(jwhited): potentially collapse into disco.LooksLikeDiscoWrapper() + // if safe to do so. + looksLikeDisco := func(msg []byte) bool { + if len(msg) >= discoHeaderLen && string(msg[:len(disco.Magic)]) == disco.Magic { + return true + } + return false + } + + // Do we have a Geneve header? + if len(msg) >= packet.GeneveFixedHeaderLength && + msg[0]&0xC0 == 0 && // version bits that we always transmit as 0s + msg[1]&0x3F == 0 && // reserved bits that we always transmit as 0s + msg[7] == 0 { // reserved byte that we always transmit as 0 + switch binary.BigEndian.Uint16(msg[2:4]) { + case packet.GeneveProtocolDisco: + if looksLikeDisco(msg[packet.GeneveFixedHeaderLength:]) { + return packetLooksLikeDisco, true + } else { + // The Geneve header is well-formed, and it indicated this + // was disco, but it's not. The evaluated bytes at this point + // are always distinct from naked WireGuard (msg[2:4] are always + // 0x0000) and naked Disco (msg[2:4] are always 0xf09f), but + // maintain pre-Geneve behavior and fall back to assuming it's + // naked WireGuard. + return packetLooksLikeWireGuard, false + } + case packet.GeneveProtocolWireGuard: + return packetLooksLikeWireGuard, true + default: + // The Geneve header is well-formed, but the protocol field value is + // unknown to us. The evaluated bytes at this point are not + // necessarily distinct from naked WireGuard or naked Disco, fall + // through. + } + } + + if looksLikeDisco(msg) { + return packetLooksLikeDisco, false + } else { + return packetLooksLikeWireGuard, false + } +} + +// handleDiscoMessage handles a discovery message. The caller is assumed to have +// verified 'msg' returns [packetLooksLikeDisco] from packetLooksLike(). // // A discovery message has the form: // @@ -1792,34 +1888,17 @@ func isDiscoMaybeGeneve(msg []byte) (isDiscoMsg bool, isGeneveEncap bool) { // - nonce [24]byte // - naclbox of payload (see tailscale.com/disco package for inner payload format) // -// For messages received over DERP, the src.Addr() will be derpMagicIP (with -// src.Port() being the region ID) and the derpNodeSrc will be the node key +// For messages received over DERP, the src.ap.Addr() will be derpMagicIP (with +// src.ap.Port() being the region ID) and the derpNodeSrc will be the node key // it was received from at the DERP layer. derpNodeSrc is zero when received // over UDP. -func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc key.NodePublic, via discoRXPath) (isDiscoMsg bool) { - isDiscoMsg, isGeneveEncap := isDiscoMaybeGeneve(msg) - if !isDiscoMsg { - return - } - var geneve packet.GeneveHeader - var vni virtualNetworkID - if isGeneveEncap { - err := geneve.Decode(msg) - if err != nil { - // Decode only returns an error when 'msg' is too short, and - // 'isGeneveEncap' indicates it's a sufficient length. - c.logf("[unexpected] geneve header decoding error: %v", err) - return - } - vni.set(geneve.VNI) - msg = msg[packet.GeneveFixedHeaderLength:] - } - // The control bit should only be set for relay handshake messages - // terminating on or originating from a UDP relay server. We have yet to - // open the encrypted payload to determine the [disco.MessageType], but - // we assert it should be handshake-related. - shouldBeRelayHandshakeMsg := isGeneveEncap && geneve.Control - +// +// If 'msg' was encapsulated by a Geneve header it is assumed to have already +// been stripped. +// +// 'shouldBeRelayHandshakeMsg' will be true if 'msg' was encapsulated +// by a Geneve header with the control bit set. +func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshakeMsg bool, derpNodeSrc key.NodePublic, via discoRXPath) { sender := key.DiscoPublicFromRaw32(mem.B(msg[len(disco.Magic):discoHeaderLen])) c.mu.Lock() @@ -1833,7 +1912,6 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke } if c.privateKey.IsZero() { // Ignore disco messages when we're stopped. - // Still return true, to not pass it down to wireguard. return } @@ -1844,7 +1922,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke di, ok = c.relayManager.discoInfo(sender) if !ok { if debugDisco() { - c.logf("magicsock: disco: ignoring disco-looking relay handshake frame, no active handshakes with key %v over VNI %d", sender.ShortString(), geneve.VNI) + c.logf("magicsock: disco: ignoring disco-looking relay handshake frame, no active handshakes with key %v over %v", sender.ShortString(), src) } return } @@ -1858,10 +1936,10 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke return } - isDERP := src.Addr() == tailcfg.DerpMagicIPAddr + isDERP := src.ap.Addr() == tailcfg.DerpMagicIPAddr if !isDERP && !shouldBeRelayHandshakeMsg { // Record receive time for UDP transport packets. - pi, ok := c.peerMap.byIPPort[src] + pi, ok := c.peerMap.byEpAddr[src] if ok { pi.ep.lastRecvUDPAny.StoreAtomic(mono.Now()) } @@ -1893,7 +1971,8 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke // Emit information about the disco frame into the pcap stream // if a capture hook is installed. if cb := c.captureHook.Load(); cb != nil { - cb(packet.PathDisco, time.Now(), disco.ToPCAPFrame(src, derpNodeSrc, payload), packet.CaptureMeta{}) + // TODO(jwhited): include VNI context? + cb(packet.PathDisco, time.Now(), disco.ToPCAPFrame(src.ap, derpNodeSrc, payload), packet.CaptureMeta{}) } dm, err := disco.Parse(payload) @@ -1925,14 +2004,14 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke c.logf("[unexpected] %T packets should not come from a relay server with Geneve control bit set", dm) return } - c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(challenge, di, src, geneve.VNI) + c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(challenge, di, src) return } switch dm := dm.(type) { case *disco.Ping: metricRecvDiscoPing.Add(1) - c.handlePingLocked(dm, src, vni, di, derpNodeSrc) + c.handlePingLocked(dm, src, di, derpNodeSrc) case *disco.Pong: metricRecvDiscoPong.Add(1) // There might be multiple nodes for the sender's DiscoKey. @@ -1940,14 +2019,14 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke // the Pong's TxID was theirs. knownTxID := false c.peerMap.forEachEndpointWithDiscoKey(sender, func(ep *endpoint) (keepGoing bool) { - if ep.handlePongConnLocked(dm, di, src, vni) { + if ep.handlePongConnLocked(dm, di, src) { knownTxID = true return false } return true }) - if !knownTxID && vni.isSet() { - c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(dm, di, src, vni.get()) + if !knownTxID && src.vni.isSet() { + c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(dm, di, src) } case *disco.CallMeMaybe, *disco.CallMeMaybeVia: var via *disco.CallMeMaybeVia @@ -2047,18 +2126,18 @@ func (c *Conn) unambiguousNodeKeyOfPingLocked(dm *disco.Ping, dk key.DiscoPublic // di is the discoInfo of the source of the ping. // derpNodeSrc is non-zero if the ping arrived via DERP. -func (c *Conn) handlePingLocked(dm *disco.Ping, src netip.AddrPort, vni virtualNetworkID, di *discoInfo, derpNodeSrc key.NodePublic) { +func (c *Conn) handlePingLocked(dm *disco.Ping, src epAddr, di *discoInfo, derpNodeSrc key.NodePublic) { likelyHeartBeat := src == di.lastPingFrom && time.Since(di.lastPingTime) < 5*time.Second di.lastPingFrom = src di.lastPingTime = time.Now() - isDerp := src.Addr() == tailcfg.DerpMagicIPAddr + isDerp := src.ap.Addr() == tailcfg.DerpMagicIPAddr - if vni.isSet() { + if src.vni.isSet() { // TODO(jwhited): check for matching [endpoint.bestAddr] once that data // structure is VNI-aware and [relayManager] can mutate it. We do not // need to reference any [endpointState] for Geneve-encapsulated disco, // we store nothing about them there. - c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(dm, di, src, vni.get()) + c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(dm, di, src) return } @@ -2071,7 +2150,7 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src netip.AddrPort, vni virtualN // the IP:port<>disco mapping. if nk, ok := c.unambiguousNodeKeyOfPingLocked(dm, di.discoKey, derpNodeSrc); ok { if !isDerp { - c.peerMap.setNodeKeyForIPPort(src, nk) + c.peerMap.setNodeKeyForEpAddr(src, nk) } } @@ -2087,14 +2166,14 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src netip.AddrPort, vni virtualN var dup bool if isDerp { if ep, ok := c.peerMap.endpointForNodeKey(derpNodeSrc); ok { - if ep.addCandidateEndpoint(src, dm.TxID) { + if ep.addCandidateEndpoint(src.ap, dm.TxID) { return } numNodes = 1 } } else { c.peerMap.forEachEndpointWithDiscoKey(di.discoKey, func(ep *endpoint) (keepGoing bool) { - if ep.addCandidateEndpoint(src, dm.TxID) { + if ep.addCandidateEndpoint(src.ap, dm.TxID) { dup = true return false } @@ -2129,9 +2208,9 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src netip.AddrPort, vni virtualN ipDst := src discoDest := di.discoKey - go c.sendDiscoMessage(ipDst, virtualNetworkID{}, dstKey, discoDest, &disco.Pong{ + go c.sendDiscoMessage(ipDst, dstKey, discoDest, &disco.Pong{ TxID: dm.TxID, - Src: src, + Src: src.ap, }, discoVerboseLog) } @@ -2174,12 +2253,12 @@ func (c *Conn) enqueueCallMeMaybe(derpAddr netip.AddrPort, de *endpoint) { for _, ep := range c.lastEndpoints { eps = append(eps, ep.Addr) } - go de.c.sendDiscoMessage(derpAddr, virtualNetworkID{}, de.publicKey, epDisco.key, &disco.CallMeMaybe{MyNumber: eps}, discoLog) + go de.c.sendDiscoMessage(epAddr{ap: derpAddr}, de.publicKey, epDisco.key, &disco.CallMeMaybe{MyNumber: eps}, discoLog) if debugSendCallMeUnknownPeer() { // Send a callMeMaybe packet to a non-existent peer unknownKey := key.NewNode().Public() c.logf("magicsock: sending CallMeMaybe to unknown peer per TS_DEBUG_SEND_CALLME_UNKNOWN_PEER") - go de.c.sendDiscoMessage(derpAddr, virtualNetworkID{}, unknownKey, epDisco.key, &disco.CallMeMaybe{MyNumber: eps}, discoLog) + go de.c.sendDiscoMessage(epAddr{ap: derpAddr}, unknownKey, epDisco.key, &disco.CallMeMaybe{MyNumber: eps}, discoLog) } } @@ -3275,12 +3354,12 @@ func portableTrySetSocketBuffer(pconn nettype.PacketConn, logf logger.Logf) { // derpStr replaces DERP IPs in s with "derp-". func derpStr(s string) string { return strings.ReplaceAll(s, "127.3.3.40:", "derp-") } -// ippEndpointCache is a mutex-free single-element cache, mapping from -// a single netip.AddrPort to a single endpoint. -type ippEndpointCache struct { - ipp netip.AddrPort - gen int64 - de *endpoint +// epAddrEndpointCache is a mutex-free single-element cache, mapping from +// a single [epAddr] to a single [*endpoint]. +type epAddrEndpointCache struct { + epAddr epAddr + gen int64 + de *endpoint } // discoInfo is the info and state for the DiscoKey @@ -3309,7 +3388,7 @@ type discoInfo struct { // Mutable fields follow, owned by Conn.mu: // lastPingFrom is the src of a ping for discoKey. - lastPingFrom netip.AddrPort + lastPingFrom epAddr // lastPingTime is the last time of a ping for discoKey. lastPingTime time.Time @@ -3444,14 +3523,14 @@ func (c *Conn) SetLastNetcheckReportForTest(ctx context.Context, report *netchec // to tell us who it is later and get the correct conn.Endpoint. type lazyEndpoint struct { c *Conn - src netip.AddrPort + src epAddr } var _ conn.PeerAwareEndpoint = (*lazyEndpoint)(nil) var _ conn.Endpoint = (*lazyEndpoint)(nil) func (le *lazyEndpoint) ClearSrc() {} -func (le *lazyEndpoint) SrcIP() netip.Addr { return le.src.Addr() } +func (le *lazyEndpoint) SrcIP() netip.Addr { return le.src.ap.Addr() } func (le *lazyEndpoint) DstIP() netip.Addr { return netip.Addr{} } func (le *lazyEndpoint) SrcToString() string { return le.src.String() } func (le *lazyEndpoint) DstToString() string { return "dst" } diff --git a/wgengine/magicsock/magicsock_linux.go b/wgengine/magicsock/magicsock_linux.go index 34c39fe62b773..07038002912f7 100644 --- a/wgengine/magicsock/magicsock_linux.go +++ b/wgengine/magicsock/magicsock_linux.go @@ -453,7 +453,13 @@ func (c *Conn) receiveDisco(pc *socket.Conn, isIPV6 bool) { metricRecvDiscoPacketIPv4.Add(1) } - c.handleDiscoMessage(payload, srcAddr, key.NodePublic{}, discoRXPathRawSocket) + pt, isGeneveEncap := packetLooksLike(payload) + if pt == packetLooksLikeDisco && !isGeneveEncap { + // The BPF program matching on disco does not currently support + // Geneve encapsulation. isGeneveEncap should not return true if + // payload is disco. + c.handleDiscoMessage(payload, epAddr{ap: srcAddr}, false, key.NodePublic{}, discoRXPathRawSocket) + } } } diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index e1801187352d5..5e71a40c9db97 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -50,6 +50,7 @@ import ( "tailscale.com/net/netmon" "tailscale.com/net/packet" "tailscale.com/net/ping" + "tailscale.com/net/stun" "tailscale.com/net/stun/stuntest" "tailscale.com/net/tstun" "tailscale.com/tailcfg" @@ -1290,41 +1291,6 @@ func assertConnStatsAndUserMetricsEqual(t *testing.T, ms *magicStack) { c.Assert(metricRecvDataPacketsDERP.Value(), qt.Equals, metricDERPRxPackets*2) } -func TestDiscoMessage(t *testing.T) { - c := newConn(t.Logf) - c.privateKey = key.NewNode() - - peer1Pub := c.DiscoPublicKey() - peer1Priv := c.discoPrivate - n := &tailcfg.Node{ - Key: key.NewNode().Public(), - DiscoKey: peer1Pub, - } - ep := &endpoint{ - nodeID: 1, - publicKey: n.Key, - } - ep.disco.Store(&endpointDisco{ - key: n.DiscoKey, - short: n.DiscoKey.ShortString(), - }) - c.peerMap.upsertEndpoint(ep, key.DiscoPublic{}) - - const payload = "why hello" - - var nonce [24]byte - crand.Read(nonce[:]) - - pkt := peer1Pub.AppendTo([]byte("TS💬")) - - box := peer1Priv.Shared(c.discoPrivate.Public()).Seal([]byte(payload)) - pkt = append(pkt, box...) - got := c.handleDiscoMessage(pkt, netip.AddrPort{}, key.NodePublic{}, discoRXPathUDP) - if !got { - t.Error("failed to open it") - } -} - // tests that having a endpoint.String prevents wireguard-go's // log.Printf("%v") of its conn.Endpoint values from using reflect to // walk into read mutex while they're being used and then causing data @@ -1358,11 +1324,11 @@ func Test32bitAlignment(t *testing.T) { t.Fatalf("endpoint.lastRecvWG is not 8-byte aligned") } - de.noteRecvActivity(netip.AddrPort{}, mono.Now()) // verify this doesn't panic on 32-bit + de.noteRecvActivity(epAddr{}, mono.Now()) // verify this doesn't panic on 32-bit if called != 1 { t.Fatal("expected call to noteRecvActivity") } - de.noteRecvActivity(netip.AddrPort{}, mono.Now()) + de.noteRecvActivity(epAddr{}, mono.Now()) if called != 1 { t.Error("expected no second call to noteRecvActivity") } @@ -1799,10 +1765,15 @@ func TestEndpointSetsEqual(t *testing.T) { func TestBetterAddr(t *testing.T) { const ms = time.Millisecond al := func(ipps string, d time.Duration) addrQuality { - return addrQuality{AddrPort: netip.MustParseAddrPort(ipps), latency: d} + return addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort(ipps)}, latency: d} } almtu := func(ipps string, d time.Duration, mtu tstun.WireMTU) addrQuality { - return addrQuality{AddrPort: netip.MustParseAddrPort(ipps), latency: d, wireMTU: mtu} + return addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort(ipps)}, latency: d, wireMTU: mtu} + } + avl := func(ipps string, vni uint32, d time.Duration) addrQuality { + q := al(ipps, d) + q.vni.set(vni) + return q } zero := addrQuality{} @@ -1908,6 +1879,18 @@ func TestBetterAddr(t *testing.T) { b: al("[::1]:555", 100*ms), want: false, }, + + // Prefer non-Geneve over Geneve-encapsulated + { + a: al(publicV4, 100*ms), + b: avl(publicV4, 1, 100*ms), + want: true, + }, + { + a: avl(publicV4, 1, 100*ms), + b: al(publicV4, 100*ms), + want: false, + }, } for i, tt := range tests { got := betterAddr(tt.a, tt.b) @@ -2019,9 +2002,9 @@ func (m *peerMap) validate() error { return fmt.Errorf("duplicate endpoint present: %v", pi.ep.publicKey) } seenEps[pi.ep] = true - for ipp := range pi.ipPorts { - if got := m.byIPPort[ipp]; got != pi { - return fmt.Errorf("m.byIPPort[%v] = %v, want %v", ipp, got, pi) + for addr := range pi.epAddrs { + if got := m.byEpAddr[addr]; got != pi { + return fmt.Errorf("m.byEpAddr[%v] = %v, want %v", addr, got, pi) } } } @@ -2037,13 +2020,13 @@ func (m *peerMap) validate() error { } } - for ipp, pi := range m.byIPPort { - if !pi.ipPorts.Contains(ipp) { - return fmt.Errorf("ipPorts[%v] for %v is false", ipp, pi.ep.publicKey) + for addr, pi := range m.byEpAddr { + if !pi.epAddrs.Contains(addr) { + return fmt.Errorf("epAddrs[%v] for %v is false", addr, pi.ep.publicKey) } pi2 := m.byNodeKey[pi.ep.publicKey] if pi != pi2 { - return fmt.Errorf("byNodeKey[%v]=%p doesn't match byIPPort[%v]=%p", pi, pi, pi.ep.publicKey, pi2) + return fmt.Errorf("byNodeKey[%v]=%p doesn't match byEpAddr[%v]=%p", pi, pi, pi.ep.publicKey, pi2) } } @@ -2444,7 +2427,7 @@ func TestIsWireGuardOnlyPickEndpointByPing(t *testing.T) { // Check that we got a valid address set on the first send - this // will be randomly selected, but because we have noV6 set to true, // it will be the IPv4 address. - if !pi.ep.bestAddr.Addr().IsValid() { + if !pi.ep.bestAddr.ap.Addr().IsValid() { t.Fatal("bestaddr was nil") } @@ -2504,12 +2487,12 @@ func TestIsWireGuardOnlyPickEndpointByPing(t *testing.T) { t.Fatal("wgkey doesn't exist in peer map") } - if !pi.ep.bestAddr.Addr().IsValid() { + if !pi.ep.bestAddr.ap.Addr().IsValid() { t.Error("no bestAddr address was set") } - if pi.ep.bestAddr.Addr() != wgEp.Addr() { - t.Errorf("bestAddr was not set to the expected IPv4 address: got %v, want %v", pi.ep.bestAddr.Addr().String(), wgEp.Addr()) + if pi.ep.bestAddr.ap.Addr() != wgEp.Addr() { + t.Errorf("bestAddr was not set to the expected IPv4 address: got %v, want %v", pi.ep.bestAddr.ap.Addr().String(), wgEp.Addr()) } if pi.ep.trustBestAddrUntil.IsZero() { @@ -2670,7 +2653,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { sendFollowUpPing bool pingTime mono.Time ep []endpointDetails - want netip.AddrPort + want epAddr }{ { name: "no endpoints", @@ -2679,7 +2662,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { sendFollowUpPing: false, pingTime: testTime, ep: []endpointDetails{}, - want: netip.AddrPort{}, + want: epAddr{}, }, { name: "singular endpoint does not request ping", @@ -2693,7 +2676,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { latency: 100 * time.Millisecond, }, }, - want: netip.MustParseAddrPort("1.1.1.1:111"), + want: epAddr{ap: netip.MustParseAddrPort("1.1.1.1:111")}, }, { name: "ping sent within wireguardPingInterval should not request ping", @@ -2711,7 +2694,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { latency: 2000 * time.Millisecond, }, }, - want: netip.MustParseAddrPort("1.1.1.1:111"), + want: epAddr{ap: netip.MustParseAddrPort("1.1.1.1:111")}, }, { name: "ping sent outside of wireguardPingInterval should request ping", @@ -2729,7 +2712,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { latency: 150 * time.Millisecond, }, }, - want: netip.MustParseAddrPort("1.1.1.1:111"), + want: epAddr{ap: netip.MustParseAddrPort("1.1.1.1:111")}, }, { name: "choose lowest latency for useable IPv4 and IPv6", @@ -2747,7 +2730,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { latency: 10 * time.Millisecond, }, }, - want: netip.MustParseAddrPort("[2345:0425:2CA1:0000:0000:0567:5673:23b5]:222"), + want: epAddr{ap: netip.MustParseAddrPort("[2345:0425:2CA1:0000:0000:0567:5673:23b5]:222")}, }, { name: "choose IPv6 address when latency is the same for v4 and v6", @@ -2765,7 +2748,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { latency: 100 * time.Millisecond, }, }, - want: netip.MustParseAddrPort("[1::1]:567"), + want: epAddr{ap: netip.MustParseAddrPort("[1::1]:567")}, }, } @@ -2785,8 +2768,8 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { endpoint.endpointState[epd.addrPort] = &endpointState{} } udpAddr, _, shouldPing := endpoint.addrForSendLocked(testTime) - if udpAddr.IsValid() != test.validAddr { - t.Errorf("udpAddr validity is incorrect; got %v, want %v", udpAddr.IsValid(), test.validAddr) + if udpAddr.ap.IsValid() != test.validAddr { + t.Errorf("udpAddr validity is incorrect; got %v, want %v", udpAddr.ap.IsValid(), test.validAddr) } if shouldPing != test.sendInitialPing { t.Errorf("addrForSendLocked did not indiciate correct ping state; got %v, want %v", shouldPing, test.sendInitialPing) @@ -2818,8 +2801,8 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { if shouldPing != test.sendFollowUpPing { t.Errorf("addrForSendLocked did not indiciate correct ping state; got %v, want %v", shouldPing, test.sendFollowUpPing) } - if endpoint.bestAddr.AddrPort != test.want { - t.Errorf("bestAddr.AddrPort is not as expected: got %v, want %v", endpoint.bestAddr.AddrPort, test.want) + if endpoint.bestAddr.epAddr != test.want { + t.Errorf("bestAddr.epAddr is not as expected: got %v, want %v", endpoint.bestAddr.epAddr, test.want) } }) } @@ -2906,7 +2889,7 @@ func TestAddrForPingSizeLocked(t *testing.T) { t.Run(test.desc, func(t *testing.T) { bestAddr := addrQuality{wireMTU: test.mtu} if test.bestAddr { - bestAddr.AddrPort = validUdpAddr + bestAddr.epAddr.ap = validUdpAddr } ep := &endpoint{ derpAddr: validDerpAddr, @@ -2918,10 +2901,10 @@ func TestAddrForPingSizeLocked(t *testing.T) { udpAddr, derpAddr := ep.addrForPingSizeLocked(testTime, test.size) - if test.wantUDP && !udpAddr.IsValid() { + if test.wantUDP && !udpAddr.ap.IsValid() { t.Errorf("%s: udpAddr returned is not valid, won't be sent to UDP address", test.desc) } - if !test.wantUDP && udpAddr.IsValid() { + if !test.wantUDP && udpAddr.ap.IsValid() { t.Errorf("%s: udpAddr returned is valid, discovery will not start", test.desc) } if test.wantDERP && !derpAddr.IsValid() { @@ -3157,7 +3140,7 @@ func TestNetworkDownSendErrors(t *testing.T) { } } -func Test_isDiscoMaybeGeneve(t *testing.T) { +func Test_packetLooksLike(t *testing.T) { discoPub := key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 30: 30, 31: 31})) nakedDisco := make([]byte, 0, 512) nakedDisco = append(nakedDisco, disco.Magic...) @@ -3240,80 +3223,92 @@ func Test_isDiscoMaybeGeneve(t *testing.T) { copy(geneveEncapDiscoNonZeroGeneveVNILSB[packet.GeneveFixedHeaderLength:], nakedDisco) tests := []struct { - name string - msg []byte - wantIsDiscoMsg bool - wantIsGeneveEncap bool + name string + msg []byte + wantPacketLooksLikeType packetLooksLikeType + wantIsGeneveEncap bool }{ { - name: "naked disco", - msg: nakedDisco, - wantIsDiscoMsg: true, - wantIsGeneveEncap: false, + name: "STUN binding success response", + msg: stun.Response(stun.NewTxID(), netip.MustParseAddrPort("127.0.0.1:1")), + wantPacketLooksLikeType: packetLooksLikeSTUNBinding, + wantIsGeneveEncap: false, + }, + { + name: "naked disco", + msg: nakedDisco, + wantPacketLooksLikeType: packetLooksLikeDisco, + wantIsGeneveEncap: false, + }, + { + name: "geneve encap disco", + msg: geneveEncapDisco, + wantPacketLooksLikeType: packetLooksLikeDisco, + wantIsGeneveEncap: true, }, { - name: "geneve encap disco", - msg: geneveEncapDisco, - wantIsDiscoMsg: true, - wantIsGeneveEncap: true, + name: "geneve encap too short disco", + msg: geneveEncapDisco[:len(geneveEncapDisco)-key.DiscoPublicRawLen], + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "geneve encap disco nonzero geneve version", - msg: geneveEncapDiscoNonZeroGeneveVersion, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "geneve encap disco nonzero geneve version", + msg: geneveEncapDiscoNonZeroGeneveVersion, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "geneve encap disco nonzero geneve reserved bits", - msg: geneveEncapDiscoNonZeroGeneveReservedBits, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "geneve encap disco nonzero geneve reserved bits", + msg: geneveEncapDiscoNonZeroGeneveReservedBits, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "geneve encap disco nonzero geneve vni lsb", - msg: geneveEncapDiscoNonZeroGeneveVNILSB, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "geneve encap disco nonzero geneve vni lsb", + msg: geneveEncapDiscoNonZeroGeneveVNILSB, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "geneve encap wireguard", - msg: geneveEncapWireGuard, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "geneve encap wireguard", + msg: geneveEncapWireGuard, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: true, }, { - name: "naked WireGuard Initiation type", - msg: nakedWireGuardInitiation, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "naked WireGuard Initiation type", + msg: nakedWireGuardInitiation, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "naked WireGuard Response type", - msg: nakedWireGuardResponse, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "naked WireGuard Response type", + msg: nakedWireGuardResponse, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "naked WireGuard Cookie Reply type", - msg: nakedWireGuardCookieReply, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "naked WireGuard Cookie Reply type", + msg: nakedWireGuardCookieReply, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "naked WireGuard Transport type", - msg: nakedWireGuardTransport, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "naked WireGuard Transport type", + msg: nakedWireGuardTransport, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotIsDiscoMsg, gotIsGeneveEncap := isDiscoMaybeGeneve(tt.msg) - if gotIsDiscoMsg != tt.wantIsDiscoMsg { - t.Errorf("isDiscoMaybeGeneve() gotIsDiscoMsg = %v, want %v", gotIsDiscoMsg, tt.wantIsDiscoMsg) + gotPacketLooksLikeType, gotIsGeneveEncap := packetLooksLike(tt.msg) + if gotPacketLooksLikeType != tt.wantPacketLooksLikeType { + t.Errorf("packetLooksLike() gotPacketLooksLikeType = %v, want %v", gotPacketLooksLikeType, tt.wantPacketLooksLikeType) } if gotIsGeneveEncap != tt.wantIsGeneveEncap { - t.Errorf("isDiscoMaybeGeneve() gotIsGeneveEncap = %v, want %v", gotIsGeneveEncap, tt.wantIsGeneveEncap) + t.Errorf("packetLooksLike() gotIsGeneveEncap = %v, want %v", gotIsGeneveEncap, tt.wantIsGeneveEncap) } }) } diff --git a/wgengine/magicsock/peermap.go b/wgengine/magicsock/peermap.go index e1c7db1f6c632..04d5de8c97e06 100644 --- a/wgengine/magicsock/peermap.go +++ b/wgengine/magicsock/peermap.go @@ -4,8 +4,6 @@ package magicsock import ( - "net/netip" - "tailscale.com/tailcfg" "tailscale.com/types/key" "tailscale.com/util/set" @@ -15,17 +13,17 @@ import ( // peer. type peerInfo struct { ep *endpoint // always non-nil. - // ipPorts is an inverted version of peerMap.byIPPort (below), so + // epAddrs is an inverted version of peerMap.byEpAddr (below), so // that when we're deleting this node, we can rapidly find out the - // keys that need deleting from peerMap.byIPPort without having to - // iterate over every IPPort known for any peer. - ipPorts set.Set[netip.AddrPort] + // keys that need deleting from peerMap.byEpAddr without having to + // iterate over every epAddr known for any peer. + epAddrs set.Set[epAddr] } func newPeerInfo(ep *endpoint) *peerInfo { return &peerInfo{ ep: ep, - ipPorts: set.Set[netip.AddrPort]{}, + epAddrs: set.Set[epAddr]{}, } } @@ -35,7 +33,7 @@ func newPeerInfo(ep *endpoint) *peerInfo { // It doesn't do any locking; all access must be done with Conn.mu held. type peerMap struct { byNodeKey map[key.NodePublic]*peerInfo - byIPPort map[netip.AddrPort]*peerInfo + byEpAddr map[epAddr]*peerInfo byNodeID map[tailcfg.NodeID]*peerInfo // nodesOfDisco contains the set of nodes that are using a @@ -46,7 +44,7 @@ type peerMap struct { func newPeerMap() peerMap { return peerMap{ byNodeKey: map[key.NodePublic]*peerInfo{}, - byIPPort: map[netip.AddrPort]*peerInfo{}, + byEpAddr: map[epAddr]*peerInfo{}, byNodeID: map[tailcfg.NodeID]*peerInfo{}, nodesOfDisco: map[key.DiscoPublic]set.Set[key.NodePublic]{}, } @@ -88,10 +86,10 @@ func (m *peerMap) endpointForNodeID(nodeID tailcfg.NodeID) (ep *endpoint, ok boo return nil, false } -// endpointForIPPort returns the endpoint for the peer we -// believe to be at ipp, or nil if we don't know of any such peer. -func (m *peerMap) endpointForIPPort(ipp netip.AddrPort) (ep *endpoint, ok bool) { - if info, ok := m.byIPPort[ipp]; ok { +// endpointForEpAddr returns the endpoint for the peer we +// believe to be at addr, or nil if we don't know of any such peer. +func (m *peerMap) endpointForEpAddr(addr epAddr) (ep *endpoint, ok bool) { + if info, ok := m.byEpAddr[addr]; ok { return info.ep, true } return nil, false @@ -148,10 +146,10 @@ func (m *peerMap) upsertEndpoint(ep *endpoint, oldDiscoKey key.DiscoPublic) { // TODO(raggi,catzkorn): this could mean that if a "isWireguardOnly" // peer has, say, 192.168.0.2 and so does a tailscale peer, the // wireguard one will win. That may not be the outcome that we want - - // perhaps we should prefer bestAddr.AddrPort if it is set? + // perhaps we should prefer bestAddr.epAddr.ap if it is set? // see tailscale/tailscale#7994 for ipp := range ep.endpointState { - m.setNodeKeyForIPPort(ipp, ep.publicKey) + m.setNodeKeyForEpAddr(epAddr{ap: ipp}, ep.publicKey) } return } @@ -163,20 +161,20 @@ func (m *peerMap) upsertEndpoint(ep *endpoint, oldDiscoKey key.DiscoPublic) { discoSet.Add(ep.publicKey) } -// setNodeKeyForIPPort makes future peer lookups by ipp return the +// setNodeKeyForEpAddr makes future peer lookups by addr return the // same endpoint as a lookup by nk. // -// This should only be called with a fully verified mapping of ipp to +// This should only be called with a fully verified mapping of addr to // nk, because calling this function defines the endpoint we hand to -// WireGuard for packets received from ipp. -func (m *peerMap) setNodeKeyForIPPort(ipp netip.AddrPort, nk key.NodePublic) { - if pi := m.byIPPort[ipp]; pi != nil { - delete(pi.ipPorts, ipp) - delete(m.byIPPort, ipp) +// WireGuard for packets received from addr. +func (m *peerMap) setNodeKeyForEpAddr(addr epAddr, nk key.NodePublic) { + if pi := m.byEpAddr[addr]; pi != nil { + delete(pi.epAddrs, addr) + delete(m.byEpAddr, addr) } if pi, ok := m.byNodeKey[nk]; ok { - pi.ipPorts.Add(ipp) - m.byIPPort[ipp] = pi + pi.epAddrs.Add(addr) + m.byEpAddr[addr] = pi } } @@ -203,7 +201,7 @@ func (m *peerMap) deleteEndpoint(ep *endpoint) { // Unexpected. But no logger plumbed here to log so. return } - for ip := range pi.ipPorts { - delete(m.byIPPort, ip) + for ip := range pi.epAddrs { + delete(m.byEpAddr, ip) } } diff --git a/wgengine/magicsock/rebinding_conn.go b/wgengine/magicsock/rebinding_conn.go index 7a9dd1821f306..51e97c8ccae2e 100644 --- a/wgengine/magicsock/rebinding_conn.go +++ b/wgengine/magicsock/rebinding_conn.go @@ -5,6 +5,7 @@ package magicsock import ( "errors" + "fmt" "net" "net/netip" "sync" @@ -13,6 +14,7 @@ import ( "golang.org/x/net/ipv6" "tailscale.com/net/netaddr" + "tailscale.com/net/packet" "tailscale.com/types/nettype" ) @@ -71,14 +73,28 @@ func (c *RebindingUDPConn) ReadFromUDPAddrPort(b []byte) (int, netip.AddrPort, e } // WriteBatchTo writes buffs to addr. -func (c *RebindingUDPConn) WriteBatchTo(buffs [][]byte, addr netip.AddrPort, offset int) error { +func (c *RebindingUDPConn) WriteBatchTo(buffs [][]byte, addr epAddr, offset int) error { + if offset != packet.GeneveFixedHeaderLength { + return fmt.Errorf("RebindingUDPConn.WriteBatchTo: [unexpected] offset (%d) != Geneve header length (%d)", offset, packet.GeneveFixedHeaderLength) + } for { pconn := *c.pconnAtomic.Load() b, ok := pconn.(batchingConn) if !ok { + vniIsSet := addr.vni.isSet() + var gh packet.GeneveHeader + if vniIsSet { + gh = packet.GeneveHeader{ + VNI: addr.vni.get(), + } + } for _, buf := range buffs { - buf = buf[offset:] - _, err := c.writeToUDPAddrPortWithInitPconn(pconn, buf, addr) + if vniIsSet { + gh.Encode(buf) + } else { + buf = buf[offset:] + } + _, err := c.writeToUDPAddrPortWithInitPconn(pconn, buf, addr.ap) if err != nil { return err } diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index d9fd1fa24ba2f..177eed3556dfb 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -279,8 +279,8 @@ func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, dm *disco.CallMeMaybeV // handleGeneveEncapDiscoMsgNotBestAddr handles reception of Geneve-encapsulated // disco messages if they are not associated with any known // [*endpoint.bestAddr]. -func (r *relayManager) handleGeneveEncapDiscoMsgNotBestAddr(dm disco.Message, di *discoInfo, src netip.AddrPort, vni uint32) { - relayManagerInputEvent(r, nil, &r.rxHandshakeDiscoMsgCh, relayHandshakeDiscoMsgEvent{msg: dm, disco: di.discoKey, from: src, vni: vni, at: time.Now()}) +func (r *relayManager) handleGeneveEncapDiscoMsgNotBestAddr(dm disco.Message, di *discoInfo, src epAddr) { + relayManagerInputEvent(r, nil, &r.rxHandshakeDiscoMsgCh, relayHandshakeDiscoMsgEvent{msg: dm, disco: di.discoKey, from: src.ap, vni: src.vni.get(), at: time.Now()}) } // relayManagerInputEvent initializes [relayManager] if necessary, starts @@ -437,6 +437,8 @@ func (r *relayManager) handleHandshakeWorkDoneRunLoop(done relayEndpointHandshak } // This relay endpoint is functional. // TODO(jwhited): Set it on done.work.ep.bestAddr if it is a betterAddr(). + // We also need to conn.peerMap.setNodeKeyForEpAddr(), and ensure we clean + // it up when bestAddr changes, too. } func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelayServerEndpointEvent) { @@ -540,7 +542,7 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { for _, addrPort := range work.se.AddrPorts { if addrPort.IsValid() { sentBindAny = true - go work.ep.c.sendDiscoMessage(addrPort, vni, key.NodePublic{}, work.se.ServerDisco, bind, discoVerboseLog) + go work.ep.c.sendDiscoMessage(epAddr{ap: addrPort, vni: vni}, key.NodePublic{}, work.se.ServerDisco, bind, discoVerboseLog) } } if !sentBindAny { @@ -580,9 +582,9 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { go func() { if withAnswer != nil { answer := &disco.BindUDPRelayEndpointAnswer{Answer: *withAnswer} - work.ep.c.sendDiscoMessage(to, vni, key.NodePublic{}, work.se.ServerDisco, answer, discoVerboseLog) + work.ep.c.sendDiscoMessage(epAddr{ap: to, vni: vni}, key.NodePublic{}, work.se.ServerDisco, answer, discoVerboseLog) } - work.ep.c.sendDiscoMessage(to, vni, key.NodePublic{}, epDisco.key, ping, discoVerboseLog) + work.ep.c.sendDiscoMessage(epAddr{ap: to, vni: vni}, key.NodePublic{}, epDisco.key, ping, discoVerboseLog) }() } diff --git a/wgengine/magicsock/relaymanager_test.go b/wgengine/magicsock/relaymanager_test.go index 8276849aafd8b..be0582669c964 100644 --- a/wgengine/magicsock/relaymanager_test.go +++ b/wgengine/magicsock/relaymanager_test.go @@ -4,7 +4,6 @@ package magicsock import ( - "net/netip" "testing" "tailscale.com/disco" @@ -25,6 +24,6 @@ func TestRelayManagerInitAndIdle(t *testing.T) { <-rm.runLoopStoppedCh rm = relayManager{} - rm.handleGeneveEncapDiscoMsgNotBestAddr(&disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, netip.AddrPort{}, 0) + rm.handleGeneveEncapDiscoMsgNotBestAddr(&disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, epAddr{}) <-rm.runLoopStoppedCh } From 7b06532ea108bd7d12785125c2423a1a4e9987b3 Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Fri, 6 Jun 2025 15:20:23 -0400 Subject: [PATCH 038/263] ipn/ipnlocal: Update hostinfo to control on service config change (#16146) This commit fixes the bug that c2n requests are skiped when updating vipServices in serveConfig. This then resulted netmap update being skipped which caused inaccuracy of Capmap info on client side. After this fix, client always inform control about it's vipServices config changes. Fixes tailscale/corp#29219 Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- ipn/ipnlocal/local.go | 15 ++++++--- ipn/ipnlocal/local_test.go | 69 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 0efec6b9fff29..4b5accd857da7 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -6194,17 +6194,17 @@ func (b *LocalBackend) setTCPPortsInterceptedFromNetmapAndPrefsLocked(prefs ipn. } } - // Update funnel info in hostinfo and kick off control update if needed. - b.updateIngressLocked() + // Update funnel and service hash info in hostinfo and kick off control update if needed. + b.updateIngressAndServiceHashLocked(prefs) b.setTCPPortsIntercepted(handlePorts) b.setVIPServicesTCPPortsInterceptedLocked(vipServicesPorts) } -// updateIngressLocked updates the hostinfo.WireIngress and hostinfo.IngressEnabled fields and kicks off a Hostinfo -// update if the values have changed. +// updateIngressAndServiceHashLocked updates the hostinfo.ServicesHash, hostinfo.WireIngress and +// hostinfo.IngressEnabled fields and kicks off a Hostinfo update if the values have changed. // // b.mu must be held. -func (b *LocalBackend) updateIngressLocked() { +func (b *LocalBackend) updateIngressAndServiceHashLocked(prefs ipn.PrefsView) { if b.hostinfo == nil { return } @@ -6219,6 +6219,11 @@ func (b *LocalBackend) updateIngressLocked() { b.hostinfo.WireIngress = wire hostInfoChanged = true } + latestHash := b.vipServiceHash(b.vipServicesFromPrefsLocked(prefs)) + if b.hostinfo.ServicesHash != latestHash { + b.hostinfo.ServicesHash = latestHash + hostInfoChanged = true + } // Kick off a Hostinfo update to control if ingress status has changed. if hostInfoChanged { b.goTracker.Go(b.doSetHostinfoFilterServices) diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 8f9b6ee68f62b..f14ac037cfde5 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -5134,10 +5134,17 @@ func TestUpdatePrefsOnSysPolicyChange(t *testing.T) { } } -func TestUpdateIngressLocked(t *testing.T) { +func TestUpdateIngressAndServiceHashLocked(t *testing.T) { + prefs := ipn.NewPrefs().View() + previousSC := &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:abc": {Tun: true}, + }, + } tests := []struct { name string hi *tailcfg.Hostinfo + hasPreviousSC bool // whether to overwrite the ServeConfig hash in the Hostinfo using previousSC sc *ipn.ServeConfig wantIngress bool wantWireIngress bool @@ -5163,6 +5170,16 @@ func TestUpdateIngressLocked(t *testing.T) { wantWireIngress: false, // implied by wantIngress wantControlUpdate: true, }, + { + name: "empty_hostinfo_service_configured", + hi: &tailcfg.Hostinfo{}, + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:abc": {Tun: true}, + }, + }, + wantControlUpdate: true, + }, { name: "empty_hostinfo_funnel_disabled", hi: &tailcfg.Hostinfo{}, @@ -5175,7 +5192,7 @@ func TestUpdateIngressLocked(t *testing.T) { wantControlUpdate: true, }, { - name: "empty_hostinfo_no_funnel", + name: "empty_hostinfo_no_funnel_no_service", hi: &tailcfg.Hostinfo{}, sc: &ipn.ServeConfig{ TCP: map[uint16]*ipn.TCPPortHandler{ @@ -5196,6 +5213,16 @@ func TestUpdateIngressLocked(t *testing.T) { wantIngress: true, wantWireIngress: false, // implied by wantIngress }, + { + name: "service_hash_no_change", + hi: &tailcfg.Hostinfo{}, + hasPreviousSC: true, + sc: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:abc": {Tun: true}, + }, + }, + }, { name: "funnel_disabled_no_change", hi: &tailcfg.Hostinfo{ @@ -5208,6 +5235,13 @@ func TestUpdateIngressLocked(t *testing.T) { }, wantWireIngress: true, // true if there is any AllowFunnel block }, + { + name: "service_got_removed", + hi: &tailcfg.Hostinfo{}, + hasPreviousSC: true, + sc: &ipn.ServeConfig{}, + wantControlUpdate: true, + }, { name: "funnel_changes_to_disabled", hi: &tailcfg.Hostinfo{ @@ -5235,12 +5269,35 @@ func TestUpdateIngressLocked(t *testing.T) { wantWireIngress: false, // implied by wantIngress wantControlUpdate: true, }, + { + name: "both_funnel_and_service_changes", + hi: &tailcfg.Hostinfo{ + IngressEnabled: true, + }, + sc: &ipn.ServeConfig{ + AllowFunnel: map[ipn.HostPort]bool{ + "tailnet.xyz:443": false, + }, + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:abc": {Tun: true}, + }, + }, + wantWireIngress: true, // true if there is any AllowFunnel block + wantControlUpdate: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() b := newTestLocalBackend(t) b.hostinfo = tt.hi + if tt.hasPreviousSC { + b.mu.Lock() + b.serveConfig = previousSC.View() + b.hostinfo.ServicesHash = b.vipServiceHash(b.vipServicesFromPrefsLocked(prefs)) + b.mu.Unlock() + } b.serveConfig = tt.sc.View() allDone := make(chan bool, 1) defer b.goTracker.AddDoneCallback(func() { @@ -5256,7 +5313,7 @@ func TestUpdateIngressLocked(t *testing.T) { })() was := b.goTracker.StartedGoroutines() - b.updateIngressLocked() + b.updateIngressAndServiceHashLocked(prefs) if tt.hi != nil { if tt.hi.IngressEnabled != tt.wantIngress { @@ -5265,6 +5322,12 @@ func TestUpdateIngressLocked(t *testing.T) { if tt.hi.WireIngress != tt.wantWireIngress { t.Errorf("WireIngress = %v, want %v", tt.hi.WireIngress, tt.wantWireIngress) } + b.mu.Lock() + svcHash := b.vipServiceHash(b.vipServicesFromPrefsLocked(prefs)) + b.mu.Unlock() + if tt.hi.ServicesHash != svcHash { + t.Errorf("ServicesHash = %v, want %v", tt.hi.ServicesHash, svcHash) + } } startedGoroutine := b.goTracker.StartedGoroutines() != was From 5716d0977d9ae0e3a5e4bc2071c01f8926c87912 Mon Sep 17 00:00:00 2001 From: James Sanderson Date: Fri, 6 Jun 2025 15:53:30 +0100 Subject: [PATCH 039/263] health: prefix Warnables received from the control plane Updates tailscale/corp#27759 Signed-off-by: James Sanderson --- control/controlclient/map.go | 2 +- control/controlclient/map_test.go | 12 ++++++------ health/health_test.go | 18 +++++++++--------- health/state.go | 2 +- ipn/ipnlocal/local_test.go | 26 +++++++++++++++++--------- 5 files changed, 34 insertions(+), 26 deletions(-) diff --git a/control/controlclient/map.go b/control/controlclient/map.go index f346e19d494f6..22cea5acaa2f7 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -853,7 +853,7 @@ func (ms *mapSession) netmap() *netmap.NetworkMap { } else if len(ms.lastHealth) > 0 { // Convert all ms.lastHealth to the new [netmap.NetworkMap.DisplayMessages] for _, h := range ms.lastHealth { - id := "control-health-" + strhash(h) // Unique ID in case there is more than one health message + id := "health-" + strhash(h) // Unique ID in case there is more than one health message mak.Set(&msgs, tailcfg.DisplayMessageID(id), tailcfg.DisplayMessage{ Title: "Coordination server reports an issue", Severity: tailcfg.SeverityMedium, diff --git a/control/controlclient/map_test.go b/control/controlclient/map_test.go index 013640f47b08a..7e42f6f6a8b25 100644 --- a/control/controlclient/map_test.go +++ b/control/controlclient/map_test.go @@ -1340,14 +1340,14 @@ func TestNetmapHealthIntegration(t *testing.T) { ht.SetControlHealth(nm.DisplayMessages) want := map[health.WarnableCode]health.UnhealthyState{ - "control-health-c0719e9a8d5d838d861dc6f675c899d2b309a3a65bb9fe6b11e5afcbf9a2c0b1": { - WarnableCode: "control-health-c0719e9a8d5d838d861dc6f675c899d2b309a3a65bb9fe6b11e5afcbf9a2c0b1", + "control-health.health-c0719e9a8d5d838d861dc6f675c899d2b309a3a65bb9fe6b11e5afcbf9a2c0b1": { + WarnableCode: "control-health.health-c0719e9a8d5d838d861dc6f675c899d2b309a3a65bb9fe6b11e5afcbf9a2c0b1", Title: "Coordination server reports an issue", Severity: health.SeverityMedium, Text: "The coordination server is reporting a health issue: Test message", }, - "control-health-1dc7017a73a3c55c0d6a8423e3813c7ab6562d9d3064c2ec6ac7822f61b1db9c": { - WarnableCode: "control-health-1dc7017a73a3c55c0d6a8423e3813c7ab6562d9d3064c2ec6ac7822f61b1db9c", + "control-health.health-1dc7017a73a3c55c0d6a8423e3813c7ab6562d9d3064c2ec6ac7822f61b1db9c": { + WarnableCode: "control-health.health-1dc7017a73a3c55c0d6a8423e3813c7ab6562d9d3064c2ec6ac7822f61b1db9c", Title: "Coordination server reports an issue", Severity: health.SeverityMedium, Text: "The coordination server is reporting a health issue: Another message", @@ -1401,8 +1401,8 @@ func TestNetmapDisplayMessageIntegration(t *testing.T) { } want := map[health.WarnableCode]health.UnhealthyState{ - "test-message": { - WarnableCode: "test-message", + "control-health.test-message": { + WarnableCode: "control-health.test-message", Title: "Testing", Text: "This is a test message", Severity: health.SeverityHigh, diff --git a/health/health_test.go b/health/health_test.go index aa519e92c081e..0f1140f621bb4 100644 --- a/health/health_test.go +++ b/health/health_test.go @@ -468,14 +468,14 @@ func TestControlHealth(t *testing.T) { baseStrs := ht.Strings() msgs := map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ - "control-health-test": { + "test": { Title: "Control health message", Text: "Extra help.", }, - "control-health-title": { + "title": { Title: "Control health title only", }, - "control-health-with-action": { + "with-action": { Title: "Control health message", Text: "Extra help.", PrimaryAction: &tailcfg.DisplayMessageAction{ @@ -488,19 +488,19 @@ func TestControlHealth(t *testing.T) { t.Run("Warnings", func(t *testing.T) { wantWarns := map[WarnableCode]UnhealthyState{ - "control-health-test": { - WarnableCode: "control-health-test", + "control-health.test": { + WarnableCode: "control-health.test", Severity: SeverityMedium, Title: "Control health message", Text: "Extra help.", }, - "control-health-title": { - WarnableCode: "control-health-title", + "control-health.title": { + WarnableCode: "control-health.title", Severity: SeverityMedium, Title: "Control health title only", }, - "control-health-with-action": { - WarnableCode: "control-health-with-action", + "control-health.with-action": { + WarnableCode: "control-health.with-action", Severity: SeverityMedium, Title: "Control health message", Text: "Extra help.", diff --git a/health/state.go b/health/state.go index cec967931af92..b5e6a8a3894d8 100644 --- a/health/state.go +++ b/health/state.go @@ -112,7 +112,7 @@ func (t *Tracker) CurrentState() *State { for id, msg := range t.lastNotifiedControlMessages { state := UnhealthyState{ - WarnableCode: WarnableCode(id), + WarnableCode: WarnableCode("control-health." + id), Severity: severityFromTailcfg(msg.Severity), Title: msg.Title, Text: msg.Text, diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index f14ac037cfde5..281d0e9c4eb20 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "math" "net" "net/http" @@ -5422,10 +5423,11 @@ func TestDisplayMessages(t *testing.T) { }) state := ht.CurrentState() - _, ok := state.Warnings["test-message"] + wantID := health.WarnableCode("control-health.test-message") + _, ok := state.Warnings[wantID] if !ok { - t.Error("no warning found with id 'test-message'") + t.Errorf("no warning found with id %q", wantID) } } @@ -5455,14 +5457,15 @@ func TestDisplayMessagesURLFilter(t *testing.T) { }) state := ht.CurrentState() - got, ok := state.Warnings["test-message"] + wantID := health.WarnableCode("control-health.test-message") + got, ok := state.Warnings[wantID] if !ok { - t.Fatal("no warning found with id 'test-message'") + t.Fatalf("no warning found with id %q", wantID) } want := health.UnhealthyState{ - WarnableCode: "test-message", + WarnableCode: wantID, Title: "Testing", Severity: health.SeverityHigh, } @@ -5494,12 +5497,14 @@ func TestDisplayMessageIPNBus(t *testing.T) { }, } + wantID := health.WarnableCode("control-health.test-message") + for _, tt := range []test{ { name: "older-client-no-actions", mask: 0, wantWarning: health.UnhealthyState{ - WarnableCode: "test-message", + WarnableCode: wantID, Severity: health.SeverityMedium, Title: "Message title", Text: "Message text. Learn more: https://example.com", // PrimaryAction appended to text @@ -5510,7 +5515,7 @@ func TestDisplayMessageIPNBus(t *testing.T) { name: "new-client-with-actions", mask: ipn.NotifyHealthActions, wantWarning: health.UnhealthyState{ - WarnableCode: "test-message", + WarnableCode: wantID, Severity: health.SeverityMedium, Title: "Message title", Text: "Message text.", @@ -5530,17 +5535,20 @@ func TestDisplayMessageIPNBus(t *testing.T) { ipnWatcher := newNotificationWatcher(t, lb, nil) ipnWatcher.watch(tt.mask, []wantedNotification{{ - name: "test", + name: fmt.Sprintf("warning with ID %q", wantID), cond: func(_ testing.TB, _ ipnauth.Actor, n *ipn.Notify) bool { if n.Health == nil { return false } - got, ok := n.Health.Warnings["test-message"] + got, ok := n.Health.Warnings[wantID] if ok { if diff := cmp.Diff(tt.wantWarning, got); diff != "" { t.Errorf("unexpected warning details (-want/+got):\n%s", diff) return true // we failed the test so tell the watcher we've seen what we need to to stop it waiting } + } else { + got := slices.Collect(maps.Keys(n.Health.Warnings)) + t.Logf("saw warnings: %v", got) } return ok }, From 4456f77af71367e52565a76dd58a796fa108e3f8 Mon Sep 17 00:00:00 2001 From: Tom Meadows Date: Mon, 9 Jun 2025 11:13:03 +0100 Subject: [PATCH 040/263] cmd/k8s-operator: explicitly set tcp on VIPService port configuration for Ingress with ProxyGroup (#16199) Updates tailscale/corp#24795 Signed-off-by: chaosinthecrd --- cmd/k8s-operator/ingress-for-pg.go | 4 ++-- cmd/k8s-operator/ingress-for-pg_test.go | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 4779014f37bc7..66d74292b2207 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -318,9 +318,9 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin tags = strings.Split(tstr, ",") } - tsSvcPorts := []string{"443"} // always 443 for Ingress + tsSvcPorts := []string{"tcp:443"} // always 443 for Ingress if isHTTPEndpointEnabled(ing) { - tsSvcPorts = append(tsSvcPorts, "80") + tsSvcPorts = append(tsSvcPorts, "tcp:80") } tsSvc := &tailscale.VIPService{ diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 9ce90f7716068..b487d660cfa2f 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -68,7 +68,7 @@ func TestIngressPGReconciler(t *testing.T) { populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net") expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:my-svc", false) - verifyTailscaleService(t, ft, "svc:my-svc", []string{"443"}) + verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) verifyTailscaledConfig(t, fc, []string{"svc:my-svc"}) // Verify that Role and RoleBinding have been created for the first Ingress. @@ -130,7 +130,7 @@ func TestIngressPGReconciler(t *testing.T) { populateTLSSecret(context.Background(), fc, "test-pg", "my-other-svc.ts.net") expectReconciled(t, ingPGR, "default", "my-other-ingress") verifyServeConfig(t, fc, "svc:my-other-svc", false) - verifyTailscaleService(t, ft, "svc:my-other-svc", []string{"443"}) + verifyTailscaleService(t, ft, "svc:my-other-svc", []string{"tcp:443"}) // Verify that Role and RoleBinding have been created for the first Ingress. // Do not verify the cert Secret as that was already verified implicitly above. @@ -139,7 +139,7 @@ func TestIngressPGReconciler(t *testing.T) { // Verify first Ingress is still working verifyServeConfig(t, fc, "svc:my-svc", false) - verifyTailscaleService(t, ft, "svc:my-svc", []string{"443"}) + verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) verifyTailscaledConfig(t, fc, []string{"svc:my-svc", "svc:my-other-svc"}) @@ -244,7 +244,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) { populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net") expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:my-svc", false) - verifyTailscaleService(t, ft, "svc:my-svc", []string{"443"}) + verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) verifyTailscaledConfig(t, fc, []string{"svc:my-svc"}) // Update the Ingress hostname and make sure the original Tailscale Service is deleted. @@ -255,7 +255,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) { populateTLSSecret(context.Background(), fc, "test-pg", "updated-svc.ts.net") expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:updated-svc", false) - verifyTailscaleService(t, ft, "svc:updated-svc", []string{"443"}) + verifyTailscaleService(t, ft, "svc:updated-svc", []string{"tcp:443"}) verifyTailscaledConfig(t, fc, []string{"svc:updated-svc"}) _, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName("svc:my-svc")) @@ -476,7 +476,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { expectReconciled(t, ingPGR, "default", "test-ingress") populateTLSSecret(context.Background(), fc, "test-pg", "my-svc.ts.net") expectReconciled(t, ingPGR, "default", "test-ingress") - verifyTailscaleService(t, ft, "svc:my-svc", []string{"80", "443"}) + verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:80", "tcp:443"}) verifyServeConfig(t, fc, "svc:my-svc", true) // Verify Ingress status @@ -529,7 +529,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { // Verify reconciliation after removing HTTP expectReconciled(t, ingPGR, "default", "test-ingress") - verifyTailscaleService(t, ft, "svc:my-svc", []string{"443"}) + verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) verifyServeConfig(t, fc, "svc:my-svc", false) // Verify Ingress status From 67b1693c131e9cef21d7e6a0491905a179d46eb2 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 9 Jun 2025 13:17:14 -0700 Subject: [PATCH 041/263] wgengine/magicsock: enable setting relay epAddr's as bestAddr (#16229) relayManager can now hand endpoint a relay epAddr for it to consider as bestAddr. endpoint and Conn disco ping/pong handling are now VNI-aware. Updates tailscale/corp#27502 Updates tailscale/corp#29422 Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 39 ++++++++-- wgengine/magicsock/magicsock.go | 120 ++++++++++++++++++----------- wgengine/magicsock/peermap.go | 33 +++++++- wgengine/magicsock/peermap_test.go | 36 +++++++++ wgengine/magicsock/relaymanager.go | 45 +++++++++-- 5 files changed, 212 insertions(+), 61 deletions(-) create mode 100644 wgengine/magicsock/peermap_test.go diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index faae49a97c83c..bf7758fb8bfab 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -99,6 +99,27 @@ type endpoint struct { relayCapable bool // whether the node is capable of speaking via a [tailscale.com/net/udprelay.Server] } +// relayEndpointReady determines whether the given relay addr should be +// installed as de.bestAddr. It is only called by [relayManager] once it has +// determined addr is functional via [disco.Pong] reception. +func (de *endpoint) relayEndpointReady(addr epAddr, latency time.Duration) { + de.c.mu.Lock() + defer de.c.mu.Unlock() + de.mu.Lock() + defer de.mu.Unlock() + + maybeBetter := addrQuality{addr, latency, pingSizeToPktLen(0, addr)} + if !betterAddr(maybeBetter, de.bestAddr) { + return + } + + // Promote maybeBetter to bestAddr. + // TODO(jwhited): collapse path change logging with endpoint.handlePongConnLocked() + de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v", de.publicKey.ShortString(), de.discoShort(), maybeBetter.epAddr, maybeBetter.wireMTU) + de.setBestAddrLocked(maybeBetter) + de.c.peerMap.setNodeKeyForEpAddr(addr, de.publicKey) +} + func (de *endpoint) setBestAddrLocked(v addrQuality) { if v.epAddr != de.bestAddr.epAddr { de.probeUDPLifetime.resetCycleEndpointLocked() @@ -1575,11 +1596,10 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd de.mu.Lock() defer de.mu.Unlock() - if src.vni.isSet() { - // TODO(jwhited): fall through once [relayManager] is able to set an - // [epAddr] as de.bestAddr. We do not need to reference any - // [endpointState] for Geneve-encapsulated disco, we store nothing - // about them there. + if src.vni.isSet() && src != de.bestAddr.epAddr { + // "src" is not our bestAddr, but [relayManager] might be in the + // middle of probing it, awaiting pong reception. Make it aware. + de.c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(m, di, src) return false } @@ -1605,7 +1625,9 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd now := mono.Now() latency := now.Sub(sp.at) - if !isDerp { + if !isDerp && !src.vni.isSet() { + // Note: we check vni.isSet() as relay [epAddr]'s are not stored in + // endpointState, they are either de.bestAddr or not. st, ok := de.endpointState[sp.to.ap] if !ok { // This is no longer an endpoint we care about. @@ -1643,6 +1665,11 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd if !isDerp { thisPong := addrQuality{sp.to, latency, tstun.WireMTU(pingSizeToPktLen(sp.size, sp.to))} if betterAddr(thisPong, de.bestAddr) { + if src.vni.isSet() { + // This would be unexpected. Switching to a Geneve-encapsulated + // path should only happen in de.relayEndpointReady(). + de.c.logf("[unexpected] switching to Geneve-encapsulated path %v from %v", thisPong, de.bestAddr) + } de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v tx=%x", de.publicKey.ShortString(), de.discoShort(), sp.to, thisPong.wireMTU, m.TxID[:6]) de.debugUpdates.Add(EndpointChange{ When: time.Now(), diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index c446cff2cfd14..2e288211032ce 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2132,28 +2132,10 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src epAddr, di *discoInfo, derpN di.lastPingTime = time.Now() isDerp := src.ap.Addr() == tailcfg.DerpMagicIPAddr - if src.vni.isSet() { - // TODO(jwhited): check for matching [endpoint.bestAddr] once that data - // structure is VNI-aware and [relayManager] can mutate it. We do not - // need to reference any [endpointState] for Geneve-encapsulated disco, - // we store nothing about them there. - c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(dm, di, src) - return - } - - // If we can figure out with certainty which node key this disco - // message is for, eagerly update our IP:port<>node and disco<>node - // mappings to make p2p path discovery faster in simple - // cases. Without this, disco would still work, but would be - // reliant on DERP call-me-maybe to establish the disco<>node - // mapping, and on subsequent disco handlePongConnLocked to establish - // the IP:port<>disco mapping. - if nk, ok := c.unambiguousNodeKeyOfPingLocked(dm, di.discoKey, derpNodeSrc); ok { - if !isDerp { - c.peerMap.setNodeKeyForEpAddr(src, nk) - } - } - + // numNodes tracks how many nodes (node keys) are associated with the disco + // key tied to this inbound ping. Multiple nodes may share the same disco + // key in the case of node sharing and users switching accounts. + var numNodes int // If we got a ping over DERP, then derpNodeSrc is non-zero and we reply // over DERP (in which case ipDst is also a DERP address). // But if the ping was over UDP (ipDst is not a DERP address), then dstKey @@ -2161,35 +2143,81 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src epAddr, di *discoInfo, derpN // a dstKey if the dst ip:port is DERP. dstKey := derpNodeSrc - // Remember this route if not present. - var numNodes int - var dup bool - if isDerp { - if ep, ok := c.peerMap.endpointForNodeKey(derpNodeSrc); ok { - if ep.addCandidateEndpoint(src.ap, dm.TxID) { - return + switch { + case src.vni.isSet(): + if isDerp { + c.logf("[unexpected] got Geneve-encapsulated disco ping from %v/%v over DERP", src, derpNodeSrc) + return + } + + var bestEpAddr epAddr + var discoKey key.DiscoPublic + ep, ok := c.peerMap.endpointForEpAddr(src) + if ok { + ep.mu.Lock() + bestEpAddr = ep.bestAddr.epAddr + ep.mu.Unlock() + disco := ep.disco.Load() + if disco != nil { + discoKey = disco.key } + } + + if src == bestEpAddr && discoKey == di.discoKey { + // We have an associated endpoint with src as its bestAddr. Set + // numNodes so we TX a pong further down. numNodes = 1 + } else { + // We have no [endpoint] in the [peerMap] for this relay [epAddr] + // using it as a bestAddr. [relayManager] might be in the middle of + // probing it or attempting to set it as best via + // [endpoint.relayEndpointReady()]. Make [relayManager] aware. + c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(dm, di, src) + return } - } else { - c.peerMap.forEachEndpointWithDiscoKey(di.discoKey, func(ep *endpoint) (keepGoing bool) { - if ep.addCandidateEndpoint(src.ap, dm.TxID) { - dup = true - return false - } - numNodes++ - if numNodes == 1 && dstKey.IsZero() { - dstKey = ep.publicKey + default: // no VNI + // If we can figure out with certainty which node key this disco + // message is for, eagerly update our [epAddr]<>node and disco<>node + // mappings to make p2p path discovery faster in simple + // cases. Without this, disco would still work, but would be + // reliant on DERP call-me-maybe to establish the disco<>node + // mapping, and on subsequent disco handlePongConnLocked to establish + // the IP:port<>disco mapping. + if nk, ok := c.unambiguousNodeKeyOfPingLocked(dm, di.discoKey, derpNodeSrc); ok { + if !isDerp { + c.peerMap.setNodeKeyForEpAddr(src, nk) } - return true - }) - if dup { - return } - if numNodes > 1 { - // Zero it out if it's ambiguous, so sendDiscoMessage logging - // isn't confusing. - dstKey = key.NodePublic{} + + // Remember this route if not present. + var dup bool + if isDerp { + if ep, ok := c.peerMap.endpointForNodeKey(derpNodeSrc); ok { + if ep.addCandidateEndpoint(src.ap, dm.TxID) { + return + } + numNodes = 1 + } + } else { + c.peerMap.forEachEndpointWithDiscoKey(di.discoKey, func(ep *endpoint) (keepGoing bool) { + if ep.addCandidateEndpoint(src.ap, dm.TxID) { + dup = true + return false + } + numNodes++ + if numNodes == 1 && dstKey.IsZero() { + dstKey = ep.publicKey + } + return true + }) + if dup { + return + } + if numNodes > 1 { + // Zero it out if it's ambiguous, so sendDiscoMessage logging + // isn't confusing. + dstKey = key.NodePublic{} + } } } diff --git a/wgengine/magicsock/peermap.go b/wgengine/magicsock/peermap.go index 04d5de8c97e06..838905396002d 100644 --- a/wgengine/magicsock/peermap.go +++ b/wgengine/magicsock/peermap.go @@ -36,6 +36,18 @@ type peerMap struct { byEpAddr map[epAddr]*peerInfo byNodeID map[tailcfg.NodeID]*peerInfo + // relayEpAddrByNodeKey ensures we only hold a single relay + // [epAddr] (vni.isSet()) for a given node key in byEpAddr, vs letting them + // grow unbounded. Relay [epAddr]'s are dynamically created by + // [relayManager] during path discovery, and are only useful to track in + // peerMap so long as they are the endpoint.bestAddr. [relayManager] handles + // all creation and initial probing responsibilities otherwise, and it does + // not depend on [peerMap]. + // + // Note: This doesn't address unbounded growth of non-relay epAddr's in + // byEpAddr. That issue is being tracked in http://go/corp/29422. + relayEpAddrByNodeKey map[key.NodePublic]epAddr + // nodesOfDisco contains the set of nodes that are using a // DiscoKey. Usually those sets will be just one node. nodesOfDisco map[key.DiscoPublic]set.Set[key.NodePublic] @@ -43,10 +55,11 @@ type peerMap struct { func newPeerMap() peerMap { return peerMap{ - byNodeKey: map[key.NodePublic]*peerInfo{}, - byEpAddr: map[epAddr]*peerInfo{}, - byNodeID: map[tailcfg.NodeID]*peerInfo{}, - nodesOfDisco: map[key.DiscoPublic]set.Set[key.NodePublic]{}, + byNodeKey: map[key.NodePublic]*peerInfo{}, + byEpAddr: map[epAddr]*peerInfo{}, + byNodeID: map[tailcfg.NodeID]*peerInfo{}, + relayEpAddrByNodeKey: map[key.NodePublic]epAddr{}, + nodesOfDisco: map[key.DiscoPublic]set.Set[key.NodePublic]{}, } } @@ -171,8 +184,19 @@ func (m *peerMap) setNodeKeyForEpAddr(addr epAddr, nk key.NodePublic) { if pi := m.byEpAddr[addr]; pi != nil { delete(pi.epAddrs, addr) delete(m.byEpAddr, addr) + if addr.vni.isSet() { + delete(m.relayEpAddrByNodeKey, pi.ep.publicKey) + } } if pi, ok := m.byNodeKey[nk]; ok { + if addr.vni.isSet() { + relay, ok := m.relayEpAddrByNodeKey[nk] + if ok { + delete(pi.epAddrs, relay) + delete(m.byEpAddr, relay) + } + m.relayEpAddrByNodeKey[nk] = addr + } pi.epAddrs.Add(addr) m.byEpAddr[addr] = pi } @@ -204,4 +228,5 @@ func (m *peerMap) deleteEndpoint(ep *endpoint) { for ip := range pi.epAddrs { delete(m.byEpAddr, ip) } + delete(m.relayEpAddrByNodeKey, ep.publicKey) } diff --git a/wgengine/magicsock/peermap_test.go b/wgengine/magicsock/peermap_test.go new file mode 100644 index 0000000000000..52504272ff8e2 --- /dev/null +++ b/wgengine/magicsock/peermap_test.go @@ -0,0 +1,36 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package magicsock + +import ( + "net/netip" + "testing" + + "tailscale.com/types/key" +) + +func Test_peerMap_oneRelayEpAddrPerNK(t *testing.T) { + pm := newPeerMap() + nk := key.NewNode().Public() + ep := &endpoint{ + nodeID: 1, + publicKey: nk, + } + ed := &endpointDisco{key: key.NewDisco().Public()} + ep.disco.Store(ed) + pm.upsertEndpoint(ep, key.DiscoPublic{}) + vni := virtualNetworkID{} + vni.set(1) + relayEpAddrA := epAddr{ap: netip.MustParseAddrPort("127.0.0.1:1"), vni: vni} + relayEpAddrB := epAddr{ap: netip.MustParseAddrPort("127.0.0.1:2"), vni: vni} + pm.setNodeKeyForEpAddr(relayEpAddrA, nk) + pm.setNodeKeyForEpAddr(relayEpAddrB, nk) + if len(pm.byEpAddr) != 1 { + t.Fatalf("expected 1 epAddr in byEpAddr, got: %d", len(pm.byEpAddr)) + } + got := pm.relayEpAddrByNodeKey[nk] + if got != relayEpAddrB { + t.Fatalf("expected relay epAddr %v, got: %v", relayEpAddrB, got) + } +} diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 177eed3556dfb..5cb43cd856d39 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -24,6 +24,11 @@ import ( // relayManager manages allocation, handshaking, and initial probing (disco // ping/pong) of [tailscale.com/net/udprelay.Server] endpoints. The zero value // is ready for use. +// +// [relayManager] methods can be called by [Conn] and [endpoint] while their .mu +// mutexes are held. Therefore, in order to avoid deadlocks, [relayManager] must +// never attempt to acquire those mutexes, including synchronous calls back +// towards [Conn] or [endpoint] methods that acquire them. type relayManager struct { initOnce sync.Once @@ -164,6 +169,7 @@ func (r *relayManager) runLoop() { } type relayHandshakeDiscoMsgEvent struct { + conn *Conn // for access to [Conn] if there is no associated [relayHandshakeWork] msg disco.Message disco key.DiscoPublic from netip.AddrPort @@ -366,7 +372,7 @@ func (r *relayManager) handleRxHandshakeDiscoMsgRunLoop(event relayHandshakeDisc ok bool ) apv := addrPortVNI{event.from, event.vni} - switch event.msg.(type) { + switch msg := event.msg.(type) { case *disco.BindUDPRelayEndpointChallenge: work, ok = r.handshakeWorkByServerDiscoVNI[serverDiscoVNI{event.disco, event.vni}] if !ok { @@ -392,7 +398,29 @@ func (r *relayManager) handleRxHandshakeDiscoMsgRunLoop(event relayHandshakeDisc // Update state so that future ping/pong will route to 'work'. r.handshakeWorkAwaitingPong[work] = apv r.addrPortVNIToHandshakeWork[apv] = work - case *disco.Ping, *disco.Pong: + case *disco.Ping: + // Always TX a pong. We might not have any associated work if ping + // reception raced with our call to [endpoint.relayEndpointReady()], so + // err on the side of enabling the remote side to use this path. + // + // Conn.handlePingLocked() makes efforts to suppress duplicate pongs + // where the same ping can be received both via raw socket and UDP + // socket on Linux. We make no such efforts here as the raw socket BPF + // program does not support Geneve-encapsulated disco, and is also + // disabled by default. + vni := virtualNetworkID{} + vni.set(event.vni) + go event.conn.sendDiscoMessage(epAddr{ap: event.from, vni: vni}, key.NodePublic{}, event.disco, &disco.Pong{ + TxID: msg.TxID, + Src: event.from, + }, discoVerboseLog) + + work, ok = r.addrPortVNIToHandshakeWork[apv] + if !ok { + // No outstanding work tied to this [addrPortVNI], return early. + return + } + case *disco.Pong: work, ok = r.addrPortVNIToHandshakeWork[apv] if !ok { // No outstanding work tied to this [addrPortVNI], discard. @@ -436,9 +464,13 @@ func (r *relayManager) handleHandshakeWorkDoneRunLoop(done relayEndpointHandshak return } // This relay endpoint is functional. - // TODO(jwhited): Set it on done.work.ep.bestAddr if it is a betterAddr(). - // We also need to conn.peerMap.setNodeKeyForEpAddr(), and ensure we clean - // it up when bestAddr changes, too. + vni := virtualNetworkID{} + vni.set(done.work.se.VNI) + addr := epAddr{ap: done.pongReceivedFrom, vni: vni} + // ep.relayEndpointReady() must be called in a new goroutine to prevent + // deadlocks as it acquires [endpoint] & [Conn] mutexes. See [relayManager] + // docs for details. + go done.work.ep.relayEndpointReady(addr, done.latency) } func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelayServerEndpointEvent) { @@ -613,6 +645,9 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { // latency, so send another ping. Since the handshake is // complete we do not need to send an answer in front of this // one. + // + // We don't need to TX a pong, that was already handled for us + // in handleRxHandshakeDiscoMsgRunLoop(). txPing(msgEvent.from, nil) case *disco.Pong: at, ok := sentPingAt[msg.TxID] From c343bffa72cacc583d2c39dafa9f51ded8a106f7 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 9 Jun 2025 14:49:00 -0700 Subject: [PATCH 042/263] wgengine/relaymanager: don't start runLoop() on init() (#16231) This is simply for consistency with relayManagerInputEvent(), which should be the sole launcher of runLoop(). Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/relaymanager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 5cb43cd856d39..fd3f19dfbe9f8 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -206,7 +206,7 @@ func (r *relayManager) init() { r.newServerEndpointCh = make(chan newRelayServerEndpointEvent) r.rxHandshakeDiscoMsgCh = make(chan relayHandshakeDiscoMsgEvent) r.runLoopStoppedCh = make(chan struct{}, 1) - go r.runLoop() + r.runLoopStoppedCh <- struct{}{} }) } From 9501f66985e9ac375cba7cf427453cc66a55dae8 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 9 Jun 2025 15:37:58 -0700 Subject: [PATCH 043/263] wgengine/magicsock: don't cancel in-progress relayManager work (#16233) It might complete, interrupting it reduces the chances of establishing a relay path. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/relaymanager.go | 62 +++++++++++++++--------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index fd3f19dfbe9f8..2b636dc5758d1 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -112,12 +112,21 @@ type relayEndpointHandshakeWorkDoneEvent struct { latency time.Duration // only relevant if pongReceivedFrom.IsValid() } -// activeWorkRunLoop returns true if there is outstanding allocation or -// handshaking work, otherwise it returns false. -func (r *relayManager) activeWorkRunLoop() bool { +// hasActiveWorkRunLoop returns true if there is outstanding allocation or +// handshaking work for any endpoint, otherwise it returns false. +func (r *relayManager) hasActiveWorkRunLoop() bool { return len(r.allocWorkByEndpoint) > 0 || len(r.handshakeWorkByEndpointByServerDisco) > 0 } +// hasActiveWorkForEndpointRunLoop returns true if there is outstanding +// allocation or handshaking work for the provided endpoint, otherwise it +// returns false. +func (r *relayManager) hasActiveWorkForEndpointRunLoop(ep *endpoint) bool { + _, handshakeWork := r.handshakeWorkByEndpointByServerDisco[ep] + _, allocWork := r.allocWorkByEndpoint[ep] + return handshakeWork || allocWork +} + // runLoop is a form of event loop. It ensures exclusive access to most of // [relayManager] state. func (r *relayManager) runLoop() { @@ -128,9 +137,10 @@ func (r *relayManager) runLoop() { for { select { case ep := <-r.allocateHandshakeCh: - r.stopWorkRunLoop(ep, stopHandshakeWorkOnlyKnownServers) - r.allocateAllServersRunLoop(ep) - if !r.activeWorkRunLoop() { + if !r.hasActiveWorkForEndpointRunLoop(ep) { + r.allocateAllServersRunLoop(ep) + } + if !r.hasActiveWorkRunLoop() { return } case done := <-r.allocateWorkDoneCh: @@ -141,27 +151,27 @@ func (r *relayManager) runLoop() { // overwrite pre-existing keys. delete(r.allocWorkByEndpoint, done.work.ep) } - if !r.activeWorkRunLoop() { + if !r.hasActiveWorkRunLoop() { return } case ep := <-r.cancelWorkCh: - r.stopWorkRunLoop(ep, stopHandshakeWorkAllServers) - if !r.activeWorkRunLoop() { + r.stopWorkRunLoop(ep) + if !r.hasActiveWorkRunLoop() { return } case newServerEndpoint := <-r.newServerEndpointCh: r.handleNewServerEndpointRunLoop(newServerEndpoint) - if !r.activeWorkRunLoop() { + if !r.hasActiveWorkRunLoop() { return } case done := <-r.handshakeWorkDoneCh: r.handleHandshakeWorkDoneRunLoop(done) - if !r.activeWorkRunLoop() { + if !r.hasActiveWorkRunLoop() { return } case discoMsgEvent := <-r.rxHandshakeDiscoMsgCh: r.handleRxHandshakeDiscoMsgRunLoop(discoMsgEvent) - if !r.activeWorkRunLoop() { + if !r.hasActiveWorkRunLoop() { return } } @@ -317,8 +327,8 @@ func relayManagerInputEvent[T any](r *relayManager, ctx context.Context, eventCh } // allocateAndHandshakeAllServers kicks off allocation and handshaking of relay -// endpoints for 'ep' on all known relay servers, canceling any existing -// in-progress work. +// endpoints for 'ep' on all known relay servers if there is no outstanding +// work. func (r *relayManager) allocateAndHandshakeAllServers(ep *endpoint) { relayManagerInputEvent(r, nil, &r.allocateHandshakeCh, ep) } @@ -328,18 +338,9 @@ func (r *relayManager) stopWork(ep *endpoint) { relayManagerInputEvent(r, nil, &r.cancelWorkCh, ep) } -// stopHandshakeWorkFilter represents filters for handshake work cancellation -type stopHandshakeWorkFilter bool - -const ( - stopHandshakeWorkAllServers stopHandshakeWorkFilter = false - stopHandshakeWorkOnlyKnownServers = true -) - // stopWorkRunLoop cancels & clears outstanding allocation and handshaking -// work for 'ep'. Handshake work cancellation is subject to the filter supplied -// in 'f'. -func (r *relayManager) stopWorkRunLoop(ep *endpoint, f stopHandshakeWorkFilter) { +// work for 'ep'. +func (r *relayManager) stopWorkRunLoop(ep *endpoint) { allocWork, ok := r.allocWorkByEndpoint[ep] if ok { allocWork.cancel() @@ -348,13 +349,10 @@ func (r *relayManager) stopWorkRunLoop(ep *endpoint, f stopHandshakeWorkFilter) } byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[ep] if ok { - for disco, handshakeWork := range byServerDisco { - _, knownServer := r.serversByDisco[disco] - if knownServer || f == stopHandshakeWorkAllServers { - handshakeWork.cancel() - done := <-handshakeWork.doneCh - r.handleHandshakeWorkDoneRunLoop(done) - } + for _, handshakeWork := range byServerDisco { + handshakeWork.cancel() + done := <-handshakeWork.doneCh + r.handleHandshakeWorkDoneRunLoop(done) } } } From cc8dc9e4dcc83fced123d0268e62d4530c515ac6 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 9 Jun 2025 16:12:12 -0700 Subject: [PATCH 044/263] types/netmap: fix NodeMutationEndpoints docs typo (#16234) Updates #cleanup Signed-off-by: Jordan Whited --- types/netmap/nodemut.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/netmap/nodemut.go b/types/netmap/nodemut.go index ccbdeae3f5ec7..f4de1bf0b8f02 100644 --- a/types/netmap/nodemut.go +++ b/types/netmap/nodemut.go @@ -37,7 +37,7 @@ func (m NodeMutationDERPHome) Apply(n *tailcfg.Node) { n.HomeDERP = m.DERPRegion } -// NodeMutation is a NodeMutation that says a node's endpoints have changed. +// NodeMutationEndpoints is a NodeMutation that says a node's endpoints have changed. type NodeMutationEndpoints struct { mutatingNodeID Endpoints []netip.AddrPort From db34cdcfe7d4825ed8a7edec3f6c0164b3c85b5a Mon Sep 17 00:00:00 2001 From: Anton Tolchanov Date: Thu, 22 May 2025 20:12:59 +0100 Subject: [PATCH 045/263] cmd/tailscale/cli: add a risk message about rp_filter We already present a health warning about this, but it is easy to miss on a server when blackholing traffic makes it unreachable. In addition to a health warning, present a risk message when exit node is enabled. Example: ``` $ tailscale up --exit-node=lizard The following issues on your machine will likely make usage of exit nodes impossible: - interface "ens4" has strict reverse-path filtering enabled - interface "tailscale0" has strict reverse-path filtering enabled Please set rp_filter=2 instead of rp_filter=1; see https://github.com/tailscale/tailscale/issues/3310 To skip this warning, use --accept-risk=linux-strict-rp-filter $ ``` Updates #3310 Signed-off-by: Anton Tolchanov --- client/local/local.go | 19 ++++ cmd/k8s-operator/depaware.txt | 2 +- cmd/tailscale/cli/risks.go | 21 ++++- cmd/tailscale/cli/set.go | 3 + cmd/tailscale/cli/up.go | 3 + cmd/tailscaled/depaware.txt | 2 +- health/healthmsg/healthmsg.go | 1 + ipn/ipnlocal/local.go | 3 +- ipn/localapi/localapi.go | 158 ++++++++++++++++++++-------------- tsnet/depaware.txt | 2 +- 10 files changed, 143 insertions(+), 71 deletions(-) diff --git a/client/local/local.go b/client/local/local.go index 0e4d495d3dd18..7a3a4b703a75e 100644 --- a/client/local/local.go +++ b/client/local/local.go @@ -788,6 +788,25 @@ func (lc *Client) CheckUDPGROForwarding(ctx context.Context) error { return nil } +// CheckReversePathFiltering asks the local Tailscale daemon whether strict +// reverse path filtering is enabled, which would break exit node usage on Linux. +func (lc *Client) CheckReversePathFiltering(ctx context.Context) error { + body, err := lc.get200(ctx, "/localapi/v0/check-reverse-path-filtering") + if err != nil { + return err + } + var jres struct { + Warning string + } + if err := json.Unmarshal(body, &jres); err != nil { + return fmt.Errorf("invalid JSON from check-reverse-path-filtering: %w", err) + } + if jres.Warning != "" { + return errors.New(jres.Warning) + } + return nil +} + // SetUDPGROForwarding enables UDP GRO forwarding for the main interface of this // node. This can be done to improve performance of tailnet nodes acting as exit // nodes or subnet routers. diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 2e467843ac07e..36c5184c3b44a 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -796,7 +796,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/ipn/ipnext+ tailscale.com/health from tailscale.com/control/controlclient+ - tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ tailscale.com/hostinfo from tailscale.com/client/web+ tailscale.com/internal/client/tailscale from tailscale.com/cmd/k8s-operator tailscale.com/internal/noiseconn from tailscale.com/control/controlclient diff --git a/cmd/tailscale/cli/risks.go b/cmd/tailscale/cli/risks.go index c36ffafaeb11a..9b03025a83a0b 100644 --- a/cmd/tailscale/cli/risks.go +++ b/cmd/tailscale/cli/risks.go @@ -4,15 +4,18 @@ package cli import ( + "context" "errors" "flag" "fmt" "os" "os/signal" + "runtime" "strings" "syscall" "time" + "tailscale.com/ipn" "tailscale.com/util/testenv" ) @@ -20,11 +23,12 @@ var ( riskTypes []string riskLoseSSH = registerRiskType("lose-ssh") riskMacAppConnector = registerRiskType("mac-app-connector") + riskStrictRPFilter = registerRiskType("linux-strict-rp-filter") riskAll = registerRiskType("all") ) const riskMacAppConnectorMessage = ` -You are trying to configure an app connector on macOS, which is not officially supported due to system limitations. This may result in performance and reliability issues. +You are trying to configure an app connector on macOS, which is not officially supported due to system limitations. This may result in performance and reliability issues. Do not use a macOS app connector for any mission-critical purposes. For the best experience, Linux is the only recommended platform for app connectors. ` @@ -89,3 +93,18 @@ func presentRiskToUser(riskType, riskMessage, acceptedRisks string) error { printf("\r%s\r", strings.Repeat(" ", msgLen)) return errAborted } + +// checkExitNodeRisk checks if the user is using an exit node on Linux and +// whether reverse path filtering is enabled. If so, it presents a risk message. +func checkExitNodeRisk(ctx context.Context, prefs *ipn.Prefs, acceptedRisks string) error { + if runtime.GOOS != "linux" { + return nil + } + if !prefs.ExitNodeIP.IsValid() && prefs.ExitNodeID == "" { + return nil + } + if err := localClient.CheckReversePathFiltering(ctx); err != nil { + return presentRiskToUser(riskStrictRPFilter, err.Error(), acceptedRisks) + } + return nil +} diff --git a/cmd/tailscale/cli/set.go b/cmd/tailscale/cli/set.go index aa59666987532..66e74d77ff5a5 100644 --- a/cmd/tailscale/cli/set.go +++ b/cmd/tailscale/cli/set.go @@ -183,6 +183,9 @@ func runSet(ctx context.Context, args []string) (retErr error) { } warnOnAdvertiseRouts(ctx, &maskedPrefs.Prefs) + if err := checkExitNodeRisk(ctx, &maskedPrefs.Prefs, setArgs.acceptedRisks); err != nil { + return err + } var advertiseExitNodeSet, advertiseRoutesSet bool setFlagSet.Visit(func(f *flag.Flag) { updateMaskedPrefsFromUpOrSetFlag(maskedPrefs, f.Name) diff --git a/cmd/tailscale/cli/up.go b/cmd/tailscale/cli/up.go index e4bb6f576759e..37cdab754db18 100644 --- a/cmd/tailscale/cli/up.go +++ b/cmd/tailscale/cli/up.go @@ -481,6 +481,9 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE } warnOnAdvertiseRouts(ctx, prefs) + if err := checkExitNodeRisk(ctx, prefs, upArgs.acceptedRisks); err != nil { + return err + } curPrefs, err := localClient.GetPrefs(ctx) if err != nil { diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index c6011a12cc7ef..387b944c1f4ce 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -281,7 +281,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/feature/tpm from tailscale.com/feature/condregister tailscale.com/feature/wakeonlan from tailscale.com/feature/condregister tailscale.com/health from tailscale.com/control/controlclient+ - tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ tailscale.com/hostinfo from tailscale.com/client/web+ tailscale.com/internal/noiseconn from tailscale.com/control/controlclient tailscale.com/ipn from tailscale.com/client/local+ diff --git a/health/healthmsg/healthmsg.go b/health/healthmsg/healthmsg.go index 6c237678eaf62..2384103738cf3 100644 --- a/health/healthmsg/healthmsg.go +++ b/health/healthmsg/healthmsg.go @@ -12,4 +12,5 @@ const ( TailscaleSSHOnBut = "Tailscale SSH enabled, but " // + ... something from caller LockedOut = "this node is locked out; it will not have connectivity until it is signed. For more info, see https://tailscale.com/s/locked-out" WarnExitNodeUsage = "The following issues on your machine will likely make usage of exit nodes impossible" + DisableRPFilter = "Please set rp_filter=2 instead of rp_filter=1; see https://github.com/tailscale/tailscale/issues/3310" ) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 4b5accd857da7..88adb397354a8 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -4112,9 +4112,8 @@ func updateExitNodeUsageWarning(p ipn.PrefsView, state *netmon.State, healthTrac var msg string if p.ExitNodeIP().IsValid() || p.ExitNodeID() != "" { warn, _ := netutil.CheckReversePathFiltering(state) - const comment = "please set rp_filter=2 instead of rp_filter=1; see https://github.com/tailscale/tailscale/issues/3310" if len(warn) > 0 { - msg = fmt.Sprintf("%s: %v, %s", healthmsg.WarnExitNodeUsage, warn, comment) + msg = fmt.Sprintf("%s: %v, %s", healthmsg.WarnExitNodeUsage, warn, healthmsg.DisableRPFilter) } } if len(msg) > 0 { diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index 99cb7c95b4dff..78f95b2b113dc 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -32,6 +32,7 @@ import ( "tailscale.com/clientupdate" "tailscale.com/drive" "tailscale.com/envknob" + "tailscale.com/health/healthmsg" "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/ipnauth" @@ -82,71 +83,72 @@ var handler = map[string]LocalAPIHandler{ // The other /localapi/v0/NAME handlers are exact matches and contain only NAME // without a trailing slash: - "alpha-set-device-attrs": (*Handler).serveSetDeviceAttrs, // see tailscale/corp#24690 - "bugreport": (*Handler).serveBugReport, - "check-ip-forwarding": (*Handler).serveCheckIPForwarding, - "check-prefs": (*Handler).serveCheckPrefs, - "check-udp-gro-forwarding": (*Handler).serveCheckUDPGROForwarding, - "component-debug-logging": (*Handler).serveComponentDebugLogging, - "debug": (*Handler).serveDebug, - "debug-derp-region": (*Handler).serveDebugDERPRegion, - "debug-dial-types": (*Handler).serveDebugDialTypes, - "debug-log": (*Handler).serveDebugLog, - "debug-packet-filter-matches": (*Handler).serveDebugPacketFilterMatches, - "debug-packet-filter-rules": (*Handler).serveDebugPacketFilterRules, - "debug-peer-endpoint-changes": (*Handler).serveDebugPeerEndpointChanges, - "debug-portmap": (*Handler).serveDebugPortmap, - "derpmap": (*Handler).serveDERPMap, - "dev-set-state-store": (*Handler).serveDevSetStateStore, - "dial": (*Handler).serveDial, - "disconnect-control": (*Handler).disconnectControl, - "dns-osconfig": (*Handler).serveDNSOSConfig, - "dns-query": (*Handler).serveDNSQuery, - "drive/fileserver-address": (*Handler).serveDriveServerAddr, - "drive/shares": (*Handler).serveShares, - "goroutines": (*Handler).serveGoroutines, - "handle-push-message": (*Handler).serveHandlePushMessage, - "id-token": (*Handler).serveIDToken, - "login-interactive": (*Handler).serveLoginInteractive, - "logout": (*Handler).serveLogout, - "logtap": (*Handler).serveLogTap, - "metrics": (*Handler).serveMetrics, - "ping": (*Handler).servePing, - "pprof": (*Handler).servePprof, - "prefs": (*Handler).servePrefs, - "query-feature": (*Handler).serveQueryFeature, - "reload-config": (*Handler).reloadConfig, - "reset-auth": (*Handler).serveResetAuth, - "serve-config": (*Handler).serveServeConfig, - "set-dns": (*Handler).serveSetDNS, - "set-expiry-sooner": (*Handler).serveSetExpirySooner, - "set-gui-visible": (*Handler).serveSetGUIVisible, - "set-push-device-token": (*Handler).serveSetPushDeviceToken, - "set-udp-gro-forwarding": (*Handler).serveSetUDPGROForwarding, - "set-use-exit-node-enabled": (*Handler).serveSetUseExitNodeEnabled, - "start": (*Handler).serveStart, - "status": (*Handler).serveStatus, - "suggest-exit-node": (*Handler).serveSuggestExitNode, - "tka/affected-sigs": (*Handler).serveTKAAffectedSigs, - "tka/cosign-recovery-aum": (*Handler).serveTKACosignRecoveryAUM, - "tka/disable": (*Handler).serveTKADisable, - "tka/force-local-disable": (*Handler).serveTKALocalDisable, - "tka/generate-recovery-aum": (*Handler).serveTKAGenerateRecoveryAUM, - "tka/init": (*Handler).serveTKAInit, - "tka/log": (*Handler).serveTKALog, - "tka/modify": (*Handler).serveTKAModify, - "tka/sign": (*Handler).serveTKASign, - "tka/status": (*Handler).serveTKAStatus, - "tka/submit-recovery-aum": (*Handler).serveTKASubmitRecoveryAUM, - "tka/verify-deeplink": (*Handler).serveTKAVerifySigningDeeplink, - "tka/wrap-preauth-key": (*Handler).serveTKAWrapPreauthKey, - "update/check": (*Handler).serveUpdateCheck, - "update/install": (*Handler).serveUpdateInstall, - "update/progress": (*Handler).serveUpdateProgress, - "upload-client-metrics": (*Handler).serveUploadClientMetrics, - "usermetrics": (*Handler).serveUserMetrics, - "watch-ipn-bus": (*Handler).serveWatchIPNBus, - "whois": (*Handler).serveWhoIs, + "alpha-set-device-attrs": (*Handler).serveSetDeviceAttrs, // see tailscale/corp#24690 + "bugreport": (*Handler).serveBugReport, + "check-ip-forwarding": (*Handler).serveCheckIPForwarding, + "check-prefs": (*Handler).serveCheckPrefs, + "check-reverse-path-filtering": (*Handler).serveCheckReversePathFiltering, + "check-udp-gro-forwarding": (*Handler).serveCheckUDPGROForwarding, + "component-debug-logging": (*Handler).serveComponentDebugLogging, + "debug": (*Handler).serveDebug, + "debug-derp-region": (*Handler).serveDebugDERPRegion, + "debug-dial-types": (*Handler).serveDebugDialTypes, + "debug-log": (*Handler).serveDebugLog, + "debug-packet-filter-matches": (*Handler).serveDebugPacketFilterMatches, + "debug-packet-filter-rules": (*Handler).serveDebugPacketFilterRules, + "debug-peer-endpoint-changes": (*Handler).serveDebugPeerEndpointChanges, + "debug-portmap": (*Handler).serveDebugPortmap, + "derpmap": (*Handler).serveDERPMap, + "dev-set-state-store": (*Handler).serveDevSetStateStore, + "dial": (*Handler).serveDial, + "disconnect-control": (*Handler).disconnectControl, + "dns-osconfig": (*Handler).serveDNSOSConfig, + "dns-query": (*Handler).serveDNSQuery, + "drive/fileserver-address": (*Handler).serveDriveServerAddr, + "drive/shares": (*Handler).serveShares, + "goroutines": (*Handler).serveGoroutines, + "handle-push-message": (*Handler).serveHandlePushMessage, + "id-token": (*Handler).serveIDToken, + "login-interactive": (*Handler).serveLoginInteractive, + "logout": (*Handler).serveLogout, + "logtap": (*Handler).serveLogTap, + "metrics": (*Handler).serveMetrics, + "ping": (*Handler).servePing, + "pprof": (*Handler).servePprof, + "prefs": (*Handler).servePrefs, + "query-feature": (*Handler).serveQueryFeature, + "reload-config": (*Handler).reloadConfig, + "reset-auth": (*Handler).serveResetAuth, + "serve-config": (*Handler).serveServeConfig, + "set-dns": (*Handler).serveSetDNS, + "set-expiry-sooner": (*Handler).serveSetExpirySooner, + "set-gui-visible": (*Handler).serveSetGUIVisible, + "set-push-device-token": (*Handler).serveSetPushDeviceToken, + "set-udp-gro-forwarding": (*Handler).serveSetUDPGROForwarding, + "set-use-exit-node-enabled": (*Handler).serveSetUseExitNodeEnabled, + "start": (*Handler).serveStart, + "status": (*Handler).serveStatus, + "suggest-exit-node": (*Handler).serveSuggestExitNode, + "tka/affected-sigs": (*Handler).serveTKAAffectedSigs, + "tka/cosign-recovery-aum": (*Handler).serveTKACosignRecoveryAUM, + "tka/disable": (*Handler).serveTKADisable, + "tka/force-local-disable": (*Handler).serveTKALocalDisable, + "tka/generate-recovery-aum": (*Handler).serveTKAGenerateRecoveryAUM, + "tka/init": (*Handler).serveTKAInit, + "tka/log": (*Handler).serveTKALog, + "tka/modify": (*Handler).serveTKAModify, + "tka/sign": (*Handler).serveTKASign, + "tka/status": (*Handler).serveTKAStatus, + "tka/submit-recovery-aum": (*Handler).serveTKASubmitRecoveryAUM, + "tka/verify-deeplink": (*Handler).serveTKAVerifySigningDeeplink, + "tka/wrap-preauth-key": (*Handler).serveTKAWrapPreauthKey, + "update/check": (*Handler).serveUpdateCheck, + "update/install": (*Handler).serveUpdateInstall, + "update/progress": (*Handler).serveUpdateProgress, + "upload-client-metrics": (*Handler).serveUploadClientMetrics, + "usermetrics": (*Handler).serveUserMetrics, + "watch-ipn-bus": (*Handler).serveWatchIPNBus, + "whois": (*Handler).serveWhoIs, } // Register registers a new LocalAPI handler for the given name. @@ -1175,6 +1177,32 @@ func (h *Handler) serveCheckIPForwarding(w http.ResponseWriter, r *http.Request) }) } +func (h *Handler) serveCheckReversePathFiltering(w http.ResponseWriter, r *http.Request) { + if !h.PermitRead { + http.Error(w, "reverse path filtering check access denied", http.StatusForbidden) + return + } + var warning string + + state := h.b.Sys().NetMon.Get().InterfaceState() + warn, err := netutil.CheckReversePathFiltering(state) + if err == nil && len(warn) > 0 { + var msg strings.Builder + msg.WriteString(healthmsg.WarnExitNodeUsage + ":\n") + for _, w := range warn { + msg.WriteString("- " + w + "\n") + } + msg.WriteString(healthmsg.DisableRPFilter) + warning = msg.String() + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(struct { + Warning string + }{ + Warning: warning, + }) +} + func (h *Handler) serveCheckUDPGROForwarding(w http.ResponseWriter, r *http.Request) { if !h.PermitRead { http.Error(w, "UDP GRO forwarding check access denied", http.StatusForbidden) diff --git a/tsnet/depaware.txt b/tsnet/depaware.txt index 242cd8f1b36d5..da3175b8c42d2 100644 --- a/tsnet/depaware.txt +++ b/tsnet/depaware.txt @@ -237,7 +237,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/ipn/ipnext+ tailscale.com/health from tailscale.com/control/controlclient+ - tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ tailscale.com/hostinfo from tailscale.com/client/web+ tailscale.com/internal/noiseconn from tailscale.com/control/controlclient tailscale.com/ipn from tailscale.com/client/local+ From e72c528a5fec1abda8a933e35b6c06ebd25d0175 Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Tue, 10 Jun 2025 15:29:42 -0400 Subject: [PATCH 046/263] cmd/{derp,derpprobe},prober,derp: add mesh support to derpprobe (#15414) Add mesh key support to derpprobe for probing derpers with verify set to true. Move MeshKey checking to central point for code reuse. Fix a bad error fmt msg. Fixes tailscale/corp#27294 Fixes tailscale/corp#25756 Signed-off-by: Mike O'Driscoll --- cmd/derper/derper.go | 2 +- cmd/derpprobe/derpprobe.go | 69 +++++++++++++++++++++++++++++ cmd/tsidp/depaware.txt | 2 +- derp/derp_client.go | 15 ++++++- derp/derp_server.go | 9 ++-- derp/derp_test.go | 89 ++++++++++++++++++++++++++------------ prober/derp.go | 42 ++++++++++-------- types/key/derp.go | 22 ++++++++++ 8 files changed, 195 insertions(+), 55 deletions(-) diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 840de3fba671e..7ea404beb50af 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -68,7 +68,7 @@ var ( runDERP = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.") flagHome = flag.String("home", "", "what to serve at the root path. It may be left empty (the default, for a default homepage), \"blank\" for a blank page, or a URL to redirect to") - meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.") + meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It must be 64 lowercase hexadecimal characters; whitespace is trimmed.") meshWith = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.") secretsURL = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key") secretPrefix = flag.String("secrets-path-prefix", "prod/derp", "setec path prefix for \""+setecMeshKeyName+"\" secret for DERP mesh key") diff --git a/cmd/derpprobe/derpprobe.go b/cmd/derpprobe/derpprobe.go index 2723a31aea471..25159d649408e 100644 --- a/cmd/derpprobe/derpprobe.go +++ b/cmd/derpprobe/derpprobe.go @@ -5,23 +5,36 @@ package main import ( + "context" "flag" "fmt" "log" "net/http" "os" + "path" + "path/filepath" "sort" "time" + "github.com/tailscale/setec/client/setec" "tailscale.com/prober" "tailscale.com/tsweb" + "tailscale.com/types/key" "tailscale.com/version" // Support for prometheus varz in tsweb _ "tailscale.com/tsweb/promvarz" ) +const meshKeyEnvVar = "TAILSCALE_DERPER_MESH_KEY" +const setecMeshKeyName = "meshkey" + +func defaultSetecCacheDir() string { + return filepath.Join(os.Getenv("HOME"), ".cache", "derper-secrets") +} + var ( + dev = flag.Bool("dev", false, "run in localhost development mode") derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map") versionFlag = flag.Bool("version", false, "print version and exit") listen = flag.String("listen", ":8030", "HTTP listen address") @@ -37,6 +50,10 @@ var ( qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second") qdPacketTimeout = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings") regionCodeOrID = flag.String("region-code", "", "probe only this region (e.g. 'lax' or '17'); if left blank, all regions will be probed") + meshPSKFile = flag.String("mesh-psk-file", "", "if non-empty, path to file containing the mesh pre-shared key file. It must be 64 lowercase hexadecimal characters; whitespace is trimmed.") + secretsURL = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key") + secretPrefix = flag.String("secrets-path-prefix", "prod/derp", fmt.Sprintf("setec path prefix for \"%s\" secret for DERP mesh key", setecMeshKeyName)) + secretsCacheDir = flag.String("secrets-cache-dir", defaultSetecCacheDir(), "directory to cache setec secrets in (required if --secrets-url is set)") ) func main() { @@ -47,11 +64,16 @@ func main() { } p := prober.New().WithSpread(*spread).WithOnce(*probeOnce).WithMetricNamespace("derpprobe") + meshKey, err := getMeshKey() + if err != nil { + log.Fatalf("failed to get mesh key: %v", err) + } opts := []prober.DERPOpt{ prober.WithMeshProbing(*meshInterval), prober.WithSTUNProbing(*stunInterval), prober.WithTLSProbing(*tlsInterval), prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout), + prober.WithMeshKey(meshKey), } if *bwInterval > 0 { opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address)) @@ -99,6 +121,53 @@ func main() { log.Fatal(http.ListenAndServe(*listen, mux)) } +func getMeshKey() (key.DERPMesh, error) { + var meshKey string + + if *dev { + meshKey = os.Getenv(meshKeyEnvVar) + if meshKey == "" { + log.Printf("No mesh key specified for dev via %s\n", meshKeyEnvVar) + } else { + log.Printf("Set mesh key from %s\n", meshKeyEnvVar) + } + } else if *secretsURL != "" { + meshKeySecret := path.Join(*secretPrefix, setecMeshKeyName) + fc, err := setec.NewFileCache(*secretsCacheDir) + if err != nil { + log.Fatalf("NewFileCache: %v", err) + } + log.Printf("Setting up setec store from %q", *secretsURL) + st, err := setec.NewStore(context.Background(), + setec.StoreConfig{ + Client: setec.Client{Server: *secretsURL}, + Secrets: []string{ + meshKeySecret, + }, + Cache: fc, + }) + if err != nil { + log.Fatalf("NewStore: %v", err) + } + meshKey = st.Secret(meshKeySecret).GetString() + log.Println("Got mesh key from setec store") + st.Close() + } else if *meshPSKFile != "" { + b, err := setec.StaticFile(*meshPSKFile) + if err != nil { + log.Fatalf("StaticFile failed to get key: %v", err) + } + log.Println("Got mesh key from static file") + meshKey = b.GetString() + } + if meshKey == "" { + log.Printf("No mesh key found, mesh key is empty") + return key.DERPMesh{}, nil + } + + return key.ParseDERPMesh(meshKey) +} + type overallStatus struct { good, bad []string } diff --git a/cmd/tsidp/depaware.txt b/cmd/tsidp/depaware.txt index 1ea4b3d885987..b28460352bb9f 100644 --- a/cmd/tsidp/depaware.txt +++ b/cmd/tsidp/depaware.txt @@ -241,7 +241,7 @@ tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depawar tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/ipn/ipnext+ tailscale.com/health from tailscale.com/control/controlclient+ - tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ tailscale.com/hostinfo from tailscale.com/client/web+ tailscale.com/internal/noiseconn from tailscale.com/control/controlclient tailscale.com/ipn from tailscale.com/client/local+ diff --git a/derp/derp_client.go b/derp/derp_client.go index a9b92299c291f..69f35db1e2791 100644 --- a/derp/derp_client.go +++ b/derp/derp_client.go @@ -165,7 +165,7 @@ type clientInfo struct { // trusted clients. It's required to subscribe to the // connection list & forward packets. It's empty for regular // users. - MeshKey string `json:"meshKey,omitempty"` + MeshKey key.DERPMesh `json:"meshKey,omitempty,omitzero"` // Version is the DERP protocol version that the client was built with. // See the ProtocolVersion const. @@ -179,10 +179,21 @@ type clientInfo struct { IsProber bool `json:",omitempty"` } +// Equal reports if two clientInfo values are equal. +func (c *clientInfo) Equal(other *clientInfo) bool { + if c == nil || other == nil { + return c == other + } + if c.Version != other.Version || c.CanAckPings != other.CanAckPings || c.IsProber != other.IsProber { + return false + } + return c.MeshKey.Equal(other.MeshKey) +} + func (c *Client) sendClientKey() error { msg, err := json.Marshal(clientInfo{ Version: ProtocolVersion, - MeshKey: c.meshKey.String(), + MeshKey: c.meshKey, CanAckPings: c.canAckPings, IsProber: c.isProber, }) diff --git a/derp/derp_server.go b/derp/derp_server.go index 6f86c3ea458cf..c6a7494850ec8 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -1364,14 +1364,11 @@ func (s *Server) isMeshPeer(info *clientInfo) bool { // Since mesh keys are a fixed length, we don’t need to be concerned // about timing attacks on client mesh keys that are the wrong length. // See https://github.com/tailscale/corp/issues/28720 - if info == nil || info.MeshKey == "" { + if info == nil || info.MeshKey.IsZero() { return false } - k, err := key.ParseDERPMesh(info.MeshKey) - if err != nil { - return false - } - return s.meshKey.Equal(k) + + return s.meshKey.Equal(info.MeshKey) } // verifyClient checks whether the client is allowed to connect to the derper, diff --git a/derp/derp_test.go b/derp/derp_test.go index 0093ee2b15372..9d07e159b4584 100644 --- a/derp/derp_test.go +++ b/derp/derp_test.go @@ -20,6 +20,7 @@ import ( "os" "reflect" "strconv" + "strings" "sync" "testing" "time" @@ -33,21 +34,53 @@ import ( "tailscale.com/tstest" "tailscale.com/types/key" "tailscale.com/types/logger" + "tailscale.com/util/must" ) func TestClientInfoUnmarshal(t *testing.T) { - for i, in := range []string{ - `{"Version":5,"MeshKey":"abc"}`, - `{"version":5,"meshKey":"abc"}`, + for i, in := range map[string]struct { + json string + want *clientInfo + wantErr string + }{ + "empty": { + json: `{}`, + want: &clientInfo{}, + }, + "valid": { + json: `{"Version":5,"MeshKey":"6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8"}`, + want: &clientInfo{MeshKey: must.Get(key.ParseDERPMesh("6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8")), Version: 5}, + }, + "validLowerMeshKey": { + json: `{"version":5,"meshKey":"6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8"}`, + want: &clientInfo{MeshKey: must.Get(key.ParseDERPMesh("6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8")), Version: 5}, + }, + "invalidMeshKeyToShort": { + json: `{"version":5,"meshKey":"abcdefg"}`, + wantErr: "invalid mesh key", + }, + "invalidMeshKeyToLong": { + json: `{"version":5,"meshKey":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}`, + wantErr: "invalid mesh key", + }, } { - var got clientInfo - if err := json.Unmarshal([]byte(in), &got); err != nil { - t.Fatalf("[%d]: %v", i, err) - } - want := clientInfo{Version: 5, MeshKey: "abc"} - if got != want { - t.Errorf("[%d]: got %+v; want %+v", i, got, want) - } + t.Run(i, func(t *testing.T) { + t.Parallel() + var got clientInfo + err := json.Unmarshal([]byte(in.json), &got) + if in.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), in.wantErr) { + t.Errorf("Unmarshal(%q) = %v, want error containing %q", in.json, err, in.wantErr) + } + return + } + if err != nil { + t.Fatalf("Unmarshal(%q) = %v, want no error", in.json, err) + } + if !got.Equal(in.want) { + t.Errorf("Unmarshal(%q) = %+v, want %+v", in.json, got, in.want) + } + }) } } @@ -1681,43 +1714,43 @@ func TestIsMeshPeer(t *testing.T) { t.Fatal(err) } for name, tt := range map[string]struct { - info *clientInfo want bool + meshKey string wantAllocs float64 }{ "nil": { - info: nil, - want: false, - wantAllocs: 0, - }, - "empty": { - info: &clientInfo{MeshKey: ""}, want: false, wantAllocs: 0, }, - "invalid": { - info: &clientInfo{MeshKey: "invalid"}, - want: false, - wantAllocs: 2, // error message - }, "mismatch": { - info: &clientInfo{MeshKey: "0badf00d00000000000000000000000000000000000000000000000000000000"}, + meshKey: "6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8", want: false, wantAllocs: 1, }, "match": { - info: &clientInfo{MeshKey: testMeshKey}, + meshKey: testMeshKey, want: true, - wantAllocs: 1, + wantAllocs: 0, }, } { t.Run(name, func(t *testing.T) { var got bool + var mKey key.DERPMesh + if tt.meshKey != "" { + mKey, err = key.ParseDERPMesh(tt.meshKey) + if err != nil { + t.Fatalf("ParseDERPMesh(%q) failed: %v", tt.meshKey, err) + } + } + + info := clientInfo{ + MeshKey: mKey, + } allocs := testing.AllocsPerRun(1, func() { - got = s.isMeshPeer(tt.info) + got = s.isMeshPeer(&info) }) if got != tt.want { - t.Fatalf("got %t, want %t: info = %#v", got, tt.want, tt.info) + t.Fatalf("got %t, want %t: info = %#v", got, tt.want, info) } if allocs != tt.wantAllocs && tt.want { diff --git a/prober/derp.go b/prober/derp.go index 98e61ff54b89a..e21c8ce76668f 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -47,6 +47,7 @@ import ( type derpProber struct { p *Prober derpMapURL string // or "local" + meshKey key.DERPMesh udpInterval time.Duration meshInterval time.Duration tlsInterval time.Duration @@ -71,7 +72,7 @@ type derpProber struct { udpProbeFn func(string, int) ProbeClass meshProbeFn func(string, string) ProbeClass bwProbeFn func(string, string, int64) ProbeClass - qdProbeFn func(string, string, int, time.Duration) ProbeClass + qdProbeFn func(string, string, int, time.Duration, key.DERPMesh) ProbeClass sync.Mutex lastDERPMap *tailcfg.DERPMap @@ -143,6 +144,12 @@ func WithRegionCodeOrID(regionCode string) DERPOpt { } } +func WithMeshKey(meshKey key.DERPMesh) DERPOpt { + return func(d *derpProber) { + d.meshKey = meshKey + } +} + // DERP creates a new derpProber. // // If derpMapURL is "local", the DERPMap is fetched via @@ -250,7 +257,7 @@ func (d *derpProber) probeMapFn(ctx context.Context) error { wantProbes[n] = true if d.probes[n] == nil { log.Printf("adding DERP queuing delay probe for %s->%s (%s)", server.Name, to.Name, region.RegionName) - d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout)) + d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout, d.meshKey)) } } } @@ -284,7 +291,7 @@ func (d *derpProber) probeMesh(from, to string) ProbeClass { } dm := d.lastDERPMap - return derpProbeNodePair(ctx, dm, fromN, toN) + return derpProbeNodePair(ctx, dm, fromN, toN, d.meshKey) }, Class: "derp_mesh", Labels: Labels{"derp_path": derpPath}, @@ -308,7 +315,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { if err != nil { return err } - return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix) + return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix, d.meshKey) }, Class: "derp_bw", Labels: Labels{ @@ -336,7 +343,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { // to the queuing delay measurement and are recorded as dropped. 'from' and 'to' are // expected to be names (DERPNode.Name) of two DERP servers in the same region, // and may refer to the same server. -func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration) ProbeClass { +func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration, meshKey key.DERPMesh) ProbeClass { derpPath := "mesh" if from == to { derpPath = "single" @@ -349,7 +356,7 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa if err != nil { return err } - return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh) + return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh, meshKey) }, Class: "derp_qd", Labels: Labels{"derp_path": derpPath}, @@ -368,15 +375,15 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa // derpProbeQueuingDelay continuously sends data between two local DERP clients // connected to two DERP servers in order to measure queuing delays. From and to // can be the same server. -func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram) (err error) { +func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram, meshKey key.DERPMesh) (err error) { // This probe uses clients with isProber=false to avoid spamming the derper // logs with every packet sent by the queuing delay probe. - fromc, err := newConn(ctx, dm, from, false) + fromc, err := newConn(ctx, dm, from, false, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, false) + toc, err := newConn(ctx, dm, to, false, meshKey) if err != nil { return err } @@ -674,15 +681,15 @@ func derpProbeUDP(ctx context.Context, ipStr string, port int) error { // DERP clients connected to two DERP servers.If tunIPv4Address is specified, // probes will use a TCP connection over a TUN device at this address in order // to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP. -func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix) (err error) { +func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix, meshKey key.DERPMesh) (err error) { // This probe uses clients with isProber=false to avoid spamming the derper logs with every packet // sent by the bandwidth probe. - fromc, err := newConn(ctx, dm, from, false) + fromc, err := newConn(ctx, dm, from, false, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, false) + toc, err := newConn(ctx, dm, to, false, meshKey) if err != nil { return err } @@ -712,13 +719,13 @@ func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tail // derpProbeNodePair sends a small packet between two local DERP clients // connected to two DERP servers. -func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode) (err error) { - fromc, err := newConn(ctx, dm, from, true) +func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, meshKey key.DERPMesh) (err error) { + fromc, err := newConn(ctx, dm, from, true, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, true) + toc, err := newConn(ctx, dm, to, true, meshKey) if err != nil { return err } @@ -1116,7 +1123,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT return nil } -func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool) (*derphttp.Client, error) { +func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool, meshKey key.DERPMesh) (*derphttp.Client, error) { // To avoid spamming the log with regular connection messages. l := logger.Filtered(log.Printf, func(s string) bool { return !strings.Contains(s, "derphttp.Client.Connect: connecting to") @@ -1132,6 +1139,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr } }) dc.IsProber = isProber + dc.MeshKey = meshKey err := dc.Connect(ctx) if err != nil { return nil, err @@ -1165,7 +1173,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr case derp.ServerInfoMessage: errc <- nil default: - errc <- fmt.Errorf("unexpected first message type %T", errc) + errc <- fmt.Errorf("unexpected first message type %T", m) } }() select { diff --git a/types/key/derp.go b/types/key/derp.go index 1fe690189c7be..1466b85bc5288 100644 --- a/types/key/derp.go +++ b/types/key/derp.go @@ -6,6 +6,7 @@ package key import ( "crypto/subtle" "encoding/hex" + "encoding/json" "errors" "fmt" "strings" @@ -23,6 +24,27 @@ type DERPMesh struct { k [32]byte // 64-digit hexadecimal numbers fit in 32 bytes } +// MarshalJSON implements the [encoding/json.Marshaler] interface. +func (k DERPMesh) MarshalJSON() ([]byte, error) { + return json.Marshal(k.String()) +} + +// UnmarshalJSON implements the [encoding/json.Unmarshaler] interface. +func (k *DERPMesh) UnmarshalJSON(data []byte) error { + var s string + json.Unmarshal(data, &s) + + if hex.DecodedLen(len(s)) != len(k.k) { + return fmt.Errorf("types/key/derp: cannot unmarshal, incorrect size mesh key len: %d, must be %d, %w", hex.DecodedLen(len(s)), len(k.k), ErrInvalidMeshKey) + } + _, err := hex.Decode(k.k[:], []byte(s)) + if err != nil { + return fmt.Errorf("types/key/derp: cannot unmarshal, invalid mesh key: %w", err) + } + + return nil +} + // DERPMeshFromRaw32 parses a 32-byte raw value as a DERP mesh key. func DERPMeshFromRaw32(raw mem.RO) DERPMesh { if raw.Len() != 32 { From 811426001907a93209ac811d79f09d5c8bc61988 Mon Sep 17 00:00:00 2001 From: Patrick O'Doherty Date: Tue, 10 Jun 2025 14:39:27 -0700 Subject: [PATCH 047/263] go.toolchain.rev: bump to go 1.24.4 (#16230) Updates #cleanup Signed-off-by: Patrick O'Doherty --- go.toolchain.rev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.toolchain.rev b/go.toolchain.rev index a5d73929c61b0..33aa564236c3e 100644 --- a/go.toolchain.rev +++ b/go.toolchain.rev @@ -1 +1 @@ -98e8c99c256a5aeaa13725d2e43fdd7f465ba200 +1cd3bf1a6eaf559aa8c00e749289559c884cef09 From 6a93b17c8cafc1d8e1c52e133511e52ed9086355 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 10 Jun 2025 17:31:14 -0700 Subject: [PATCH 048/263] types/netmap,wgengine/magicsock: propagate CapVer to magicsock.endpoint (#16244) This enables us to mark nodes as relay capable or not. We don't actually do that yet, as we haven't established a relay CapVer. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- types/netmap/nodemut.go | 13 +++++++++++++ types/netmap/nodemut_test.go | 9 +++++++++ wgengine/magicsock/endpoint.go | 2 ++ wgengine/magicsock/magicsock.go | 9 +++++++++ 4 files changed, 33 insertions(+) diff --git a/types/netmap/nodemut.go b/types/netmap/nodemut.go index f4de1bf0b8f02..ab30ef1e6a286 100644 --- a/types/netmap/nodemut.go +++ b/types/netmap/nodemut.go @@ -69,6 +69,17 @@ func (m NodeMutationLastSeen) Apply(n *tailcfg.Node) { n.LastSeen = ptr.To(m.LastSeen) } +// NodeMutationCap is a NodeMutation that says a node's +// [tailcfg.CapabilityVersion] value has changed. +type NodeMutationCap struct { + mutatingNodeID + Cap tailcfg.CapabilityVersion +} + +func (m NodeMutationCap) Apply(n *tailcfg.Node) { + n.Cap = m.Cap +} + var peerChangeFields = sync.OnceValue(func() []reflect.StructField { var fields []reflect.StructField rt := reflect.TypeFor[tailcfg.PeerChange]() @@ -105,6 +116,8 @@ func NodeMutationsFromPatch(p *tailcfg.PeerChange) (_ []NodeMutation, ok bool) { ret = append(ret, NodeMutationOnline{mutatingNodeID(p.NodeID), *p.Online}) case "LastSeen": ret = append(ret, NodeMutationLastSeen{mutatingNodeID(p.NodeID), *p.LastSeen}) + case "Cap": + ret = append(ret, NodeMutationCap{mutatingNodeID(p.NodeID), p.Cap}) } } return ret, true diff --git a/types/netmap/nodemut_test.go b/types/netmap/nodemut_test.go index 374f8623ad564..0f1cac6b247bc 100644 --- a/types/netmap/nodemut_test.go +++ b/types/netmap/nodemut_test.go @@ -177,6 +177,14 @@ func TestMutationsFromMapResponse(t *testing.T) { }, want: nil, }, + { + name: "patch-cap", + mr: fromChanges(&tailcfg.PeerChange{ + NodeID: 1, + Cap: 2, + }), + want: muts(NodeMutationCap{1, 2}), + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -195,6 +203,7 @@ func TestMutationsFromMapResponse(t *testing.T) { NodeMutationDERPHome{}, NodeMutationOnline{}, NodeMutationLastSeen{}, + NodeMutationCap{}, )); diff != "" { t.Errorf("wrong result (-want +got):\n%s", diff) } diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index bf7758fb8bfab..23316dcb454cf 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1423,6 +1423,8 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p } de.setEndpointsLocked(n.Endpoints()) + + de.relayCapable = capVerIsRelayCapable(n.Cap()) } func (de *endpoint) setEndpointsLocked(eps interface { diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 2e288211032ce..e5cc87dc3810a 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2507,6 +2507,11 @@ func (c *Conn) SetProbeUDPLifetime(v bool) { }) } +func capVerIsRelayCapable(version tailcfg.CapabilityVersion) bool { + // TODO(jwhited): implement once capVer is bumped + return false +} + // SetNetworkMap is called when the control client gets a new network // map from the control server. It must always be non-nil. // @@ -3203,6 +3208,10 @@ func (c *Conn) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) { ep.mu.Lock() ep.setEndpointsLocked(views.SliceOf(m.Endpoints)) ep.mu.Unlock() + case netmap.NodeMutationCap: + ep.mu.Lock() + ep.relayCapable = capVerIsRelayCapable(m.Cap) + ep.mu.Unlock() } } return true From 3b25e94352b7db6c6028a6b97c425aa01dcf3aa3 Mon Sep 17 00:00:00 2001 From: Fran Bull Date: Fri, 6 Jun 2025 09:38:34 -0700 Subject: [PATCH 049/263] cmd/natc: allow specifying the tsnet state dir Which can make operating the service more convenient. It makes sense to put the cluster state with this if specified, so rearrange the logic to handle that. Updates #14667 Signed-off-by: Fran Bull --- cmd/natc/ippool/consensusippool.go | 32 +-------------------------- cmd/natc/natc.go | 35 ++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/cmd/natc/ippool/consensusippool.go b/cmd/natc/ippool/consensusippool.go index adf2090d1b76a..3bc21bd0357dd 100644 --- a/cmd/natc/ippool/consensusippool.go +++ b/cmd/natc/ippool/consensusippool.go @@ -10,8 +10,6 @@ import ( "fmt" "log" "net/netip" - "os" - "path/filepath" "time" "github.com/hashicorp/raft" @@ -155,11 +153,7 @@ func (ipp *ConsensusIPPool) domainLookup(from tailcfg.NodeID, addr netip.Addr) ( func (ipp *ConsensusIPPool) StartConsensus(ctx context.Context, ts *tsnet.Server, clusterTag string, clusterStateDir string) error { cfg := tsconsensus.DefaultConfig() cfg.ServeDebugMonitor = true - var err error - cfg.StateDirPath, err = getStatePath(clusterStateDir) - if err != nil { - return err - } + cfg.StateDirPath = clusterStateDir cns, err := tsconsensus.Start(ctx, ts, ipp, clusterTag, cfg) if err != nil { return err @@ -211,30 +205,6 @@ func (ps *consensusPerPeerState) unusedIPV4(ipset *netipx.IPSet, reuseDeadline t return netip.Addr{}, false, "", errors.New("ip pool exhausted") } -func getStatePath(pathFromFlag string) (string, error) { - var dirPath string - if pathFromFlag != "" { - dirPath = pathFromFlag - } else { - confDir, err := os.UserConfigDir() - if err != nil { - return "", err - } - dirPath = filepath.Join(confDir, "nat-connector-cluster-state") - } - - if err := os.MkdirAll(dirPath, 0700); err != nil { - return "", err - } - if fi, err := os.Stat(dirPath); err != nil { - return "", err - } else if !fi.IsDir() { - return "", fmt.Errorf("%v is not a directory", dirPath) - } - - return dirPath, nil -} - // isCloseToExpiry returns true if the lastUsed and now times are more than // half the lifetime apart func isCloseToExpiry(lastUsed, now time.Time, lifetime time.Duration) bool { diff --git a/cmd/natc/natc.go b/cmd/natc/natc.go index 719d5d20d6b38..247bb2101f976 100644 --- a/cmd/natc/natc.go +++ b/cmd/natc/natc.go @@ -18,6 +18,7 @@ import ( "net/http" "net/netip" "os" + "path/filepath" "strings" "time" @@ -59,7 +60,7 @@ func main() { wgPort = fs.Uint("wg-port", 0, "udp port for wireguard and peer to peer traffic") clusterTag = fs.String("cluster-tag", "", "optionally run in a consensus cluster with other nodes with this tag") server = fs.String("login-server", ipn.DefaultControlURL, "the base URL of control server") - clusterStateDir = fs.String("cluster-state-dir", "", "path to directory in which to store raft state") + stateDir = fs.String("state-dir", "", "path to directory in which to store app state") ) ff.Parse(fs, os.Args[1:], ff.WithEnvVarPrefix("TS_NATC")) @@ -96,6 +97,7 @@ func main() { } ts := &tsnet.Server{ Hostname: *hostname, + Dir: *stateDir, } ts.ControlURL = *server if *wgPort != 0 { @@ -156,7 +158,11 @@ func main() { var ipp ippool.IPPool if *clusterTag != "" { cipp := ippool.NewConsensusIPPool(addrPool) - err = cipp.StartConsensus(ctx, ts, *clusterTag, *clusterStateDir) + clusterStateDir, err := getClusterStatePath(*stateDir) + if err != nil { + log.Fatalf("Creating cluster state dir failed: %v", err) + } + err = cipp.StartConsensus(ctx, ts, *clusterTag, clusterStateDir) if err != nil { log.Fatalf("StartConsensus: %v", err) } @@ -570,3 +576,28 @@ func proxyTCPConn(c net.Conn, dest string, ctor *connector) { p.Start() } + +func getClusterStatePath(stateDirFlag string) (string, error) { + var dirPath string + if stateDirFlag != "" { + dirPath = stateDirFlag + } else { + confDir, err := os.UserConfigDir() + if err != nil { + return "", err + } + dirPath = filepath.Join(confDir, "nat-connector-state") + } + dirPath = filepath.Join(dirPath, "cluster") + + if err := os.MkdirAll(dirPath, 0700); err != nil { + return "", err + } + if fi, err := os.Stat(dirPath); err != nil { + return "", err + } else if !fi.IsDir() { + return "", fmt.Errorf("%v is not a directory", dirPath) + } + + return dirPath, nil +} From 6010812f0c033caed665ef66daca7d103d8399af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus=20Lensb=C3=B8l?= Date: Wed, 11 Jun 2025 14:22:30 -0400 Subject: [PATCH 050/263] ipn/localapi,client/local: add debug watcher for bus events (#16239) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates: #15160 Signed-off-by: Claus Lensbøl --- client/local/local.go | 20 +++++ cmd/tailscale/cli/debug.go | 24 ++++++ cmd/tailscaled/tailscaled.go | 1 + ipn/localapi/localapi.go | 137 +++++++++++++++++++++++++---------- util/eventbus/debug.go | 9 +++ 5 files changed, 154 insertions(+), 37 deletions(-) diff --git a/client/local/local.go b/client/local/local.go index 7a3a4b703a75e..bc643ad7932db 100644 --- a/client/local/local.go +++ b/client/local/local.go @@ -414,6 +414,26 @@ func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) { return res.Body, nil } +// StreamBusEvents returns a stream of the Tailscale bus events as they arrive. +// Close the context to stop the stream. +// Expected response from the server is newline-delimited JSON. +// The caller must close the reader when it is finished reading. +func (lc *Client) StreamBusEvents(ctx context.Context) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, "GET", + "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-bus-events", nil) + if err != nil { + return nil, err + } + res, err := lc.doLocalRequestNiceError(req) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + return nil, errors.New(res.Status) + } + return res.Body, nil +} + // Pprof returns a pprof profile of the Tailscale daemon. func (lc *Client) Pprof(ctx context.Context, pprofType string, sec int) ([]byte, error) { var secArg string diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index 213a0166e2aa5..025382ca9a7ea 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -102,6 +102,12 @@ func debugCmd() *ffcli.Command { return fs })(), }, + { + Name: "daemon-bus-events", + ShortUsage: "tailscale debug daemon-bus-events", + Exec: runDaemonBusEvents, + ShortHelp: "Watch events on the tailscaled bus", + }, { Name: "metrics", ShortUsage: "tailscale debug metrics", @@ -784,6 +790,24 @@ func runDaemonLogs(ctx context.Context, args []string) error { } } +func runDaemonBusEvents(ctx context.Context, args []string) error { + logs, err := localClient.StreamBusEvents(ctx) + if err != nil { + return err + } + defer logs.Close() + d := json.NewDecoder(bufio.NewReader(logs)) + for { + var line eventbus.DebugEvent + err := d.Decode(&line) + if err != nil { + return err + } + fmt.Printf("[%d][%q][from: %q][to: %q] %s\n", line.Count, line.Type, + line.From, line.To, line.Event) + } +} + var metricsArgs struct { watch bool } diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index 87750bc5d5241..61b811c129454 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -719,6 +719,7 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo Dialer: sys.Dialer.Get(), SetSubsystem: sys.Set, ControlKnobs: sys.ControlKnobs(), + EventBus: sys.Bus.Get(), DriveForLocal: driveimpl.NewFileSystemForLocal(logf), } diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index 78f95b2b113dc..6344da42d2e54 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -20,6 +20,7 @@ import ( "net/url" "os" "path" + "reflect" "runtime" "slices" "strconv" @@ -91,6 +92,7 @@ var handler = map[string]LocalAPIHandler{ "check-udp-gro-forwarding": (*Handler).serveCheckUDPGROForwarding, "component-debug-logging": (*Handler).serveComponentDebugLogging, "debug": (*Handler).serveDebug, + "debug-bus-events": (*Handler).serveDebugBusEvents, "debug-derp-region": (*Handler).serveDebugDERPRegion, "debug-dial-types": (*Handler).serveDebugDialTypes, "debug-log": (*Handler).serveDebugLog, @@ -332,7 +334,7 @@ func (h *Handler) serveIDToken(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - httpReq, err := http.NewRequest("POST", "https://unused/machine/id-token", bytes.NewReader(b)) + httpReq, err := http.NewRequest(httpm.POST, "https://unused/machine/id-token", bytes.NewReader(b)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -355,7 +357,7 @@ func (h *Handler) serveBugReport(w http.ResponseWriter, r *http.Request) { http.Error(w, "bugreport access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "only POST allowed", http.StatusMethodNotAllowed) return } @@ -482,7 +484,7 @@ func (h *Handler) serveSetDeviceAttrs(w http.ResponseWriter, r *http.Request) { http.Error(w, "set-device-attrs access denied", http.StatusForbidden) return } - if r.Method != "PATCH" { + if r.Method != httpm.PATCH { http.Error(w, "only PATCH allowed", http.StatusMethodNotAllowed) return } @@ -587,7 +589,7 @@ func (h *Handler) serveLogTap(w http.ResponseWriter, r *http.Request) { http.Error(w, "logtap access denied", http.StatusForbidden) return } - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "GET required", http.StatusMethodNotAllowed) return } @@ -639,7 +641,7 @@ func (h *Handler) serveDebug(w http.ResponseWriter, r *http.Request) { http.Error(w, "debug access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "POST required", http.StatusMethodNotAllowed) return } @@ -712,7 +714,7 @@ func (h *Handler) serveDevSetStateStore(w http.ResponseWriter, r *http.Request) http.Error(w, "debug access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "POST required", http.StatusMethodNotAllowed) return } @@ -917,6 +919,68 @@ func (h *Handler) serveDebugPortmap(w http.ResponseWriter, r *http.Request) { } } +// serveDebugBusEvents taps into the tailscaled/utils/eventbus and streams +// events to the client. +func (h *Handler) serveDebugBusEvents(w http.ResponseWriter, r *http.Request) { + // Require write access (~root) as the logs could contain something + // sensitive. + if !h.PermitWrite { + http.Error(w, "event bus access denied", http.StatusForbidden) + return + } + if r.Method != httpm.GET { + http.Error(w, "GET required", http.StatusMethodNotAllowed) + return + } + + bus, ok := h.LocalBackend().Sys().Bus.GetOK() + if !ok { + http.Error(w, "event bus not running", http.StatusNoContent) + return + } + + f, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", http.StatusInternalServerError) + return + } + + io.WriteString(w, `{"Event":"[event listener connected]\n"}`+"\n") + f.Flush() + + mon := bus.Debugger().WatchBus() + defer mon.Close() + + i := 0 + for { + select { + case <-r.Context().Done(): + fmt.Fprintf(w, `{"Event":"[event listener closed]\n"}`) + return + case <-mon.Done(): + return + case event := <-mon.Events(): + data := eventbus.DebugEvent{ + Count: i, + Type: reflect.TypeOf(event.Event).String(), + Event: event.Event, + From: event.From.Name(), + } + for _, client := range event.To { + data.To = append(data.To, client.Name()) + } + + if msg, err := json.Marshal(data); err != nil { + fmt.Fprintf(w, `{"Event":"[ERROR] failed to marshal JSON for %T"}\n`, event.Event) + } else { + w.Write(msg) + } + f.Flush() + i++ + } + } +} + func (h *Handler) serveComponentDebugLogging(w http.ResponseWriter, r *http.Request) { if !h.PermitWrite { http.Error(w, "debug access denied", http.StatusForbidden) @@ -1078,7 +1142,7 @@ func (h *Handler) serveResetAuth(w http.ResponseWriter, r *http.Request) { func (h *Handler) serveServeConfig(w http.ResponseWriter, r *http.Request) { switch r.Method { - case "GET": + case httpm.GET: if !h.PermitRead { http.Error(w, "serve config denied", http.StatusForbidden) return @@ -1094,7 +1158,7 @@ func (h *Handler) serveServeConfig(w http.ResponseWriter, r *http.Request) { w.Header().Set("Etag", etag) w.Header().Set("Content-Type", "application/json") w.Write(bts) - case "POST": + case httpm.POST: if !h.PermitWrite { http.Error(w, "serve config denied", http.StatusForbidden) return @@ -1157,7 +1221,6 @@ func authorizeServeConfigForGOOSAndUserContext(goos string, configIn *ipn.ServeC // should never happen. panic("unreachable") } - } func (h *Handler) serveCheckIPForwarding(w http.ResponseWriter, r *http.Request) { @@ -1291,7 +1354,7 @@ func (h *Handler) serveDebugPeerEndpointChanges(w http.ResponseWriter, r *http.R // (in ipnserver.Server) provides the blocking until the connection is no longer // in use. func InUseOtherUserIPNStream(w http.ResponseWriter, r *http.Request, err error) (handled bool) { - if r.Method != "GET" || r.URL.Path != "/localapi/v0/watch-ipn-bus" { + if r.Method != httpm.GET || r.URL.Path != "/localapi/v0/watch-ipn-bus" { return false } js, err := json.Marshal(&ipn.Notify{ @@ -1356,7 +1419,7 @@ func (h *Handler) serveLoginInteractive(w http.ResponseWriter, r *http.Request) http.Error(w, "login access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "want POST", http.StatusBadRequest) return } @@ -1370,7 +1433,7 @@ func (h *Handler) serveStart(w http.ResponseWriter, r *http.Request) { http.Error(w, "access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "want POST", http.StatusBadRequest) return } @@ -1393,7 +1456,7 @@ func (h *Handler) serveLogout(w http.ResponseWriter, r *http.Request) { http.Error(w, "logout access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "want POST", http.StatusBadRequest) return } @@ -1412,7 +1475,7 @@ func (h *Handler) servePrefs(w http.ResponseWriter, r *http.Request) { } var prefs ipn.PrefsView switch r.Method { - case "PATCH": + case httpm.PATCH: if !h.PermitWrite { http.Error(w, "prefs write access denied", http.StatusForbidden) return @@ -1436,7 +1499,7 @@ func (h *Handler) servePrefs(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(resJSON{Error: err.Error()}) return } - case "GET", "HEAD": + case httpm.GET, httpm.HEAD: prefs = h.b.Prefs() default: http.Error(w, "unsupported method", http.StatusMethodNotAllowed) @@ -1476,9 +1539,9 @@ func (h *Handler) servePolicy(w http.ResponseWriter, r *http.Request) { var effectivePolicy *setting.Snapshot switch r.Method { - case "GET": + case httpm.GET: effectivePolicy = policy.Get() - case "POST": + case httpm.POST: effectivePolicy, err = policy.Reload() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1504,7 +1567,7 @@ func (h *Handler) serveCheckPrefs(w http.ResponseWriter, r *http.Request) { http.Error(w, "checkprefs access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "unsupported method", http.StatusMethodNotAllowed) return } @@ -1542,7 +1605,7 @@ func (h *Handler) serveSetDNS(w http.ResponseWriter, r *http.Request) { http.Error(w, "access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "want POST", http.StatusBadRequest) return } @@ -1557,7 +1620,7 @@ func (h *Handler) serveSetDNS(w http.ResponseWriter, r *http.Request) { } func (h *Handler) serveDERPMap(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "want GET", http.StatusBadRequest) return } @@ -1574,7 +1637,7 @@ func (h *Handler) serveSetExpirySooner(w http.ResponseWriter, r *http.Request) { http.Error(w, "access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "POST required", http.StatusMethodNotAllowed) return } @@ -1602,7 +1665,7 @@ func (h *Handler) serveSetExpirySooner(w http.ResponseWriter, r *http.Request) { func (h *Handler) servePing(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "want POST", http.StatusBadRequest) return } @@ -1648,7 +1711,7 @@ func (h *Handler) servePing(w http.ResponseWriter, r *http.Request) { } func (h *Handler) serveDial(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "POST required", http.StatusMethodNotAllowed) return } @@ -1711,7 +1774,7 @@ func (h *Handler) serveSetPushDeviceToken(w http.ResponseWriter, r *http.Request http.Error(w, "set push device token access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "unsupported method", http.StatusMethodNotAllowed) return } @@ -1729,7 +1792,7 @@ func (h *Handler) serveHandlePushMessage(w http.ResponseWriter, r *http.Request) http.Error(w, "handle push message not allowed", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "unsupported method", http.StatusMethodNotAllowed) return } @@ -1746,7 +1809,7 @@ func (h *Handler) serveHandlePushMessage(w http.ResponseWriter, r *http.Request) } func (h *Handler) serveUploadClientMetrics(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "unsupported method", http.StatusMethodNotAllowed) return } @@ -2337,7 +2400,7 @@ func (h *Handler) serveQueryFeature(w http.ResponseWriter, r *http.Request) { } req, err := http.NewRequestWithContext(r.Context(), - "POST", "https://unused/machine/feature/query", bytes.NewReader(b)) + httpm.POST, "https://unused/machine/feature/query", bytes.NewReader(b)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -2416,7 +2479,7 @@ func (h *Handler) serveDebugLog(w http.ResponseWriter, r *http.Request) { // Effectively, it tells us whether serveUpdateInstall will be able to install // an update for us. func (h *Handler) serveUpdateCheck(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "only GET allowed", http.StatusMethodNotAllowed) return } @@ -2445,7 +2508,7 @@ func (h *Handler) serveUpdateCheck(w http.ResponseWriter, r *http.Request) { // serveUpdateProgress after pinging this endpoint to check how the update is // going. func (h *Handler) serveUpdateInstall(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "only POST allowed", http.StatusMethodNotAllowed) return } @@ -2460,7 +2523,7 @@ func (h *Handler) serveUpdateInstall(w http.ResponseWriter, r *http.Request) { // log messages in order from oldest to newest. If an update is not in progress, // the returned slice will be empty. func (h *Handler) serveUpdateProgress(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "only GET allowed", http.StatusMethodNotAllowed) return } @@ -2516,7 +2579,7 @@ func (h *Handler) serveDNSOSConfig(w http.ResponseWriter, r *http.Request) { // // The response if successful is a DNSQueryResponse JSON object. func (h *Handler) serveDNSQuery(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "only GET allowed", http.StatusMethodNotAllowed) return } @@ -2553,7 +2616,7 @@ func (h *Handler) serveDNSQuery(w http.ResponseWriter, r *http.Request) { // serveDriveServerAddr handles updates of the Taildrive file server address. func (h *Handler) serveDriveServerAddr(w http.ResponseWriter, r *http.Request) { - if r.Method != "PUT" { + if r.Method != httpm.PUT { http.Error(w, "only PUT allowed", http.StatusMethodNotAllowed) return } @@ -2580,7 +2643,7 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) { return } switch r.Method { - case "PUT": + case httpm.PUT: var share drive.Share err := json.NewDecoder(r.Body).Decode(&share) if err != nil { @@ -2616,7 +2679,7 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) { return } w.WriteHeader(http.StatusCreated) - case "DELETE": + case httpm.DELETE: b, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -2632,7 +2695,7 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) { return } w.WriteHeader(http.StatusNoContent) - case "POST": + case httpm.POST: var names [2]string err := json.NewDecoder(r.Body).Decode(&names) if err != nil { @@ -2657,7 +2720,7 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) { return } w.WriteHeader(http.StatusNoContent) - case "GET": + case httpm.GET: shares := h.b.DriveGetShares() err := json.NewEncoder(w).Encode(shares) if err != nil { @@ -2671,7 +2734,7 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) { // serveSuggestExitNode serves a POST endpoint for returning a suggested exit node. func (h *Handler) serveSuggestExitNode(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "only GET allowed", http.StatusMethodNotAllowed) return } diff --git a/util/eventbus/debug.go b/util/eventbus/debug.go index 832d72ac07dda..b6264f82fd0eb 100644 --- a/util/eventbus/debug.go +++ b/util/eventbus/debug.go @@ -186,3 +186,12 @@ type hookFn[T any] struct { ID uint64 Fn func(T) } + +// DebugEvent is a representation of an event used for debug clients. +type DebugEvent struct { + Count int + Type string + From string + To []string + Event any +} From 8baa016a23a3960463b47472d5d1dd8b18f3a6cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Jun 2025 23:17:14 -0600 Subject: [PATCH 051/263] .github: Bump github/codeql-action from 3.28.15 to 3.28.19 (#16227) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.28.15 to 3.28.19. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/45775bd8235c68ba998cffa5171334d58593da47...fca7ace96b7d713c7035871441bd52efbe39e27e) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.28.19 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 311f539e1978f..124a165610bec 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -55,7 +55,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@45775bd8235c68ba998cffa5171334d58593da47 # v3.28.15 + uses: github/codeql-action/init@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@45775bd8235c68ba998cffa5171334d58593da47 # v3.28.15 + uses: github/codeql-action/autobuild@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -80,4 +80,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@45775bd8235c68ba998cffa5171334d58593da47 # v3.28.15 + uses: github/codeql-action/analyze@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 From 75a42977c7312706496bb7ade24a5b65d488b45a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Jun 2025 23:18:14 -0600 Subject: [PATCH 052/263] .github: Bump slackapi/slack-github-action from 2.0.0 to 2.1.0 (#15948) Bumps [slackapi/slack-github-action](https://github.com/slackapi/slack-github-action) from 2.0.0 to 2.1.0. - [Release notes](https://github.com/slackapi/slack-github-action/releases) - [Commits](https://github.com/slackapi/slack-github-action/compare/485a9d42d3a73031f12ec201c457e2162c45d02d...b0fa283ad8fea605de13dc3f449259339835fc52) --- updated-dependencies: - dependency-name: slackapi/slack-github-action dependency-version: 2.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/govulncheck.yml | 2 +- .github/workflows/installer.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index 10269ff0bf078..36ed1fe9bf603 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -24,7 +24,7 @@ jobs: - name: Post to slack if: failure() && github.event_name == 'schedule' - uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 + uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 with: method: chat.postMessage token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }} diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml index 7888d9ba5d3e2..0ca16ae9fa6c1 100644 --- a/.github/workflows/installer.yml +++ b/.github/workflows/installer.yml @@ -108,7 +108,7 @@ jobs: steps: - name: Notify Slack of failure on scheduled runs if: failure() && github.event_name == 'schedule' - uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 + uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL }} webhook-type: incoming-webhook diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2aad005ae6ee0..9f89fae010404 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -586,7 +586,7 @@ jobs: # By having the job always run, but skipping its only step as needed, we # let the CI output collapse nicely in PRs. if: failure() && github.event_name == 'push' - uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 + uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL }} webhook-type: incoming-webhook From 7c05811af03ec69ce28d1cc92c6412d5b714bdae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Jun 2025 23:18:58 -0600 Subject: [PATCH 053/263] .github: Bump actions/setup-go from 5.4.0 to 5.5.0 (#15947) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5.4.0 to 5.5.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/0aaccfd150d50ccaeb58ebd88d36e91967a5f35b...d35c59abb061a4a6fb18e82ac0862c26744d6ab5) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: 5.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/golangci-lint.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 124a165610bec..8bd72d80d6bdb 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,7 +49,7 @@ jobs: # Install a more recent Go that understands modern go.mod content. - name: Install Go - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: go-version-file: go.mod diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 04a2e042db27d..60eb6852a0ab2 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: go-version-file: go.mod cache: false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f89fae010404..1776653f4eb4c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -151,7 +151,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Go - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: go-version-file: go.mod cache: false From 3219de4cb8f136456b211dc8bdf69aa4750a8c34 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Thu, 12 Jun 2025 13:47:34 +0100 Subject: [PATCH 054/263] cmd/k8s-operator: ensure status update errors are displayed to users (#16251) Updates#cleanup Signed-off-by: Irbe Krumina --- cmd/k8s-operator/ingress_test.go | 2 +- cmd/k8s-operator/nameserver.go | 4 ++-- cmd/k8s-operator/proxygroup.go | 4 ++-- cmd/k8s-operator/proxygroup_test.go | 37 +++++++++++++++-------------- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index a975fec7a6329..dbd6961d7d7ff 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -427,7 +427,7 @@ func TestIngressLetsEncryptStaging(t *testing.T) { pcLEStaging, pcLEStagingFalse, pcOther := proxyClassesForLEStagingTest() - testCases := testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther) + testCases := testCasesForLEStagingTests() for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/k8s-operator/nameserver.go b/cmd/k8s-operator/nameserver.go index ef0762a1234e6..20d66f7d0766a 100644 --- a/cmd/k8s-operator/nameserver.go +++ b/cmd/k8s-operator/nameserver.go @@ -7,6 +7,7 @@ package main import ( "context" + "errors" "fmt" "slices" "strings" @@ -14,7 +15,6 @@ import ( _ "embed" - "github.com/pkg/errors" "go.uber.org/zap" xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" @@ -106,7 +106,7 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ if !apiequality.Semantic.DeepEqual(oldCnStatus, &dnsCfg.Status) { // An error encountered here should get returned by the Reconcile function. if updateErr := a.Client.Status().Update(ctx, dnsCfg); updateErr != nil { - err = errors.Wrap(err, updateErr.Error()) + err = errors.Join(err, updateErr) } } return res, err diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index f263829d73963..e7c0590b0dbb9 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -9,13 +9,13 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "net/http" "slices" "strings" "sync" - "github.com/pkg/errors" "go.uber.org/zap" xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" @@ -122,7 +122,7 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) { // An error encountered here should get returned by the Reconcile function. if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { - err = errors.Wrap(err, updateErr.Error()) + err = errors.Join(err, updateErr) } } return reconcile.Result{}, err diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 159329eda2335..f3f87aaacf663 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -257,10 +257,12 @@ func TestProxyGroupTypes(t *testing.T) { }, Spec: tsapi.ProxyClassSpec{}, } + // Passing ProxyGroup as status subresource is a way to get around fake + // client's limitations for updating resource statuses. fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). WithObjects(pc). - WithStatusSubresource(pc). + WithStatusSubresource(pc, &tsapi.ProxyGroup{}). Build() mustUpdateStatus(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { p.Status.Conditions = []metav1.Condition{{ @@ -450,6 +452,7 @@ func TestProxyGroupTypes(t *testing.T) { func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.ProxyGroup{}). Build() reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, @@ -693,7 +696,7 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { pgType tsapi.ProxyGroupType } pcLEStaging, pcLEStagingFalse, pcOther := proxyClassesForLEStagingTest() - sharedTestCases := testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther) + sharedTestCases := testCasesForLEStagingTests() var tests []proxyGroupLETestCase for _, tt := range sharedTestCases { tests = append(tests, proxyGroupLETestCase{ @@ -715,9 +718,20 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { builder := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme) + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tt.pgType, + Replicas: ptr.To[int32](1), + ProxyClass: tt.proxyClassPerResource, + }, + } + // Pre-populate the fake client with ProxyClasses. - builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse, pcOther). - WithStatusSubresource(pcLEStaging, pcLEStagingFalse, pcOther) + builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse, pcOther, pg). + WithStatusSubresource(pcLEStaging, pcLEStagingFalse, pcOther, pg) fc := builder.Build() @@ -730,19 +744,6 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { setProxyClassReady(t, fc, cl, name) } - // Create ProxyGroup - pg := &tsapi.ProxyGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - Spec: tsapi.ProxyGroupSpec{ - Type: tt.pgType, - Replicas: ptr.To[int32](1), - ProxyClass: tt.proxyClassPerResource, - }, - } - mustCreate(t, fc, pg) - reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, proxyImage: testProxyImage, @@ -783,7 +784,7 @@ type leStagingTestCase struct { // Shared test cases for LE staging endpoint configuration for ProxyGroup and // non-HA Ingress. -func testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther *tsapi.ProxyClass) []leStagingTestCase { +func testCasesForLEStagingTests() []leStagingTestCase { return []leStagingTestCase{ { name: "with_staging_proxyclass", From 3b5ce9d1bcc30a6d400e5f57e79d710b05bbceb9 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 11 Jun 2025 19:15:20 -0700 Subject: [PATCH 055/263] tsweb/varz: add binary name to version metric Fixes tailscale/corp#29530 Change-Id: Iae04456d7ac5527897f060370e90c9517c00a818 Signed-off-by: Brad Fitzpatrick --- tsweb/varz/varz.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tsweb/varz/varz.go b/tsweb/varz/varz.go index c6d66fbe2beda..aca2878b74f29 100644 --- a/tsweb/varz/varz.go +++ b/tsweb/varz/varz.go @@ -11,6 +11,8 @@ import ( "fmt" "io" "net/http" + "os" + "path/filepath" "reflect" "runtime" "sort" @@ -189,7 +191,11 @@ func writePromExpVar(w io.Writer, prefix string, kv expvar.KeyValue) { return } if vs, ok := v.(string); ok && strings.HasSuffix(name, "version") { - fmt.Fprintf(w, "%s{version=%q} 1\n", name, vs) + if name == "version" { + fmt.Fprintf(w, "%s{version=%q,binary=%q} 1\n", name, vs, binaryName()) + } else { + fmt.Fprintf(w, "%s{version=%q} 1\n", name, vs) + } return } switch v := v.(type) { @@ -308,6 +314,18 @@ func ExpvarDoHandler(expvarDoFunc func(f func(expvar.KeyValue))) func(http.Respo } } +var binaryName = sync.OnceValue(func() string { + exe, err := os.Executable() + if err != nil { + return "" + } + exe2, err := filepath.EvalSymlinks(exe) + if err != nil { + return filepath.Base(exe) + } + return filepath.Base(exe2) +}) + // PrometheusMetricsReflectRooter is an optional interface that expvar.Var implementations // can implement to indicate that they should be walked recursively with reflect to find // sets of fields to export. From 3ed76ceed34e2fbff6eeee59facdcd72a8b5b795 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 12 Jun 2025 09:57:45 -0700 Subject: [PATCH 056/263] feature/relayserver,net/{netcheck,udprelay}: implement addr discovery (#16253) The relay server now fetches IPs from local interfaces and external perspective IP:port's via netcheck (STUN). Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- feature/relayserver/relayserver.go | 3 +- net/netcheck/netcheck.go | 19 ++- net/udprelay/server.go | 189 +++++++++++++++++++++++------ net/udprelay/server_test.go | 2 +- 4 files changed, 170 insertions(+), 43 deletions(-) diff --git a/feature/relayserver/relayserver.go b/feature/relayserver/relayserver.go index 96d21138edfbc..a38587aa37b3a 100644 --- a/feature/relayserver/relayserver.go +++ b/feature/relayserver/relayserver.go @@ -10,7 +10,6 @@ import ( "errors" "io" "net/http" - "net/netip" "sync" "tailscale.com/envknob" @@ -136,7 +135,7 @@ func (e *extension) relayServerOrInit() (relayServer, error) { return nil, errors.New("TAILSCALE_USE_WIP_CODE envvar is not set") } var err error - e.server, _, err = udprelay.NewServer(*e.port, []netip.Addr{netip.MustParseAddr("127.0.0.1")}) + e.server, _, err = udprelay.NewServer(e.logf, *e.port, nil) if err != nil { return nil, err } diff --git a/net/netcheck/netcheck.go b/net/netcheck/netcheck.go index c9f03966beedd..54627f71383d1 100644 --- a/net/netcheck/netcheck.go +++ b/net/netcheck/netcheck.go @@ -753,6 +753,7 @@ func newReport() *Report { // GetReportOpts contains options that can be passed to GetReport. Unless // specified, all fields are optional and can be left as their zero value. +// At most one of OnlyTCP443 or OnlySTUN may be set. type GetReportOpts struct { // GetLastDERPActivity is a callback that, if provided, should return // the absolute time that the calling code last communicated with a @@ -765,6 +766,8 @@ type GetReportOpts struct { // OnlyTCP443 constrains netcheck reporting to measurements over TCP port // 443. OnlyTCP443 bool + // OnlySTUN constrains netcheck reporting to STUN measurements over UDP. + OnlySTUN bool } // getLastDERPActivity calls o.GetLastDERPActivity if both o and @@ -790,6 +793,13 @@ func (c *Client) SetForcePreferredDERP(region int) { // // It may not be called concurrently with itself. func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetReportOpts) (_ *Report, reterr error) { + onlySTUN := false + if opts != nil && opts.OnlySTUN { + if opts.OnlyTCP443 { + return nil, errors.New("netcheck: only one of OnlySTUN or OnlyTCP443 may be set in opts") + } + onlySTUN = true + } defer func() { if reterr != nil { metricNumGetReportError.Add(1) @@ -865,6 +875,9 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe }() if runtime.GOOS == "js" || runtime.GOOS == "tamago" || (runtime.GOOS == "plan9" && hostinfo.IsInVM86()) { + if onlySTUN { + return nil, errors.New("platform is restricted to HTTP, but OnlySTUN is set in opts") + } if err := c.runHTTPOnlyChecks(ctx, last, rs, dm); err != nil { return nil, err } @@ -896,7 +909,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe // it's unnecessary. captivePortalDone := syncs.ClosedChan() captivePortalStop := func() {} - if !rs.incremental { + if !rs.incremental && !onlySTUN { // NOTE(andrew): we can't simply add this goroutine to the // `NewWaitGroupChan` below, since we don't wait for that // waitgroup to finish when exiting this function and thus get @@ -970,9 +983,9 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe rs.stopTimers() // Try HTTPS and ICMP latency check if all STUN probes failed due to - // UDP presumably being blocked. + // UDP presumably being blocked, and we are not constrained to only STUN. // TODO: this should be moved into the probePlan, using probeProto probeHTTPS. - if !rs.anyUDP() && ctx.Err() == nil { + if !rs.anyUDP() && ctx.Err() == nil && !onlySTUN { var wg sync.WaitGroup var need []*tailcfg.DERPRegion for rid, reg := range dm.Regions { diff --git a/net/udprelay/server.go b/net/udprelay/server.go index 7b63ec95e77c4..f7f5868c06f21 100644 --- a/net/udprelay/server.go +++ b/net/udprelay/server.go @@ -8,6 +8,7 @@ package udprelay import ( "bytes" + "context" "crypto/rand" "errors" "fmt" @@ -19,11 +20,18 @@ import ( "time" "go4.org/mem" + "tailscale.com/client/local" "tailscale.com/disco" + "tailscale.com/net/netcheck" + "tailscale.com/net/netmon" "tailscale.com/net/packet" + "tailscale.com/net/stun" "tailscale.com/net/udprelay/endpoint" "tailscale.com/tstime" "tailscale.com/types/key" + "tailscale.com/types/logger" + "tailscale.com/util/eventbus" + "tailscale.com/util/set" ) const ( @@ -42,25 +50,22 @@ const ( // Server implements an experimental UDP relay server. type Server struct { - // disco keypair used as part of 3-way bind handshake - disco key.DiscoPrivate - discoPublic key.DiscoPublic - + // The following fields are initialized once and never mutated. + logf logger.Logf + disco key.DiscoPrivate + discoPublic key.DiscoPublic bindLifetime time.Duration steadyStateLifetime time.Duration - - // addrPorts contains the ip:port pairs returned as candidate server - // endpoints in response to an allocation request. - addrPorts []netip.AddrPort - - uc *net.UDPConn - - closeOnce sync.Once - wg sync.WaitGroup - closeCh chan struct{} + bus *eventbus.Bus + uc *net.UDPConn + closeOnce sync.Once + wg sync.WaitGroup + closeCh chan struct{} + netChecker *netcheck.Client + + mu sync.Mutex // guards the following fields + addrPorts []netip.AddrPort // the ip:port pairs returned as candidate endpoints closed bool - - mu sync.Mutex // guards the following fields lamportID uint64 vniPool []uint32 // the pool of available VNIs byVNI map[uint32]*serverEndpoint @@ -270,14 +275,13 @@ func (e *serverEndpoint) isBound() bool { // NewServer constructs a [Server] listening on 0.0.0.0:'port'. IPv6 is not yet // supported. Port may be 0, and what ultimately gets bound is returned as -// 'boundPort'. Supplied 'addrs' are joined with 'boundPort' and returned as -// [endpoint.ServerEndpoint.AddrPorts] in response to Server.AllocateEndpoint() -// requests. +// 'boundPort'. If len(overrideAddrs) > 0 these will be used in place of dynamic +// discovery, which is useful to override in tests. // // TODO: IPv6 support -// TODO: dynamic addrs:port discovery -func NewServer(port int, addrs []netip.Addr) (s *Server, boundPort int, err error) { +func NewServer(logf logger.Logf, port int, overrideAddrs []netip.Addr) (s *Server, boundPort uint16, err error) { s = &Server{ + logf: logger.WithPrefix(logf, "relayserver"), disco: key.NewDisco(), bindLifetime: defaultBindLifetime, steadyStateLifetime: defaultSteadyStateLifetime, @@ -292,26 +296,120 @@ func NewServer(port int, addrs []netip.Addr) (s *Server, boundPort int, err erro for i := 1; i < 1<<24; i++ { s.vniPool = append(s.vniPool, uint32(i)) } - boundPort, err = s.listenOn(port) + + bus := eventbus.New() + s.bus = bus + netMon, err := netmon.New(s.bus, logf) if err != nil { return nil, 0, err } - addrPorts := make([]netip.AddrPort, 0, len(addrs)) - for _, addr := range addrs { - addrPort, err := netip.ParseAddrPort(net.JoinHostPort(addr.String(), strconv.Itoa(boundPort))) - if err != nil { - return nil, 0, err - } - addrPorts = append(addrPorts, addrPort) + s.netChecker = &netcheck.Client{ + NetMon: netMon, + Logf: logger.WithPrefix(logf, "relayserver: netcheck:"), + SendPacket: func(b []byte, addrPort netip.AddrPort) (int, error) { + return s.uc.WriteToUDPAddrPort(b, addrPort) + }, + } + + boundPort, err = s.listenOn(port) + if err != nil { + return nil, 0, err } - s.addrPorts = addrPorts - s.wg.Add(2) + + s.wg.Add(1) go s.packetReadLoop() + s.wg.Add(1) go s.endpointGCLoop() + if len(overrideAddrs) > 0 { + var addrPorts set.Set[netip.AddrPort] + addrPorts.Make() + for _, addr := range overrideAddrs { + if addr.IsValid() { + addrPorts.Add(netip.AddrPortFrom(addr, boundPort)) + } + } + s.addrPorts = addrPorts.Slice() + } else { + s.wg.Add(1) + go s.addrDiscoveryLoop() + } return s, boundPort, nil } -func (s *Server) listenOn(port int) (int, error) { +func (s *Server) addrDiscoveryLoop() { + defer s.wg.Done() + + timer := time.NewTimer(0) // fire immediately + defer timer.Stop() + + getAddrPorts := func() ([]netip.AddrPort, error) { + var addrPorts set.Set[netip.AddrPort] + addrPorts.Make() + + // get local addresses + localPort := s.uc.LocalAddr().(*net.UDPAddr).Port + ips, _, err := netmon.LocalAddresses() + if err != nil { + return nil, err + } + for _, ip := range ips { + if ip.IsValid() { + addrPorts.Add(netip.AddrPortFrom(ip, uint16(localPort))) + } + } + + // fetch DERPMap to feed to netcheck + derpMapCtx, derpMapCancel := context.WithTimeout(context.Background(), time.Second) + defer derpMapCancel() + localClient := &local.Client{} + // TODO(jwhited): We are in-process so use eventbus or similar. + // local.Client gets us going. + dm, err := localClient.CurrentDERPMap(derpMapCtx) + if err != nil { + return nil, err + } + + // get addrPorts as visible from DERP + netCheckerCtx, netCheckerCancel := context.WithTimeout(context.Background(), netcheck.ReportTimeout) + defer netCheckerCancel() + rep, err := s.netChecker.GetReport(netCheckerCtx, dm, &netcheck.GetReportOpts{ + OnlySTUN: true, + }) + if err != nil { + return nil, err + } + if rep.GlobalV4.IsValid() { + addrPorts.Add(rep.GlobalV4) + } + if rep.GlobalV6.IsValid() { + addrPorts.Add(rep.GlobalV6) + } + // TODO(jwhited): consider logging if rep.MappingVariesByDestIP as + // that's a hint we are not well-positioned to operate as a UDP relay. + return addrPorts.Slice(), nil + } + + for { + select { + case <-timer.C: + // Mirror magicsock behavior for duration between STUN. We consider + // 30s a min bound for NAT timeout. + timer.Reset(tstime.RandomDurationBetween(20*time.Second, 26*time.Second)) + addrPorts, err := getAddrPorts() + if err != nil { + s.logf("error discovering IP:port candidates: %v", err) + } + s.mu.Lock() + s.addrPorts = addrPorts + s.mu.Unlock() + case <-s.closeCh: + return + } + } + +} + +func (s *Server) listenOn(port int) (uint16, error) { uc, err := net.ListenUDP("udp4", &net.UDPAddr{Port: port}) if err != nil { return 0, err @@ -322,13 +420,13 @@ func (s *Server) listenOn(port int) (int, error) { s.uc.Close() return 0, err } - boundPort, err := strconv.Atoi(boundPortStr) + boundPort, err := strconv.ParseUint(boundPortStr, 10, 16) if err != nil { s.uc.Close() return 0, err } s.uc = uc - return boundPort, nil + return uint16(boundPort), nil } // Close closes the server. @@ -343,6 +441,7 @@ func (s *Server) Close() error { clear(s.byDisco) s.vniPool = nil s.closed = true + s.bus.Close() }) return nil } @@ -378,6 +477,13 @@ func (s *Server) endpointGCLoop() { } func (s *Server) handlePacket(from netip.AddrPort, b []byte, uw udpWriter) { + if stun.Is(b) && b[1] == 0x01 { + // A b[1] value of 0x01 (STUN method binding) is sufficiently + // non-overlapping with the Geneve header where the LSB is always 0 + // (part of 6 "reserved" bits). + s.netChecker.ReceiveSTUNPacket(b, from) + return + } gh := packet.GeneveHeader{} err := gh.Decode(b) if err != nil { @@ -426,6 +532,10 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv return endpoint.ServerEndpoint{}, ErrServerClosed } + if len(s.addrPorts) == 0 { + return endpoint.ServerEndpoint{}, errors.New("server addrPorts are not yet known") + } + if discoA.Compare(s.discoPublic) == 0 || discoB.Compare(s.discoPublic) == 0 { return endpoint.ServerEndpoint{}, fmt.Errorf("client disco equals server disco: %s", s.discoPublic.ShortString()) } @@ -439,8 +549,13 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv // TODO: consider ServerEndpoint.BindLifetime -= time.Now()-e.allocatedAt // to give the client a more accurate picture of the bind window. return endpoint.ServerEndpoint{ - ServerDisco: s.discoPublic, - AddrPorts: s.addrPorts, + ServerDisco: s.discoPublic, + // Returning the "latest" addrPorts for an existing allocation is + // the simple choice. It may not be the best depending on client + // behaviors and endpoint state (bound or not). We might want to + // consider storing them (maybe interning) in the [*serverEndpoint] + // at allocation time. + AddrPorts: slices.Clone(s.addrPorts), VNI: e.vni, LamportID: e.lamportID, BindLifetime: tstime.GoDuration{Duration: s.bindLifetime}, @@ -469,7 +584,7 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv return endpoint.ServerEndpoint{ ServerDisco: s.discoPublic, - AddrPorts: s.addrPorts, + AddrPorts: slices.Clone(s.addrPorts), VNI: e.vni, LamportID: e.lamportID, BindLifetime: tstime.GoDuration{Duration: s.bindLifetime}, diff --git a/net/udprelay/server_test.go b/net/udprelay/server_test.go index 38c7ae5d9a749..a4e5ca451af2e 100644 --- a/net/udprelay/server_test.go +++ b/net/udprelay/server_test.go @@ -156,7 +156,7 @@ func TestServer(t *testing.T) { ipv4LoopbackAddr := netip.MustParseAddr("127.0.0.1") - server, _, err := NewServer(0, []netip.Addr{ipv4LoopbackAddr}) + server, _, err := NewServer(t.Logf, 0, []netip.Addr{ipv4LoopbackAddr}) if err != nil { t.Fatal(err) } From b0f7b23efe1c7d02e8caec2a5ad74ab2d5cb138a Mon Sep 17 00:00:00 2001 From: James Tucker Date: Wed, 11 Jun 2025 15:57:55 -0700 Subject: [PATCH 057/263] net/netcheck: preserve live home DERP through packet loss During a short period of packet loss, a TCP connection to the home DERP may be maintained. If no other regions emerge as winners, such as when all regions but one are avoided/disallowed as candidates, ensure that the current home region, if still active, is not dropped as the preferred region until it has failed two keepalives. Relatedly apply avoid and no measure no home to ICMP and HTTP checks as intended. Updates tailscale/corp#12894 Updates tailscale/corp#29491 Signed-off-by: James Tucker --- cmd/tailscale/depaware.txt | 2 +- derp/derp.go | 6 +++++- derp/derp_server.go | 2 +- net/netcheck/netcheck.go | 38 ++++++++++++++++++++++------------- net/netcheck/netcheck_test.go | 34 +++++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 8c3b404b1fadc..69d054ea42fb6 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -85,7 +85,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/control/controlhttp from tailscale.com/cmd/tailscale/cli tailscale.com/control/controlhttp/controlhttpcommon from tailscale.com/control/controlhttp tailscale.com/control/controlknobs from tailscale.com/net/portmapper - tailscale.com/derp from tailscale.com/derp/derphttp + tailscale.com/derp from tailscale.com/derp/derphttp+ tailscale.com/derp/derpconst from tailscale.com/derp+ tailscale.com/derp/derphttp from tailscale.com/net/netcheck tailscale.com/disco from tailscale.com/derp diff --git a/derp/derp.go b/derp/derp.go index 65acd43210234..24c1ca65cfb3c 100644 --- a/derp/derp.go +++ b/derp/derp.go @@ -36,9 +36,13 @@ const ( frameHeaderLen = 1 + 4 // frameType byte + 4 byte length keyLen = 32 maxInfoLen = 1 << 20 - keepAlive = 60 * time.Second ) +// KeepAlive is the minimum frequency at which the DERP server sends +// keep alive frames. The server adds some jitter, so this timing is not +// exact, but 2x this value can be considered a missed keep alive. +const KeepAlive = 60 * time.Second + // ProtocolVersion is bumped whenever there's a wire-incompatible change. // - version 1 (zero on wire): consistent box headers, in use by employee dev nodes a bit // - version 2: received packets have src addrs in frameRecvPacket at beginning diff --git a/derp/derp_server.go b/derp/derp_server.go index c6a7494850ec8..bd67e7eeca22f 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -1789,7 +1789,7 @@ func (c *sclient) sendLoop(ctx context.Context) error { defer c.onSendLoopDone() jitter := rand.N(5 * time.Second) - keepAliveTick, keepAliveTickChannel := c.s.clock.NewTicker(keepAlive + jitter) + keepAliveTick, keepAliveTickChannel := c.s.clock.NewTicker(KeepAlive + jitter) defer keepAliveTick.Stop() var werr error // last write error diff --git a/net/netcheck/netcheck.go b/net/netcheck/netcheck.go index 54627f71383d1..cb622a339944d 100644 --- a/net/netcheck/netcheck.go +++ b/net/netcheck/netcheck.go @@ -23,6 +23,7 @@ import ( "syscall" "time" + "tailscale.com/derp" "tailscale.com/derp/derphttp" "tailscale.com/envknob" "tailscale.com/hostinfo" @@ -449,7 +450,7 @@ func makeProbePlan(dm *tailcfg.DERPMap, ifState *netmon.State, last *Report, pre // restoration back to the home DERP on the next full netcheck ~5 minutes later // - which is highly disruptive when it causes shifts in geo routed subnet // routers. By always including the home DERP in the incremental netcheck, we - // ensure that the home DERP is always probed, even if it observed a recenet + // ensure that the home DERP is always probed, even if it observed a recent // poor latency sample. This inclusion enables the latency history checks in // home DERP selection to still take effect. // planContainsHome indicates whether the home DERP has been added to the probePlan, @@ -989,7 +990,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe var wg sync.WaitGroup var need []*tailcfg.DERPRegion for rid, reg := range dm.Regions { - if !rs.haveRegionLatency(rid) && regionHasDERPNode(reg) { + if !rs.haveRegionLatency(rid) && regionHasDERPNode(reg) && !reg.Avoid && !reg.NoMeasureNoHome { need = append(need, reg) } } @@ -1371,6 +1372,15 @@ const ( // even without receiving a STUN response. // Note: must remain higher than the derp package frameReceiveRecordRate PreferredDERPFrameTime = 8 * time.Second + // PreferredDERPKeepAliveTimeout is 2x the DERP Keep Alive timeout. If there + // is no latency data to make judgements from, but we have heard from our + // current DERP region inside of 2x the KeepAlive window, don't switch DERP + // regions yet, keep the current region. This prevents region flapping / + // home DERP removal during short periods of packet loss where the DERP TCP + // connection may itself naturally recover. + // TODO(raggi): expose shared time bounds from the DERP package rather than + // duplicating them here. + PreferredDERPKeepAliveTimeout = 2 * derp.KeepAlive ) // addReportHistoryAndSetPreferredDERP adds r to the set of recent Reports @@ -1455,13 +1465,10 @@ func (c *Client) addReportHistoryAndSetPreferredDERP(rs *reportState, r *Report, // the STUN probe) since we started the netcheck, or in the past 2s, as // another signal for "this region is still working". heardFromOldRegionRecently := false + prevRegionLastHeard := rs.opts.getLastDERPActivity(prevDERP) if changingPreferred { - if lastHeard := rs.opts.getLastDERPActivity(prevDERP); !lastHeard.IsZero() { - now := c.timeNow() - - heardFromOldRegionRecently = lastHeard.After(rs.start) - heardFromOldRegionRecently = heardFromOldRegionRecently || lastHeard.After(now.Add(-PreferredDERPFrameTime)) - } + heardFromOldRegionRecently = prevRegionLastHeard.After(rs.start) + heardFromOldRegionRecently = heardFromOldRegionRecently || prevRegionLastHeard.After(now.Add(-PreferredDERPFrameTime)) } // The old region is accessible if we've heard from it via a non-STUN @@ -1488,17 +1495,20 @@ func (c *Client) addReportHistoryAndSetPreferredDERP(rs *reportState, r *Report, // If the forced DERP region probed successfully, or has recent traffic, // use it. _, haveLatencySample := r.RegionLatency[c.ForcePreferredDERP] - var recentActivity bool - if lastHeard := rs.opts.getLastDERPActivity(c.ForcePreferredDERP); !lastHeard.IsZero() { - now := c.timeNow() - recentActivity = lastHeard.After(rs.start) - recentActivity = recentActivity || lastHeard.After(now.Add(-PreferredDERPFrameTime)) - } + lastHeard := rs.opts.getLastDERPActivity(c.ForcePreferredDERP) + recentActivity := lastHeard.After(rs.start) + recentActivity = recentActivity || lastHeard.After(now.Add(-PreferredDERPFrameTime)) if haveLatencySample || recentActivity { r.PreferredDERP = c.ForcePreferredDERP } } + // If there was no latency data to make judgements on, but there is an + // active DERP connection that has at least been doing KeepAlive recently, + // keep it, rather than dropping it. + if r.PreferredDERP == 0 && prevRegionLastHeard.After(now.Add(-PreferredDERPKeepAliveTimeout)) { + r.PreferredDERP = prevDERP + } } func updateLatency(m map[int]time.Duration, regionID int, d time.Duration) { diff --git a/net/netcheck/netcheck_test.go b/net/netcheck/netcheck_test.go index 3affa614dae88..6830e7f27075c 100644 --- a/net/netcheck/netcheck_test.go +++ b/net/netcheck/netcheck_test.go @@ -18,6 +18,7 @@ import ( "testing" "time" + "tailscale.com/derp" "tailscale.com/net/netmon" "tailscale.com/net/stun/stuntest" "tailscale.com/tailcfg" @@ -419,6 +420,39 @@ func TestAddReportHistoryAndSetPreferredDERP(t *testing.T) { wantPrevLen: 2, wantDERP: 1, }, + { + name: "no_data_keep_home", + steps: []step{ + {0, report("d1", 2, "d2", 3)}, + {30 * time.Second, report()}, + {2 * time.Second, report()}, + {2 * time.Second, report()}, + {2 * time.Second, report()}, + {2 * time.Second, report()}, + }, + opts: &GetReportOpts{ + GetLastDERPActivity: mkLDAFunc(map[int]time.Time{ + 1: startTime, + }), + }, + wantPrevLen: 6, + wantDERP: 1, + }, + { + name: "no_data_home_expires", + steps: []step{ + {0, report("d1", 2, "d2", 3)}, + {30 * time.Second, report()}, + {2 * derp.KeepAlive, report()}, + }, + opts: &GetReportOpts{ + GetLastDERPActivity: mkLDAFunc(map[int]time.Time{ + 1: startTime, + }), + }, + wantPrevLen: 3, + wantDERP: 0, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 9206e766edc0492a3899633c5ea3a9a8ace12fe0 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Tue, 3 Jun 2025 15:24:31 -0700 Subject: [PATCH 058/263] net/packet: cleanup IPv4 fragment guards The first packet fragment guard had an additional guard clause that was incorrectly comparing a length in bytes to a length in octets, and was also comparing what should have been an entire IPv4 through transport header length to a subprotocol payload length. The subprotocol header size guards were otherwise protecting against short transport headers, as is the conservative non-first fragment minimum offset size. Add an explicit disallowing of fragmentation for TSMP for the avoidance of doubt. Updates #cleanup Updates #5727 Signed-off-by: James Tucker --- net/packet/header.go | 1 + net/packet/packet.go | 34 +++++++---- net/packet/packet_test.go | 122 ++++++++++++++++++++++++++++++++++++++ net/packet/tsmp.go | 2 + 4 files changed, 149 insertions(+), 10 deletions(-) diff --git a/net/packet/header.go b/net/packet/header.go index dbe84429adbd8..fa66a8641c6c4 100644 --- a/net/packet/header.go +++ b/net/packet/header.go @@ -8,6 +8,7 @@ import ( "math" ) +const igmpHeaderLength = 8 const tcpHeaderLength = 20 const sctpHeaderLength = 12 diff --git a/net/packet/packet.go b/net/packet/packet.go index 876a653ed4c79..34b63aadd2c2e 100644 --- a/net/packet/packet.go +++ b/net/packet/packet.go @@ -161,14 +161,8 @@ func (q *Parsed) decode4(b []byte) { if fragOfs == 0 { // This is the first fragment - if moreFrags && len(sub) < minFragBlks { - // Suspiciously short first fragment, dump it. - q.IPProto = unknown - return - } - // otherwise, this is either non-fragmented (the usual case) - // or a big enough initial fragment that we can read the - // whole subprotocol header. + // Every protocol below MUST check that it has at least one entire + // transport header in order to protect against fragment confusion. switch q.IPProto { case ipproto.ICMPv4: if len(sub) < icmp4HeaderLength { @@ -180,6 +174,10 @@ func (q *Parsed) decode4(b []byte) { q.dataofs = q.subofs + icmp4HeaderLength return case ipproto.IGMP: + if len(sub) < igmpHeaderLength { + q.IPProto = unknown + return + } // Keep IPProto, but don't parse anything else // out. return @@ -212,6 +210,15 @@ func (q *Parsed) decode4(b []byte) { q.Dst = withPort(q.Dst, binary.BigEndian.Uint16(sub[2:4])) return case ipproto.TSMP: + // Strictly disallow fragmented TSMP + if moreFrags { + q.IPProto = unknown + return + } + if len(sub) < minTSMPSize { + q.IPProto = unknown + return + } // Inter-tailscale messages. q.dataofs = q.subofs return @@ -224,8 +231,11 @@ func (q *Parsed) decode4(b []byte) { } else { // This is a fragment other than the first one. if fragOfs < minFragBlks { - // First frag was suspiciously short, so we can't - // trust the followup either. + // disallow fragment offsets that are potentially inside of a + // transport header. This is notably asymmetric with the + // first-packet limit, that may allow a first-packet that requires a + // shorter offset than this limit, but without state to tie this + // to the first fragment we can not allow shorter packets. q.IPProto = unknown return } @@ -315,6 +325,10 @@ func (q *Parsed) decode6(b []byte) { q.Dst = withPort(q.Dst, binary.BigEndian.Uint16(sub[2:4])) return case ipproto.TSMP: + if len(sub) < minTSMPSize { + q.IPProto = unknown + return + } // Inter-tailscale messages. q.dataofs = q.subofs return diff --git a/net/packet/packet_test.go b/net/packet/packet_test.go index 4fc804a4fea21..09c2c101d66d9 100644 --- a/net/packet/packet_test.go +++ b/net/packet/packet_test.go @@ -385,6 +385,124 @@ var sctpDecode = Parsed{ Dst: mustIPPort("100.74.70.3:456"), } +var ipv4ShortFirstFragmentBuffer = []byte{ + // IP header (20 bytes) + 0x45, 0x00, 0x00, 0x4f, // Total length 79 bytes + 0x00, 0x01, 0x20, 0x00, // ID, Flags (MoreFragments set, offset 0) + 0x40, 0x06, 0x00, 0x00, // TTL, Protocol (TCP), Checksum + 0x01, 0x02, 0x03, 0x04, // Source IP + 0x05, 0x06, 0x07, 0x08, // Destination IP + // TCP header (20 bytes), but packet is truncated to 59 bytes of TCP data + // (total 79 bytes, 20 for IP) + 0x00, 0x7b, 0x02, 0x37, 0x00, 0x00, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + // Payload (39 bytes) + 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, +} + +var ipv4ShortFirstFragmentDecode = Parsed{ + b: ipv4ShortFirstFragmentBuffer, + subofs: 20, + dataofs: 40, + length: len(ipv4ShortFirstFragmentBuffer), + IPVersion: 4, + IPProto: ipproto.TCP, + Src: mustIPPort("1.2.3.4:123"), + Dst: mustIPPort("5.6.7.8:567"), + TCPFlags: 0x12, // SYN + ACK +} + +var ipv4SmallOffsetFragmentBuffer = []byte{ + // IP header (20 bytes) + 0x45, 0x00, 0x00, 0x28, // Total length 40 bytes + 0x00, 0x01, 0x20, 0x08, // ID, Flags (MoreFragments set, offset 8 bytes (0x08 / 8 = 1)) + 0x40, 0x06, 0x00, 0x00, // TTL, Protocol (TCP), Checksum + 0x01, 0x02, 0x03, 0x04, // Source IP + 0x05, 0x06, 0x07, 0x08, // Destination IP + // Payload (20 bytes) - this would be part of the TCP header in a real scenario + 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x61, 0x61, 0x61, 0x61, +} + +var ipv4SmallOffsetFragmentDecode = Parsed{ + b: ipv4SmallOffsetFragmentBuffer, + subofs: 20, // subofs will still be set based on IHL + dataofs: 0, // It's unknown, so dataofs should be 0 + length: len(ipv4SmallOffsetFragmentBuffer), + IPVersion: 4, + IPProto: ipproto.Unknown, // Expected to be Unknown + Src: mustIPPort("1.2.3.4:0"), + Dst: mustIPPort("5.6.7.8:0"), +} + +// First fragment packet missing exactly one byte of the TCP header +var ipv4OneByteShortTCPHeaderBuffer = []byte{ + // IP header (20 bytes) + 0x45, 0x00, 0x00, 0x27, // Total length 51 bytes (20 IP + 19 TCP) + 0x00, 0x01, 0x20, 0x00, // ID, Flags (MoreFragments set, offset 0) + 0x40, 0x06, 0x00, 0x00, // TTL, Protocol (TCP), Checksum + 0x01, 0x02, 0x03, 0x04, // Source IP + 0x05, 0x06, 0x07, 0x08, // Destination IP + // TCP header - only 19 bytes (one byte short of the required 20) + 0x00, 0x7b, 0x02, 0x37, // Source port, Destination port + 0x00, 0x00, 0x12, 0x34, // Sequence number + 0x00, 0x00, 0x00, 0x00, // Acknowledgment number + 0x50, 0x12, 0x01, 0x00, // Data offset, flags, window size + 0x00, 0x00, 0x00, // Checksum (missing the last byte of urgent pointer) +} + +// IPv4 packet with maximum header length (60 bytes = 15 words) and a TCP header that's +// one byte short of being complete +var ipv4MaxHeaderShortTCPBuffer = []byte{ + // IP header with max options (60 bytes) + 0x4F, 0x00, 0x00, 0x4F, // Version (4) + IHL (15), ToS, Total length 79 bytes (60 IP + 19 TCP) + 0x00, 0x01, 0x20, 0x00, // ID, Flags (MoreFragments set, offset 0) + 0x40, 0x06, 0x00, 0x00, // TTL, Protocol (TCP), Checksum + 0x01, 0x02, 0x03, 0x04, // Source IP + 0x05, 0x06, 0x07, 0x08, // Destination IP + // IPv4 options (40 bytes) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + // TCP header - only 19 bytes (one byte short of the required 20) + 0x00, 0x7b, 0x02, 0x37, // Source port, Destination port + 0x00, 0x00, 0x12, 0x34, // Sequence number + 0x00, 0x00, 0x00, 0x00, // Acknowledgment number + 0x50, 0x12, 0x01, 0x00, // Data offset, flags, window size + 0x00, 0x00, 0x00, // Checksum (missing the last byte of urgent pointer) +} + +var ipv4MaxHeaderShortTCPDecode = Parsed{ + b: ipv4MaxHeaderShortTCPBuffer, + subofs: 60, // 60 bytes for full IPv4 header with max options + dataofs: 0, // It's unknown, so dataofs should be 0 + length: len(ipv4MaxHeaderShortTCPBuffer), + IPVersion: 4, + IPProto: ipproto.Unknown, // Expected to be Unknown + Src: mustIPPort("1.2.3.4:0"), + Dst: mustIPPort("5.6.7.8:0"), +} + +var ipv4OneByteShortTCPHeaderDecode = Parsed{ + b: ipv4OneByteShortTCPHeaderBuffer, + subofs: 20, + dataofs: 0, // It's unknown, so dataofs should be 0 + length: len(ipv4OneByteShortTCPHeaderBuffer), + IPVersion: 4, + IPProto: ipproto.Unknown, // Expected to be Unknown + Src: mustIPPort("1.2.3.4:0"), + Dst: mustIPPort("5.6.7.8:0"), +} + func TestParsedString(t *testing.T) { tests := []struct { name string @@ -450,6 +568,10 @@ func TestDecode(t *testing.T) { {"ipv4_sctp", sctpBuffer, sctpDecode}, {"ipv4_frag", tcp4MediumFragmentBuffer, tcp4MediumFragmentDecode}, {"ipv4_fragtooshort", tcp4ShortFragmentBuffer, tcp4ShortFragmentDecode}, + {"ipv4_short_first_fragment", ipv4ShortFirstFragmentBuffer, ipv4ShortFirstFragmentDecode}, + {"ipv4_small_offset_fragment", ipv4SmallOffsetFragmentBuffer, ipv4SmallOffsetFragmentDecode}, + {"ipv4_one_byte_short_tcp_header", ipv4OneByteShortTCPHeaderBuffer, ipv4OneByteShortTCPHeaderDecode}, + {"ipv4_max_header_short_tcp", ipv4MaxHeaderShortTCPBuffer, ipv4MaxHeaderShortTCPDecode}, {"ip97", mustHexDecode("4500 0019 d186 4000 4061 751d 644a 4603 6449 e549 6865 6c6c 6f"), Parsed{ IPVersion: 4, diff --git a/net/packet/tsmp.go b/net/packet/tsmp.go index 4e004cca2cb7a..d78d10d36d3bb 100644 --- a/net/packet/tsmp.go +++ b/net/packet/tsmp.go @@ -19,6 +19,8 @@ import ( "tailscale.com/types/ipproto" ) +const minTSMPSize = 7 // the rejected body is 7 bytes + // TailscaleRejectedHeader is a TSMP message that says that one // Tailscale node has rejected the connection from another. Unlike a // TCP RST, this includes a reason. From 923bbd696fabd6a2a11f380cf4368974461d9690 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Wed, 11 Jun 2025 13:56:46 -0700 Subject: [PATCH 059/263] prober: record DERP dropped packets as they occur Record dropped packets as soon as they time out, rather than after tx record queues spill over, this will more accurately capture small amounts of packet loss in a timely fashion. Updates tailscale/corp#24522 Signed-off-by: James Tucker --- prober/derp.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/prober/derp.go b/prober/derp.go index e21c8ce76668f..c7a82317dcabc 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -425,6 +425,24 @@ func runDerpProbeQueuingDelayContinously(ctx context.Context, from, to *tailcfg. txRecords := make([]txRecord, 0, packetsPerSecond*int(packetTimeout.Seconds())) var txRecordsMu sync.Mutex + // applyTimeouts walks over txRecords and expires any records that are older + // than packetTimeout, recording in metrics that they were removed. + applyTimeouts := func() { + txRecordsMu.Lock() + defer txRecordsMu.Unlock() + + now := time.Now() + recs := txRecords[:0] + for _, r := range txRecords { + if now.Sub(r.at) > packetTimeout { + packetsDropped.Add(1) + } else { + recs = append(recs, r) + } + } + txRecords = recs + } + // Send the packets. sendErrC := make(chan error, 1) // TODO: construct a disco CallMeMaybe in the same fashion as magicsock, e.g. magic bytes, src pub, seal payload. @@ -445,10 +463,12 @@ func runDerpProbeQueuingDelayContinously(ctx context.Context, from, to *tailcfg. case <-ctx.Done(): return case <-t.C: + applyTimeouts() txRecordsMu.Lock() if len(txRecords) == cap(txRecords) { txRecords = slices.Delete(txRecords, 0, 1) packetsDropped.Add(1) + log.Printf("unexpected: overflow in txRecords") } txRecords = append(txRecords, txRecord{time.Now(), seq}) txRecordsMu.Unlock() From dac00e99163d88895bdfe1c1606e62938580d89f Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Fri, 13 Jun 2025 11:30:55 -0700 Subject: [PATCH 060/263] go.mod: bump github.com/cloudflare/circl (#16264) See https://github.com/cloudflare/circl/security/advisories/GHSA-2x5j-vhc8-9cwm This dependency is used in our release builder indirectly via https://github.com/ProtonMail/go-crypto/blob/3b22d8539b95b3b7e76a911053023e6ef9ef51d6/go.mod#L6 We should not be affected, since this is used indirectly for pgp signatures on our .deb releases, where we use only trusted inputs. Updates #cleanup Signed-off-by: Andrew Lytvynov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ec98275e599f0..0c3d05d590c62 100644 --- a/go.mod +++ b/go.mod @@ -225,7 +225,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charithe/durationcheck v0.0.10 // indirect github.com/chavacava/garif v0.1.0 // indirect - github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudflare/circl v1.6.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/curioswitch/go-reassign v0.2.0 // indirect github.com/daixiang0/gci v0.12.3 // indirect diff --git a/go.sum b/go.sum index 0b521da8ca4c4..6f44cd86eb068 100644 --- a/go.sum +++ b/go.sum @@ -226,8 +226,8 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp github.com/ckaznocha/intrange v0.1.0 h1:ZiGBhvrdsKpoEfzh9CjBfDSZof6QB0ORY5tXasUtiew= github.com/ckaznocha/intrange v0.1.0/go.mod h1:Vwa9Ekex2BrEQMg6zlrWwbs/FtYw7eS5838Q7UjK7TQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= From 6a4d92ecef284229bf86286de0bb15e086a79d2c Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Fri, 13 Jun 2025 14:39:35 -0500 Subject: [PATCH 061/263] ipn/ipnlocal: replace nodeContext with nodeBackend in comments We renamed the type in #15866 but didn't update the comments at the time. Updates #cleanup Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 48 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 88adb397354a8..7b7893bc1a9e6 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -200,14 +200,14 @@ type LocalBackend struct { portpollOnce sync.Once // guards starting readPoller varRoot string // or empty if SetVarRoot never called logFlushFunc func() // or nil if SetLogFlusher wasn't called - em *expiryManager // non-nil; TODO(nickkhyl): move to nodeContext - sshAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeContext + em *expiryManager // non-nil; TODO(nickkhyl): move to nodeBackend + sshAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeBackend // webClientAtomicBool controls whether the web client is running. This should // be true unless the disable-web-client node attribute has been set. - webClientAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeContext + webClientAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeBackend // exposeRemoteWebClientAtomicBool controls whether the web client is exposed over // Tailscale on port 5252. - exposeRemoteWebClientAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeContext + exposeRemoteWebClientAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeBackend shutdownCalled bool // if Shutdown has been called debugSink packet.CaptureSink sockstatLogger *sockstatlog.Logger @@ -228,10 +228,10 @@ type LocalBackend struct { // is never called. getTCPHandlerForFunnelFlow func(srcAddr netip.AddrPort, dstPort uint16) (handler func(net.Conn)) - containsViaIPFuncAtomic syncs.AtomicValue[func(netip.Addr) bool] // TODO(nickkhyl): move to nodeContext - shouldInterceptTCPPortAtomic syncs.AtomicValue[func(uint16) bool] // TODO(nickkhyl): move to nodeContext - shouldInterceptVIPServicesTCPPortAtomic syncs.AtomicValue[func(netip.AddrPort) bool] // TODO(nickkhyl): move to nodeContext - numClientStatusCalls atomic.Uint32 // TODO(nickkhyl): move to nodeContext + containsViaIPFuncAtomic syncs.AtomicValue[func(netip.Addr) bool] // TODO(nickkhyl): move to nodeBackend + shouldInterceptTCPPortAtomic syncs.AtomicValue[func(uint16) bool] // TODO(nickkhyl): move to nodeBackend + shouldInterceptVIPServicesTCPPortAtomic syncs.AtomicValue[func(netip.AddrPort) bool] // TODO(nickkhyl): move to nodeBackend + numClientStatusCalls atomic.Uint32 // TODO(nickkhyl): move to nodeBackend // goTracker accounts for all goroutines started by LocalBacked, primarily // for testing and graceful shutdown purposes. @@ -256,7 +256,7 @@ type LocalBackend struct { // // It is safe for reading with or without holding b.mu, but mutating it in place // or creating a new one must be done with b.mu held. If both mutexes must be held, - // the LocalBackend's mutex must be acquired first before acquiring the nodeContext's mutex. + // the LocalBackend's mutex must be acquired first before acquiring the nodeBackend's mutex. // // We intend to relax this in the future and only require holding b.mu when replacing it, // but that requires a better (strictly ordered?) state machine and better management @@ -265,30 +265,30 @@ type LocalBackend struct { conf *conffile.Config // latest parsed config, or nil if not in declarative mode pm *profileManager // mu guards access - filterHash deephash.Sum // TODO(nickkhyl): move to nodeContext + filterHash deephash.Sum // TODO(nickkhyl): move to nodeBackend httpTestClient *http.Client // for controlclient. nil by default, used by tests. ccGen clientGen // function for producing controlclient; lazily populated sshServer SSHServer // or nil, initialized lazily. appConnector *appc.AppConnector // or nil, initialized when configured. // notifyCancel cancels notifications to the current SetNotifyCallback. notifyCancel context.CancelFunc - cc controlclient.Client // TODO(nickkhyl): move to nodeContext - ccAuto *controlclient.Auto // if cc is of type *controlclient.Auto; TODO(nickkhyl): move to nodeContext + cc controlclient.Client // TODO(nickkhyl): move to nodeBackend + ccAuto *controlclient.Auto // if cc is of type *controlclient.Auto; TODO(nickkhyl): move to nodeBackend machinePrivKey key.MachinePrivate - tka *tkaState // TODO(nickkhyl): move to nodeContext - state ipn.State // TODO(nickkhyl): move to nodeContext + tka *tkaState // TODO(nickkhyl): move to nodeBackend + state ipn.State // TODO(nickkhyl): move to nodeBackend capTailnetLock bool // whether netMap contains the tailnet lock capability // hostinfo is mutated in-place while mu is held. - hostinfo *tailcfg.Hostinfo // TODO(nickkhyl): move to nodeContext - nmExpiryTimer tstime.TimerController // for updating netMap on node expiry; can be nil; TODO(nickkhyl): move to nodeContext - activeLogin string // last logged LoginName from netMap; TODO(nickkhyl): move to nodeContext (or remove? it's in [ipn.LoginProfile]). + hostinfo *tailcfg.Hostinfo // TODO(nickkhyl): move to nodeBackend + nmExpiryTimer tstime.TimerController // for updating netMap on node expiry; can be nil; TODO(nickkhyl): move to nodeBackend + activeLogin string // last logged LoginName from netMap; TODO(nickkhyl): move to nodeBackend (or remove? it's in [ipn.LoginProfile]). engineStatus ipn.EngineStatus endpoints []tailcfg.Endpoint blocked bool - keyExpired bool // TODO(nickkhyl): move to nodeContext - authURL string // non-empty if not Running; TODO(nickkhyl): move to nodeContext - authURLTime time.Time // when the authURL was received from the control server; TODO(nickkhyl): move to nodeContext - authActor ipnauth.Actor // an actor who called [LocalBackend.StartLoginInteractive] last, or nil; TODO(nickkhyl): move to nodeContext + keyExpired bool // TODO(nickkhyl): move to nodeBackend + authURL string // non-empty if not Running; TODO(nickkhyl): move to nodeBackend + authURLTime time.Time // when the authURL was received from the control server; TODO(nickkhyl): move to nodeBackend + authActor ipnauth.Actor // an actor who called [LocalBackend.StartLoginInteractive] last, or nil; TODO(nickkhyl): move to nodeBackend egg bool prevIfState *netmon.State peerAPIServer *peerAPIServer // or nil @@ -305,7 +305,7 @@ type LocalBackend struct { lastSelfUpdateState ipnstate.SelfUpdateStatus // capForcedNetfilter is the netfilter that control instructs Linux clients // to use, unless overridden locally. - capForcedNetfilter string // TODO(nickkhyl): move to nodeContext + capForcedNetfilter string // TODO(nickkhyl): move to nodeBackend // offlineAutoUpdateCancel stops offline auto-updates when called. It // should be used via stopOfflineAutoUpdate and // maybeStartOfflineAutoUpdate. It is nil when offline auto-updates are @@ -317,7 +317,7 @@ type LocalBackend struct { // ServeConfig fields. (also guarded by mu) lastServeConfJSON mem.RO // last JSON that was parsed into serveConfig serveConfig ipn.ServeConfigView // or !Valid if none - ipVIPServiceMap netmap.IPServiceMappings // map of VIPService IPs to their corresponding service names; TODO(nickkhyl): move to nodeContext + ipVIPServiceMap netmap.IPServiceMappings // map of VIPService IPs to their corresponding service names; TODO(nickkhyl): move to nodeBackend webClient webClient webClientListeners map[netip.AddrPort]*localListener // listeners for local web client traffic @@ -332,7 +332,7 @@ type LocalBackend struct { // dialPlan is any dial plan that we've received from the control // server during a previous connection; it is cleared on logout. - dialPlan atomic.Pointer[tailcfg.ControlDialPlan] // TODO(nickkhyl): maybe move to nodeContext? + dialPlan atomic.Pointer[tailcfg.ControlDialPlan] // TODO(nickkhyl): maybe move to nodeBackend? // tkaSyncLock is used to make tkaSyncIfNeeded an exclusive // section. This is needed to stop two map-responses in quick succession From fe391d569442283ed4f10e1d57c9e845290275bb Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Fri, 13 Jun 2025 15:47:35 -0700 Subject: [PATCH 062/263] client/local: use an iterator to stream bus events (#16269) This means the caller does not have to remember to close the reader, and avoids having to duplicate the logic to decode JSON into events. Updates #15160 Change-Id: I20186fabb02f72522f61d5908c4cc80b86b8936b Signed-off-by: M. J. Fromberger --- client/local/local.go | 57 +++++++++++++++++++++++++------------- cmd/derper/depaware.txt | 2 +- cmd/tailscale/cli/debug.go | 11 ++------ 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/client/local/local.go b/client/local/local.go index bc643ad7932db..12bf2f7d6fef3 100644 --- a/client/local/local.go +++ b/client/local/local.go @@ -1,12 +1,11 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build go1.22 - // Package local contains a Go client for the Tailscale LocalAPI. package local import ( + "bufio" "bytes" "cmp" "context" @@ -16,6 +15,7 @@ import ( "errors" "fmt" "io" + "iter" "net" "net/http" "net/http/httptrace" @@ -42,6 +42,7 @@ import ( "tailscale.com/types/dnstype" "tailscale.com/types/key" "tailscale.com/types/tkatype" + "tailscale.com/util/eventbus" "tailscale.com/util/syspolicy/setting" ) @@ -414,24 +415,42 @@ func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) { return res.Body, nil } -// StreamBusEvents returns a stream of the Tailscale bus events as they arrive. -// Close the context to stop the stream. -// Expected response from the server is newline-delimited JSON. -// The caller must close the reader when it is finished reading. -func (lc *Client) StreamBusEvents(ctx context.Context) (io.ReadCloser, error) { - req, err := http.NewRequestWithContext(ctx, "GET", - "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-bus-events", nil) - if err != nil { - return nil, err - } - res, err := lc.doLocalRequestNiceError(req) - if err != nil { - return nil, err - } - if res.StatusCode != http.StatusOK { - return nil, errors.New(res.Status) +// StreamBusEvents returns an iterator of Tailscale bus events as they arrive. +// Each pair is a valid event and a nil error, or a zero event a non-nil error. +// In case of error, the iterator ends after the pair reporting the error. +// Iteration stops if ctx ends. +func (lc *Client) StreamBusEvents(ctx context.Context) iter.Seq2[eventbus.DebugEvent, error] { + return func(yield func(eventbus.DebugEvent, error) bool) { + req, err := http.NewRequestWithContext(ctx, "GET", + "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-bus-events", nil) + if err != nil { + yield(eventbus.DebugEvent{}, err) + return + } + res, err := lc.doLocalRequestNiceError(req) + if err != nil { + yield(eventbus.DebugEvent{}, err) + return + } + if res.StatusCode != http.StatusOK { + yield(eventbus.DebugEvent{}, errors.New(res.Status)) + return + } + defer res.Body.Close() + dec := json.NewDecoder(bufio.NewReader(res.Body)) + for { + var evt eventbus.DebugEvent + if err := dec.Decode(&evt); err == io.EOF { + return + } else if err != nil { + yield(eventbus.DebugEvent{}, err) + return + } + if !yield(evt, nil) { + return + } + } } - return res.Body, nil } // Pprof returns a pprof profile of the Tailscale daemon. diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index 640e64d6cab21..7adbf397f2f4f 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -157,7 +157,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa 💣 tailscale.com/util/deephash from tailscale.com/util/syspolicy/setting L 💣 tailscale.com/util/dirwalk from tailscale.com/metrics tailscale.com/util/dnsname from tailscale.com/hostinfo+ - tailscale.com/util/eventbus from tailscale.com/net/netmon + tailscale.com/util/eventbus from tailscale.com/net/netmon+ 💣 tailscale.com/util/hashx from tailscale.com/util/deephash tailscale.com/util/httpm from tailscale.com/client/tailscale tailscale.com/util/lineiter from tailscale.com/hostinfo+ diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index 025382ca9a7ea..ec8a0700dec19 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -791,21 +791,14 @@ func runDaemonLogs(ctx context.Context, args []string) error { } func runDaemonBusEvents(ctx context.Context, args []string) error { - logs, err := localClient.StreamBusEvents(ctx) - if err != nil { - return err - } - defer logs.Close() - d := json.NewDecoder(bufio.NewReader(logs)) - for { - var line eventbus.DebugEvent - err := d.Decode(&line) + for line, err := range localClient.StreamBusEvents(ctx) { if err != nil { return err } fmt.Printf("[%d][%q][from: %q][to: %q] %s\n", line.Count, line.Type, line.From, line.To, line.Event) } + return nil } var metricsArgs struct { From 733bfaeffed5c7cc3a05478b7671e85fafd5689c Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Fri, 13 Jun 2025 12:51:40 -0500 Subject: [PATCH 063/263] ipn/ipnlocal: signal nodeBackend readiness and shutdown We update LocalBackend to shut down the current nodeBackend when switching to a different node, and to mark the new node's nodeBackend as ready when the switch completes. Updates tailscale/corp#28014 Updates tailscale/corp#29543 Updates #12614 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 43 ++++++++--- ipn/ipnlocal/node_backend.go | 82 ++++++++++++++++++-- ipn/ipnlocal/node_backend_test.go | 121 ++++++++++++++++++++++++++++++ 3 files changed, 230 insertions(+), 16 deletions(-) create mode 100644 ipn/ipnlocal/node_backend_test.go diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 7b7893bc1a9e6..daedb1e1924a0 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -168,6 +168,17 @@ type watchSession struct { var metricCaptivePortalDetected = clientmetric.NewCounter("captiveportal_detected") +var ( + // errShutdown indicates that the [LocalBackend.Shutdown] was called. + errShutdown = errors.New("shutting down") + + // errNodeContextChanged indicates that [LocalBackend] has switched + // to a different [localNodeContext], usually due to a profile change. + // It is used as a context cancellation cause for the old context + // and can be returned when an operation is performed on it. + errNodeContextChanged = errors.New("profile changed") +) + // LocalBackend is the glue between the major pieces of the Tailscale // network software: the cloud control plane (via controlclient), the // network data plane (via wgengine), and the user-facing UIs and CLIs @@ -180,11 +191,11 @@ var metricCaptivePortalDetected = clientmetric.NewCounter("captiveportal_detecte // state machine generates events back out to zero or more components. type LocalBackend struct { // Elements that are thread-safe or constant after construction. - ctx context.Context // canceled by [LocalBackend.Shutdown] - ctxCancel context.CancelFunc // cancels ctx - logf logger.Logf // general logging - keyLogf logger.Logf // for printing list of peers on change - statsLogf logger.Logf // for printing peers stats on change + ctx context.Context // canceled by [LocalBackend.Shutdown] + ctxCancel context.CancelCauseFunc // cancels ctx + logf logger.Logf // general logging + keyLogf logger.Logf // for printing list of peers on change + statsLogf logger.Logf // for printing peers stats on change sys *tsd.System health *health.Tracker // always non-nil metrics metrics @@ -463,7 +474,7 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo envknob.LogCurrent(logf) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancelCause(context.Background()) clock := tstime.StdClock{} // Until we transition to a Running state, use a canceled context for @@ -503,7 +514,10 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo captiveCancel: nil, // so that we start checkCaptivePortalLoop when Running needsCaptiveDetection: make(chan bool), } - b.currentNodeAtomic.Store(newNodeBackend()) + nb := newNodeBackend(ctx) + b.currentNodeAtomic.Store(nb) + nb.ready() + mConn.SetNetInfoCallback(b.setNetInfo) if sys.InitialConfig != nil { @@ -586,8 +600,10 @@ func (b *LocalBackend) currentNode() *nodeBackend { return v } // Auto-init one in tests for LocalBackend created without the NewLocalBackend constructor... - v := newNodeBackend() - b.currentNodeAtomic.CompareAndSwap(nil, v) + v := newNodeBackend(cmp.Or(b.ctx, context.Background())) + if b.currentNodeAtomic.CompareAndSwap(nil, v) { + v.ready() + } return b.currentNodeAtomic.Load() } @@ -1089,8 +1105,9 @@ func (b *LocalBackend) Shutdown() { if cc != nil { cc.Shutdown() } + b.ctxCancel(errShutdown) + b.currentNode().shutdown(errShutdown) extHost.Shutdown() - b.ctxCancel() b.e.Close() <-b.e.Done() b.awaitNoGoroutinesInTest() @@ -6992,7 +7009,11 @@ func (b *LocalBackend) resetForProfileChangeLockedOnEntry(unlock unlockOnce) err // down, so no need to do any work. return nil } - b.currentNodeAtomic.Store(newNodeBackend()) + newNode := newNodeBackend(b.ctx) + if oldNode := b.currentNodeAtomic.Swap(newNode); oldNode != nil { + oldNode.shutdown(errNodeContextChanged) + } + defer newNode.ready() b.setNetMapLocked(nil) // Reset netmap. b.updateFilterLocked(ipn.PrefsView{}) // Reset the NetworkMap in the engine diff --git a/ipn/ipnlocal/node_backend.go b/ipn/ipnlocal/node_backend.go index fb77f38ebc87a..361d10bb6cfcc 100644 --- a/ipn/ipnlocal/node_backend.go +++ b/ipn/ipnlocal/node_backend.go @@ -5,6 +5,7 @@ package ipnlocal import ( "cmp" + "context" "net/netip" "slices" "sync" @@ -39,7 +40,7 @@ import ( // Two pointers to different [nodeBackend] instances represent different local nodes. // However, there's currently a bug where a new [nodeBackend] might not be created // during an implicit node switch (see tailscale/corp#28014). - +// // In the future, we might want to include at least the following in this struct (in addition to the current fields). // However, not everything should be exported or otherwise made available to the outside world (e.g. [ipnext] extensions, // peer API handlers, etc.). @@ -61,6 +62,9 @@ import ( // Even if they're tied to the local node, instead of moving them here, we should extract the entire feature // into a separate package and have it install proper hooks. type nodeBackend struct { + ctx context.Context // canceled by [nodeBackend.shutdown] + ctxCancel context.CancelCauseFunc // cancels ctx + // filterAtomic is a stateful packet filter. Immutable once created, but can be // replaced with a new one. filterAtomic atomic.Pointer[filter.Filter] @@ -68,6 +72,9 @@ type nodeBackend struct { // TODO(nickkhyl): maybe use sync.RWMutex? mu sync.Mutex // protects the following fields + shutdownOnce sync.Once // guards calling [nodeBackend.shutdown] + readyCh chan struct{} // closed by [nodeBackend.ready]; nil after shutdown + // NetMap is the most recently set full netmap from the controlclient. // It can't be mutated in place once set. Because it can't be mutated in place, // delta updates from the control server don't apply to it. Instead, use @@ -88,12 +95,24 @@ type nodeBackend struct { nodeByAddr map[netip.Addr]tailcfg.NodeID } -func newNodeBackend() *nodeBackend { - cn := &nodeBackend{} +func newNodeBackend(ctx context.Context) *nodeBackend { + ctx, ctxCancel := context.WithCancelCause(ctx) + nb := &nodeBackend{ + ctx: ctx, + ctxCancel: ctxCancel, + readyCh: make(chan struct{}), + } // Default filter blocks everything and logs nothing. noneFilter := filter.NewAllowNone(logger.Discard, &netipx.IPSet{}) - cn.filterAtomic.Store(noneFilter) - return cn + nb.filterAtomic.Store(noneFilter) + return nb +} + +// Context returns a context that is canceled when the [nodeBackend] shuts down, +// either because [LocalBackend] is switching to a different [nodeBackend] +// or is shutting down itself. +func (nb *nodeBackend) Context() context.Context { + return nb.ctx } func (nb *nodeBackend) Self() tailcfg.NodeView { @@ -475,6 +494,59 @@ func (nb *nodeBackend) exitNodeCanProxyDNS(exitNodeID tailcfg.StableNodeID) (doh return exitNodeCanProxyDNS(nb.netMap, nb.peers, exitNodeID) } +// ready signals that [LocalBackend] has completed the switch to this [nodeBackend] +// and any pending calls to [nodeBackend.Wait] must be unblocked. +func (nb *nodeBackend) ready() { + nb.mu.Lock() + defer nb.mu.Unlock() + if nb.readyCh != nil { + close(nb.readyCh) + } +} + +// Wait blocks until [LocalBackend] completes the switch to this [nodeBackend] +// and calls [nodeBackend.ready]. It returns an error if the provided context +// is canceled or if the [nodeBackend] shuts down or is already shut down. +// +// It must not be called with the [LocalBackend]'s internal mutex held as [LocalBackend] +// may need to acquire it to complete the switch. +// +// TODO(nickkhyl): Relax this restriction once [LocalBackend]'s state machine +// runs in its own goroutine, or if we decide that waiting for the state machine +// restart to finish isn't necessary for [LocalBackend] to consider the switch complete. +// We mostly need this because of [LocalBackend.Start] acquiring b.mu and the fact that +// methods like [LocalBackend.SwitchProfile] must report any errors returned by it. +// Perhaps we could report those errors asynchronously as [health.Warnable]s? +func (nb *nodeBackend) Wait(ctx context.Context) error { + nb.mu.Lock() + readyCh := nb.readyCh + nb.mu.Unlock() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-nb.ctx.Done(): + return context.Cause(nb.ctx) + case <-readyCh: + return nil + } +} + +// shutdown shuts down the [nodeBackend] and cancels its context +// with the provided cause. +func (nb *nodeBackend) shutdown(cause error) { + nb.shutdownOnce.Do(func() { + nb.doShutdown(cause) + }) +} + +func (nb *nodeBackend) doShutdown(cause error) { + nb.mu.Lock() + defer nb.mu.Unlock() + nb.ctxCancel(cause) + nb.readyCh = nil +} + // dnsConfigForNetmap returns a *dns.Config for the given netmap, // prefs, client OS version, and cloud hosting environment. // diff --git a/ipn/ipnlocal/node_backend_test.go b/ipn/ipnlocal/node_backend_test.go new file mode 100644 index 0000000000000..a82b60a9a1726 --- /dev/null +++ b/ipn/ipnlocal/node_backend_test.go @@ -0,0 +1,121 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ipnlocal + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestNodeBackendReadiness(t *testing.T) { + nb := newNodeBackend(t.Context()) + + // The node backend is not ready until [nodeBackend.ready] is called, + // and [nodeBackend.Wait] should fail with [context.DeadlineExceeded]. + ctx, cancelCtx := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancelCtx() + if err := nb.Wait(ctx); err != ctx.Err() { + t.Fatalf("Wait: got %v; want %v", err, ctx.Err()) + } + + // Start a goroutine to wait for the node backend to become ready. + waitDone := make(chan struct{}) + go func() { + if err := nb.Wait(context.Background()); err != nil { + t.Errorf("Wait: got %v; want nil", err) + } + close(waitDone) + }() + + // Call [nodeBackend.ready] to indicate that the node backend is now ready. + go nb.ready() + + // Once the backend is called, [nodeBackend.Wait] should return immediately without error. + if err := nb.Wait(context.Background()); err != nil { + t.Fatalf("Wait: got %v; want nil", err) + } + // And any pending waiters should also be unblocked. + <-waitDone +} + +func TestNodeBackendShutdown(t *testing.T) { + nb := newNodeBackend(t.Context()) + + shutdownCause := errors.New("test shutdown") + + // Start a goroutine to wait for the node backend to become ready. + // This test expects it to block until the node backend shuts down + // and then return the specified shutdown cause. + waitDone := make(chan struct{}) + go func() { + if err := nb.Wait(context.Background()); err != shutdownCause { + t.Errorf("Wait: got %v; want %v", err, shutdownCause) + } + close(waitDone) + }() + + // Call [nodeBackend.shutdown] to indicate that the node backend is shutting down. + nb.shutdown(shutdownCause) + + // Calling it again is fine, but should not change the shutdown cause. + nb.shutdown(errors.New("test shutdown again")) + + // After shutdown, [nodeBackend.Wait] should return with the specified shutdown cause. + if err := nb.Wait(context.Background()); err != shutdownCause { + t.Fatalf("Wait: got %v; want %v", err, shutdownCause) + } + // The context associated with the node backend should also be cancelled + // and its cancellation cause should match the shutdown cause. + if err := nb.Context().Err(); !errors.Is(err, context.Canceled) { + t.Fatalf("Context.Err: got %v; want %v", err, context.Canceled) + } + if cause := context.Cause(nb.Context()); cause != shutdownCause { + t.Fatalf("Cause: got %v; want %v", cause, shutdownCause) + } + // And any pending waiters should also be unblocked. + <-waitDone +} + +func TestNodeBackendReadyAfterShutdown(t *testing.T) { + nb := newNodeBackend(t.Context()) + + shutdownCause := errors.New("test shutdown") + nb.shutdown(shutdownCause) + nb.ready() // Calling ready after shutdown is a no-op, but should not panic, etc. + if err := nb.Wait(context.Background()); err != shutdownCause { + t.Fatalf("Wait: got %v; want %v", err, shutdownCause) + } +} + +func TestNodeBackendParentContextCancellation(t *testing.T) { + ctx, cancelCtx := context.WithCancel(context.Background()) + nb := newNodeBackend(ctx) + + cancelCtx() + + // Cancelling the parent context should cause [nodeBackend.Wait] + // to return with [context.Canceled]. + if err := nb.Wait(context.Background()); !errors.Is(err, context.Canceled) { + t.Fatalf("Wait: got %v; want %v", err, context.Canceled) + } + + // And the node backend's context should also be cancelled. + if err := nb.Context().Err(); !errors.Is(err, context.Canceled) { + t.Fatalf("Context.Err: got %v; want %v", err, context.Canceled) + } +} + +func TestNodeBackendConcurrentReadyAndShutdown(t *testing.T) { + nb := newNodeBackend(t.Context()) + + // Calling [nodeBackend.ready] and [nodeBackend.shutdown] concurrently + // should not cause issues, and [nodeBackend.Wait] should unblock, + // but the result of [nodeBackend.Wait] is intentionally undefined. + go nb.ready() + go nb.shutdown(errors.New("test shutdown")) + + nb.Wait(context.Background()) +} From e29e3c150ff2a8fc5ecbe016ec275a9097a02b2e Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Mon, 16 Jun 2025 12:21:59 +0100 Subject: [PATCH 064/263] cmd/k8s-operator: ensure that TLS resources are updated for HA Ingress (#16262) Ensure that if the ProxyGroup for HA Ingress changes, the TLS Secret and Role and RoleBinding that allow proxies to read/write to it are updated. Fixes #16259 Signed-off-by: Irbe Krumina --- cmd/k8s-operator/ingress-for-pg.go | 31 ++- cmd/k8s-operator/ingress-for-pg_test.go | 296 +++++++++++++----------- cmd/k8s-operator/svc-for-pg_test.go | 10 +- 3 files changed, 183 insertions(+), 154 deletions(-) diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 66d74292b2207..ea31dbd63befb 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -252,7 +252,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin return false, fmt.Errorf("error determining DNS name base: %w", err) } dnsName := hostname + "." + tcd - if err := r.ensureCertResources(ctx, pgName, dnsName, ing); err != nil { + if err := r.ensureCertResources(ctx, pg, dnsName, ing); err != nil { return false, fmt.Errorf("error ensuring cert resources: %w", err) } @@ -931,18 +931,31 @@ func ownersAreSetAndEqual(a, b *tailscale.VIPService) bool { // (domain) is a valid Kubernetes resource name. // https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99 // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names -func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string, ing *networkingv1.Ingress) error { - secret := certSecret(pgName, r.tsNamespace, domain, ing) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, nil); err != nil { +func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string, ing *networkingv1.Ingress) error { + secret := certSecret(pg.Name, r.tsNamespace, domain, ing) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) { + // Labels might have changed if the Ingress has been updated to use a + // different ProxyGroup. + s.Labels = secret.Labels + }); err != nil { return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err) } - role := certSecretRole(pgName, r.tsNamespace, domain) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, nil); err != nil { + role := certSecretRole(pg.Name, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { + // Labels might have changed if the Ingress has been updated to use a + // different ProxyGroup. + r.Labels = role.Labels + }); err != nil { return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err) } - rb := certSecretRoleBinding(pgName, r.tsNamespace, domain) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rb, nil); err != nil { - return fmt.Errorf("failed to create or update RoleBinding %s: %w", rb.Name, err) + rolebinding := certSecretRoleBinding(pg.Name, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) { + // Labels and subjects might have changed if the Ingress has been updated to use a + // different ProxyGroup. + rb.Labels = rolebinding.Labels + rb.Subjects = rolebinding.Subjects + }); err != nil { + return fmt.Errorf("failed to create or update RoleBinding %s: %w", rolebinding.Name, err) } return nil } diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index b487d660cfa2f..05f4827927923 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -69,7 +69,7 @@ func TestIngressPGReconciler(t *testing.T) { expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:my-svc", false) verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) - verifyTailscaledConfig(t, fc, []string{"svc:my-svc"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc"}) // Verify that Role and RoleBinding have been created for the first Ingress. // Do not verify the cert Secret as that was already verified implicitly above. @@ -132,7 +132,7 @@ func TestIngressPGReconciler(t *testing.T) { verifyServeConfig(t, fc, "svc:my-other-svc", false) verifyTailscaleService(t, ft, "svc:my-other-svc", []string{"tcp:443"}) - // Verify that Role and RoleBinding have been created for the first Ingress. + // Verify that Role and RoleBinding have been created for the second Ingress. // Do not verify the cert Secret as that was already verified implicitly above. expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net")) expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net")) @@ -141,7 +141,7 @@ func TestIngressPGReconciler(t *testing.T) { verifyServeConfig(t, fc, "svc:my-svc", false) verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) - verifyTailscaledConfig(t, fc, []string{"svc:my-svc", "svc:my-other-svc"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc", "svc:my-other-svc"}) // Delete second Ingress if err := fc.Delete(context.Background(), ing2); err != nil { @@ -172,11 +172,20 @@ func TestIngressPGReconciler(t *testing.T) { t.Error("second Ingress service config was not cleaned up") } - verifyTailscaledConfig(t, fc, []string{"svc:my-svc"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc"}) expectMissing[corev1.Secret](t, fc, "operator-ns", "my-other-svc.ts.net") expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-other-svc.ts.net") expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-other-svc.ts.net") + // Test Ingress ProxyGroup change + createPGResources(t, fc, "test-pg-second") + mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) { + ing.Annotations["tailscale.com/proxy-group"] = "test-pg-second" + }) + expectReconciled(t, ingPGR, "default", "test-ingress") + expectEqual(t, fc, certSecretRole("test-pg-second", "operator-ns", "my-svc.ts.net")) + expectEqual(t, fc, certSecretRoleBinding("test-pg-second", "operator-ns", "my-svc.ts.net")) + // Delete the first Ingress and verify cleanup if err := fc.Delete(context.Background(), ing); err != nil { t.Fatalf("deleting Ingress: %v", err) @@ -187,7 +196,7 @@ func TestIngressPGReconciler(t *testing.T) { // Verify the ConfigMap was cleaned up cm = &corev1.ConfigMap{} if err := fc.Get(context.Background(), types.NamespacedName{ - Name: "test-pg-ingress-config", + Name: "test-pg-second-ingress-config", Namespace: "operator-ns", }, cm); err != nil { t.Fatalf("getting ConfigMap: %v", err) @@ -201,7 +210,7 @@ func TestIngressPGReconciler(t *testing.T) { if len(cfg.Services) > 0 { t.Error("serve config not cleaned up") } - verifyTailscaledConfig(t, fc, nil) + verifyTailscaledConfig(t, fc, "test-pg-second", nil) // Add verification that cert resources were cleaned up expectMissing[corev1.Secret](t, fc, "operator-ns", "my-svc.ts.net") @@ -245,7 +254,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) { expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:my-svc", false) verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) - verifyTailscaledConfig(t, fc, []string{"svc:my-svc"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc"}) // Update the Ingress hostname and make sure the original Tailscale Service is deleted. mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) { @@ -256,7 +265,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) { expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:updated-svc", false) verifyTailscaleService(t, ft, "svc:updated-svc", []string{"tcp:443"}) - verifyTailscaledConfig(t, fc, []string{"svc:updated-svc"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:updated-svc"}) _, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName("svc:my-svc")) if err == nil { @@ -550,6 +559,117 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { } } +func TestIngressPGReconciler_MultiCluster(t *testing.T) { + ingPGR, fc, ft := setupIngressTest(t) + ingPGR.operatorID = "operator-1" + + // Create initial Ingress + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: ptr.To("tailscale"), + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"my-svc"}}, + }, + }, + } + mustCreate(t, fc, ing) + + // Simulate existing Tailscale Service from another cluster + existingVIPSvc := &tailscale.VIPService{ + Name: "svc:my-svc", + Annotations: map[string]string{ + ownerAnnotation: `{"ownerrefs":[{"operatorID":"operator-2"}]}`, + }, + } + ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{ + "svc:my-svc": existingVIPSvc, + } + + // Verify reconciliation adds our operator reference + expectReconciled(t, ingPGR, "default", "test-ingress") + + tsSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc") + if err != nil { + t.Fatalf("getting Tailscale Service: %v", err) + } + if tsSvc == nil { + t.Fatal("Tailscale Service not found") + } + + o, err := parseOwnerAnnotation(tsSvc) + if err != nil { + t.Fatalf("parsing owner annotation: %v", err) + } + + wantOwnerRefs := []OwnerRef{ + {OperatorID: "operator-2"}, + {OperatorID: "operator-1"}, + } + if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) { + t.Errorf("incorrect owner refs\ngot: %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs) + } + + // Delete the Ingress and verify Tailscale Service still exists with one owner ref + if err := fc.Delete(context.Background(), ing); err != nil { + t.Fatalf("deleting Ingress: %v", err) + } + expectRequeue(t, ingPGR, "default", "test-ingress") + + tsSvc, err = ft.GetVIPService(context.Background(), "svc:my-svc") + if err != nil { + t.Fatalf("getting Tailscale Service after deletion: %v", err) + } + if tsSvc == nil { + t.Fatal("Tailscale Service was incorrectly deleted") + } + + o, err = parseOwnerAnnotation(tsSvc) + if err != nil { + t.Fatalf("parsing owner annotation: %v", err) + } + + wantOwnerRefs = []OwnerRef{ + {OperatorID: "operator-2"}, + } + if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) { + t.Errorf("incorrect owner refs after deletion\ngot: %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs) + } +} + +func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: domain, + Namespace: "operator-ns", + Labels: map[string]string{ + kubetypes.LabelManaged: "true", + labelProxyGroup: pgName, + labelDomain: domain, + kubetypes.LabelSecretType: "certs", + }, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSCertKey: []byte("fake-cert"), + corev1.TLSPrivateKeyKey: []byte("fake-key"), + }, + } + + _, err := createOrUpdate(ctx, c, "operator-ns", secret, func(s *corev1.Secret) { + s.Data = secret.Data + }) + return err +} + func verifyTailscaleService(t *testing.T, ft *fakeTSClient, serviceName string, wantPorts []string) { t.Helper() tsSvc, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(serviceName)) @@ -618,7 +738,7 @@ func verifyServeConfig(t *testing.T, fc client.Client, serviceName string, wantH } } -func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []string) { +func verifyTailscaledConfig(t *testing.T, fc client.Client, pgName string, expectedServices []string) { t.Helper() var expected string if expectedServices != nil && len(expectedServices) > 0 { @@ -630,9 +750,9 @@ func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []s } expectEqual(t, fc, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: pgConfigSecretName("test-pg", 0), + Name: pgConfigSecretName(pgName, 0), Namespace: "operator-ns", - Labels: pgSecretLabels("test-pg", "config"), + Labels: pgSecretLabels(pgName, "config"), }, Data: map[string][]byte{ tsoperator.TailscaledConfigFileName(106): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)), @@ -640,53 +760,44 @@ func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []s }) } -func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) { - tsIngressClass := &networkingv1.IngressClass{ - ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, - Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}, - } - +func createPGResources(t *testing.T, fc client.Client, pgName string) { + t.Helper() // Pre-create the ProxyGroup pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-pg", + Name: pgName, Generation: 1, }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, }, } + mustCreate(t, fc, pg) // Pre-create the ConfigMap for the ProxyGroup pgConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-pg-ingress-config", + Name: fmt.Sprintf("%s-ingress-config", pgName), Namespace: "operator-ns", }, BinaryData: map[string][]byte{ "serve-config.json": []byte(`{"Services":{}}`), }, } + mustCreate(t, fc, pgConfigMap) // Pre-create a config Secret for the ProxyGroup pgCfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: pgConfigSecretName("test-pg", 0), + Name: pgConfigSecretName(pgName, 0), Namespace: "operator-ns", - Labels: pgSecretLabels("test-pg", "config"), + Labels: pgSecretLabels(pgName, "config"), }, Data: map[string][]byte{ tsoperator.TailscaledConfigFileName(106): []byte("{}"), }, } - - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). - WithObjects(pg, pgCfgSecret, pgConfigMap, tsIngressClass). - WithStatusSubresource(pg). - Build() - - // Set ProxyGroup status to ready + mustCreate(t, fc, pgCfgSecret) pg.Status.Conditions = []metav1.Condition{ { Type: string(tsapi.ProxyGroupReady), @@ -697,6 +808,22 @@ func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeT if err := fc.Status().Update(context.Background(), pg); err != nil { t.Fatal(err) } +} + +func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) { + tsIngressClass := &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, + Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}, + } + + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(tsIngressClass). + WithStatusSubresource(&tsapi.ProxyGroup{}). + Build() + + createPGResources(t, fc, "test-pg") + fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} ft := &fakeTSClient{} @@ -726,114 +853,3 @@ func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeT return ingPGR, fc, ft } - -func TestIngressPGReconciler_MultiCluster(t *testing.T) { - ingPGR, fc, ft := setupIngressTest(t) - ingPGR.operatorID = "operator-1" - - // Create initial Ingress - ing := &networkingv1.Ingress{ - TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "default", - UID: types.UID("1234-UID"), - Annotations: map[string]string{ - "tailscale.com/proxy-group": "test-pg", - }, - }, - Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), - TLS: []networkingv1.IngressTLS{ - {Hosts: []string{"my-svc"}}, - }, - }, - } - mustCreate(t, fc, ing) - - // Simulate existing Tailscale Service from another cluster - existingVIPSvc := &tailscale.VIPService{ - Name: "svc:my-svc", - Annotations: map[string]string{ - ownerAnnotation: `{"ownerrefs":[{"operatorID":"operator-2"}]}`, - }, - } - ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{ - "svc:my-svc": existingVIPSvc, - } - - // Verify reconciliation adds our operator reference - expectReconciled(t, ingPGR, "default", "test-ingress") - - tsSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc") - if err != nil { - t.Fatalf("getting Tailscale Service: %v", err) - } - if tsSvc == nil { - t.Fatal("Tailscale Service not found") - } - - o, err := parseOwnerAnnotation(tsSvc) - if err != nil { - t.Fatalf("parsing owner annotation: %v", err) - } - - wantOwnerRefs := []OwnerRef{ - {OperatorID: "operator-2"}, - {OperatorID: "operator-1"}, - } - if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) { - t.Errorf("incorrect owner refs\ngot: %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs) - } - - // Delete the Ingress and verify Tailscale Service still exists with one owner ref - if err := fc.Delete(context.Background(), ing); err != nil { - t.Fatalf("deleting Ingress: %v", err) - } - expectRequeue(t, ingPGR, "default", "test-ingress") - - tsSvc, err = ft.GetVIPService(context.Background(), "svc:my-svc") - if err != nil { - t.Fatalf("getting Tailscale Service after deletion: %v", err) - } - if tsSvc == nil { - t.Fatal("Tailscale Service was incorrectly deleted") - } - - o, err = parseOwnerAnnotation(tsSvc) - if err != nil { - t.Fatalf("parsing owner annotation: %v", err) - } - - wantOwnerRefs = []OwnerRef{ - {OperatorID: "operator-2"}, - } - if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) { - t.Errorf("incorrect owner refs after deletion\ngot: %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs) - } -} - -func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error { - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: domain, - Namespace: "operator-ns", - Labels: map[string]string{ - kubetypes.LabelManaged: "true", - labelProxyGroup: pgName, - labelDomain: domain, - kubetypes.LabelSecretType: "certs", - }, - }, - Type: corev1.SecretTypeTLS, - Data: map[string][]byte{ - corev1.TLSCertKey: []byte("fake-cert"), - corev1.TLSPrivateKeyKey: []byte("fake-key"), - }, - } - - _, err := createOrUpdate(ctx, c, "operator-ns", secret, func(s *corev1.Secret) { - s.Data = secret.Data - }) - return err -} diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go index ecd60af50f220..5772cd5d64e7f 100644 --- a/cmd/k8s-operator/svc-for-pg_test.go +++ b/cmd/k8s-operator/svc-for-pg_test.go @@ -46,7 +46,7 @@ func TestServicePGReconciler(t *testing.T) { config = append(config, fmt.Sprintf("svc:default-%s", svc.Name)) verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"}) - verifyTailscaledConfig(t, fc, config) + verifyTailscaledConfig(t, fc, "test-pg", config) } for i, svc := range svcs { @@ -75,7 +75,7 @@ func TestServicePGReconciler(t *testing.T) { } config = removeEl(config, fmt.Sprintf("svc:default-%s", svc.Name)) - verifyTailscaledConfig(t, fc, config) + verifyTailscaledConfig(t, fc, "test-pg", config) } } @@ -88,7 +88,7 @@ func TestServicePGReconciler_UpdateHostname(t *testing.T) { expectReconciled(t, svcPGR, "default", svc.Name) verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"}) - verifyTailscaledConfig(t, fc, []string{fmt.Sprintf("svc:default-%s", svc.Name)}) + verifyTailscaledConfig(t, fc, "test-pg", []string{fmt.Sprintf("svc:default-%s", svc.Name)}) hostname := "foobarbaz" mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) { @@ -100,7 +100,7 @@ func TestServicePGReconciler_UpdateHostname(t *testing.T) { expectReconciled(t, svcPGR, "default", svc.Name) verifyTailscaleService(t, ft, fmt.Sprintf("svc:%s", hostname), []string{"do-not-validate"}) - verifyTailscaledConfig(t, fc, []string{fmt.Sprintf("svc:%s", hostname)}) + verifyTailscaledConfig(t, fc, "test-pg", []string{fmt.Sprintf("svc:%s", hostname)}) _, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(fmt.Sprintf("svc:default-%s", svc.Name))) if err == nil { @@ -334,7 +334,7 @@ func TestIgnoreRegularService(t *testing.T) { mustCreate(t, fc, svc) expectReconciled(t, pgr, "default", "test") - verifyTailscaledConfig(t, fc, nil) + verifyTailscaledConfig(t, fc, "test-pg", nil) tsSvcs, err := ft.ListVIPServices(context.Background()) if err == nil { From 59fab8bda797fa1316c1133a9bdd0b732686dd4a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 08:02:26 -0600 Subject: [PATCH 065/263] .github: Bump github/codeql-action from 3.28.19 to 3.29.0 (#16287) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.28.19 to 3.29.0. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/fca7ace96b7d713c7035871441bd52efbe39e27e...ce28f5bb42b7a9f2c824e633a3f6ee835bab6858) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.29.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 8bd72d80d6bdb..32d2e7c2f64c5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -55,7 +55,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 + uses: github/codeql-action/init@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 + uses: github/codeql-action/autobuild@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -80,4 +80,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 + uses: github/codeql-action/analyze@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0 From 42da161b194abe7104cbc8312f913a3db296d6b5 Mon Sep 17 00:00:00 2001 From: Anton Tolchanov Date: Fri, 13 Jun 2025 14:45:28 +0100 Subject: [PATCH 066/263] tka: reject removal of the last signing key Fixes tailscale/corp#19447 Signed-off-by: Anton Tolchanov --- cmd/tailscale/cli/network-lock.go | 3 +++ tka/builder_test.go | 15 +++++++++++++++ tka/tka.go | 7 +++++++ 3 files changed, 25 insertions(+) diff --git a/cmd/tailscale/cli/network-lock.go b/cmd/tailscale/cli/network-lock.go index c7776707422ec..ae1e90bbfaea9 100644 --- a/cmd/tailscale/cli/network-lock.go +++ b/cmd/tailscale/cli/network-lock.go @@ -326,6 +326,9 @@ func runNetworkLockRemove(ctx context.Context, args []string) error { if !st.Enabled { return errors.New("tailnet lock is not enabled") } + if len(st.TrustedKeys) == 1 { + return errors.New("cannot remove the last trusted signing key; use 'tailscale lock disable' to disable tailnet lock instead, or add another signing key before removing one") + } if nlRemoveArgs.resign { // Validate we are not removing trust in ourselves while resigning. This is because diff --git a/tka/builder_test.go b/tka/builder_test.go index 666af9ad07daf..3dbd4347abf06 100644 --- a/tka/builder_test.go +++ b/tka/builder_test.go @@ -5,6 +5,7 @@ package tka import ( "crypto/ed25519" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -90,6 +91,20 @@ func TestAuthorityBuilderRemoveKey(t *testing.T) { if _, err := a.state.GetKey(key2.MustID()); err != ErrNoSuchKey { t.Errorf("GetKey(key2).err = %v, want %v", err, ErrNoSuchKey) } + + // Check that removing the remaining key errors out. + b = a.NewUpdater(signer25519(priv)) + if err := b.RemoveKey(key.MustID()); err != nil { + t.Fatalf("RemoveKey(%v) failed: %v", key, err) + } + updates, err = b.Finalize(storage) + if err != nil { + t.Fatalf("Finalize() failed: %v", err) + } + wantErr := "cannot remove the last key" + if err := a.Inform(storage, updates); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("expected Inform() to return error %q, got: %v", wantErr, err) + } } func TestAuthorityBuilderSetKeyVote(t *testing.T) { diff --git a/tka/tka.go b/tka/tka.go index 04b712660d270..ade621bc689e3 100644 --- a/tka/tka.go +++ b/tka/tka.go @@ -440,6 +440,13 @@ func aumVerify(aum AUM, state State, isGenesisAUM bool) error { return fmt.Errorf("signature %d: %v", i, err) } } + + if aum.MessageKind == AUMRemoveKey && len(state.Keys) == 1 { + if kid, err := state.Keys[0].ID(); err == nil && bytes.Equal(aum.KeyID, kid) { + return errors.New("cannot remove the last key in the state") + } + } + return nil } From 8e6f63cf110364b52e3f6c232b23196f16484473 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 16 Jun 2025 08:42:09 -0700 Subject: [PATCH 067/263] ipn/ipnlocal,wgengine/magicsock: use eventbus for node & filter updates (#16271) nodeBackend now publishes filter and node changes to eventbus topics that are consumed by magicsock.Conn Updates tailscale/corp#27502 Updates tailscale/corp#29543 Signed-off-by: Jordan Whited --- ipn/ipnlocal/local.go | 16 ++++++-- ipn/ipnlocal/node_backend.go | 42 +++++++++++++++++--- ipn/ipnlocal/node_backend_test.go | 12 +++--- wgengine/magicsock/magicsock.go | 65 +++++++++++++++++++++++++------ 4 files changed, 109 insertions(+), 26 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index daedb1e1924a0..cd30e92bb905b 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -98,6 +98,7 @@ import ( "tailscale.com/util/clientmetric" "tailscale.com/util/deephash" "tailscale.com/util/dnsname" + "tailscale.com/util/eventbus" "tailscale.com/util/goroutines" "tailscale.com/util/httpm" "tailscale.com/util/mak" @@ -514,7 +515,7 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo captiveCancel: nil, // so that we start checkCaptivePortalLoop when Running needsCaptiveDetection: make(chan bool), } - nb := newNodeBackend(ctx) + nb := newNodeBackend(ctx, b.sys.Bus.Get()) b.currentNodeAtomic.Store(nb) nb.ready() @@ -599,8 +600,15 @@ func (b *LocalBackend) currentNode() *nodeBackend { if v := b.currentNodeAtomic.Load(); v != nil || !testenv.InTest() { return v } - // Auto-init one in tests for LocalBackend created without the NewLocalBackend constructor... - v := newNodeBackend(cmp.Or(b.ctx, context.Background())) + // Auto-init [nodeBackend] in tests for LocalBackend created without the + // NewLocalBackend() constructor. Same reasoning for checking b.sys. + var bus *eventbus.Bus + if b.sys == nil { + bus = eventbus.New() + } else { + bus = b.sys.Bus.Get() + } + v := newNodeBackend(cmp.Or(b.ctx, context.Background()), bus) if b.currentNodeAtomic.CompareAndSwap(nil, v) { v.ready() } @@ -7009,7 +7017,7 @@ func (b *LocalBackend) resetForProfileChangeLockedOnEntry(unlock unlockOnce) err // down, so no need to do any work. return nil } - newNode := newNodeBackend(b.ctx) + newNode := newNodeBackend(b.ctx, b.sys.Bus.Get()) if oldNode := b.currentNodeAtomic.Swap(newNode); oldNode != nil { oldNode.shutdown(errNodeContextChanged) } diff --git a/ipn/ipnlocal/node_backend.go b/ipn/ipnlocal/node_backend.go index 361d10bb6cfcc..efa74577bc8ee 100644 --- a/ipn/ipnlocal/node_backend.go +++ b/ipn/ipnlocal/node_backend.go @@ -23,9 +23,11 @@ import ( "tailscale.com/types/ptr" "tailscale.com/types/views" "tailscale.com/util/dnsname" + "tailscale.com/util/eventbus" "tailscale.com/util/mak" "tailscale.com/util/slicesx" "tailscale.com/wgengine/filter" + "tailscale.com/wgengine/magicsock" ) // nodeBackend is node-specific [LocalBackend] state. It is usually the current node. @@ -69,6 +71,11 @@ type nodeBackend struct { // replaced with a new one. filterAtomic atomic.Pointer[filter.Filter] + // initialized once and immutable + eventClient *eventbus.Client + filterUpdates *eventbus.Publisher[magicsock.FilterUpdate] + nodeUpdates *eventbus.Publisher[magicsock.NodeAddrsHostInfoUpdate] + // TODO(nickkhyl): maybe use sync.RWMutex? mu sync.Mutex // protects the following fields @@ -95,16 +102,20 @@ type nodeBackend struct { nodeByAddr map[netip.Addr]tailcfg.NodeID } -func newNodeBackend(ctx context.Context) *nodeBackend { +func newNodeBackend(ctx context.Context, bus *eventbus.Bus) *nodeBackend { ctx, ctxCancel := context.WithCancelCause(ctx) nb := &nodeBackend{ - ctx: ctx, - ctxCancel: ctxCancel, - readyCh: make(chan struct{}), + ctx: ctx, + ctxCancel: ctxCancel, + eventClient: bus.Client("ipnlocal.nodeBackend"), + readyCh: make(chan struct{}), } // Default filter blocks everything and logs nothing. noneFilter := filter.NewAllowNone(logger.Discard, &netipx.IPSet{}) nb.filterAtomic.Store(noneFilter) + nb.filterUpdates = eventbus.Publish[magicsock.FilterUpdate](nb.eventClient) + nb.nodeUpdates = eventbus.Publish[magicsock.NodeAddrsHostInfoUpdate](nb.eventClient) + nb.filterUpdates.Publish(magicsock.FilterUpdate{Filter: nb.filterAtomic.Load()}) return nb } @@ -418,9 +429,16 @@ func (nb *nodeBackend) updatePeersLocked() { nb.peers[k] = tailcfg.NodeView{} } + changed := magicsock.NodeAddrsHostInfoUpdate{ + Complete: true, + } // Second pass, add everything wanted. for _, p := range nm.Peers { mak.Set(&nb.peers, p.ID(), p) + mak.Set(&changed.NodesByID, p.ID(), magicsock.NodeAddrsHostInfo{ + Addresses: p.Addresses(), + Hostinfo: p.Hostinfo(), + }) } // Third pass, remove deleted things. @@ -429,6 +447,7 @@ func (nb *nodeBackend) updatePeersLocked() { delete(nb.peers, k) } } + nb.nodeUpdates.Publish(changed) } func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) { @@ -443,6 +462,9 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo // call (e.g. its endpoints + online status both change) var mutableNodes map[tailcfg.NodeID]*tailcfg.Node + changed := magicsock.NodeAddrsHostInfoUpdate{ + Complete: false, + } for _, m := range muts { n, ok := mutableNodes[m.NodeIDBeingMutated()] if !ok { @@ -457,8 +479,14 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo m.Apply(n) } for nid, n := range mutableNodes { - nb.peers[nid] = n.View() - } + nv := n.View() + nb.peers[nid] = nv + mak.Set(&changed.NodesByID, nid, magicsock.NodeAddrsHostInfo{ + Addresses: nv.Addresses(), + Hostinfo: nv.Hostinfo(), + }) + } + nb.nodeUpdates.Publish(changed) return true } @@ -480,6 +508,7 @@ func (nb *nodeBackend) filter() *filter.Filter { func (nb *nodeBackend) setFilter(f *filter.Filter) { nb.filterAtomic.Store(f) + nb.filterUpdates.Publish(magicsock.FilterUpdate{Filter: f}) } func (nb *nodeBackend) dnsConfigForNetmap(prefs ipn.PrefsView, selfExpired bool, logf logger.Logf, versionOS string) *dns.Config { @@ -545,6 +574,7 @@ func (nb *nodeBackend) doShutdown(cause error) { defer nb.mu.Unlock() nb.ctxCancel(cause) nb.readyCh = nil + nb.eventClient.Close() } // dnsConfigForNetmap returns a *dns.Config for the given netmap, diff --git a/ipn/ipnlocal/node_backend_test.go b/ipn/ipnlocal/node_backend_test.go index a82b60a9a1726..dc67d327c8041 100644 --- a/ipn/ipnlocal/node_backend_test.go +++ b/ipn/ipnlocal/node_backend_test.go @@ -8,10 +8,12 @@ import ( "errors" "testing" "time" + + "tailscale.com/util/eventbus" ) func TestNodeBackendReadiness(t *testing.T) { - nb := newNodeBackend(t.Context()) + nb := newNodeBackend(t.Context(), eventbus.New()) // The node backend is not ready until [nodeBackend.ready] is called, // and [nodeBackend.Wait] should fail with [context.DeadlineExceeded]. @@ -42,7 +44,7 @@ func TestNodeBackendReadiness(t *testing.T) { } func TestNodeBackendShutdown(t *testing.T) { - nb := newNodeBackend(t.Context()) + nb := newNodeBackend(t.Context(), eventbus.New()) shutdownCause := errors.New("test shutdown") @@ -80,7 +82,7 @@ func TestNodeBackendShutdown(t *testing.T) { } func TestNodeBackendReadyAfterShutdown(t *testing.T) { - nb := newNodeBackend(t.Context()) + nb := newNodeBackend(t.Context(), eventbus.New()) shutdownCause := errors.New("test shutdown") nb.shutdown(shutdownCause) @@ -92,7 +94,7 @@ func TestNodeBackendReadyAfterShutdown(t *testing.T) { func TestNodeBackendParentContextCancellation(t *testing.T) { ctx, cancelCtx := context.WithCancel(context.Background()) - nb := newNodeBackend(ctx) + nb := newNodeBackend(ctx, eventbus.New()) cancelCtx() @@ -109,7 +111,7 @@ func TestNodeBackendParentContextCancellation(t *testing.T) { } func TestNodeBackendConcurrentReadyAndShutdown(t *testing.T) { - nb := newNodeBackend(t.Context()) + nb := newNodeBackend(t.Context(), eventbus.New()) // Calling [nodeBackend.ready] and [nodeBackend.shutdown] concurrently // should not cause issues, and [nodeBackend.Wait] should unblock, diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index e5cc87dc3810a..1042e67942791 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -63,6 +63,7 @@ import ( "tailscale.com/util/set" "tailscale.com/util/testenv" "tailscale.com/util/usermetric" + "tailscale.com/wgengine/filter" "tailscale.com/wgengine/wgint" ) @@ -502,6 +503,30 @@ func (o *Options) derpActiveFunc() func() { return o.DERPActiveFunc } +// NodeAddrsHostInfoUpdate represents an update event of the addresses and +// [tailcfg.HostInfoView] for a node set. This event is published over an +// [eventbus.Bus]. [magicsock.Conn] is the sole subscriber as of 2025-06. If +// you are adding more subscribers consider moving this type out of magicsock. +type NodeAddrsHostInfoUpdate struct { + NodesByID map[tailcfg.NodeID]NodeAddrsHostInfo + Complete bool // true if NodesByID contains all known nodes, false if it may be a subset +} + +// NodeAddrsHostInfo represents the addresses and [tailcfg.HostinfoView] for a +// Tailscale node. +type NodeAddrsHostInfo struct { + Addresses views.Slice[netip.Prefix] + Hostinfo tailcfg.HostinfoView +} + +// FilterUpdate represents an update event for a [*filter.Filter]. This event is +// signaled over an [eventbus.Bus]. [magicsock.Conn] is the sole subscriber as +// of 2025-06. If you are adding more subscribers consider moving this type out +// of magicsock. +type FilterUpdate struct { + *filter.Filter +} + // newConn is the error-free, network-listening-side-effect-free based // of NewConn. Mostly for tests. func newConn(logf logger.Logf) *Conn { @@ -535,6 +560,20 @@ func newConn(logf logger.Logf) *Conn { return c } +// consumeEventbusTopic consumes events from sub and passes them to +// handlerFn until sub.Done() is closed. +func consumeEventbusTopic[T any](sub *eventbus.Subscriber[T], handlerFn func(t T)) { + defer sub.Close() + for { + select { + case evt := <-sub.Events(): + handlerFn(evt) + case <-sub.Done(): + return + } + } +} + // NewConn creates a magic Conn listening on opts.Port. // As the set of possible endpoints for a Conn changes, the // callback opts.EndpointsFunc is called. @@ -562,17 +601,17 @@ func NewConn(opts Options) (*Conn, error) { c.eventClient = c.eventBus.Client("magicsock.Conn") pmSub := eventbus.Subscribe[portmapper.Mapping](c.eventClient) - go func() { - defer pmSub.Close() - for { - select { - case <-pmSub.Events(): - c.onPortMapChanged() - case <-pmSub.Done(): - return - } - } - }() + go consumeEventbusTopic(pmSub, func(_ portmapper.Mapping) { + c.onPortMapChanged() + }) + filterSub := eventbus.Subscribe[FilterUpdate](c.eventClient) + go consumeEventbusTopic(filterSub, func(t FilterUpdate) { + // TODO(jwhited): implement + }) + nodeSub := eventbus.Subscribe[NodeAddrsHostInfoUpdate](c.eventClient) + go consumeEventbusTopic(nodeSub, func(t NodeAddrsHostInfoUpdate) { + // TODO(jwhited): implement + }) // Disable the explicit callback from the portmapper, the subscriber handles it. onPortMapChanged = nil @@ -2798,6 +2837,10 @@ func (c *connBind) Close() error { return nil } c.closed = true + // Close the [eventbus.Client]. + if c.eventClient != nil { + c.eventClient.Close() + } // Unblock all outstanding receives. c.pconn4.Close() c.pconn6.Close() From 5b7cf7fc3681d11aaddaa9a163ed72502405c83a Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 15 Jun 2025 12:42:33 -0700 Subject: [PATCH 068/263] .github/workflows: do a go mod download & cache it before all jobs Updates tailscale/corp#28679 Change-Id: Ib0127cb2b03f781fc3187199abe4881e97074f5f Signed-off-by: Brad Fitzpatrick --- .github/workflows/test.yml | 246 ++++++++++++++++++++++++++++++++----- go.mod | 2 +- 2 files changed, 215 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1776653f4eb4c..11a851dc4ee6e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,10 @@ env: # - false: we expect fuzzing to be happy, and should report failure if it's not. # - true: we expect fuzzing is broken, and should report failure if it start working. TS_FUZZ_CURRENTLY_BROKEN: false + # GOMODCACHE is the same definition on all OSes. Within the workspace, we use + # toplevel directories "src" (for the checked out source code), and "gomodcache" + # and other caches as siblings to follow. + GOMODCACHE: ${{ github.workspace }}/gomodcache on: push: @@ -38,8 +42,42 @@ concurrency: cancel-in-progress: true jobs: + gomod-cache: + runs-on: ubuntu-24.04 + outputs: + cache-key: ${{ steps.hash.outputs.key }} + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src + - name: Compute cache key from go.{mod,sum} + id: hash + run: echo "key=gomod-cross3-${{ hashFiles('src/go.mod', 'src/go.sum') }}" >> $GITHUB_OUTPUT + # See if the cache entry already exists to avoid downloading it + # and doing the cache write again. + - id: check-cache + uses: actions/cache/restore@v4 + with: + path: gomodcache # relative to workspace; see env note at top of file + key: ${{ steps.hash.outputs.key }} + lookup-only: true + enableCrossOsArchive: true + - name: Download modules + if: steps.check-cache.outputs.cache-hit != 'true' + working-directory: src + run: go mod download + - name: Cache Go modules + if: steps.check-cache.outputs.cache-hit != 'true' + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache # relative to workspace; see env note at top of file + key: ${{ steps.hash.outputs.key }} + enableCrossOsArchive: true + race-root-integration: runs-on: ubuntu-24.04 + needs: gomod-cache strategy: fail-fast: false # don't abort the entire matrix if one element fails matrix: @@ -51,9 +89,19 @@ jobs: steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: build test wrapper + working-directory: src run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper - name: integration tests as root + working-directory: src run: PATH=$PWD/tool:$PATH /tmp/testwrapper -exec "sudo -E" -race ./tstest/integration/ env: TS_TEST_SHARD: ${{ matrix.shard }} @@ -75,9 +123,18 @@ jobs: shard: '3/3' - goarch: "386" # thanks yaml runs-on: ubuntu-24.04 + needs: gomod-cache steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: Restore Cache uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: @@ -87,7 +144,6 @@ jobs: # fetched and extracted by tar path: | ~/.cache/go-build - ~/go/pkg/mod/cache ~\AppData\Local\go-build # The -2- here should be incremented when the scheme of data to be # cached changes (e.g. path above changes). @@ -97,11 +153,13 @@ jobs: ${{ github.job }}-${{ runner.os }}-${{ matrix.goarch }}-${{ matrix.buildflags }}-go-2- - name: build all if: matrix.buildflags == '' # skip on race builder + working-directory: src run: ./tool/go build ${{matrix.buildflags}} ./... env: GOARCH: ${{ matrix.goarch }} - name: build variant CLIs if: matrix.buildflags == '' # skip on race builder + working-directory: src run: | export TS_USE_TOOLCHAIN=1 ./build_dist.sh --extra-small ./cmd/tailscaled @@ -116,19 +174,24 @@ jobs: sudo apt-get -y update sudo apt-get -y install qemu-user - name: build test wrapper + working-directory: src run: ./tool/go build -o /tmp/testwrapper ./cmd/testwrapper - name: test all + working-directory: src run: NOBASHDEBUG=true PATH=$PWD/tool:$PATH /tmp/testwrapper ./... ${{matrix.buildflags}} env: GOARCH: ${{ matrix.goarch }} TS_TEST_SHARD: ${{ matrix.shard }} - name: bench all + working-directory: src run: ./tool/go test ${{matrix.buildflags}} -bench=. -benchtime=1x -run=^$ $(for x in $(git grep -l "^func Benchmark" | xargs dirname | sort | uniq); do echo "./$x"; done) env: GOARCH: ${{ matrix.goarch }} - name: check that no tracked files changed + working-directory: src run: git diff --no-ext-diff --name-only --exit-code || (echo "Build/test modified the files above."; exit 1) - name: check that no new files were added + working-directory: src run: | # Note: The "error: pathspec..." you see below is normal! # In the success case in which there are no new untracked files, @@ -140,22 +203,33 @@ jobs: exit 1 fi - name: Tidy cache + working-directory: src shell: bash run: | find $(go env GOCACHE) -type f -mmin +90 -delete - find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete + windows: runs-on: windows-2022 + needs: gomod-cache steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src - name: Install Go uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: - go-version-file: go.mod + go-version-file: src/go.mod cache: false + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true + - name: Restore Cache uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: @@ -165,7 +239,6 @@ jobs: # fetched and extracted by tar path: | ~/.cache/go-build - ~/go/pkg/mod/cache ~\AppData\Local\go-build # The -2- here should be incremented when the scheme of data to be # cached changes (e.g. path above changes). @@ -174,19 +247,22 @@ jobs: ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }} ${{ github.job }}-${{ runner.os }}-go-2- - name: test + working-directory: src run: go run ./cmd/testwrapper ./... - name: bench all + working-directory: src # Don't use -bench=. -benchtime=1x. # Somewhere in the layers (powershell?) # the equals signs cause great confusion. run: go test ./... -bench . -benchtime 1x -run "^$" - name: Tidy cache + working-directory: src shell: bash run: | find $(go env GOCACHE) -type f -mmin +90 -delete - find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete privileged: + needs: gomod-cache runs-on: ubuntu-24.04 container: image: golang:latest @@ -194,36 +270,47 @@ jobs: steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: chown + working-directory: src run: chown -R $(id -u):$(id -g) $PWD - name: privileged tests + working-directory: src run: ./tool/go test ./util/linuxfw ./derp/xdp vm: + needs: gomod-cache runs-on: ["self-hosted", "linux", "vm"] # VM tests run with some privileges, don't let them run on 3p PRs. if: github.repository == 'tailscale/tailscale' steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: Run VM tests + working-directory: src run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2004 env: HOME: "/var/lib/ghrunner/home" TMPDIR: "/tmp" XDG_CACHE_HOME: "/var/lib/ghrunner/cache" - race-build: - runs-on: ubuntu-24.04 - steps: - - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: build all - run: ./tool/go install -race ./cmd/... - - name: build tests - run: ./tool/go test -race -exec=true ./... - cross: # cross-compile checks, build only. + needs: gomod-cache strategy: fail-fast: false # don't abort the entire matrix if one element fails matrix: @@ -262,6 +349,8 @@ jobs: steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src - name: Restore Cache uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: @@ -271,7 +360,6 @@ jobs: # fetched and extracted by tar path: | ~/.cache/go-build - ~/go/pkg/mod/cache ~\AppData\Local\go-build # The -2- here should be incremented when the scheme of data to be # cached changes (e.g. path above changes). @@ -279,7 +367,14 @@ jobs: restore-keys: | ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-${{ hashFiles('**/go.sum') }} ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2- + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: build all + working-directory: src run: ./tool/go build ./cmd/... env: GOOS: ${{ matrix.goos }} @@ -287,30 +382,42 @@ jobs: GOARM: ${{ matrix.goarm }} CGO_ENABLED: "0" - name: build tests + working-directory: src run: ./tool/go test -exec=true ./... env: GOOS: ${{ matrix.goos }} GOARCH: ${{ matrix.goarch }} CGO_ENABLED: "0" - name: Tidy cache + working-directory: src shell: bash run: | find $(go env GOCACHE) -type f -mmin +90 -delete - find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete ios: # similar to cross above, but iOS can't build most of the repo. So, just - #make it build a few smoke packages. + # make it build a few smoke packages. runs-on: ubuntu-24.04 + needs: gomod-cache steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: build some + working-directory: src run: ./tool/go build ./ipn/... ./ssh/tailssh ./wgengine/ ./types/... ./control/controlclient env: GOOS: ios GOARCH: arm64 crossmin: # cross-compile for platforms where we only check cmd/tailscale{,d} + needs: gomod-cache strategy: fail-fast: false # don't abort the entire matrix if one element fails matrix: @@ -332,6 +439,8 @@ jobs: steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src - name: Restore Cache uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: @@ -341,7 +450,6 @@ jobs: # fetched and extracted by tar path: | ~/.cache/go-build - ~/go/pkg/mod/cache ~\AppData\Local\go-build # The -2- here should be incremented when the scheme of data to be # cached changes (e.g. path above changes). @@ -349,7 +457,14 @@ jobs: restore-keys: | ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2-${{ hashFiles('**/go.sum') }} ${{ github.job }}-${{ runner.os }}-${{ matrix.goos }}-${{ matrix.goarch }}-go-2- + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: build core + working-directory: src run: ./tool/go build ./cmd/tailscale ./cmd/tailscaled env: GOOS: ${{ matrix.goos }} @@ -357,24 +472,34 @@ jobs: GOARM: ${{ matrix.goarm }} CGO_ENABLED: "0" - name: Tidy cache + working-directory: src shell: bash run: | find $(go env GOCACHE) -type f -mmin +90 -delete - find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete android: # similar to cross above, but android fails to build a few pieces of the # repo. We should fix those pieces, they're small, but as a stepping stone, # only test the subset of android that our past smoke test checked. runs-on: ubuntu-24.04 + needs: gomod-cache steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src # Super minimal Android build that doesn't even use CGO and doesn't build everything that's needed # and is only arm64. But it's a smoke build: it's not meant to catch everything. But it'll catch # some Android breakages early. # TODO(bradfitz): better; see https://github.com/tailscale/tailscale/issues/4482 + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: build some + working-directory: src run: ./tool/go install ./net/netns ./ipn/ipnlocal ./wgengine/magicsock/ ./wgengine/ ./wgengine/router/ ./wgengine/netstack ./util/dnsname/ ./ipn/ ./net/netmon ./wgengine/router/ ./tailcfg/ ./types/logger/ ./net/dns ./hostinfo ./version ./ssh/tailssh env: GOOS: android @@ -382,9 +507,12 @@ jobs: wasm: # builds tsconnect, which is the only wasm build we support runs-on: ubuntu-24.04 + needs: gomod-cache steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src - name: Restore Cache uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: @@ -394,7 +522,6 @@ jobs: # fetched and extracted by tar path: | ~/.cache/go-build - ~/go/pkg/mod/cache ~\AppData\Local\go-build # The -2- here should be incremented when the scheme of data to be # cached changes (e.g. path above changes). @@ -402,28 +529,45 @@ jobs: restore-keys: | ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }} ${{ github.job }}-${{ runner.os }}-go-2- + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: build tsconnect client + working-directory: src run: ./tool/go build ./cmd/tsconnect/wasm ./cmd/tailscale/cli env: GOOS: js GOARCH: wasm - name: build tsconnect server + working-directory: src # Note, no GOOS/GOARCH in env on this build step, we're running a build # tool that handles the build itself. run: | ./tool/go run ./cmd/tsconnect --fast-compression build ./tool/go run ./cmd/tsconnect --fast-compression build-pkg - name: Tidy cache + working-directory: src shell: bash run: | find $(go env GOCACHE) -type f -mmin +90 -delete - find $(go env GOMODCACHE)/cache -type f -mmin +90 -delete tailscale_go: # Subset of tests that depend on our custom Go toolchain. runs-on: ubuntu-24.04 + needs: gomod-cache steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set GOMODCACHE env + run: echo "GOMODCACHE=$HOME/.cache/go-mod" >> $GITHUB_ENV + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: test tailscale_go run: ./tool/go test -tags=tailscale_go,ts_enable_sockstats ./net/sockstats/... @@ -477,7 +621,7 @@ jobs: uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master with: oss-fuzz-project-name: 'tailscale' - fuzz-seconds: 300 + fuzz-seconds: 150 dry-run: false language: go - name: Set artifacts_path in env (workaround for actions/upload-artifact#176) @@ -493,19 +637,40 @@ jobs: depaware: runs-on: ubuntu-24.04 + needs: gomod-cache steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src + - name: Set GOMODCACHE env + run: echo "GOMODCACHE=$HOME/.cache/go-mod" >> $GITHUB_ENV + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: check depaware - run: | - make depaware + working-directory: src + run: make depaware go_generate: runs-on: ubuntu-24.04 + needs: gomod-cache steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: check that 'go generate' is clean + working-directory: src run: | pkgs=$(./tool/go list ./... | grep -Ev 'dnsfallback|k8s-operator|xdp') ./tool/go generate $pkgs @@ -515,10 +680,20 @@ jobs: go_mod_tidy: runs-on: ubuntu-24.04 + needs: gomod-cache steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: check that 'go mod tidy' is clean + working-directory: src run: | ./tool/go mod tidy echo @@ -535,6 +710,7 @@ jobs: staticcheck: runs-on: ubuntu-24.04 + needs: gomod-cache strategy: fail-fast: false # don't abort the entire matrix if one element fails matrix: @@ -546,16 +722,22 @@ jobs: steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: install staticcheck - run: GOBIN=~/.local/bin ./tool/go install honnef.co/go/tools/cmd/staticcheck + with: + path: src + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: run staticcheck + working-directory: src run: | export GOROOT=$(./tool/go env GOROOT) - export PATH=$GOROOT/bin:$PATH - staticcheck -- $(./tool/go list ./... | grep -v tempfork) - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} + ./tool/go run -exec \ + "env GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}" \ + honnef.co/go/tools/cmd/staticcheck -- \ + $(env GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} ./tool/go list ./... | grep -v tempfork) notify_slack: if: always() diff --git a/go.mod b/go.mod index 0c3d05d590c62..0d031d0baa6c2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module tailscale.com -go 1.24.0 +go 1.24.4 require ( filippo.io/mkcert v1.4.4 From 866614202c96fa1e5116116acf50834ee787ed6c Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Fri, 13 Jun 2025 18:08:22 -0500 Subject: [PATCH 069/263] util/eventbus: remove redundant code from eventbus.Publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eventbus.Publish() calls newPublisher(), which in turn invokes (*Client).addPublisher(). That method adds the new publisher to c.pub, so we don’t need to add it again in eventbus.Publish. Updates #cleanup Signed-off-by: Nick Khyl --- util/eventbus/client.go | 13 +++++++------ util/eventbus/publish.go | 6 +----- util/eventbus/subscribe.go | 14 +++++--------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/util/eventbus/client.go b/util/eventbus/client.go index a7a88c0a158c7..f4261b13c9f45 100644 --- a/util/eventbus/client.go +++ b/util/eventbus/client.go @@ -113,15 +113,16 @@ func (c *Client) shouldPublish(t reflect.Type) bool { // Subscribe requests delivery of events of type T through the given // Queue. Panics if the queue already has a subscriber for T. func Subscribe[T any](c *Client) *Subscriber[T] { - return newSubscriber[T](c.subscribeState()) + r := c.subscribeState() + s := newSubscriber[T](r) + r.addSubscriber(s) + return s } // Publisher returns a publisher for event type T using the given // client. func Publish[T any](c *Client) *Publisher[T] { - ret := newPublisher[T](c) - c.mu.Lock() - defer c.mu.Unlock() - c.pub.Add(ret) - return ret + p := newPublisher[T](c) + c.addPublisher(p) + return p } diff --git a/util/eventbus/publish.go b/util/eventbus/publish.go index 9897114b64973..4a4bdfb7eda11 100644 --- a/util/eventbus/publish.go +++ b/util/eventbus/publish.go @@ -21,11 +21,7 @@ type Publisher[T any] struct { } func newPublisher[T any](c *Client) *Publisher[T] { - ret := &Publisher[T]{ - client: c, - } - c.addPublisher(ret) - return ret + return &Publisher[T]{client: c} } // Close closes the publisher. diff --git a/util/eventbus/subscribe.go b/util/eventbus/subscribe.go index ba17e85484655..ee534781a2cce 100644 --- a/util/eventbus/subscribe.go +++ b/util/eventbus/subscribe.go @@ -91,7 +91,7 @@ func (q *subscribeState) pump(ctx context.Context) { } } else { // Keep the cases in this select in sync with - // Subscriber.dispatch below. The only different should be + // Subscriber.dispatch below. The only difference should be // that this select doesn't deliver queued values to // anyone, and unconditionally accepts new values. select { @@ -134,9 +134,10 @@ func (s *subscribeState) subscribeTypes() []reflect.Type { return ret } -func (s *subscribeState) addSubscriber(t reflect.Type, sub subscriber) { +func (s *subscribeState) addSubscriber(sub subscriber) { s.outputsMu.Lock() defer s.outputsMu.Unlock() + t := sub.subscribeType() if s.outputs[t] != nil { panic(fmt.Errorf("double subscription for event %s", t)) } @@ -183,15 +184,10 @@ type Subscriber[T any] struct { } func newSubscriber[T any](r *subscribeState) *Subscriber[T] { - t := reflect.TypeFor[T]() - - ret := &Subscriber[T]{ + return &Subscriber[T]{ read: make(chan T), - unregister: func() { r.deleteSubscriber(t) }, + unregister: func() { r.deleteSubscriber(reflect.TypeFor[T]()) }, } - r.addSubscriber(t, ret) - - return ret } func newMonitor[T any](attach func(fn func(T)) (cancel func())) *Subscriber[T] { From 3d6e1171c165524987cdb878bf98bc6d2bb33256 Mon Sep 17 00:00:00 2001 From: Fran Bull Date: Mon, 16 Jun 2025 07:39:02 -0700 Subject: [PATCH 070/263] tsconsensus: protect from data race lock for access to a.peers Fixes #16284 Signed-off-by: Fran Bull --- tsconsensus/authorization.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tsconsensus/authorization.go b/tsconsensus/authorization.go index 1e0b70c0759d3..bd8e2f39a014b 100644 --- a/tsconsensus/authorization.go +++ b/tsconsensus/authorization.go @@ -87,29 +87,29 @@ func (a *authorization) Refresh(ctx context.Context) error { } func (a *authorization) AllowsHost(addr netip.Addr) bool { + a.mu.Lock() + defer a.mu.Unlock() if a.peers == nil { return false } - a.mu.Lock() - defer a.mu.Unlock() return a.peers.addrs.Contains(addr) } func (a *authorization) SelfAllowed() bool { + a.mu.Lock() + defer a.mu.Unlock() if a.peers == nil { return false } - a.mu.Lock() - defer a.mu.Unlock() return a.peers.status.Self.Tags != nil && views.SliceContains(*a.peers.status.Self.Tags, a.tag) } func (a *authorization) AllowedPeers() views.Slice[*ipnstate.PeerStatus] { + a.mu.Lock() + defer a.mu.Unlock() if a.peers == nil { return views.Slice[*ipnstate.PeerStatus]{} } - a.mu.Lock() - defer a.mu.Unlock() return views.SliceOf(a.peers.statuses) } From 735f15cb49520a198cd2e063bcf9e8e511bcc691 Mon Sep 17 00:00:00 2001 From: James Sanderson Date: Mon, 16 Jun 2025 16:09:41 +0100 Subject: [PATCH 071/263] util/must: add Get2 for functions that return two values Updates #cleanup Signed-off-by: James Sanderson --- util/must/must.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/util/must/must.go b/util/must/must.go index 21965daa9b038..a292da2268c27 100644 --- a/util/must/must.go +++ b/util/must/must.go @@ -23,3 +23,11 @@ func Get[T any](v T, err error) T { } return v } + +// Get2 returns v1 and v2 as is. It panics if err is non-nil. +func Get2[T any, U any](v1 T, v2 U, err error) (T, U) { + if err != nil { + panic(err) + } + return v1, v2 +} From 86985228bcef855a8071f6989bbceeb5b21810c2 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Mon, 16 Jun 2025 10:27:00 -0700 Subject: [PATCH 072/263] cmd/natc: add a flag to use specific DNS servers If natc is running on a host with tailscale using `--accept-dns=true` then a DNS loop can occur. Provide a flag for some specific DNS upstreams for natc to use instead, to overcome such situations. Updates #14667 Signed-off-by: James Tucker --- cmd/natc/natc.go | 31 ++++++- cmd/natc/natc_test.go | 196 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+), 2 deletions(-) diff --git a/cmd/natc/natc.go b/cmd/natc/natc.go index 247bb2101f976..fdbce3da189b2 100644 --- a/cmd/natc/natc.go +++ b/cmd/natc/natc.go @@ -54,6 +54,7 @@ func main() { hostname = fs.String("hostname", "", "Hostname to register the service under") siteID = fs.Uint("site-id", 1, "an integer site ID to use for the ULA prefix which allows for multiple proxies to act in a HA configuration") v4PfxStr = fs.String("v4-pfx", "100.64.1.0/24", "comma-separated list of IPv4 prefixes to advertise") + dnsServers = fs.String("dns-servers", "", "comma separated list of upstream DNS to use, including host and port (use system if empty)") verboseTSNet = fs.Bool("verbose-tsnet", false, "enable verbose logging in tsnet") printULA = fs.Bool("print-ula", false, "print the ULA prefix and exit") ignoreDstPfxStr = fs.String("ignore-destinations", "", "comma-separated list of prefixes to ignore") @@ -78,7 +79,7 @@ func main() { } var ignoreDstTable *bart.Table[bool] - for _, s := range strings.Split(*ignoreDstPfxStr, ",") { + for s := range strings.SplitSeq(*ignoreDstPfxStr, ",") { s := strings.TrimSpace(s) if s == "" { continue @@ -185,11 +186,37 @@ func main() { ipPool: ipp, routes: routes, dnsAddr: dnsAddr, - resolver: net.DefaultResolver, + resolver: getResolver(*dnsServers), } c.run(ctx, lc) } +// getResolver parses serverFlag and returns either the default resolver, or a +// resolver that uses the provided comma-separated DNS server AddrPort's, or +// panics. +func getResolver(serverFlag string) lookupNetIPer { + if serverFlag == "" { + return net.DefaultResolver + } + var addrs []string + for s := range strings.SplitSeq(serverFlag, ",") { + s = strings.TrimSpace(s) + addr, err := netip.ParseAddrPort(s) + if err != nil { + log.Fatalf("dns server provided: %q does not parse: %v", s, err) + } + addrs = append(addrs, addr.String()) + } + return &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network string, address string) (net.Conn, error) { + var dialer net.Dialer + // TODO(raggi): perhaps something other than random? + return dialer.DialContext(ctx, network, addrs[rand.N(len(addrs))]) + }, + } +} + func calculateAddresses(prefixes []netip.Prefix) (*netipx.IPSet, netip.Addr, *netipx.IPSet) { var ipsb netipx.IPSetBuilder for _, p := range prefixes { diff --git a/cmd/natc/natc_test.go b/cmd/natc/natc_test.go index 78dec86fdf02f..c0a66deb8a4da 100644 --- a/cmd/natc/natc_test.go +++ b/cmd/natc/natc_test.go @@ -9,6 +9,7 @@ import ( "io" "net" "net/netip" + "sync" "testing" "time" @@ -480,3 +481,198 @@ func TestV6V4(t *testing.T) { } } } + +// echoServer is a simple server that just echos back data set to it. +type echoServer struct { + listener net.Listener + addr string + wg sync.WaitGroup + done chan struct{} +} + +// newEchoServer creates a new test DNS server on the specified network and address +func newEchoServer(t *testing.T, network, addr string) *echoServer { + listener, err := net.Listen(network, addr) + if err != nil { + t.Fatalf("Failed to create test DNS server: %v", err) + } + + server := &echoServer{ + listener: listener, + addr: listener.Addr().String(), + done: make(chan struct{}), + } + + server.wg.Add(1) + go server.serve() + + return server +} + +func (s *echoServer) serve() { + defer s.wg.Done() + + for { + select { + case <-s.done: + return + default: + conn, err := s.listener.Accept() + if err != nil { + select { + case <-s.done: + return + default: + continue + } + } + go s.handleConnection(conn) + } + } +} + +func (s *echoServer) handleConnection(conn net.Conn) { + defer conn.Close() + // Simple response - just echo back some data to confirm connectivity + buf := make([]byte, 1024) + n, err := conn.Read(buf) + if err != nil { + return + } + conn.Write(buf[:n]) +} + +func (s *echoServer) close() { + close(s.done) + s.listener.Close() + s.wg.Wait() +} + +func TestGetResolver(t *testing.T) { + tests := []struct { + name string + network string + addr string + }{ + { + name: "ipv4_loopback", + network: "tcp4", + addr: "127.0.0.1:0", + }, + { + name: "ipv6_loopback", + network: "tcp6", + addr: "[::1]:0", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := newEchoServer(t, tc.network, tc.addr) + defer server.close() + serverAddr := server.addr + resolver := getResolver(serverAddr) + if resolver == nil { + t.Fatal("getResolver returned nil") + } + + netResolver, ok := resolver.(*net.Resolver) + if !ok { + t.Fatal("getResolver did not return a *net.Resolver") + } + if netResolver.Dial == nil { + t.Fatal("resolver.Dial is nil") + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := netResolver.Dial(ctx, "tcp", "dummy.address:53") + if err != nil { + t.Fatalf("Failed to dial test DNS server: %v", err) + } + defer conn.Close() + + testData := []byte("test") + _, err = conn.Write(testData) + if err != nil { + t.Fatalf("Failed to write to connection: %v", err) + } + + response := make([]byte, len(testData)) + _, err = conn.Read(response) + if err != nil { + t.Fatalf("Failed to read from connection: %v", err) + } + + if string(response) != string(testData) { + t.Fatalf("Expected echo response %q, got %q", testData, response) + } + }) + } +} + +func TestGetResolverMultipleServers(t *testing.T) { + server1 := newEchoServer(t, "tcp4", "127.0.0.1:0") + defer server1.close() + server2 := newEchoServer(t, "tcp4", "127.0.0.1:0") + defer server2.close() + serverFlag := server1.addr + ", " + server2.addr + + resolver := getResolver(serverFlag) + netResolver, ok := resolver.(*net.Resolver) + if !ok { + t.Fatal("getResolver did not return a *net.Resolver") + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + servers := map[string]bool{ + server1.addr: false, + server2.addr: false, + } + + // Try up to 1000 times to hit all servers, this should be very quick, and + // if this fails randomness has regressed beyond reason. + for range 1000 { + conn, err := netResolver.Dial(ctx, "tcp", "dummy.address:53") + if err != nil { + t.Fatalf("Failed to dial test DNS server: %v", err) + } + + remoteAddr := conn.RemoteAddr().String() + + conn.Close() + + servers[remoteAddr] = true + + var allDone = true + for _, done := range servers { + if !done { + allDone = false + break + } + } + if allDone { + break + } + } + + var allDone = true + for _, done := range servers { + if !done { + allDone = false + break + } + } + if !allDone { + t.Errorf("after 1000 queries, not all servers were hit, significant lack of randomness: %#v", servers) + } +} + +func TestGetResolverEmpty(t *testing.T) { + resolver := getResolver("") + if resolver != net.DefaultResolver { + t.Fatal(`getResolver("") should return net.DefaultResolver`) + } +} From 259bab9bff0d377eae360f10943819fab8f3813b Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 16 Jun 2025 12:02:20 -0700 Subject: [PATCH 073/263] scripts/check_license_headers.sh: delete, rewrite as a Go test Updates tailscale/corp#29650 Change-Id: Iad4e4ccd9d68ebb1d1a12f335cc5295d0bd05b60 Signed-off-by: Brad Fitzpatrick --- .github/workflows/test.yml | 14 ++- chirp/chirp_test.go | 1 + cmd/cloner/cloner_test.go | 1 + cmd/gitops-pusher/gitops-pusher_test.go | 1 + cmd/proxy-to-grafana/proxy-to-grafana_test.go | 1 + cmd/tsidp/tsidp_test.go | 1 + ipn/serve_test.go | 1 + license_test.go | 117 ++++++++++++++++++ net/tstun/mtu_test.go | 1 + scripts/check_license_headers.sh | 77 ------------ tsweb/promvarz/promvarz_test.go | 1 + 11 files changed, 138 insertions(+), 78 deletions(-) create mode 100644 license_test.go delete mode 100755 scripts/check_license_headers.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11a851dc4ee6e..2d179566814c7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -702,11 +702,23 @@ jobs: licenses: runs-on: ubuntu-24.04 + needs: gomod-cache steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: src + - name: Restore Go module cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: gomodcache + key: ${{ needs.gomod-cache.outputs.cache-key }} + enableCrossOsArchive: true - name: check licenses - run: ./scripts/check_license_headers.sh . + working-directory: src + run: | + grep -q TestLicenseHeaders *.go || (echo "Expected a test named TestLicenseHeaders"; exit 1) + ./tool/go test -v -run=TestLicenseHeaders staticcheck: runs-on: ubuntu-24.04 diff --git a/chirp/chirp_test.go b/chirp/chirp_test.go index 2549c163fd819..a57ef224b2c1b 100644 --- a/chirp/chirp_test.go +++ b/chirp/chirp_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package chirp import ( diff --git a/cmd/cloner/cloner_test.go b/cmd/cloner/cloner_test.go index d8d5df3cb040c..cf1063714afda 100644 --- a/cmd/cloner/cloner_test.go +++ b/cmd/cloner/cloner_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package main import ( diff --git a/cmd/gitops-pusher/gitops-pusher_test.go b/cmd/gitops-pusher/gitops-pusher_test.go index b050761d9832d..e08b06c9cd194 100644 --- a/cmd/gitops-pusher/gitops-pusher_test.go +++ b/cmd/gitops-pusher/gitops-pusher_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package main import ( diff --git a/cmd/proxy-to-grafana/proxy-to-grafana_test.go b/cmd/proxy-to-grafana/proxy-to-grafana_test.go index 083c4bc494ad6..4831d54364943 100644 --- a/cmd/proxy-to-grafana/proxy-to-grafana_test.go +++ b/cmd/proxy-to-grafana/proxy-to-grafana_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package main import ( diff --git a/cmd/tsidp/tsidp_test.go b/cmd/tsidp/tsidp_test.go index 76a11899187f7..6932d8e29b084 100644 --- a/cmd/tsidp/tsidp_test.go +++ b/cmd/tsidp/tsidp_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package main import ( diff --git a/ipn/serve_test.go b/ipn/serve_test.go index ae1d56eef6b09..ba0a26f8c0698 100644 --- a/ipn/serve_test.go +++ b/ipn/serve_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package ipn import ( diff --git a/license_test.go b/license_test.go new file mode 100644 index 0000000000000..ec452a6e36be7 --- /dev/null +++ b/license_test.go @@ -0,0 +1,117 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package tailscaleroot + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "tailscale.com/util/set" +) + +func normalizeLineEndings(b []byte) []byte { + return bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n")) +} + +// TestLicenseHeaders checks that all Go files in the tree +// directory tree have a correct-looking Tailscale license header. +func TestLicenseHeaders(t *testing.T) { + want := normalizeLineEndings([]byte(strings.TrimLeft(` +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause +`, "\n"))) + + exceptions := set.Of( + // Subprocess test harness code + "util/winutil/testdata/testrestartableprocesses/main.go", + "util/winutil/subprocess_windows_test.go", + + // WireGuard copyright + "cmd/tailscale/cli/authenticode_windows.go", + "wgengine/router/ifconfig_windows.go", + + // noiseexplorer.com copyright + "control/controlbase/noiseexplorer_test.go", + + // Generated eBPF management code + "derp/xdp/bpf_bpfeb.go", + "derp/xdp/bpf_bpfel.go", + + // Generated kube deepcopy funcs file starts with a Go build tag + an empty line + "k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go", + ) + + err := filepath.Walk(".", func(path string, fi os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("path %s: %v", path, err) + } + if exceptions.Contains(filepath.ToSlash(path)) { + return nil + } + base := filepath.Base(path) + switch base { + case ".git", "node_modules", "tempfork": + return filepath.SkipDir + } + switch base { + case "zsyscall_windows.go": + // Generated code. + return nil + } + + if strings.HasSuffix(base, ".config.ts") { + return nil + } + if strings.HasSuffix(base, "_string.go") { + // Generated file from go:generate stringer + return nil + } + + ext := filepath.Ext(base) + switch ext { + default: + return nil + case ".go", ".ts", ".tsx": + } + + buf := make([]byte, 512) + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + if n, err := io.ReadAtLeast(f, buf, 512); err != nil && err != io.ErrUnexpectedEOF { + return err + } else { + buf = buf[:n] + } + + buf = normalizeLineEndings(buf) + + bufNoTrunc := buf + if i := bytes.Index(buf, []byte("\npackage ")); i != -1 { + buf = buf[:i] + } + + if bytes.Contains(buf, want) { + return nil + } + + if bytes.Contains(bufNoTrunc, []byte("BSD-3-Clause\npackage ")) { + t.Errorf("file %s has license header as a package doc; add a blank line before the package line", path) + return nil + } + + t.Errorf("file %s is missing Tailscale copyright header:\n\n%s", path, want) + return nil + }) + if err != nil { + t.Fatalf("Walk: %v", err) + } +} diff --git a/net/tstun/mtu_test.go b/net/tstun/mtu_test.go index 8d165bfd341a9..ec31e45ce73f5 100644 --- a/net/tstun/mtu_test.go +++ b/net/tstun/mtu_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package tstun import ( diff --git a/scripts/check_license_headers.sh b/scripts/check_license_headers.sh deleted file mode 100755 index 8345afab76508..0000000000000 --- a/scripts/check_license_headers.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh -# -# Copyright (c) Tailscale Inc & AUTHORS -# SPDX-License-Identifier: BSD-3-Clause -# -# check_license_headers.sh checks that all Go files in the given -# directory tree have a correct-looking Tailscale license header. - -check_file() { - got=$1 - - want=$(cat <&2 - exit 1 -fi - -fail=0 -for file in $(find $1 \( -name '*.go' -or -name '*.tsx' -or -name '*.ts' -not -name '*.config.ts' \) -not -path '*/.git/*' -not -path '*/node_modules/*'); do - case $file in - $1/tempfork/*) - # Skip, tempfork of third-party code - ;; - $1/wgengine/router/ifconfig_windows.go) - # WireGuard copyright. - ;; - $1/cmd/tailscale/cli/authenticode_windows.go) - # WireGuard copyright. - ;; - *_string.go) - # Generated file from go:generate stringer - ;; - $1/control/controlbase/noiseexplorer_test.go) - # Noiseexplorer.com copyright. - ;; - */zsyscall_windows.go) - # Generated syscall wrappers - ;; - $1/util/winutil/subprocess_windows_test.go) - # Subprocess test harness code - ;; - $1/util/winutil/testdata/testrestartableprocesses/main.go) - # Subprocess test harness code - ;; - *$1/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go) - # Generated kube deepcopy funcs file starts with a Go build tag + an empty line - header="$(head -5 $file | tail -n+3 )" - ;; - $1/derp/xdp/bpf_bpfe*.go) - # Generated eBPF management code - ;; - *) - header="$(head -2 $file)" - ;; - esac - if [ ! -z "$header" ]; then - if ! check_file "$header"; then - fail=1 - echo "${file#$1/} doesn't have the right copyright header:" - echo "$header" | sed -e 's/^/ /g' - fi - fi -done - -if [ $fail -ne 0 ]; then - exit 1 -fi diff --git a/tsweb/promvarz/promvarz_test.go b/tsweb/promvarz/promvarz_test.go index 9f91b5d12380e..cffbbec2273c8 100644 --- a/tsweb/promvarz/promvarz_test.go +++ b/tsweb/promvarz/promvarz_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package promvarz import ( From 5b086cd2addc694c4b59c1a827e13a31e2f04d26 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 15 Jun 2025 08:25:36 -0700 Subject: [PATCH 074/263] tool/gocross: make gocross opt-in instead of opt-out gocross is not needed like it used to be, now that Go does version stamping itself. We keep it for the xcode and Windows builds for now. This simplifies things in the build, especially with upcoming build system updates. Updates tailscale/corp#28679 Updates tailscale/corp#26717 Change-Id: Ib4bebe6f50f3b9c3d6cd27323fca603e3dfb43cc Signed-off-by: Brad Fitzpatrick --- .github/workflows/test.yml | 1 - tool/gocross/gocross-wrapper.sh | 36 +++++++++++++++++++++++++--- tool/gocross/gocross_wrapper_test.go | 2 +- version/print.go | 1 + 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d179566814c7..313ce609f19be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -161,7 +161,6 @@ jobs: if: matrix.buildflags == '' # skip on race builder working-directory: src run: | - export TS_USE_TOOLCHAIN=1 ./build_dist.sh --extra-small ./cmd/tailscaled ./build_dist.sh --box ./cmd/tailscaled ./build_dist.sh --extra-small --box ./cmd/tailscaled diff --git a/tool/gocross/gocross-wrapper.sh b/tool/gocross/gocross-wrapper.sh index 366011fefdd6b..90f308eb50968 100755 --- a/tool/gocross/gocross-wrapper.sh +++ b/tool/gocross/gocross-wrapper.sh @@ -3,8 +3,11 @@ # SPDX-License-Identifier: BSD-3-Clause # # gocross-wrapper.sh is a wrapper that can be aliased to 'go', which -# transparently builds gocross using a "bootstrap" Go toolchain, and -# then invokes gocross. +# transparently runs the version of github.com/tailscale/go as specified repo's +# go.toolchain.rev file. +# +# It also conditionally (if TS_USE_GOCROSS=1) builds gocross and uses it as a go +# wrapper to inject certain go flags. set -euo pipefail @@ -76,6 +79,14 @@ case "$REV" in ;; esac +# gocross is opt-in as of 2025-06-16. See tailscale/corp#26717. +# It's primarily used for xcode builds, and a bit still for Windows. +# In the past we needed it for git version stamping on Linux etc, but +# Go does that itself nowadays. +if [ "${TS_USE_GOCROSS:-}" != "1" ]; then + exit 0 # out of subshell +fi + if [[ -d "$toolchain" ]]; then # A toolchain exists, but is it recent enough to compile gocross? If not, # wipe it out so that the next if block fetches a usable one. @@ -119,4 +130,23 @@ if [[ "$gocross_ok" == "0" ]]; then fi ) # End of the subshell execution. -exec "${BASH_SOURCE%/*}/../../gocross" "$@" +repo_root="${BASH_SOURCE%/*}/../.." + +# gocross is opt-in as of 2025-06-16. See tailscale/corp#26717 +# and comment above in this file. +if [ "${TS_USE_GOCROSS:-}" != "1" ]; then + read -r REV <"${repo_root}/go.toolchain.rev" + case "$REV" in + /*) + toolchain="$REV" + ;; + *) + # If the prior subshell completed successfully, this toolchain location + # should be valid at this point. + toolchain="$HOME/.cache/tsgo/$REV" + ;; + esac + exec "$toolchain/bin/go" "$@" +fi + +exec "${repo_root}/gocross" "$@" diff --git a/tool/gocross/gocross_wrapper_test.go b/tool/gocross/gocross_wrapper_test.go index 2b0f016a29d57..f4dcec4292695 100644 --- a/tool/gocross/gocross_wrapper_test.go +++ b/tool/gocross/gocross_wrapper_test.go @@ -15,7 +15,7 @@ import ( func TestGocrossWrapper(t *testing.T) { for i := range 2 { // once to build gocross; second to test it's cached cmd := exec.Command("./gocross-wrapper.sh", "version") - cmd.Env = append(os.Environ(), "CI=true", "NOBASHDEBUG=false") // for "set -x" verbosity + cmd.Env = append(os.Environ(), "CI=true", "NOBASHDEBUG=false", "TS_USE_GOCROSS=1") // for "set -x" verbosity out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("gocross-wrapper.sh failed: %v\n%s", err, out) diff --git a/version/print.go b/version/print.go index be90432cc85df..43ee2b5591410 100644 --- a/version/print.go +++ b/version/print.go @@ -20,6 +20,7 @@ var stringLazy = sync.OnceValue(func() string { if gitCommit() != "" { fmt.Fprintf(&ret, " tailscale commit: %s%s\n", gitCommit(), dirtyString()) } + fmt.Fprintf(&ret, " long version: %s\n", Long()) if extraGitCommitStamp != "" { fmt.Fprintf(&ret, " other commit: %s\n", extraGitCommitStamp) } From 077d52b22f4ff35eb6e1a7427164df45f8efc1c0 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Mon, 16 Jun 2025 16:01:07 +0100 Subject: [PATCH 075/263] .github/workflows: removes extra '$' Signed-off-by: Irbe Krumina --- .github/workflows/checklocks.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/golangci-lint.yml | 2 +- .github/workflows/kubemanifests.yaml | 2 +- .github/workflows/natlab-integrationtest.yml | 2 +- .github/workflows/ssh-integrationtest.yml | 2 +- .github/workflows/update-flake.yml | 2 +- .github/workflows/update-webclient-prebuilt.yml | 2 +- .github/workflows/webclient.yml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/checklocks.yml b/.github/workflows/checklocks.yml index 7464524ce99e2..5957e69258db5 100644 --- a/.github/workflows/checklocks.yml +++ b/.github/workflows/checklocks.yml @@ -10,7 +10,7 @@ on: - '.github/workflows/checklocks.yml' concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 32d2e7c2f64c5..2b471e943318f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,7 +23,7 @@ on: - cron: '31 14 * * 5' concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 60eb6852a0ab2..ee62f04bed91c 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -15,7 +15,7 @@ permissions: pull-requests: read concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: diff --git a/.github/workflows/kubemanifests.yaml b/.github/workflows/kubemanifests.yaml index 5b100a2763e3b..4cffea02fce6b 100644 --- a/.github/workflows/kubemanifests.yaml +++ b/.github/workflows/kubemanifests.yaml @@ -9,7 +9,7 @@ on: # Cancel workflow run if there is a newer push to the same PR for which it is # running concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: diff --git a/.github/workflows/natlab-integrationtest.yml b/.github/workflows/natlab-integrationtest.yml index 1de74cdaa45f8..99d58717b7beb 100644 --- a/.github/workflows/natlab-integrationtest.yml +++ b/.github/workflows/natlab-integrationtest.yml @@ -3,7 +3,7 @@ name: "natlab-integrationtest" concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true on: diff --git a/.github/workflows/ssh-integrationtest.yml b/.github/workflows/ssh-integrationtest.yml index 829d10ab8c2c8..463f4bdd4b24f 100644 --- a/.github/workflows/ssh-integrationtest.yml +++ b/.github/workflows/ssh-integrationtest.yml @@ -3,7 +3,7 @@ name: "ssh-integrationtest" concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true on: diff --git a/.github/workflows/update-flake.yml b/.github/workflows/update-flake.yml index f695c578eca91..af7bdff1ee66d 100644 --- a/.github/workflows/update-flake.yml +++ b/.github/workflows/update-flake.yml @@ -12,7 +12,7 @@ on: workflow_dispatch: concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: diff --git a/.github/workflows/update-webclient-prebuilt.yml b/.github/workflows/update-webclient-prebuilt.yml index 412836db78e7c..f1c2b0c3b9368 100644 --- a/.github/workflows/update-webclient-prebuilt.yml +++ b/.github/workflows/update-webclient-prebuilt.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: diff --git a/.github/workflows/webclient.yml b/.github/workflows/webclient.yml index b1cfb7620f97d..e64137f2b160d 100644 --- a/.github/workflows/webclient.yml +++ b/.github/workflows/webclient.yml @@ -15,7 +15,7 @@ on: # - main concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: From d7770d2b81d5b07466d0098c637d81204779eb0b Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Mon, 16 Jun 2025 16:01:46 +0100 Subject: [PATCH 076/263] .github/workflows: test that ./go/tool version matches go mod version Tests that go mod version matches ./tool/go version. Mismatched versions result in incosistent Go versions being used i.e. in CI jobs as the version in go.mod is used to determine what Go version Github actions pull in. Updates #16283 Signed-off-by: Irbe Krumina --- version_test.go | 72 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/version_test.go b/version_test.go index 1f434e682f1d6..3d983a19d51db 100644 --- a/version_test.go +++ b/version_test.go @@ -6,21 +6,16 @@ package tailscaleroot import ( "fmt" "os" - "regexp" + "os/exec" + "runtime" "strings" "testing" + + "golang.org/x/mod/modfile" ) func TestDockerfileVersion(t *testing.T) { - goMod, err := os.ReadFile("go.mod") - if err != nil { - t.Fatal(err) - } - m := regexp.MustCompile(`(?m)^go (\d\.\d+)\r?($|\.)`).FindStringSubmatch(string(goMod)) - if m == nil { - t.Fatalf("didn't find go version in go.mod") - } - goVersion := m[1] + goVersion := mustGetGoModVersion(t, false) dockerFile, err := os.ReadFile("Dockerfile") if err != nil { @@ -31,3 +26,60 @@ func TestDockerfileVersion(t *testing.T) { t.Errorf("didn't find %q in Dockerfile", wantSub) } } + +// TestGoVersion tests that the Go version specified in go.mod matches ./tool/go version. +func TestGoVersion(t *testing.T) { + // We could special-case ./tool/go path for Windows, but really there is no + // need to run it there. + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows") + } + goModVersion := mustGetGoModVersion(t, true) + + goToolCmd := exec.Command("./tool/go", "version") + goToolOutput, err := goToolCmd.Output() + if err != nil { + t.Fatalf("Failed to get ./tool/go version: %v", err) + } + + // Version info will approximately look like 'go version go1.24.4 linux/amd64'. + parts := strings.Fields(string(goToolOutput)) + if len(parts) < 4 { + t.Fatalf("Unexpected ./tool/go version output format: %s", goToolOutput) + } + + goToolVersion := strings.TrimPrefix(parts[2], "go") + + if goModVersion != goToolVersion { + t.Errorf("Go version in go.mod (%q) does not match the version of ./tool/go (%q).\nEnsure that the go.mod refers to the same Go version as ./go.toolchain.rev.", + goModVersion, goToolVersion) + } +} + +func mustGetGoModVersion(t *testing.T, includePatchVersion bool) string { + t.Helper() + + goModBytes, err := os.ReadFile("go.mod") + if err != nil { + t.Fatal(err) + } + + modFile, err := modfile.Parse("go.mod", goModBytes, nil) + if err != nil { + t.Fatal(err) + } + + if modFile.Go == nil { + t.Fatal("no Go version found in go.mod") + } + + version := modFile.Go.Version + + parts := strings.Split(version, ".") + if !includePatchVersion { + if len(parts) >= 2 { + version = parts[0] + "." + parts[1] + } + } + return version +} From 42f71e959dff4dc55b138c358764c8fbfe8cdb7f Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 16 Jun 2025 18:18:36 -0700 Subject: [PATCH 077/263] prober: speed up TestCRL ~450x by baking in some test keys Fixes #16290 Updates tailscale/corp#28679 Change-Id: Ic90129b686779d0ed1cb40acf187cfcbdd39eb83 Signed-off-by: Brad Fitzpatrick --- prober/tls_test.go | 65 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/prober/tls_test.go b/prober/tls_test.go index 9ba17f79da911..f6ca4aeb19be6 100644 --- a/prober/tls_test.go +++ b/prober/tls_test.go @@ -6,6 +6,7 @@ package prober import ( "bytes" "context" + "crypto/ecdsa" "crypto/rand" "crypto/rsa" "crypto/tls" @@ -140,16 +141,60 @@ func (s *CRLServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Write(s.crlBytes) } -func TestCRL(t *testing.T) { - // Generate CA key and self-signed CA cert - caKey, err := rsa.GenerateKey(rand.Reader, 4096) +// someECDSAKey{1,2,3} are different EC private keys in PEM format +// as generated by: +// +// openssl ecparam -name prime256v1 -genkey -noout -out - +// +// They're used in tests to avoid burning CPU at test time to just +// to make some arbitrary test keys. +const ( + someECDSAKey1 = ` +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIDKggO47Si0/JgqF0q9m0HfQ92lbERWsBaKS5YihtuheoAoGCCqGSM49 +AwEHoUQDQgAE/JtNZkfFmAGQJHW5Xgz0Eoyi9MKVxl77sXjIFDMX233QDIWPEM/B +vmNMvdFkuYBjwbq6H+SNf1NXRNladEGU/Q== +-----END EC PRIVATE KEY----- +` + someECDSAKey2 = ` +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIPIJhRf4MpzLil1ZKcRqMx+jPeJXw96KtYYzV2AcgBzgoAoGCCqGSM49 +AwEHoUQDQgAEhA9CSWFmUvdvXMzyt+as+6f+0luydHU1x/gEksVByYIgYxahaGts +xbSKj6F2WgAN/ok1gFLqhH3UWMNVthM1wA== +-----END EC PRIVATE KEY----- +` + someECDSAKey3 = ` +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIKgZ1OJjK2St9O0i52N1K+IgSiu2/NSMk9Yt2+kDMHd7oAoGCCqGSM49 +AwEHoUQDQgAExFp80etkjy/AEUtSgJjXRA39jTU7eiEmCGRREewFQhwcEscBEfrg +6NN31r9YlEs+hZ8gXE1L3Deu6jn5jW3pig== +-----END EC PRIVATE KEY----- +` +) + +// parseECKey parses an EC private key from a PEM-encoded string. +func parseECKey(t *testing.T, pemPriv string) *ecdsa.PrivateKey { + t.Helper() + block, _ := pem.Decode([]byte(pemPriv)) + if block == nil { + t.Fatal("failed to decode PEM") + } + key, err := x509.ParseECPrivateKey(block.Bytes) if err != nil { - t.Fatal(err) + t.Fatalf("failed to parse EC key: %v", err) } + return key +} + +func TestCRL(t *testing.T) { + // Generate CA key and self-signed CA cert + caKey := parseECKey(t, someECDSAKey1) + caTpl := issuerCertTpl caTpl.BasicConstraintsValid = true caTpl.IsCA = true caTpl.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature + caTpl.SignatureAlgorithm = x509.ECDSAWithSHA256 caBytes, err := x509.CreateCertificate(rand.Reader, &caTpl, &caTpl, &caKey.PublicKey, caKey) if err != nil { t.Fatal(err) @@ -162,11 +207,9 @@ func TestCRL(t *testing.T) { // Issue a leaf cert signed by the CA leaf := leafCert leaf.SerialNumber = big.NewInt(20001) + leaf.SignatureAlgorithm = x509.ECDSAWithSHA256 leaf.Issuer = caCert.Subject - leafKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - t.Fatal(err) - } + leafKey := parseECKey(t, someECDSAKey2) leafBytes, err := x509.CreateCertificate(rand.Reader, &leaf, caCert, &leafKey.PublicKey, caKey) if err != nil { t.Fatal(err) @@ -182,10 +225,8 @@ func TestCRL(t *testing.T) { noCRLCert.CRLDistributionPoints = []string{} noCRLCert.NotBefore = time.Unix(letsEncryptStartedStaplingCRL, 0).Add(-48 * time.Hour) noCRLCert.Issuer = caCert.Subject - noCRLCertKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - t.Fatal(err) - } + noCRLCert.SignatureAlgorithm = x509.ECDSAWithSHA256 + noCRLCertKey := parseECKey(t, someECDSAKey3) noCRLStapledBytes, err := x509.CreateCertificate(rand.Reader, &noCRLCert, caCert, &noCRLCertKey.PublicKey, caKey) if err != nil { t.Fatal(err) From d37e8d0bfaaf3a40ef2432f6bed7bab2004e36eb Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 16 Jun 2025 21:10:59 -0700 Subject: [PATCH 078/263] .github/workflows: remove redundant work between staticcheck jobs Make the OS-specific staticcheck jobs only test stuff that's specialized for that OS. Do that using a new ./tool/listpkgs program that's a fancy 'go list' with more filtering flags. Updates tailscale/corp#28679 Change-Id: I790be2e3a0b42b105bd39f68c4b20e217a26de60 Signed-off-by: Brad Fitzpatrick --- .github/workflows/test.yml | 87 ++++++++++++++-- Makefile | 2 +- tool/listpkgs/listpkgs.go | 206 +++++++++++++++++++++++++++++++++++++ 3 files changed, 283 insertions(+), 12 deletions(-) create mode 100644 tool/listpkgs/listpkgs.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 313ce609f19be..6d8ab863ce422 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -232,10 +232,6 @@ jobs: - name: Restore Cache uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - # Note: unlike the other setups, this is only grabbing the mod download - # cache, rather than the whole mod directory, as the download cache - # contains zips that can be unpacked in parallel faster than they can be - # fetched and extracted by tar path: | ~/.cache/go-build ~\AppData\Local\go-build @@ -722,14 +718,40 @@ jobs: staticcheck: runs-on: ubuntu-24.04 needs: gomod-cache + name: staticcheck (${{ matrix.name }}) strategy: fail-fast: false # don't abort the entire matrix if one element fails matrix: - goos: ["linux", "windows", "darwin"] - goarch: ["amd64"] include: - - goos: "windows" - goarch: "386" + - name: "macOS" + goos: "darwin" + goarch: "arm64" + flags: "--with-tags-all=darwin" + - name: "Windows" + goos: "windows" + goarch: "amd64" + flags: "--with-tags-all=windows" + - name: "Linux" + goos: "linux" + goarch: "amd64" + flags: "--with-tags-all=linux" + - name: "Portable (1/4)" + goos: "linux" + goarch: "amd64" + flags: "--without-tags-any=windows,darwin,linux --shard=1/4" + - name: "Portable (2/4)" + goos: "linux" + goarch: "amd64" + flags: "--without-tags-any=windows,darwin,linux --shard=2/4" + - name: "Portable (3/4)" + goos: "linux" + goarch: "amd64" + flags: "--without-tags-any=windows,darwin,linux --shard=3/4" + - name: "Portable (4/4)" + goos: "linux" + goarch: "amd64" + flags: "--without-tags-any=windows,darwin,linux --shard=4/4" + steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -741,14 +763,14 @@ jobs: path: gomodcache key: ${{ needs.gomod-cache.outputs.cache-key }} enableCrossOsArchive: true - - name: run staticcheck + - name: run staticcheck (${{ matrix.name }}) working-directory: src run: | export GOROOT=$(./tool/go env GOROOT) ./tool/go run -exec \ "env GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}" \ honnef.co/go/tools/cmd/staticcheck -- \ - $(env GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} ./tool/go list ./... | grep -v tempfork) + $(./tool/go run ./tool/listpkgs --ignore-3p --goos=${{ matrix.goos }} --goarch=${{ matrix.goarch }} ${{ matrix.flags }} ./...) notify_slack: if: always() @@ -795,7 +817,7 @@ jobs: }] } - check_mergeability: + merge_blocker: if: always() runs-on: ubuntu-24.04 needs: @@ -819,3 +841,46 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 with: jobs: ${{ toJSON(needs) }} + + # This waits on all the jobs which must never fail. Branch protection rules + # enforce these. No flaky tests are allowed in these jobs. (We don't want flaky + # tests anywhere, really, but a flaky test here prevents merging.) + check_mergeability_strict: + if: always() + runs-on: ubuntu-24.04 + needs: + - android + - cross + - crossmin + - ios + - tailscale_go + - depaware + - go_generate + - go_mod_tidy + - licenses + - staticcheck + steps: + - name: Decide if change is okay to merge + if: github.event_name != 'push' + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 + with: + jobs: ${{ toJSON(needs) }} + + check_mergeability: + if: always() + runs-on: ubuntu-24.04 + needs: + - check_mergeability_strict + - test + - windows + - vm + - wasm + - fuzz + - race-root-integration + - privileged + steps: + - name: Decide if change is okay to merge + if: github.event_name != 'push' + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/Makefile b/Makefile index 1978af90d259a..41c67c711791d 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ buildmultiarchimage: ## Build (and optionally push) multiarch docker image check: staticcheck vet depaware buildwindows build386 buildlinuxarm buildwasm ## Perform basic checks and compilation tests staticcheck: ## Run staticcheck.io checks - ./tool/go run honnef.co/go/tools/cmd/staticcheck -- $$(./tool/go list ./... | grep -v tempfork) + ./tool/go run honnef.co/go/tools/cmd/staticcheck -- $$(./tool/go run ./tool/listpkgs --ignore-3p ./...) kube-generate-all: kube-generate-deepcopy ## Refresh generated files for Tailscale Kubernetes Operator ./tool/go generate ./cmd/k8s-operator diff --git a/tool/listpkgs/listpkgs.go b/tool/listpkgs/listpkgs.go new file mode 100644 index 0000000000000..400bf90c18315 --- /dev/null +++ b/tool/listpkgs/listpkgs.go @@ -0,0 +1,206 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// listpkgs prints the import paths that match the Go package patterns +// given on the command line and conditionally filters them in various ways. +package main + +import ( + "bufio" + "flag" + "fmt" + "go/build/constraint" + "log" + "os" + "slices" + "strings" + "sync" + + "golang.org/x/tools/go/packages" +) + +var ( + ignore3p = flag.Bool("ignore-3p", false, "ignore third-party packages forked/vendored into Tailscale") + goos = flag.String("goos", "", "GOOS to use for loading packages (default: current OS)") + goarch = flag.String("goarch", "", "GOARCH to use for loading packages (default: current architecture)") + withTagsAllStr = flag.String("with-tags-all", "", "if non-empty, a comma-separated list of builds tags to require (a package will only be listed if it contains all of these build tags)") + withoutTagsAnyStr = flag.String("without-tags-any", "", "if non-empty, a comma-separated list of build constraints to exclude (a package will be omitted if it contains any of these build tags)") + shard = flag.String("shard", "", "if non-empty, a string of the form 'N/M' to only print packages in shard N of M (e.g. '1/3', '2/3', '3/3/' for different thirds of the list)") +) + +func main() { + flag.Parse() + + patterns := flag.Args() + if len(patterns) == 0 { + flag.Usage() + os.Exit(1) + } + + cfg := &packages.Config{ + Mode: packages.LoadFiles, + Env: os.Environ(), + } + if *goos != "" { + cfg.Env = append(cfg.Env, "GOOS="+*goos) + } + if *goarch != "" { + cfg.Env = append(cfg.Env, "GOARCH="+*goarch) + } + + pkgs, err := packages.Load(cfg, patterns...) + if err != nil { + log.Fatalf("loading packages: %v", err) + } + + var withoutAny []string + if *withoutTagsAnyStr != "" { + withoutAny = strings.Split(*withoutTagsAnyStr, ",") + } + var withAll []string + if *withTagsAllStr != "" { + withAll = strings.Split(*withTagsAllStr, ",") + } + + seen := map[string]bool{} + matches := 0 +Pkg: + for _, pkg := range pkgs { + if pkg.PkgPath == "" { // malformed (shouldn’t happen) + continue + } + if seen[pkg.PkgPath] { + continue // suppress duplicates when patterns overlap + } + seen[pkg.PkgPath] = true + + pkgPath := pkg.PkgPath + + if *ignore3p && isThirdParty(pkgPath) { + continue + } + if withAll != nil { + for _, t := range withAll { + if !hasBuildTag(pkg, t) { + continue Pkg + } + } + } + for _, t := range withoutAny { + if hasBuildTag(pkg, t) { + continue Pkg + } + } + matches++ + + if *shard != "" { + var n, m int + if _, err := fmt.Sscanf(*shard, "%d/%d", &n, &m); err != nil || n < 1 || m < 1 { + log.Fatalf("invalid shard format %q; expected 'N/M'", *shard) + } + if m > 0 && (matches-1)%m != n-1 { + continue // not in this shard + } + } + fmt.Println(pkgPath) + } + + // If any package had errors (e.g. missing deps) report them via packages.PrintErrors. + // This mirrors `go list` behaviour when -e is *not* supplied. + if packages.PrintErrors(pkgs) > 0 { + os.Exit(1) + } +} + +func isThirdParty(pkg string) bool { + return strings.HasPrefix(pkg, "tailscale.com/tempfork/") +} + +// hasBuildTag reports whether any source file in pkg mentions `tag` +// in a //go:build constraint. +func hasBuildTag(pkg *packages.Package, tag string) bool { + all := slices.Concat(pkg.CompiledGoFiles, pkg.OtherFiles, pkg.IgnoredFiles) + suffix := "_" + tag + ".go" + for _, name := range all { + if strings.HasSuffix(name, suffix) { + return true + } + ok, err := fileMentionsTag(name, tag) + if err != nil { + log.Printf("reading %s: %v", name, err) + continue + } + if ok { + return true + } + } + return false +} + +// tagSet is a set of build tags. +// The values are always true. We avoid non-std set types +// to make this faster to "go run" on empty caches. +type tagSet map[string]bool + +var ( + mu sync.Mutex + fileTags = map[string]tagSet{} // abs path -> set of build tags mentioned in file +) + +func getFileTags(filename string) (tagSet, error) { + mu.Lock() + tags, ok := fileTags[filename] + mu.Unlock() + if ok { + return tags, nil + } + + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + ts := make(tagSet) + s := bufio.NewScanner(f) + for s.Scan() { + line := s.Text() + if strings.TrimSpace(line) == "" { + continue // still in leading blank lines + } + if !strings.HasPrefix(line, "//") { + // hit real code – done with header comments + // TODO(bradfitz): care about /* */ comments? + break + } + if !strings.HasPrefix(line, "//go:build") { + continue // some other comment + } + expr, err := constraint.Parse(line) + if err != nil { + return nil, fmt.Errorf("parsing %q: %w", line, err) + } + // Call Eval to populate ts with the tags mentioned in the expression. + // We don't care about the result, just the side effect of populating ts. + expr.Eval(func(tag string) bool { + ts[tag] = true + return true // arbitrary + }) + } + if err := s.Err(); err != nil { + return nil, fmt.Errorf("reading %s: %w", filename, err) + } + + mu.Lock() + defer mu.Unlock() + fileTags[filename] = ts + return tags, nil +} + +func fileMentionsTag(filename, tag string) (bool, error) { + tags, err := getFileTags(filename) + if err != nil { + return false, err + } + return tags[tag], nil +} From e7f5c9a01583b6d26977216d93c676ee21cb84eb Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Tue, 17 Jun 2025 13:05:05 -0400 Subject: [PATCH 079/263] derp/derphttp: add error notify for RunWatchConnectionLoop (#16261) The caller of client.RunWatchConnectionLoop may need to be aware of errors that occur within loop. Add a channel that notifies of errors to the caller to allow for decisions to be make as to the state of the client. Updates tailscale/corp#25756 Signed-off-by: Mike O'Driscoll --- cmd/derper/mesh.go | 3 +- derp/derphttp/derphttp_test.go | 73 ++++++++++++++++++++++++++++++++-- derp/derphttp/mesh_client.go | 15 ++++++- 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/cmd/derper/mesh.go b/cmd/derper/mesh.go index 1d8e3ef93c8b3..cbb2fa59ac030 100644 --- a/cmd/derper/mesh.go +++ b/cmd/derper/mesh.go @@ -72,6 +72,7 @@ func startMeshWithHost(s *derp.Server, hostTuple string) error { add := func(m derp.PeerPresentMessage) { s.AddPacketForwarder(m.Key, c) } remove := func(m derp.PeerGoneMessage) { s.RemovePacketForwarder(m.Peer, c) } - go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove) + notifyError := func(err error) {} + go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove, notifyError) return nil } diff --git a/derp/derphttp/derphttp_test.go b/derp/derphttp/derphttp_test.go index 8d02db922605b..25254966084ee 100644 --- a/derp/derphttp/derphttp_test.go +++ b/derp/derphttp/derphttp_test.go @@ -11,12 +11,14 @@ import ( "net" "net/http" "net/http/httptest" + "strings" "sync" "testing" "time" "tailscale.com/derp" "tailscale.com/net/netmon" + "tailscale.com/net/netx" "tailscale.com/types/key" ) @@ -298,6 +300,7 @@ func TestBreakWatcherConnRecv(t *testing.T) { defer cancel() watcherChan := make(chan int, 1) + errChan := make(chan error, 1) // Start the watcher thread (which connects to the watched server) wg.Add(1) // To avoid using t.Logf after the test ends. See https://golang.org/issue/40343 @@ -311,8 +314,11 @@ func TestBreakWatcherConnRecv(t *testing.T) { watcherChan <- peers } remove := func(m derp.PeerGoneMessage) { t.Logf("remove: %v", m.Peer.ShortString()); peers-- } + notifyErr := func(err error) { + errChan <- err + } - watcher1.RunWatchConnectionLoop(ctx, serverPrivateKey1.Public(), t.Logf, add, remove) + watcher1.RunWatchConnectionLoop(ctx, serverPrivateKey1.Public(), t.Logf, add, remove, notifyErr) }() timer := time.NewTimer(5 * time.Second) @@ -326,6 +332,10 @@ func TestBreakWatcherConnRecv(t *testing.T) { if peers != 1 { t.Fatal("wrong number of peers added during watcher connection") } + case err := <-errChan: + if !strings.Contains(err.Error(), "use of closed network connection") { + t.Fatalf("expected notifyError connection error to contain 'use of closed network connection', got %v", err) + } case <-timer.C: t.Fatalf("watcher did not process the peer update") } @@ -369,6 +379,7 @@ func TestBreakWatcherConn(t *testing.T) { watcherChan := make(chan int, 1) breakerChan := make(chan bool, 1) + errorChan := make(chan error, 1) // Start the watcher thread (which connects to the watched server) wg.Add(1) // To avoid using t.Logf after the test ends. See https://golang.org/issue/40343 @@ -384,8 +395,11 @@ func TestBreakWatcherConn(t *testing.T) { <-breakerChan } remove := func(m derp.PeerGoneMessage) { t.Logf("remove: %v", m.Peer.ShortString()); peers-- } + notifyError := func(err error) { + errorChan <- err + } - watcher1.RunWatchConnectionLoop(ctx, serverPrivateKey1.Public(), t.Logf, add, remove) + watcher1.RunWatchConnectionLoop(ctx, serverPrivateKey1.Public(), t.Logf, add, remove, notifyError) }() timer := time.NewTimer(5 * time.Second) @@ -399,6 +413,10 @@ func TestBreakWatcherConn(t *testing.T) { if peers != 1 { t.Fatal("wrong number of peers added during watcher connection") } + case err := <-errorChan: + if !strings.Contains(err.Error(), "use of closed network connection") { + t.Fatalf("expected notifyError connection error to contain 'use of closed network connection', got %v", err) + } case <-timer.C: t.Fatalf("watcher did not process the peer update") } @@ -414,6 +432,7 @@ func TestBreakWatcherConn(t *testing.T) { func noopAdd(derp.PeerPresentMessage) {} func noopRemove(derp.PeerGoneMessage) {} +func noopNotifyError(error) {} func TestRunWatchConnectionLoopServeConnect(t *testing.T) { defer func() { testHookWatchLookConnectResult = nil }() @@ -441,7 +460,7 @@ func TestRunWatchConnectionLoopServeConnect(t *testing.T) { } return false } - watcher.RunWatchConnectionLoop(ctx, pub, t.Logf, noopAdd, noopRemove) + watcher.RunWatchConnectionLoop(ctx, pub, t.Logf, noopAdd, noopRemove, noopNotifyError) // Test connecting to the server with a zero value for ignoreServerKey, // so we should always connect. @@ -455,7 +474,7 @@ func TestRunWatchConnectionLoopServeConnect(t *testing.T) { } return false } - watcher.RunWatchConnectionLoop(ctx, key.NodePublic{}, t.Logf, noopAdd, noopRemove) + watcher.RunWatchConnectionLoop(ctx, key.NodePublic{}, t.Logf, noopAdd, noopRemove, noopNotifyError) } // verify that the LocalAddr method doesn't acquire the mutex. @@ -491,3 +510,49 @@ func TestProbe(t *testing.T) { } } } + +func TestNotifyError(t *testing.T) { + defer func() { testHookWatchLookConnectResult = nil }() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + priv := key.NewNode() + serverURL, s := newTestServer(t, priv) + defer s.Close() + + pub := priv.Public() + + // Test early error notification when c.connect fails. + watcher := newWatcherClient(t, priv, serverURL) + watcher.SetURLDialer(netx.DialFunc(func(ctx context.Context, network, addr string) (net.Conn, error) { + t.Helper() + return nil, fmt.Errorf("test error: %s", addr) + })) + defer watcher.Close() + + testHookWatchLookConnectResult = func(err error, wasSelfConnect bool) bool { + t.Helper() + if err == nil { + t.Fatal("expected error connecting to server, got nil") + } + if wasSelfConnect { + t.Error("wanted normal connect; got self connect") + } + return false + } + + errChan := make(chan error, 1) + notifyError := func(err error) { + errChan <- err + } + watcher.RunWatchConnectionLoop(ctx, pub, t.Logf, noopAdd, noopRemove, notifyError) + + select { + case err := <-errChan: + if !strings.Contains(err.Error(), "test") { + t.Errorf("expected test error, got %v", err) + } + case <-ctx.Done(): + t.Fatalf("context done before receiving error: %v", ctx.Err()) + } +} diff --git a/derp/derphttp/mesh_client.go b/derp/derphttp/mesh_client.go index 66b8c166eeb37..c14a9a7e11111 100644 --- a/derp/derphttp/mesh_client.go +++ b/derp/derphttp/mesh_client.go @@ -31,6 +31,9 @@ var testHookWatchLookConnectResult func(connectError error, wasSelfConnect bool) // This behavior will likely change. Callers should do their own accounting // and dup suppression as needed. // +// If set the notifyError func is called with any error that occurs within the ctx +// main loop connection setup, or the inner loop receiving messages via RecvDetail. +// // infoLogf, if non-nil, is the logger to write periodic status updates about // how many peers are on the server. Error log output is set to the c's logger, // regardless of infoLogf's value. @@ -42,10 +45,11 @@ var testHookWatchLookConnectResult func(connectError error, wasSelfConnect bool) // initialized Client.WatchConnectionChanges to true. // // If the DERP connection breaks and reconnects, remove will be called for all -// previously seen peers, with Reason type PeerGoneReasonSynthetic. Those +// previously seen peers, with Reason type PeerGoneReasonMeshConnBroke. Those // clients are likely still connected and their add message will appear after // reconnect. -func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key.NodePublic, infoLogf logger.Logf, add func(derp.PeerPresentMessage), remove func(derp.PeerGoneMessage)) { +func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key.NodePublic, infoLogf logger.Logf, + add func(derp.PeerPresentMessage), remove func(derp.PeerGoneMessage), notifyError func(error)) { if !c.WatchConnectionChanges { if c.isStarted() { panic("invalid use of RunWatchConnectionLoop on already-started Client without setting Client.RunWatchConnectionLoop") @@ -121,6 +125,10 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key // Make sure we're connected before calling s.ServerPublicKey. _, _, err := c.connect(ctx, "RunWatchConnectionLoop") if err != nil { + logf("mesh connect: %v", err) + if notifyError != nil { + notifyError(err) + } if f := testHookWatchLookConnectResult; f != nil && !f(err, false) { return } @@ -141,6 +149,9 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key if err != nil { clear() logf("Recv: %v", err) + if notifyError != nil { + notifyError(err) + } sleep(retryInterval) break } From 939355f66727bb86819f90e74b25a6ed11ff5ad7 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 15 Jun 2025 08:20:48 -0700 Subject: [PATCH 080/263] tool/gocross: put the synthetic GOROOTs outside of the tsgo directory We aim to make the tsgo directories be read-only mounts on builders. But gocross was previously writing within the ~/.cache/tsgo/$HASH/ directories to make the synthetic GOROOT directories. This moves them to ~/.cache/tsgoroot/$HASH/ instead. Updates tailscale/corp#28679 Updates tailscale/corp#26717 Change-Id: I0d17730bbdce3d6374e79d49486826575d4690af Signed-off-by: Brad Fitzpatrick --- tool/gocross/gocross-wrapper.sh | 1 + tool/gocross/toolchain.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tool/gocross/gocross-wrapper.sh b/tool/gocross/gocross-wrapper.sh index 90f308eb50968..e9fca2aea71b5 100755 --- a/tool/gocross/gocross-wrapper.sh +++ b/tool/gocross/gocross-wrapper.sh @@ -74,6 +74,7 @@ case "$REV" in echo "# Cleaning up old Go toolchain $hash" >&2 rm -rf "$HOME/.cache/tsgo/$hash" rm -rf "$HOME/.cache/tsgo/$hash.extracted" + rm -rf "$HOME/.cache/tsgoroot/$hash" done fi ;; diff --git a/tool/gocross/toolchain.go b/tool/gocross/toolchain.go index e701662f5b1e8..f422e289e3571 100644 --- a/tool/gocross/toolchain.go +++ b/tool/gocross/toolchain.go @@ -62,7 +62,7 @@ func getToolchain() (toolchainDir, gorootDir string, err error) { cache := filepath.Join(os.Getenv("HOME"), ".cache") toolchainDir = filepath.Join(cache, "tsgo", rev) - gorootDir = filepath.Join(toolchainDir, "gocross-goroot") + gorootDir = filepath.Join(cache, "tsgoroot", rev) // You might wonder why getting the toolchain also provisions and returns a // path suitable for use as GOROOT. Wonder no longer! From 4431fb89c2191fc501e14b2fb92e934feaaf264e Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Tue, 17 Jun 2025 13:38:17 -0500 Subject: [PATCH 081/263] ipn/ipnlocal: add some verbose logging to taildrive peerapi handler Updates tailscale/corp#29702 Signed-off-by: Percy Wegmann --- ipn/ipnlocal/peerapi.go | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/ipn/ipnlocal/peerapi.go b/ipn/ipnlocal/peerapi.go index 60dd4102413f0..89554f0ff9eb1 100644 --- a/ipn/ipnlocal/peerapi.go +++ b/ipn/ipnlocal/peerapi.go @@ -247,6 +247,10 @@ func (h *peerAPIHandler) logf(format string, a ...any) { h.ps.b.logf("peerapi: "+format, a...) } +func (h *peerAPIHandler) logfv1(format string, a ...any) { + h.ps.b.logf("[v1] peerapi: "+format, a...) +} + // isAddressValid reports whether addr is a valid destination address for this // node originating from the peer. func (h *peerAPIHandler) isAddressValid(addr netip.Addr) bool { @@ -1015,6 +1019,7 @@ func (rbw *requestBodyWrapper) Read(b []byte) (int, error) { } func (h *peerAPIHandler) handleServeDrive(w http.ResponseWriter, r *http.Request) { + h.logfv1("taildrive: got %s request from %s", r.Method, h.peerNode.Key().ShortString()) if !h.ps.b.DriveSharingEnabled() { h.logf("taildrive: not enabled") http.Error(w, "taildrive not enabled", http.StatusNotFound) @@ -1055,21 +1060,23 @@ func (h *peerAPIHandler) handleServeDrive(w http.ResponseWriter, r *http.Request } r.Body = bw - if r.Method == httpm.PUT || r.Method == httpm.GET { - defer func() { - switch wr.statusCode { - case 304: - // 304s are particularly chatty so skip logging. - default: - contentType := "unknown" - if ct := wr.Header().Get("Content-Type"); ct != "" { - contentType = ct - } - - h.logf("taildrive: share: %s from %s to %s: status-code=%d ext=%q content-type=%q tx=%.f rx=%.f", r.Method, h.peerNode.Key().ShortString(), h.selfNode.Key().ShortString(), wr.statusCode, parseDriveFileExtensionForLog(r.URL.Path), contentType, roundTraffic(wr.contentLength), roundTraffic(bw.bytesRead)) + defer func() { + switch wr.statusCode { + case 304: + // 304s are particularly chatty so skip logging. + default: + log := h.logf + if r.Method != httpm.PUT && r.Method != httpm.GET { + log = h.logfv1 } - }() - } + contentType := "unknown" + if ct := wr.Header().Get("Content-Type"); ct != "" { + contentType = ct + } + + log("taildrive: share: %s from %s to %s: status-code=%d ext=%q content-type=%q tx=%.f rx=%.f", r.Method, h.peerNode.Key().ShortString(), h.selfNode.Key().ShortString(), wr.statusCode, parseDriveFileExtensionForLog(r.URL.Path), contentType, roundTraffic(wr.contentLength), roundTraffic(bw.bytesRead)) + } + }() r.URL.Path = strings.TrimPrefix(r.URL.Path, taildrivePrefix) fs.ServeHTTPWithPerms(p, wr, r) From cbc14bd3b07f8367a4062033b701dbaa18c2c22a Mon Sep 17 00:00:00 2001 From: Juan Francisco Cantero Hurtado Date: Tue, 17 Jun 2025 20:22:42 +0200 Subject: [PATCH 082/263] ipn: add missing entries for OpenBSD Signed-off-by: Juan Francisco Cantero Hurtado --- ipn/ipnlocal/c2n.go | 2 +- ipn/ipnserver/actor.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ipn/ipnlocal/c2n.go b/ipn/ipnlocal/c2n.go index 876c130641b81..4b91c3cb9453d 100644 --- a/ipn/ipnlocal/c2n.go +++ b/ipn/ipnlocal/c2n.go @@ -443,7 +443,7 @@ func findCmdTailscale() (string, error) { } case "windows": ts = filepath.Join(filepath.Dir(self), "tailscale.exe") - case "freebsd": + case "freebsd", "openbsd": if self == "/usr/local/bin/tailscaled" { ts = "/usr/local/bin/tailscale" } diff --git a/ipn/ipnserver/actor.go b/ipn/ipnserver/actor.go index dd40924bbf542..9d86d2c825fda 100644 --- a/ipn/ipnserver/actor.go +++ b/ipn/ipnserver/actor.go @@ -144,7 +144,7 @@ func (a *actor) Username() (string, error) { } defer tok.Close() return tok.Username() - case "darwin", "linux", "illumos", "solaris": + case "darwin", "linux", "illumos", "solaris", "openbsd": uid, ok := a.ci.Creds().UserID() if !ok { return "", errors.New("missing user ID") From 49ae66c10c95f5f6a1c33e1b021e4c8b2a95cd9f Mon Sep 17 00:00:00 2001 From: Simon Law Date: Tue, 17 Jun 2025 20:39:59 -0700 Subject: [PATCH 083/263] cmd/tailscale: clean up dns --help messages (#16306) This patch contains the following cleanups: 1. Simplify `ffcli.Command` definitions; 2. Word-wrap help text, consistent with other commands; 3. `tailscale dns --help` usage makes subcommand usage more obvious; 4. `tailscale dns query --help` describes DNS record types. Updates #cleanup Signed-off-by: Simon Law --- cmd/tailscale/cli/dns-query.go | 19 +++++++ cmd/tailscale/cli/dns-status.go | 94 ++++++++++++++++++++++----------- cmd/tailscale/cli/dns.go | 48 ++++++----------- 3 files changed, 98 insertions(+), 63 deletions(-) diff --git a/cmd/tailscale/cli/dns-query.go b/cmd/tailscale/cli/dns-query.go index da2d9d2a56d77..11f64453732fa 100644 --- a/cmd/tailscale/cli/dns-query.go +++ b/cmd/tailscale/cli/dns-query.go @@ -9,12 +9,31 @@ import ( "fmt" "net/netip" "os" + "strings" "text/tabwriter" + "github.com/peterbourgon/ff/v3/ffcli" "golang.org/x/net/dns/dnsmessage" "tailscale.com/types/dnstype" ) +var dnsQueryCmd = &ffcli.Command{ + Name: "query", + ShortUsage: "tailscale dns query [a|aaaa|cname|mx|ns|opt|ptr|srv|txt]", + Exec: runDNSQuery, + ShortHelp: "Perform a DNS query", + LongHelp: strings.TrimSpace(` +The 'tailscale dns query' subcommand performs a DNS query for the specified name +using the internal DNS forwarder (100.100.100.100). + +By default, the DNS query will request an A record. Another DNS record type can +be specified as the second parameter. + +The output also provides information about the resolver(s) used to resolve the +query. +`), +} + func runDNSQuery(ctx context.Context, args []string) error { if len(args) < 1 { return flag.ErrHelp diff --git a/cmd/tailscale/cli/dns-status.go b/cmd/tailscale/cli/dns-status.go index e487c66bc331c..8c18622ce45af 100644 --- a/cmd/tailscale/cli/dns-status.go +++ b/cmd/tailscale/cli/dns-status.go @@ -5,15 +5,77 @@ package cli import ( "context" + "flag" "fmt" "maps" "slices" "strings" + "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/ipn" "tailscale.com/types/netmap" ) +var dnsStatusCmd = &ffcli.Command{ + Name: "status", + ShortUsage: "tailscale dns status [--all]", + Exec: runDNSStatus, + ShortHelp: "Print the current DNS status and configuration", + LongHelp: strings.TrimSpace(` +The 'tailscale dns status' subcommand prints the current DNS status and +configuration, including: + +- Whether the built-in DNS forwarder is enabled. + +- The MagicDNS configuration provided by the coordination server. + +- Details on which resolver(s) Tailscale believes the system is using by + default. + +The --all flag can be used to output advanced debugging information, including +fallback resolvers, nameservers, certificate domains, extra records, and the +exit node filtered set. + +=== Contents of the MagicDNS configuration === + +The MagicDNS configuration is provided by the coordination server to the client +and includes the following components: + +- MagicDNS enablement status: Indicates whether MagicDNS is enabled across the + entire tailnet. + +- MagicDNS Suffix: The DNS suffix used for devices within your tailnet. + +- DNS Name: The DNS name that other devices in the tailnet can use to reach this + device. + +- Resolvers: The preferred DNS resolver(s) to be used for resolving queries, in + order of preference. If no resolvers are listed here, the system defaults are + used. + +- Split DNS Routes: Custom DNS resolvers may be used to resolve hostnames in + specific domains, this is also known as a 'Split DNS' configuration. The + mapping of domains to their respective resolvers is provided here. + +- Certificate Domains: The DNS names for which the coordination server will + assist in provisioning TLS certificates. + +- Extra Records: Additional DNS records that the coordination server might + provide to the internal DNS resolver. + +- Exit Node Filtered Set: DNS suffixes that the node, when acting as an exit + node DNS proxy, will not answer. + +For more information about the DNS functionality built into Tailscale, refer to +https://tailscale.com/kb/1054/dns. +`), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("status") + fs.BoolVar(&dnsStatusArgs.all, "all", false, "outputs advanced debugging information") + return fs + })(), +} + // dnsStatusArgs are the arguments for the "dns status" subcommand. var dnsStatusArgs struct { all bool @@ -208,35 +270,3 @@ func fetchNetMap() (netMap *netmap.NetworkMap, err error) { } return notify.NetMap, nil } - -func dnsStatusLongHelp() string { - return `The 'tailscale dns status' subcommand prints the current DNS status and configuration, including: - -- Whether the built-in DNS forwarder is enabled. -- The MagicDNS configuration provided by the coordination server. -- Details on which resolver(s) Tailscale believes the system is using by default. - -The --all flag can be used to output advanced debugging information, including fallback resolvers, nameservers, certificate domains, extra records, and the exit node filtered set. - -=== Contents of the MagicDNS configuration === - -The MagicDNS configuration is provided by the coordination server to the client and includes the following components: - -- MagicDNS enablement status: Indicates whether MagicDNS is enabled across the entire tailnet. - -- MagicDNS Suffix: The DNS suffix used for devices within your tailnet. - -- DNS Name: The DNS name that other devices in the tailnet can use to reach this device. - -- Resolvers: The preferred DNS resolver(s) to be used for resolving queries, in order of preference. If no resolvers are listed here, the system defaults are used. - -- Split DNS Routes: Custom DNS resolvers may be used to resolve hostnames in specific domains, this is also known as a 'Split DNS' configuration. The mapping of domains to their respective resolvers is provided here. - -- Certificate Domains: The DNS names for which the coordination server will assist in provisioning TLS certificates. - -- Extra Records: Additional DNS records that the coordination server might provide to the internal DNS resolver. - -- Exit Node Filtered Set: DNS suffixes that the node, when acting as an exit node DNS proxy, will not answer. - -For more information about the DNS functionality built into Tailscale, refer to https://tailscale.com/kb/1054/dns.` -} diff --git a/cmd/tailscale/cli/dns.go b/cmd/tailscale/cli/dns.go index 402f0cedf0a1e..086abefd6b2bf 100644 --- a/cmd/tailscale/cli/dns.go +++ b/cmd/tailscale/cli/dns.go @@ -4,46 +4,32 @@ package cli import ( - "flag" + "strings" "github.com/peterbourgon/ff/v3/ffcli" ) var dnsCmd = &ffcli.Command{ - Name: "dns", - ShortHelp: "Diagnose the internal DNS forwarder", - LongHelp: dnsCmdLongHelp(), - ShortUsage: "tailscale dns [flags]", - UsageFunc: usageFuncNoDefaultValues, + Name: "dns", + ShortHelp: "Diagnose the internal DNS forwarder", + LongHelp: strings.TrimSpace(` +The 'tailscale dns' subcommand provides tools for diagnosing the internal DNS +forwarder (100.100.100.100). + +For more information about the DNS functionality built into Tailscale, refer to +https://tailscale.com/kb/1054/dns. +`), + ShortUsage: strings.Join([]string{ + dnsStatusCmd.ShortUsage, + dnsQueryCmd.ShortUsage, + }, "\n"), + UsageFunc: usageFuncNoDefaultValues, Subcommands: []*ffcli.Command{ - { - Name: "status", - ShortUsage: "tailscale dns status [--all]", - Exec: runDNSStatus, - ShortHelp: "Print the current DNS status and configuration", - LongHelp: dnsStatusLongHelp(), - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("status") - fs.BoolVar(&dnsStatusArgs.all, "all", false, "outputs advanced debugging information (fallback resolvers, nameservers, cert domains, extra records, and exit node filtered set)") - return fs - })(), - }, - { - Name: "query", - ShortUsage: "tailscale dns query [a|aaaa|cname|mx|ns|opt|ptr|srv|txt]", - Exec: runDNSQuery, - ShortHelp: "Perform a DNS query", - LongHelp: "The 'tailscale dns query' subcommand performs a DNS query for the specified name using the internal DNS forwarder (100.100.100.100).\n\nIt also provides information about the resolver(s) used to resolve the query.", - }, + dnsStatusCmd, + dnsQueryCmd, // TODO: implement `tailscale log` here // The above work is tracked in https://github.com/tailscale/tailscale/issues/13326 }, } - -func dnsCmdLongHelp() string { - return `The 'tailscale dns' subcommand provides tools for diagnosing the internal DNS forwarder (100.100.100.100). - -For more information about the DNS functionality built into Tailscale, refer to https://tailscale.com/kb/1054/dns.` -} From a91fcc88138cceb4f891f5338f2b28d80bb81b9c Mon Sep 17 00:00:00 2001 From: Anton Tolchanov Date: Wed, 18 Jun 2025 11:38:18 +0100 Subject: [PATCH 084/263] ipn/ipnlocal: make pricing restriction message for Tailnet Lock clearer Fixes tailscale/corp#24417 Signed-off-by: Anton Tolchanov --- ipn/ipnlocal/network-lock.go | 15 +++++++++------ ipn/localapi/localapi.go | 5 +++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/ipn/ipnlocal/network-lock.go b/ipn/ipnlocal/network-lock.go index 36d39a465a654..10f0cc8278109 100644 --- a/ipn/ipnlocal/network-lock.go +++ b/ipn/ipnlocal/network-lock.go @@ -600,18 +600,14 @@ func (b *LocalBackend) NetworkLockInit(keys []tka.Key, disablementValues [][]byt var ourNodeKey key.NodePublic var nlPriv key.NLPrivate - b.mu.Lock() - - if !b.capTailnetLock { - b.mu.Unlock() - return errors.New("not permitted to enable tailnet lock") - } + b.mu.Lock() if p := b.pm.CurrentPrefs(); p.Valid() && p.Persist().Valid() && !p.Persist().PrivateNodeKey().IsZero() { ourNodeKey = p.Persist().PublicNodeKey() nlPriv = p.Persist().NetworkLockKey() } b.mu.Unlock() + if ourNodeKey.IsZero() || nlPriv.IsZero() { return errors.New("no node-key: is tailscale logged in?") } @@ -671,6 +667,13 @@ func (b *LocalBackend) NetworkLockInit(keys []tka.Key, disablementValues [][]byt return err } +// NetworkLockAllowed reports whether the node is allowed to use Tailnet Lock. +func (b *LocalBackend) NetworkLockAllowed() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.capTailnetLock +} + // Only use is in tests. func (b *LocalBackend) NetworkLockVerifySignatureForTest(nks tkatype.MarshaledSignature, nodeKey key.NodePublic) error { b.mu.Lock() diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index 6344da42d2e54..a90ae5d844b90 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -1970,6 +1970,11 @@ func (h *Handler) serveTKAInit(w http.ResponseWriter, r *http.Request) { return } + if !h.b.NetworkLockAllowed() { + http.Error(w, "Tailnet Lock is not supported on your pricing plan", http.StatusForbidden) + return + } + if err := h.b.NetworkLockInit(req.Keys, req.DisablementValues, req.SupportDisablement); err != nil { http.Error(w, "initialization failed: "+err.Error(), http.StatusInternalServerError) return From 45a4b69ce01f3529728bb523ac348794d9abc14a Mon Sep 17 00:00:00 2001 From: Raj Singh Date: Wed, 18 Jun 2025 10:43:19 -0500 Subject: [PATCH 085/263] cmd/tsidp: fix OIDC client persistence across restarts Fixes #16088 Signed-off-by: Raj Singh --- cmd/tsidp/tsidp.go | 19 +++--- cmd/tsidp/tsidp_test.go | 138 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 9 deletions(-) diff --git a/cmd/tsidp/tsidp.go b/cmd/tsidp/tsidp.go index 5df99e1b82232..43020eaf73e63 100644 --- a/cmd/tsidp/tsidp.go +++ b/cmd/tsidp/tsidp.go @@ -161,16 +161,17 @@ func main() { } else { srv.serverURL = fmt.Sprintf("https://%s", strings.TrimSuffix(st.Self.DNSName, ".")) } - if *flagFunnel { - f, err := os.Open(funnelClientsFile) - if err == nil { - srv.funnelClients = make(map[string]*funnelClient) - if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil { - log.Fatalf("could not parse %s: %v", funnelClientsFile, err) - } - } else if !errors.Is(err, os.ErrNotExist) { - log.Fatalf("could not open %s: %v", funnelClientsFile, err) + + // Load funnel clients from disk if they exist, regardless of whether funnel is enabled + // This ensures OIDC clients persist across restarts + f, err := os.Open(funnelClientsFile) + if err == nil { + if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil { + log.Fatalf("could not parse %s: %v", funnelClientsFile, err) } + f.Close() + } else if !errors.Is(err, os.ErrNotExist) { + log.Fatalf("could not open %s: %v", funnelClientsFile, err) } log.Printf("Running tsidp at %s ...", srv.serverURL) diff --git a/cmd/tsidp/tsidp_test.go b/cmd/tsidp/tsidp_test.go index 6932d8e29b084..e5465d3cfbf62 100644 --- a/cmd/tsidp/tsidp_test.go +++ b/cmd/tsidp/tsidp_test.go @@ -7,6 +7,7 @@ import ( "crypto/rand" "crypto/rsa" "encoding/json" + "errors" "fmt" "io" "log" @@ -14,6 +15,7 @@ import ( "net/http/httptest" "net/netip" "net/url" + "os" "reflect" "sort" "strings" @@ -825,3 +827,139 @@ func TestExtraUserInfo(t *testing.T) { }) } } + +func TestFunnelClientsPersistence(t *testing.T) { + testClients := map[string]*funnelClient{ + "test-client-1": { + ID: "test-client-1", + Secret: "test-secret-1", + Name: "Test Client 1", + RedirectURI: "https://example.com/callback", + }, + "test-client-2": { + ID: "test-client-2", + Secret: "test-secret-2", + Name: "Test Client 2", + RedirectURI: "https://example2.com/callback", + }, + } + + testData, err := json.Marshal(testClients) + if err != nil { + t.Fatalf("failed to marshal test data: %v", err) + } + + tmpFile := t.TempDir() + "/oidc-funnel-clients.json" + if err := os.WriteFile(tmpFile, testData, 0600); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + t.Run("step1_load_from_existing_file", func(t *testing.T) { + srv := &idpServer{} + + // Simulate the funnel clients loading logic from main() + srv.funnelClients = make(map[string]*funnelClient) + f, err := os.Open(tmpFile) + if err == nil { + if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil { + t.Fatalf("could not parse %s: %v", tmpFile, err) + } + f.Close() + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("could not open %s: %v", tmpFile, err) + } + + // Verify clients were loaded correctly + if len(srv.funnelClients) != 2 { + t.Errorf("expected 2 clients, got %d", len(srv.funnelClients)) + } + + client1, ok := srv.funnelClients["test-client-1"] + if !ok { + t.Error("expected test-client-1 to be loaded") + } else { + if client1.Name != "Test Client 1" { + t.Errorf("expected client name 'Test Client 1', got '%s'", client1.Name) + } + if client1.Secret != "test-secret-1" { + t.Errorf("expected client secret 'test-secret-1', got '%s'", client1.Secret) + } + } + }) + + t.Run("step2_initialize_empty_when_no_file", func(t *testing.T) { + nonExistentFile := t.TempDir() + "/non-existent.json" + + srv := &idpServer{} + + // Simulate the funnel clients loading logic from main() + srv.funnelClients = make(map[string]*funnelClient) + f, err := os.Open(nonExistentFile) + if err == nil { + if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil { + t.Fatalf("could not parse %s: %v", nonExistentFile, err) + } + f.Close() + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("could not open %s: %v", nonExistentFile, err) + } + + // Verify map is initialized but empty + if srv.funnelClients == nil { + t.Error("expected funnelClients map to be initialized") + } + if len(srv.funnelClients) != 0 { + t.Errorf("expected empty map, got %d clients", len(srv.funnelClients)) + } + }) + + t.Run("step3_persist_and_reload_clients", func(t *testing.T) { + tmpFile2 := t.TempDir() + "/test-persistence.json" + + // Create initial server with one client + srv1 := &idpServer{ + funnelClients: make(map[string]*funnelClient), + } + srv1.funnelClients["new-client"] = &funnelClient{ + ID: "new-client", + Secret: "new-secret", + Name: "New Client", + RedirectURI: "https://new.example.com/callback", + } + + // Save clients to file (simulating saveFunnelClients) + data, err := json.Marshal(srv1.funnelClients) + if err != nil { + t.Fatalf("failed to marshal clients: %v", err) + } + if err := os.WriteFile(tmpFile2, data, 0600); err != nil { + t.Fatalf("failed to write clients file: %v", err) + } + + // Create new server instance and load clients + srv2 := &idpServer{} + srv2.funnelClients = make(map[string]*funnelClient) + f, err := os.Open(tmpFile2) + if err == nil { + if err := json.NewDecoder(f).Decode(&srv2.funnelClients); err != nil { + t.Fatalf("could not parse %s: %v", tmpFile2, err) + } + f.Close() + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("could not open %s: %v", tmpFile2, err) + } + + // Verify the client was persisted correctly + loadedClient, ok := srv2.funnelClients["new-client"] + if !ok { + t.Error("expected new-client to be loaded after persistence") + } else { + if loadedClient.Name != "New Client" { + t.Errorf("expected client name 'New Client', got '%s'", loadedClient.Name) + } + if loadedClient.Secret != "new-secret" { + t.Errorf("expected client secret 'new-secret', got '%s'", loadedClient.Secret) + } + } + }) +} From fcab50b2763a1c7cd51f3c5d9cf8d2198eb7fa90 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 18 Jun 2025 10:31:00 -0700 Subject: [PATCH 086/263] ipn/ipnlocal,wgengine{/magicsock}: replace SetNetworkMap with eventbus (#16299) Same with UpdateNetmapDelta. Updates tailscale/corp#27502 Updates #15160 Signed-off-by: Jordan Whited --- ipn/ipnlocal/local.go | 4 - ipn/ipnlocal/local_test.go | 49 +++++++- ipn/ipnlocal/node_backend.go | 46 ++++---- ipn/ipnlocal/serve_test.go | 2 + wgengine/magicsock/magicsock.go | 161 ++++++++++++++------------- wgengine/magicsock/magicsock_test.go | 62 +++++++---- wgengine/userspace.go | 1 - 7 files changed, 195 insertions(+), 130 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index cd30e92bb905b..908418d4aad2c 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1946,10 +1946,6 @@ var _ controlclient.NetmapDeltaUpdater = (*LocalBackend)(nil) // UpdateNetmapDelta implements controlclient.NetmapDeltaUpdater. func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) { - if !b.MagicConn().UpdateNetmapDelta(muts) { - return false - } - var notify *ipn.Notify // non-nil if we need to send a Notify defer func() { if notify != nil { diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 281d0e9c4eb20..6e24f43006bd4 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -5,6 +5,7 @@ package ipnlocal import ( "context" + "encoding/binary" "encoding/json" "errors" "fmt" @@ -23,6 +24,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + memro "go4.org/mem" "go4.org/netipx" "golang.org/x/net/dns/dnsmessage" "tailscale.com/appc" @@ -77,6 +79,12 @@ func inRemove(ip netip.Addr) bool { return false } +func makeNodeKeyFromID(nodeID tailcfg.NodeID) key.NodePublic { + raw := make([]byte, 32) + binary.BigEndian.PutUint64(raw[24:], uint64(nodeID)) + return key.NodePublicFromRaw32(memro.B(raw)) +} + func TestShrinkDefaultRoute(t *testing.T) { tests := []struct { route string @@ -794,6 +802,7 @@ func TestStatusPeerCapabilities(t *testing.T) { (&tailcfg.Node{ ID: 1, StableID: "foo", + Key: makeNodeKeyFromID(1), IsWireGuardOnly: true, Hostinfo: (&tailcfg.Hostinfo{}).View(), Capabilities: []tailcfg.NodeCapability{tailcfg.CapabilitySSH}, @@ -804,6 +813,7 @@ func TestStatusPeerCapabilities(t *testing.T) { (&tailcfg.Node{ ID: 2, StableID: "bar", + Key: makeNodeKeyFromID(2), Hostinfo: (&tailcfg.Hostinfo{}).View(), Capabilities: []tailcfg.NodeCapability{tailcfg.CapabilityAdmin}, CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{ @@ -830,12 +840,14 @@ func TestStatusPeerCapabilities(t *testing.T) { (&tailcfg.Node{ ID: 1, StableID: "foo", + Key: makeNodeKeyFromID(1), IsWireGuardOnly: true, Hostinfo: (&tailcfg.Hostinfo{}).View(), }).View(), (&tailcfg.Node{ ID: 2, StableID: "bar", + Key: makeNodeKeyFromID(2), Hostinfo: (&tailcfg.Hostinfo{}).View(), }).View(), }, @@ -927,7 +939,11 @@ func TestUpdateNetmapDelta(t *testing.T) { nm := &netmap.NetworkMap{} for i := range 5 { - nm.Peers = append(nm.Peers, (&tailcfg.Node{ID: (tailcfg.NodeID(i) + 1)}).View()) + id := tailcfg.NodeID(i + 1) + nm.Peers = append(nm.Peers, (&tailcfg.Node{ + ID: id, + Key: makeNodeKeyFromID(id), + }).View()) } b.currentNode().SetNetMap(nm) @@ -963,18 +979,22 @@ func TestUpdateNetmapDelta(t *testing.T) { wants := []*tailcfg.Node{ { ID: 1, + Key: makeNodeKeyFromID(1), HomeDERP: 1, }, { ID: 2, + Key: makeNodeKeyFromID(2), Online: ptr.To(true), }, { ID: 3, + Key: makeNodeKeyFromID(3), Online: ptr.To(false), }, { ID: 4, + Key: makeNodeKeyFromID(4), LastSeen: ptr.To(someTime), }, } @@ -998,12 +1018,14 @@ func TestWhoIs(t *testing.T) { SelfNode: (&tailcfg.Node{ ID: 1, User: 10, + Key: makeNodeKeyFromID(1), Addresses: []netip.Prefix{netip.MustParsePrefix("100.101.102.103/32")}, }).View(), Peers: []tailcfg.NodeView{ (&tailcfg.Node{ ID: 2, User: 20, + Key: makeNodeKeyFromID(2), Addresses: []netip.Prefix{netip.MustParsePrefix("100.200.200.200/32")}, }).View(), }, @@ -1593,6 +1615,7 @@ func dnsResponse(domain, address string) []byte { } func TestSetExitNodeIDPolicy(t *testing.T) { + zeroValHostinfoView := new(tailcfg.Hostinfo).View() pfx := netip.MustParsePrefix tests := []struct { name string @@ -1669,14 +1692,18 @@ func TestSetExitNodeIDPolicy(t *testing.T) { }).View(), Peers: []tailcfg.NodeView{ (&tailcfg.Node{ + ID: 201, Name: "a.tailnet", + Key: makeNodeKeyFromID(201), Addresses: []netip.Prefix{ pfx("100.0.0.201/32"), pfx("100::201/128"), }, }).View(), (&tailcfg.Node{ + ID: 202, Name: "b.tailnet", + Key: makeNodeKeyFromID(202), Addresses: []netip.Prefix{ pfx("100::202/128"), }, @@ -1702,18 +1729,24 @@ func TestSetExitNodeIDPolicy(t *testing.T) { }).View(), Peers: []tailcfg.NodeView{ (&tailcfg.Node{ + ID: 123, Name: "a.tailnet", StableID: tailcfg.StableNodeID("123"), + Key: makeNodeKeyFromID(123), Addresses: []netip.Prefix{ pfx("127.0.0.1/32"), pfx("100::201/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), (&tailcfg.Node{ + ID: 202, Name: "b.tailnet", + Key: makeNodeKeyFromID(202), Addresses: []netip.Prefix{ pfx("100::202/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), }, }, @@ -1734,18 +1767,24 @@ func TestSetExitNodeIDPolicy(t *testing.T) { }).View(), Peers: []tailcfg.NodeView{ (&tailcfg.Node{ + ID: 123, Name: "a.tailnet", StableID: tailcfg.StableNodeID("123"), + Key: makeNodeKeyFromID(123), Addresses: []netip.Prefix{ pfx("127.0.0.1/32"), pfx("100::201/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), (&tailcfg.Node{ + ID: 202, Name: "b.tailnet", + Key: makeNodeKeyFromID(202), Addresses: []netip.Prefix{ pfx("100::202/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), }, }, @@ -1768,18 +1807,24 @@ func TestSetExitNodeIDPolicy(t *testing.T) { }).View(), Peers: []tailcfg.NodeView{ (&tailcfg.Node{ + ID: 123, Name: "a.tailnet", StableID: tailcfg.StableNodeID("123"), + Key: makeNodeKeyFromID(123), Addresses: []netip.Prefix{ pfx("100.64.5.6/32"), pfx("100::201/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), (&tailcfg.Node{ + ID: 202, Name: "b.tailnet", + Key: makeNodeKeyFromID(202), Addresses: []netip.Prefix{ pfx("100::202/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), }, }, @@ -1827,7 +1872,6 @@ func TestSetExitNodeIDPolicy(t *testing.T) { b.currentNode().SetNetMap(test.nm) b.pm = pm b.lastSuggestedExitNode = test.lastSuggestedExitNode - prefs := b.pm.prefs.AsStruct() if changed := applySysPolicy(prefs, test.lastSuggestedExitNode, false) || setExitNodeID(prefs, test.nm); changed != test.prefsChanged { t.Errorf("wanted prefs changed %v, got prefs changed %v", test.prefsChanged, changed) @@ -3218,6 +3262,7 @@ type peerOptFunc func(*tailcfg.Node) func makePeer(id tailcfg.NodeID, opts ...peerOptFunc) tailcfg.NodeView { node := &tailcfg.Node{ ID: id, + Key: makeNodeKeyFromID(id), StableID: tailcfg.StableNodeID(fmt.Sprintf("stable%d", id)), Name: fmt.Sprintf("peer%d", id), HomeDERP: int(id), diff --git a/ipn/ipnlocal/node_backend.go b/ipn/ipnlocal/node_backend.go index efa74577bc8ee..05389a677d4f5 100644 --- a/ipn/ipnlocal/node_backend.go +++ b/ipn/ipnlocal/node_backend.go @@ -72,9 +72,10 @@ type nodeBackend struct { filterAtomic atomic.Pointer[filter.Filter] // initialized once and immutable - eventClient *eventbus.Client - filterUpdates *eventbus.Publisher[magicsock.FilterUpdate] - nodeUpdates *eventbus.Publisher[magicsock.NodeAddrsHostInfoUpdate] + eventClient *eventbus.Client + filterPub *eventbus.Publisher[magicsock.FilterUpdate] + nodeViewsPub *eventbus.Publisher[magicsock.NodeViewsUpdate] + nodeMutsPub *eventbus.Publisher[magicsock.NodeMutationsUpdate] // TODO(nickkhyl): maybe use sync.RWMutex? mu sync.Mutex // protects the following fields @@ -113,9 +114,10 @@ func newNodeBackend(ctx context.Context, bus *eventbus.Bus) *nodeBackend { // Default filter blocks everything and logs nothing. noneFilter := filter.NewAllowNone(logger.Discard, &netipx.IPSet{}) nb.filterAtomic.Store(noneFilter) - nb.filterUpdates = eventbus.Publish[magicsock.FilterUpdate](nb.eventClient) - nb.nodeUpdates = eventbus.Publish[magicsock.NodeAddrsHostInfoUpdate](nb.eventClient) - nb.filterUpdates.Publish(magicsock.FilterUpdate{Filter: nb.filterAtomic.Load()}) + nb.filterPub = eventbus.Publish[magicsock.FilterUpdate](nb.eventClient) + nb.nodeViewsPub = eventbus.Publish[magicsock.NodeViewsUpdate](nb.eventClient) + nb.nodeMutsPub = eventbus.Publish[magicsock.NodeMutationsUpdate](nb.eventClient) + nb.filterPub.Publish(magicsock.FilterUpdate{Filter: nb.filterAtomic.Load()}) return nb } @@ -379,6 +381,12 @@ func (nb *nodeBackend) SetNetMap(nm *netmap.NetworkMap) { nb.netMap = nm nb.updateNodeByAddrLocked() nb.updatePeersLocked() + nv := magicsock.NodeViewsUpdate{} + if nm != nil { + nv.SelfNode = nm.SelfNode + nv.Peers = nm.Peers + } + nb.nodeViewsPub.Publish(nv) } func (nb *nodeBackend) updateNodeByAddrLocked() { @@ -429,16 +437,9 @@ func (nb *nodeBackend) updatePeersLocked() { nb.peers[k] = tailcfg.NodeView{} } - changed := magicsock.NodeAddrsHostInfoUpdate{ - Complete: true, - } // Second pass, add everything wanted. for _, p := range nm.Peers { mak.Set(&nb.peers, p.ID(), p) - mak.Set(&changed.NodesByID, p.ID(), magicsock.NodeAddrsHostInfo{ - Addresses: p.Addresses(), - Hostinfo: p.Hostinfo(), - }) } // Third pass, remove deleted things. @@ -447,7 +448,6 @@ func (nb *nodeBackend) updatePeersLocked() { delete(nb.peers, k) } } - nb.nodeUpdates.Publish(changed) } func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) { @@ -462,8 +462,8 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo // call (e.g. its endpoints + online status both change) var mutableNodes map[tailcfg.NodeID]*tailcfg.Node - changed := magicsock.NodeAddrsHostInfoUpdate{ - Complete: false, + update := magicsock.NodeMutationsUpdate{ + Mutations: make([]netmap.NodeMutation, 0, len(muts)), } for _, m := range muts { n, ok := mutableNodes[m.NodeIDBeingMutated()] @@ -475,18 +475,14 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo } n = nv.AsStruct() mak.Set(&mutableNodes, nv.ID(), n) + update.Mutations = append(update.Mutations, m) } m.Apply(n) } for nid, n := range mutableNodes { - nv := n.View() - nb.peers[nid] = nv - mak.Set(&changed.NodesByID, nid, magicsock.NodeAddrsHostInfo{ - Addresses: nv.Addresses(), - Hostinfo: nv.Hostinfo(), - }) - } - nb.nodeUpdates.Publish(changed) + nb.peers[nid] = n.View() + } + nb.nodeMutsPub.Publish(update) return true } @@ -508,7 +504,7 @@ func (nb *nodeBackend) filter() *filter.Filter { func (nb *nodeBackend) setFilter(f *filter.Filter) { nb.filterAtomic.Store(f) - nb.filterUpdates.Publish(magicsock.FilterUpdate{Filter: f}) + nb.filterPub.Publish(magicsock.FilterUpdate{Filter: f}) } func (nb *nodeBackend) dnsConfigForNetmap(prefs ipn.PrefsView, selfExpired bool, logf logger.Logf, versionOS string) *dns.Config { diff --git a/ipn/ipnlocal/serve_test.go b/ipn/ipnlocal/serve_test.go index b9370f8778e6b..57d1a4745a4a3 100644 --- a/ipn/ipnlocal/serve_test.go +++ b/ipn/ipnlocal/serve_test.go @@ -918,6 +918,7 @@ func newTestBackend(t *testing.T) *LocalBackend { ID: 152, ComputedName: "some-peer", User: tailcfg.UserID(1), + Key: makeNodeKeyFromID(152), Addresses: []netip.Prefix{ netip.MustParsePrefix("100.150.151.152/32"), }, @@ -927,6 +928,7 @@ func newTestBackend(t *testing.T) *LocalBackend { ComputedName: "some-tagged-peer", Tags: []string{"tag:server", "tag:test"}, User: tailcfg.UserID(1), + Key: makeNodeKeyFromID(153), Addresses: []netip.Prefix{ netip.MustParsePrefix("100.150.151.153/32"), }, diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 1042e67942791..a6c6a3fb62206 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -160,6 +160,14 @@ type Conn struct { connCtxCancel func() // closes connCtx donec <-chan struct{} // connCtx.Done()'s to avoid context.cancelCtx.Done()'s mutex per call + // These [eventbus.Subscriber] fields are solely accessed by + // consumeEventbusTopics once initialized. + pmSub *eventbus.Subscriber[portmapper.Mapping] + filterSub *eventbus.Subscriber[FilterUpdate] + nodeViewsSub *eventbus.Subscriber[NodeViewsUpdate] + nodeMutsSub *eventbus.Subscriber[NodeMutationsUpdate] + subsDoneCh chan struct{} // closed when consumeEventbusTopics returns + // pconn4 and pconn6 are the underlying UDP sockets used to // send/receive packets for wireguard and other magicsock // protocols. @@ -341,9 +349,9 @@ type Conn struct { netInfoLast *tailcfg.NetInfo derpMap *tailcfg.DERPMap // nil (or zero regions/nodes) means DERP is disabled - peers views.Slice[tailcfg.NodeView] // from last SetNetworkMap update - lastFlags debugFlags // at time of last SetNetworkMap - firstAddrForTest netip.Addr // from last SetNetworkMap update; for tests only + peers views.Slice[tailcfg.NodeView] // from last onNodeViewsUpdate update + lastFlags debugFlags // at time of last onNodeViewsUpdate + firstAddrForTest netip.Addr // from last onNodeViewsUpdate update; for tests only privateKey key.NodePrivate // WireGuard private key for this node everHadKey bool // whether we ever had a non-zero private key myDerp int // nearest DERP region ID; 0 means none/unknown @@ -411,10 +419,8 @@ func (c *Conn) dlogf(format string, a ...any) { // Options contains options for Listen. type Options struct { // EventBus, if non-nil, is used for event publication and subscription by - // each Conn created from these Options. - // - // TODO(creachadair): As of 2025-03-19 this is optional, but is intended to - // become required non-nil. + // each Conn created from these Options. It must not be nil outside of + // tests. EventBus *eventbus.Bus // Logf provides a log function to use. It must not be nil. @@ -503,20 +509,22 @@ func (o *Options) derpActiveFunc() func() { return o.DERPActiveFunc } -// NodeAddrsHostInfoUpdate represents an update event of the addresses and -// [tailcfg.HostInfoView] for a node set. This event is published over an -// [eventbus.Bus]. [magicsock.Conn] is the sole subscriber as of 2025-06. If -// you are adding more subscribers consider moving this type out of magicsock. -type NodeAddrsHostInfoUpdate struct { - NodesByID map[tailcfg.NodeID]NodeAddrsHostInfo - Complete bool // true if NodesByID contains all known nodes, false if it may be a subset +// NodeViewsUpdate represents an update event of [tailcfg.NodeView] for all +// nodes. This event is published over an [eventbus.Bus]. It may be published +// with an invalid SelfNode, and/or zero/nil Peers. [magicsock.Conn] is the sole +// subscriber as of 2025-06. If you are adding more subscribers consider moving +// this type out of magicsock. +type NodeViewsUpdate struct { + SelfNode tailcfg.NodeView + Peers []tailcfg.NodeView } -// NodeAddrsHostInfo represents the addresses and [tailcfg.HostinfoView] for a -// Tailscale node. -type NodeAddrsHostInfo struct { - Addresses views.Slice[netip.Prefix] - Hostinfo tailcfg.HostinfoView +// NodeMutationsUpdate represents an update event of one or more +// [netmap.NodeMutation]. This event is published over an [eventbus.Bus]. +// [magicsock.Conn] is the sole subscriber as of 2025-06. If you are adding more +// subscribers consider moving this type out of magicsock. +type NodeMutationsUpdate struct { + Mutations []netmap.NodeMutation } // FilterUpdate represents an update event for a [*filter.Filter]. This event is @@ -560,16 +568,28 @@ func newConn(logf logger.Logf) *Conn { return c } -// consumeEventbusTopic consumes events from sub and passes them to -// handlerFn until sub.Done() is closed. -func consumeEventbusTopic[T any](sub *eventbus.Subscriber[T], handlerFn func(t T)) { - defer sub.Close() +// consumeEventbusTopics consumes events from all [Conn]-relevant +// [eventbus.Subscriber]'s and passes them to their related handler. Events are +// always handled in the order they are received, i.e. the next event is not +// read until the previous event's handler has returned. It returns when the +// [portmapper.Mapping] subscriber is closed, which is interpreted to be the +// same as the [eventbus.Client] closing ([eventbus.Subscribers] are either +// all open or all closed). +func (c *Conn) consumeEventbusTopics() { + defer close(c.subsDoneCh) + for { select { - case evt := <-sub.Events(): - handlerFn(evt) - case <-sub.Done(): + case <-c.pmSub.Done(): return + case <-c.pmSub.Events(): + c.onPortMapChanged() + case filterUpdate := <-c.filterSub.Events(): + c.onFilterUpdate(filterUpdate) + case nodeViews := <-c.nodeViewsSub.Events(): + c.onNodeViewsUpdate(nodeViews) + case nodeMuts := <-c.nodeMutsSub.Events(): + c.onNodeMutationsUpdate(nodeMuts) } } } @@ -592,29 +612,17 @@ func NewConn(opts Options) (*Conn, error) { c.testOnlyPacketListener = opts.TestOnlyPacketListener c.noteRecvActivity = opts.NoteRecvActivity - // If an event bus is enabled, subscribe to portmapping changes; otherwise - // use the callback mechanism of portmapper.Client. - // - // TODO(creachadair): Remove the switch once the event bus is mandatory. - onPortMapChanged := c.onPortMapChanged if c.eventBus != nil { c.eventClient = c.eventBus.Client("magicsock.Conn") - pmSub := eventbus.Subscribe[portmapper.Mapping](c.eventClient) - go consumeEventbusTopic(pmSub, func(_ portmapper.Mapping) { - c.onPortMapChanged() - }) - filterSub := eventbus.Subscribe[FilterUpdate](c.eventClient) - go consumeEventbusTopic(filterSub, func(t FilterUpdate) { - // TODO(jwhited): implement - }) - nodeSub := eventbus.Subscribe[NodeAddrsHostInfoUpdate](c.eventClient) - go consumeEventbusTopic(nodeSub, func(t NodeAddrsHostInfoUpdate) { - // TODO(jwhited): implement - }) - - // Disable the explicit callback from the portmapper, the subscriber handles it. - onPortMapChanged = nil + // Subscribe calls must return before NewConn otherwise published + // events can be missed. + c.pmSub = eventbus.Subscribe[portmapper.Mapping](c.eventClient) + c.filterSub = eventbus.Subscribe[FilterUpdate](c.eventClient) + c.nodeViewsSub = eventbus.Subscribe[NodeViewsUpdate](c.eventClient) + c.nodeMutsSub = eventbus.Subscribe[NodeMutationsUpdate](c.eventClient) + c.subsDoneCh = make(chan struct{}) + go c.consumeEventbusTopics() } // Don't log the same log messages possibly every few seconds in our @@ -630,7 +638,6 @@ func NewConn(opts Options) (*Conn, error) { NetMon: opts.NetMon, DebugKnobs: portMapOpts, ControlKnobs: opts.ControlKnobs, - OnChange: onPortMapChanged, }) c.portMapper.SetGatewayLookupFunc(opts.NetMon.GatewayAndSelfIP) c.netMon = opts.NetMon @@ -2551,12 +2558,13 @@ func capVerIsRelayCapable(version tailcfg.CapabilityVersion) bool { return false } -// SetNetworkMap is called when the control client gets a new network -// map from the control server. It must always be non-nil. -// -// It should not use the DERPMap field of NetworkMap; that's -// conditionally sent to SetDERPMap instead. -func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) { +func (c *Conn) onFilterUpdate(f FilterUpdate) { + // TODO(jwhited): implement +} + +// onNodeViewsUpdate is called when a [NodeViewsUpdate] is received over the +// [eventbus.Bus]. +func (c *Conn) onNodeViewsUpdate(update NodeViewsUpdate) { c.mu.Lock() defer c.mu.Unlock() @@ -2565,15 +2573,15 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) { } priorPeers := c.peers - metricNumPeers.Set(int64(len(nm.Peers))) + metricNumPeers.Set(int64(len(update.Peers))) // Update c.netMap regardless, before the following early return. - curPeers := views.SliceOf(nm.Peers) + curPeers := views.SliceOf(update.Peers) c.peers = curPeers flags := c.debugFlagsLocked() - if addrs := nm.GetAddresses(); addrs.Len() > 0 { - c.firstAddrForTest = addrs.At(0).Addr() + if update.SelfNode.Valid() && update.SelfNode.Addresses().Len() > 0 { + c.firstAddrForTest = update.SelfNode.Addresses().At(0).Addr() } else { c.firstAddrForTest = netip.Addr{} } @@ -2588,16 +2596,16 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) { c.lastFlags = flags - c.logf("[v1] magicsock: got updated network map; %d peers", len(nm.Peers)) + c.logf("[v1] magicsock: got updated network map; %d peers", len(update.Peers)) - entriesPerBuffer := debugRingBufferSize(len(nm.Peers)) + entriesPerBuffer := debugRingBufferSize(len(update.Peers)) // Try a pass of just upserting nodes and creating missing // endpoints. If the set of nodes is the same, this is an // efficient alloc-free update. If the set of nodes is different, // we'll fall through to the next pass, which allocates but can // handle full set updates. - for _, n := range nm.Peers { + for _, n := range update.Peers { if n.ID() == 0 { devPanicf("node with zero ID") continue @@ -2697,14 +2705,14 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) { c.peerMap.upsertEndpoint(ep, key.DiscoPublic{}) } - // If the set of nodes changed since the last SetNetworkMap, the + // If the set of nodes changed since the last onNodeViewsUpdate, the // upsert loop just above made c.peerMap contain the union of the // old and new peers - which will be larger than the set from the // current netmap. If that happens, go through the allocful // deletion path to clean up moribund nodes. - if c.peerMap.nodeCount() != len(nm.Peers) { + if c.peerMap.nodeCount() != len(update.Peers) { keep := set.Set[key.NodePublic]{} - for _, n := range nm.Peers { + for _, n := range update.Peers { keep.Add(n.Key()) } c.peerMap.forEachEndpoint(func(ep *endpoint) { @@ -2837,10 +2845,6 @@ func (c *connBind) Close() error { return nil } c.closed = true - // Close the [eventbus.Client]. - if c.eventClient != nil { - c.eventClient.Close() - } // Unblock all outstanding receives. c.pconn4.Close() c.pconn6.Close() @@ -2850,9 +2854,6 @@ func (c *connBind) Close() error { if c.closeDisco6 != nil { c.closeDisco6.Close() } - if c.eventClient != nil { - c.eventClient.Close() - } // Send an empty read result to unblock receiveDERP, // which will then check connBind.Closed. // connBind.Closed takes c.mu, but c.derpRecvCh is buffered. @@ -2871,6 +2872,17 @@ func (c *connBind) isClosed() bool { // // Only the first close does anything. Any later closes return nil. func (c *Conn) Close() error { + // Close the [eventbus.Client] and wait for Conn.consumeEventbusTopics to + // return. Do this before acquiring c.mu: + // 1. Conn.consumeEventbusTopics event handlers also acquire c.mu, they can + // deadlock with c.Close(). + // 2. Conn.consumeEventbusTopics event handlers may not guard against + // undesirable post/in-progress Conn.Close() behaviors. + if c.eventClient != nil { + c.eventClient.Close() + <-c.subsDoneCh + } + c.mu.Lock() defer c.mu.Unlock() if c.closed { @@ -2901,7 +2913,6 @@ func (c *Conn) Close() error { if c.closeDisco6 != nil { c.closeDisco6.Close() } - // Wait on goroutines updating right at the end, once everything is // already closed. We want everything else in the Conn to be // consistently in the closed state before we release mu to wait @@ -3233,12 +3244,13 @@ func simpleDur(d time.Duration) time.Duration { return d.Round(time.Minute) } -// UpdateNetmapDelta implements controlclient.NetmapDeltaUpdater. -func (c *Conn) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) { +// onNodeMutationsUpdate is called when a [NodeMutationsUpdate] is received over +// the [eventbus.Bus]. +func (c *Conn) onNodeMutationsUpdate(update NodeMutationsUpdate) { c.mu.Lock() defer c.mu.Unlock() - for _, m := range muts { + for _, m := range update.Mutations { nodeID := m.NodeIDBeingMutated() ep, ok := c.peerMap.endpointForNodeID(nodeID) if !ok { @@ -3257,7 +3269,6 @@ func (c *Conn) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) { ep.mu.Unlock() } } - return true } // UpdateStatus implements the interface nede by ipnstate.StatusBuilder. diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 5e71a40c9db97..7fa062fa87df2 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -166,7 +166,7 @@ type magicStack struct { } // newMagicStack builds and initializes an idle magicsock and -// friends. You need to call conn.SetNetworkMap and dev.Reconfig +// friends. You need to call conn.onNodeViewsUpdate and dev.Reconfig // before anything interesting happens. func newMagicStack(t testing.TB, logf logger.Logf, l nettype.PacketListener, derpMap *tailcfg.DERPMap) *magicStack { privateKey := key.NewNode() @@ -339,9 +339,13 @@ func meshStacks(logf logger.Logf, mutateNetmap func(idx int, nm *netmap.NetworkM for i, m := range ms { nm := buildNetmapLocked(i) - m.conn.SetNetworkMap(nm) - peerSet := make(set.Set[key.NodePublic], len(nm.Peers)) - for _, peer := range nm.Peers { + nv := NodeViewsUpdate{ + SelfNode: nm.SelfNode, + Peers: nm.Peers, + } + m.conn.onNodeViewsUpdate(nv) + peerSet := make(set.Set[key.NodePublic], len(nv.Peers)) + for _, peer := range nv.Peers { peerSet.Add(peer.Key()) } m.conn.UpdatePeers(peerSet) @@ -1366,7 +1370,7 @@ func newTestConn(t testing.TB) *Conn { return conn } -// addTestEndpoint sets conn's network map to a single peer expected +// addTestEndpoint sets conn's node views to a single peer expected // to receive packets from sendConn (or DERP), and returns that peer's // nodekey and discokey. func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (key.NodePublic, key.DiscoPublic) { @@ -1375,7 +1379,7 @@ func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (key.No // codepath. discoKey := key.DiscoPublicFromRaw32(mem.B([]byte{31: 1})) nodeKey := key.NodePublicFromRaw32(mem.B([]byte{0: 'N', 1: 'K', 31: 0})) - conn.SetNetworkMap(&netmap.NetworkMap{ + conn.onNodeViewsUpdate(NodeViewsUpdate{ Peers: nodeViews([]*tailcfg.Node{ { ID: 1, @@ -1564,11 +1568,11 @@ func nodeViews(v []*tailcfg.Node) []tailcfg.NodeView { return nv } -// Test that a netmap update where node changes its node key but +// Test that a node views update where node changes its node key but // doesn't change its disco key doesn't result in a broken state. // // https://github.com/tailscale/tailscale/issues/1391 -func TestSetNetworkMapChangingNodeKey(t *testing.T) { +func TestOnNodeViewsUpdateChangingNodeKey(t *testing.T) { conn := newTestConn(t) t.Cleanup(func() { conn.Close() }) var buf tstest.MemLogger @@ -1580,7 +1584,7 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) { nodeKey1 := key.NodePublicFromRaw32(mem.B([]byte{0: 'N', 1: 'K', 2: '1', 31: 0})) nodeKey2 := key.NodePublicFromRaw32(mem.B([]byte{0: 'N', 1: 'K', 2: '2', 31: 0})) - conn.SetNetworkMap(&netmap.NetworkMap{ + conn.onNodeViewsUpdate(NodeViewsUpdate{ Peers: nodeViews([]*tailcfg.Node{ { ID: 1, @@ -1596,7 +1600,7 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) { } for range 3 { - conn.SetNetworkMap(&netmap.NetworkMap{ + conn.onNodeViewsUpdate(NodeViewsUpdate{ Peers: nodeViews([]*tailcfg.Node{ { ID: 2, @@ -1921,7 +1925,7 @@ func eps(s ...string) []netip.AddrPort { return eps } -func TestStressSetNetworkMap(t *testing.T) { +func TestStressOnNodeViewsUpdate(t *testing.T) { t.Parallel() conn := newTestConn(t) @@ -1969,15 +1973,15 @@ func TestStressSetNetworkMap(t *testing.T) { allPeers[j].Key = randNodeKey() } } - // Clone existing peers into a new netmap. + // Clone existing peers. peers := make([]*tailcfg.Node, 0, len(allPeers)) for peerIdx, p := range allPeers { if present[peerIdx] { peers = append(peers, p.Clone()) } } - // Set the netmap. - conn.SetNetworkMap(&netmap.NetworkMap{ + // Set the node views. + conn.onNodeViewsUpdate(NodeViewsUpdate{ Peers: nodeViews(peers), }) // Check invariants. @@ -2102,10 +2106,10 @@ func TestRebindingUDPConn(t *testing.T) { } // https://github.com/tailscale/tailscale/issues/6680: don't ignore -// SetNetworkMap calls when there are no peers. (A too aggressive fast path was +// onNodeViewsUpdate calls when there are no peers. (A too aggressive fast path was // previously bailing out early, thinking there were no changes since all zero -// peers didn't change, but the netmap has non-peer info in it too we shouldn't discard) -func TestSetNetworkMapWithNoPeers(t *testing.T) { +// peers didn't change, but the node views has non-peer info in it too we shouldn't discard) +func TestOnNodeViewsUpdateWithNoPeers(t *testing.T) { var c Conn knobs := &controlknobs.Knobs{} c.logf = logger.Discard @@ -2114,9 +2118,9 @@ func TestSetNetworkMapWithNoPeers(t *testing.T) { for i := 1; i <= 3; i++ { v := !debugEnableSilentDisco() envknob.Setenv("TS_DEBUG_ENABLE_SILENT_DISCO", fmt.Sprint(v)) - nm := &netmap.NetworkMap{} - c.SetNetworkMap(nm) - t.Logf("ptr %d: %p", i, nm) + nv := NodeViewsUpdate{} + c.onNodeViewsUpdate(nv) + t.Logf("ptr %d: %p", i, nv) if c.lastFlags.heartbeatDisabled != v { t.Fatalf("call %d: didn't store netmap", i) } @@ -2213,7 +2217,11 @@ func TestIsWireGuardOnlyPeer(t *testing.T) { }, }), } - m.conn.SetNetworkMap(nm) + nv := NodeViewsUpdate{ + SelfNode: nm.SelfNode, + Peers: nm.Peers, + } + m.conn.onNodeViewsUpdate(nv) cfg, err := nmcfg.WGCfg(nm, t.Logf, netmap.AllowSubnetRoutes, "") if err != nil { @@ -2275,7 +2283,11 @@ func TestIsWireGuardOnlyPeerWithMasquerade(t *testing.T) { }, }), } - m.conn.SetNetworkMap(nm) + nv := NodeViewsUpdate{ + SelfNode: nm.SelfNode, + Peers: nm.Peers, + } + m.conn.onNodeViewsUpdate(nv) cfg, err := nmcfg.WGCfg(nm, t.Logf, netmap.AllowSubnetRoutes, "") if err != nil { @@ -2312,7 +2324,11 @@ func TestIsWireGuardOnlyPeerWithMasquerade(t *testing.T) { // configures WG. func applyNetworkMap(t *testing.T, m *magicStack, nm *netmap.NetworkMap) { t.Helper() - m.conn.SetNetworkMap(nm) + nv := NodeViewsUpdate{ + SelfNode: nm.SelfNode, + Peers: nm.Peers, + } + m.conn.onNodeViewsUpdate(nv) // Make sure we can't use v6 to avoid test failures. m.conn.noV6.Store(true) diff --git a/wgengine/userspace.go b/wgengine/userspace.go index b1b82032b2a6c..4a9f321430c12 100644 --- a/wgengine/userspace.go +++ b/wgengine/userspace.go @@ -1300,7 +1300,6 @@ func (e *userspaceEngine) linkChange(delta *netmon.ChangeDelta) { } func (e *userspaceEngine) SetNetworkMap(nm *netmap.NetworkMap) { - e.magicConn.SetNetworkMap(nm) e.mu.Lock() e.netMap = nm e.mu.Unlock() From ad0dfcb1857105597b1bed3422c9057aafd7b22f Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 17 Jun 2025 20:25:09 -0700 Subject: [PATCH 087/263] net/*: remove Windows exceptions for when Resolver.PreferGo didn't work Resolver.PreferGo didn't used to work on Windows. It was fixed in 2022, though. (https://github.com/golang/go/issues/33097) Updates #5161 Change-Id: I4e1aeff220ebd6adc8a14f781664fa6a2068b48c Signed-off-by: Brad Fitzpatrick --- net/dns/resolver/tsdns_test.go | 7 ------- net/dnscache/messagecache_test.go | 9 --------- net/tsdial/tsdial.go | 2 +- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/net/dns/resolver/tsdns_test.go b/net/dns/resolver/tsdns_test.go index de08450d2e3eb..4bbfd4d6a417e 100644 --- a/net/dns/resolver/tsdns_test.go +++ b/net/dns/resolver/tsdns_test.go @@ -1106,10 +1106,6 @@ type linkSelFunc func(ip netip.Addr) string func (f linkSelFunc) PickLink(ip netip.Addr) string { return f(ip) } func TestHandleExitNodeDNSQueryWithNetPkg(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("skipping test on Windows; waiting for golang.org/issue/33097") - } - records := []any{ "no-records.test.", dnsHandler(), @@ -1405,9 +1401,6 @@ func TestHandleExitNodeDNSQueryWithNetPkg(t *testing.T) { // newWrapResolver returns a resolver that uses r (via handleExitNodeDNSQueryWithNetPkg) // to make DNS requests. func newWrapResolver(r *net.Resolver) *net.Resolver { - if runtime.GOOS == "windows" { - panic("doesn't work on Windows") // golang.org/issue/33097 - } return &net.Resolver{ PreferGo: true, Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { diff --git a/net/dnscache/messagecache_test.go b/net/dnscache/messagecache_test.go index 41fc334483f78..0bedfa5ad78e7 100644 --- a/net/dnscache/messagecache_test.go +++ b/net/dnscache/messagecache_test.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "net" - "runtime" "testing" "time" @@ -249,14 +248,6 @@ func TestGetDNSQueryCacheKey(t *testing.T) { } func getGoNetPacketDNSQuery(name string) []byte { - if runtime.GOOS == "windows" { - // On Windows, Go's net.Resolver doesn't use the DNS client. - // See https://github.com/golang/go/issues/33097 which - // was approved but not yet implemented. - // For now just pretend it's implemented to make this test - // pass on Windows with complicated the caller. - return makeQ(123, name) - } res := make(chan []byte, 1) r := &net.Resolver{ PreferGo: true, diff --git a/net/tsdial/tsdial.go b/net/tsdial/tsdial.go index 2492f666cf063..e4e4e9e8b0f92 100644 --- a/net/tsdial/tsdial.go +++ b/net/tsdial/tsdial.go @@ -322,7 +322,7 @@ func (d *Dialer) userDialResolve(ctx context.Context, network, addr string) (net } var r net.Resolver - if exitDNSDoH != "" && runtime.GOOS != "windows" { // Windows: https://github.com/golang/go/issues/33097 + if exitDNSDoH != "" { r.PreferGo = true r.Dial = func(ctx context.Context, network, address string) (net.Conn, error) { return &dohConn{ From 4979ce7a94cd023db5cd03cbb556934d9652dfd2 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Wed, 18 Jun 2025 14:17:12 -0700 Subject: [PATCH 088/263] feature/tpm: implement ipn.StateStore using TPM sealing (#16030) Updates #15830 Signed-off-by: Andrew Lytvynov --- cmd/tailscaled/depaware.txt | 2 +- feature/tpm/tpm.go | 322 +++++++++++++++++++++++++++++++++++- feature/tpm/tpm_linux.go | 11 +- feature/tpm/tpm_other.go | 10 +- feature/tpm/tpm_test.go | 165 +++++++++++++++++- feature/tpm/tpm_windows.go | 11 +- ipn/store/stores.go | 2 + 7 files changed, 500 insertions(+), 23 deletions(-) diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 387b944c1f4ce..7c4885a4be4c4 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -474,7 +474,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ golang.org/x/crypto/nacl/box from tailscale.com/types/key - golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box + golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box+ golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ LD golang.org/x/crypto/ssh from github.com/pkg/sftp+ diff --git a/feature/tpm/tpm.go b/feature/tpm/tpm.go index 18e56ae891ee1..6feac85e35ecb 100644 --- a/feature/tpm/tpm.go +++ b/feature/tpm/tpm.go @@ -5,14 +5,29 @@ package tpm import ( + "bytes" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "path/filepath" "slices" + "strings" "sync" "github.com/google/go-tpm/tpm2" "github.com/google/go-tpm/tpm2/transport" + "golang.org/x/crypto/nacl/secretbox" + "tailscale.com/atomicfile" "tailscale.com/feature" "tailscale.com/hostinfo" + "tailscale.com/ipn" + "tailscale.com/ipn/store" + "tailscale.com/paths" "tailscale.com/tailcfg" + "tailscale.com/types/logger" ) var infoOnce = sync.OnceValue(info) @@ -22,10 +37,16 @@ func init() { hostinfo.RegisterHostinfoNewHook(func(hi *tailcfg.Hostinfo) { hi.TPM = infoOnce() }) + store.Register(storePrefix, newStore) } -//lint:ignore U1000 used in Linux and Windows builds only -func infoFromCapabilities(tpm transport.TPM) *tailcfg.TPMInfo { +func info() *tailcfg.TPMInfo { + tpm, err := open() + if err != nil { + return nil + } + defer tpm.Close() + info := new(tailcfg.TPMInfo) toStr := func(s *string) func(*tailcfg.TPMInfo, uint32) { return func(info *tailcfg.TPMInfo, value uint32) { @@ -81,3 +102,300 @@ func propToString(v uint32) string { // Delete any non-printable ASCII characters. return string(slices.DeleteFunc(chars, func(b byte) bool { return b < ' ' || b > '~' })) } + +const storePrefix = "tpmseal:" + +func newStore(logf logger.Logf, path string) (ipn.StateStore, error) { + path = strings.TrimPrefix(path, storePrefix) + if err := paths.MkStateDir(filepath.Dir(path)); err != nil { + return nil, fmt.Errorf("creating state directory: %w", err) + } + var parsed map[ipn.StateKey][]byte + bs, err := os.ReadFile(path) + if err != nil { + if !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to open %q: %w", path, err) + } + logf("tpm.newStore: initializing state file") + + var key [32]byte + // crypto/rand.Read never returns an error. + rand.Read(key[:]) + + store := &tpmStore{ + logf: logf, + path: path, + key: key, + cache: make(map[ipn.StateKey][]byte), + } + if err := store.writeSealed(); err != nil { + return nil, fmt.Errorf("failed to write initial state file: %w", err) + } + return store, nil + } + + // State file exists, unseal and parse it. + var sealed encryptedData + if err := json.Unmarshal(bs, &sealed); err != nil { + return nil, fmt.Errorf("failed to unmarshal state file: %w", err) + } + if len(sealed.Data) == 0 || sealed.Key == nil || len(sealed.Nonce) == 0 { + return nil, fmt.Errorf("state file %q has not been TPM-sealed or is corrupt", path) + } + data, err := unseal(logf, sealed) + if err != nil { + return nil, fmt.Errorf("failed to unseal state file: %w", err) + } + if err := json.Unmarshal(data.Data, &parsed); err != nil { + return nil, fmt.Errorf("failed to parse state file: %w", err) + } + return &tpmStore{ + logf: logf, + path: path, + key: data.Key, + cache: parsed, + }, nil +} + +// tpmStore is an ipn.StateStore that stores the state in a secretbox-encrypted +// file using a TPM-sealed symmetric key. +type tpmStore struct { + logf logger.Logf + path string + key [32]byte + + mu sync.RWMutex + cache map[ipn.StateKey][]byte +} + +func (s *tpmStore) ReadState(k ipn.StateKey) ([]byte, error) { + s.mu.RLock() + defer s.mu.RUnlock() + v, ok := s.cache[k] + if !ok { + return nil, ipn.ErrStateNotExist + } + return bytes.Clone(v), nil +} + +func (s *tpmStore) WriteState(k ipn.StateKey, bs []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + if bytes.Equal(s.cache[k], bs) { + return nil + } + s.cache[k] = bytes.Clone(bs) + + return s.writeSealed() +} + +func (s *tpmStore) writeSealed() error { + bs, err := json.Marshal(s.cache) + if err != nil { + return err + } + sealed, err := seal(s.logf, decryptedData{Key: s.key, Data: bs}) + if err != nil { + return fmt.Errorf("failed to seal state file: %w", err) + } + buf, err := json.Marshal(sealed) + if err != nil { + return err + } + return atomicfile.WriteFile(s.path, buf, 0600) +} + +// The nested levels of encoding and encryption are confusing, so here's what's +// going on in plain English. +// +// Not all TPM devices support symmetric encryption (TPM2_EncryptDecrypt2) +// natively, but they do support "sealing" small values (see +// tpmSeal/tpmUnseal). The size limit is too small for the actual state file, +// so we seal a symmetric key instead. This symmetric key is then used to seal +// the actual data using nacl/secretbox. +// Confusingly, both TPMs and secretbox use "seal" terminology. +// +// tpmSeal/tpmUnseal do the lower-level sealing of small []byte blobs, which we +// use to seal a 32-byte secretbox key. +// +// seal/unseal do the higher-level sealing of store data using secretbox, and +// also sealing of the symmetric key using TPM. + +// decryptedData contains the fully decrypted raw data along with the symmetric +// key used for secretbox. This struct should only live in memory and never get +// stored to disk! +type decryptedData struct { + Key [32]byte + Data []byte +} + +func (decryptedData) MarshalJSON() ([]byte, error) { + return nil, errors.New("[unexpected]: decryptedData should never get JSON-marshaled!") +} + +// encryptedData contains the secretbox-sealed data and nonce, along with a +// TPM-sealed key. All fields are required. +type encryptedData struct { + Key *tpmSealedData `json:"key"` + Nonce []byte `json:"nonce"` + Data []byte `json:"data"` +} + +func seal(logf logger.Logf, dec decryptedData) (*encryptedData, error) { + var nonce [24]byte + // crypto/rand.Read never returns an error. + rand.Read(nonce[:]) + + sealedData := secretbox.Seal(nil, dec.Data, &nonce, &dec.Key) + sealedKey, err := tpmSeal(logf, dec.Key[:]) + if err != nil { + return nil, fmt.Errorf("failed to seal encryption key to TPM: %w", err) + } + + return &encryptedData{ + Key: sealedKey, + Nonce: nonce[:], + Data: sealedData, + }, nil +} + +func unseal(logf logger.Logf, data encryptedData) (*decryptedData, error) { + if len(data.Nonce) != 24 { + return nil, fmt.Errorf("nonce should be 24 bytes long, got %d", len(data.Nonce)) + } + + unsealedKey, err := tpmUnseal(logf, data.Key) + if err != nil { + return nil, fmt.Errorf("failed to unseal encryption key with TPM: %w", err) + } + if len(unsealedKey) != 32 { + return nil, fmt.Errorf("unsealed key should be 32 bytes long, got %d", len(unsealedKey)) + } + unsealedData, ok := secretbox.Open(nil, data.Data, (*[24]byte)(data.Nonce), (*[32]byte)(unsealedKey)) + if !ok { + return nil, errors.New("failed to unseal data") + } + + return &decryptedData{ + Key: *(*[32]byte)(unsealedKey), + Data: unsealedData, + }, nil +} + +type tpmSealedData struct { + Private []byte + Public []byte +} + +// withSRK runs fn with the loaded Storage Root Key (SRK) handle. The SRK is +// flushed after fn returns. +func withSRK(logf logger.Logf, tpm transport.TPM, fn func(srk tpm2.AuthHandle) error) error { + srkCmd := tpm2.CreatePrimary{ + PrimaryHandle: tpm2.TPMRHOwner, + InPublic: tpm2.New2B(tpm2.ECCSRKTemplate), + } + srkRes, err := srkCmd.Execute(tpm) + if err != nil { + return fmt.Errorf("tpm2.CreatePrimary: %w", err) + } + defer func() { + cmd := tpm2.FlushContext{FlushHandle: srkRes.ObjectHandle} + if _, err := cmd.Execute(tpm); err != nil { + logf("tpm2.FlushContext: failed to flush SRK handle: %v", err) + } + }() + + return fn(tpm2.AuthHandle{ + Handle: srkRes.ObjectHandle, + Name: srkRes.Name, + Auth: tpm2.HMAC(tpm2.TPMAlgSHA256, 32), + }) +} + +// tpmSeal seals the data using SRK of the local TPM. +func tpmSeal(logf logger.Logf, data []byte) (*tpmSealedData, error) { + tpm, err := open() + if err != nil { + return nil, fmt.Errorf("opening TPM: %w", err) + } + defer tpm.Close() + + var res *tpmSealedData + err = withSRK(logf, tpm, func(srk tpm2.AuthHandle) error { + sealCmd := tpm2.Create{ + ParentHandle: srk, + InSensitive: tpm2.TPM2BSensitiveCreate{ + Sensitive: &tpm2.TPMSSensitiveCreate{ + Data: tpm2.NewTPMUSensitiveCreate(&tpm2.TPM2BSensitiveData{ + Buffer: data, + }), + }, + }, + InPublic: tpm2.New2B(tpm2.TPMTPublic{ + Type: tpm2.TPMAlgKeyedHash, + NameAlg: tpm2.TPMAlgSHA256, + ObjectAttributes: tpm2.TPMAObject{ + FixedTPM: true, + FixedParent: true, + UserWithAuth: true, + }, + }), + } + sealRes, err := sealCmd.Execute(tpm) + if err != nil { + return fmt.Errorf("tpm2.Create: %w", err) + } + + res = &tpmSealedData{ + Private: sealRes.OutPrivate.Buffer, + Public: sealRes.OutPublic.Bytes(), + } + return nil + }) + return res, err +} + +// tpmUnseal unseals the data using SRK of the local TPM. +func tpmUnseal(logf logger.Logf, data *tpmSealedData) ([]byte, error) { + tpm, err := open() + if err != nil { + return nil, fmt.Errorf("opening TPM: %w", err) + } + defer tpm.Close() + + var res []byte + err = withSRK(logf, tpm, func(srk tpm2.AuthHandle) error { + // Load the sealed object into the TPM first under SRK. + loadCmd := tpm2.Load{ + ParentHandle: srk, + InPrivate: tpm2.TPM2BPrivate{Buffer: data.Private}, + InPublic: tpm2.BytesAs2B[tpm2.TPMTPublic](data.Public), + } + loadRes, err := loadCmd.Execute(tpm) + if err != nil { + return fmt.Errorf("tpm2.Load: %w", err) + } + defer func() { + cmd := tpm2.FlushContext{FlushHandle: loadRes.ObjectHandle} + if _, err := cmd.Execute(tpm); err != nil { + log.Printf("tpm2.FlushContext: failed to flush loaded sealed blob handle: %v", err) + } + }() + + // Then unseal the object. + unsealCmd := tpm2.Unseal{ + ItemHandle: tpm2.NamedHandle{ + Handle: loadRes.ObjectHandle, + Name: loadRes.Name, + }, + } + unsealRes, err := unsealCmd.Execute(tpm) + if err != nil { + return fmt.Errorf("tpm2.Unseal: %w", err) + } + res = unsealRes.OutData.Buffer + + return nil + }) + return res, err +} diff --git a/feature/tpm/tpm_linux.go b/feature/tpm/tpm_linux.go index a90c0e153962f..f2d0f1402c16e 100644 --- a/feature/tpm/tpm_linux.go +++ b/feature/tpm/tpm_linux.go @@ -4,15 +4,10 @@ package tpm import ( + "github.com/google/go-tpm/tpm2/transport" "github.com/google/go-tpm/tpm2/transport/linuxtpm" - "tailscale.com/tailcfg" ) -func info() *tailcfg.TPMInfo { - t, err := linuxtpm.Open("/dev/tpm0") - if err != nil { - return nil - } - defer t.Close() - return infoFromCapabilities(t) +func open() (transport.TPMCloser, error) { + return linuxtpm.Open("/dev/tpm0") } diff --git a/feature/tpm/tpm_other.go b/feature/tpm/tpm_other.go index ba7c67621eafb..108b2c057e4bd 100644 --- a/feature/tpm/tpm_other.go +++ b/feature/tpm/tpm_other.go @@ -5,8 +5,12 @@ package tpm -import "tailscale.com/tailcfg" +import ( + "errors" -func info() *tailcfg.TPMInfo { - return nil + "github.com/google/go-tpm/tpm2/transport" +) + +func open() (transport.TPMCloser, error) { + return nil, errors.New("TPM not supported on this platform") } diff --git a/feature/tpm/tpm_test.go b/feature/tpm/tpm_test.go index fc0fc178c1a79..a022b69b2bf04 100644 --- a/feature/tpm/tpm_test.go +++ b/feature/tpm/tpm_test.go @@ -3,7 +3,17 @@ package tpm -import "testing" +import ( + "bytes" + "crypto/rand" + "errors" + "path/filepath" + "strconv" + "testing" + + "tailscale.com/ipn" + "tailscale.com/ipn/store" +) func TestPropToString(t *testing.T) { for prop, want := range map[uint32]string{ @@ -17,3 +27,156 @@ func TestPropToString(t *testing.T) { } } } + +func skipWithoutTPM(t testing.TB) { + tpm, err := open() + if err != nil { + t.Skip("TPM not available") + } + tpm.Close() +} + +func TestSealUnseal(t *testing.T) { + skipWithoutTPM(t) + + data := make([]byte, 100*1024) + rand.Read(data) + var key [32]byte + rand.Read(key[:]) + + sealed, err := seal(t.Logf, decryptedData{Key: key, Data: data}) + if err != nil { + t.Fatalf("seal: %v", err) + } + if bytes.Contains(sealed.Data, data) { + t.Fatalf("sealed data %q contains original input %q", sealed.Data, data) + } + + unsealed, err := unseal(t.Logf, *sealed) + if err != nil { + t.Fatalf("unseal: %v", err) + } + if !bytes.Equal(data, unsealed.Data) { + t.Errorf("got unsealed data: %q, want: %q", unsealed, data) + } + if key != unsealed.Key { + t.Errorf("got unsealed key: %q, want: %q", unsealed.Key, key) + } +} + +func TestStore(t *testing.T) { + skipWithoutTPM(t) + + path := storePrefix + filepath.Join(t.TempDir(), "state") + store, err := newStore(t.Logf, path) + if err != nil { + t.Fatal(err) + } + + checkState := func(t *testing.T, store ipn.StateStore, k ipn.StateKey, want []byte) { + got, err := store.ReadState(k) + if err != nil { + t.Errorf("ReadState(%q): %v", k, err) + } + if !bytes.Equal(want, got) { + t.Errorf("ReadState(%q): got %q, want %q", k, got, want) + } + } + + k1, k2 := ipn.StateKey("k1"), ipn.StateKey("k2") + v1, v2 := []byte("v1"), []byte("v2") + + t.Run("read-non-existent-key", func(t *testing.T) { + _, err := store.ReadState(k1) + if !errors.Is(err, ipn.ErrStateNotExist) { + t.Errorf("ReadState succeeded, want %v", ipn.ErrStateNotExist) + } + }) + + t.Run("read-write-k1", func(t *testing.T) { + if err := store.WriteState(k1, v1); err != nil { + t.Errorf("WriteState(%q, %q): %v", k1, v1, err) + } + checkState(t, store, k1, v1) + }) + + t.Run("read-write-k2", func(t *testing.T) { + if err := store.WriteState(k2, v2); err != nil { + t.Errorf("WriteState(%q, %q): %v", k2, v2, err) + } + checkState(t, store, k2, v2) + }) + + t.Run("update-k2", func(t *testing.T) { + v2 = []byte("new v2") + if err := store.WriteState(k2, v2); err != nil { + t.Errorf("WriteState(%q, %q): %v", k2, v2, err) + } + checkState(t, store, k2, v2) + }) + + t.Run("reopen-store", func(t *testing.T) { + store, err := newStore(t.Logf, path) + if err != nil { + t.Fatal(err) + } + checkState(t, store, k1, v1) + checkState(t, store, k2, v2) + }) +} + +func BenchmarkStore(b *testing.B) { + skipWithoutTPM(b) + b.StopTimer() + + stores := make(map[string]ipn.StateStore) + key := ipn.StateKey(b.Name()) + + // Set up tpmStore + tpmStore, err := newStore(b.Logf, filepath.Join(b.TempDir(), "tpm.store")) + if err != nil { + b.Fatal(err) + } + if err := tpmStore.WriteState(key, []byte("-1")); err != nil { + b.Fatal(err) + } + stores["tpmStore"] = tpmStore + + // Set up FileStore + fileStore, err := store.NewFileStore(b.Logf, filepath.Join(b.TempDir(), "file.store")) + if err != nil { + b.Fatal(err) + } + if err := fileStore.WriteState(key, []byte("-1")); err != nil { + b.Fatal(err) + } + stores["fileStore"] = fileStore + + b.StartTimer() + + for name, store := range stores { + b.Run(name, func(b *testing.B) { + b.Run("write-noop", func(b *testing.B) { + for range b.N { + if err := store.WriteState(key, []byte("-1")); err != nil { + b.Fatal(err) + } + } + }) + b.Run("write", func(b *testing.B) { + for i := range b.N { + if err := store.WriteState(key, []byte(strconv.Itoa(i))); err != nil { + b.Fatal(err) + } + } + }) + b.Run("read", func(b *testing.B) { + for range b.N { + if _, err := store.ReadState(key); err != nil { + b.Fatal(err) + } + } + }) + }) + } +} diff --git a/feature/tpm/tpm_windows.go b/feature/tpm/tpm_windows.go index 578d687af5739..429d20cb879f7 100644 --- a/feature/tpm/tpm_windows.go +++ b/feature/tpm/tpm_windows.go @@ -4,15 +4,10 @@ package tpm import ( + "github.com/google/go-tpm/tpm2/transport" "github.com/google/go-tpm/tpm2/transport/windowstpm" - "tailscale.com/tailcfg" ) -func info() *tailcfg.TPMInfo { - t, err := windowstpm.Open() - if err != nil { - return nil - } - defer t.Close() - return infoFromCapabilities(t) +func open() (transport.TPMCloser, error) { + return windowstpm.Open() } diff --git a/ipn/store/stores.go b/ipn/store/stores.go index 1f98891bff248..1a98574c91cf9 100644 --- a/ipn/store/stores.go +++ b/ipn/store/stores.go @@ -45,6 +45,8 @@ var knownStores map[string]Provider // the suffix an AWS ARN for an SSM. // - (Linux-only) if the string begins with "kube:", // the suffix is a Kubernetes secret name +// - (Linux or Windows) if the string begins with "tpmseal:", the suffix is +// filepath that is sealed with the local TPM device. // - In all other cases, the path is treated as a filepath. func New(logf logger.Logf, path string) (ipn.StateStore, error) { for prefix, sf := range knownStores { From e92eb6b17bb59cd66cd78c90db3b285015ed5e11 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 8 Jun 2025 18:51:41 -0700 Subject: [PATCH 089/263] net/tlsdial: fix TLS cert validation of HTTPS proxies If you had HTTPS_PROXY=https://some-valid-cert.example.com running a CONNECT proxy, we should've been able to do a TLS CONNECT request to e.g. controlplane.tailscale.com:443 through that, and I'm pretty sure it used to work, but refactorings and lack of integration tests made it regress. It probably regressed when we added the baked-in LetsEncrypt root cert validation fallback code, which was testing against the wrong hostname (the ultimate one, not the one which we were being asked to validate) Fixes #16222 Change-Id: If014e395f830e2f87f056f588edacad5c15e91bc Signed-off-by: Brad Fitzpatrick --- cmd/proxy-test-server/proxy-test-server.go | 81 +++++++ control/controlclient/controlclient_test.go | 225 ++++++++++++++++++ control/controlclient/direct.go | 7 +- control/controlhttp/client.go | 2 +- derp/derphttp/derphttp_client.go | 3 +- derp/derphttp/derphttp_test.go | 34 +++ logpolicy/logpolicy.go | 4 +- net/bakedroots/bakedroots.go | 5 +- net/connectproxy/connectproxy.go | 93 ++++++++ net/dnscache/dnscache.go | 13 +- net/dnsfallback/dnsfallback.go | 2 +- net/tlsdial/tlsdial.go | 68 +++--- net/tlsdial/tlsdial_test.go | 2 +- .../tlstest/testdata/controlplane.tstest.key | 5 + tstest/tlstest/testdata/proxy.tstest.key | 5 + tstest/tlstest/testdata/root-ca.key | 5 + tstest/tlstest/tlstest.go | 167 +++++++++++++ 17 files changed, 672 insertions(+), 49 deletions(-) create mode 100644 cmd/proxy-test-server/proxy-test-server.go create mode 100644 net/connectproxy/connectproxy.go create mode 100644 tstest/tlstest/testdata/controlplane.tstest.key create mode 100644 tstest/tlstest/testdata/proxy.tstest.key create mode 100644 tstest/tlstest/testdata/root-ca.key create mode 100644 tstest/tlstest/tlstest.go diff --git a/cmd/proxy-test-server/proxy-test-server.go b/cmd/proxy-test-server/proxy-test-server.go new file mode 100644 index 0000000000000..9f8c94a384ea5 --- /dev/null +++ b/cmd/proxy-test-server/proxy-test-server.go @@ -0,0 +1,81 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// The proxy-test-server command is a simple HTTP proxy server for testing +// Tailscale's client proxy functionality. +package main + +import ( + "crypto/tls" + "flag" + "fmt" + "log" + "net" + "net/http" + "os" + "strings" + + "golang.org/x/crypto/acme/autocert" + "tailscale.com/net/connectproxy" + "tailscale.com/tempfork/acme" +) + +var ( + listen = flag.String("listen", ":8080", "Address to listen on for HTTPS proxy requests") + hostname = flag.String("hostname", "localhost", "Hostname for the proxy server") + tailscaleOnly = flag.Bool("tailscale-only", true, "Restrict proxy to Tailscale targets only") + extraAllowedHosts = flag.String("allow-hosts", "", "Comma-separated list of allowed target hosts to additionally allow if --tailscale-only is true") +) + +func main() { + flag.Parse() + + am := &autocert.Manager{ + HostPolicy: autocert.HostWhitelist(*hostname), + Prompt: autocert.AcceptTOS, + Cache: autocert.DirCache(os.ExpandEnv("$HOME/.cache/autocert/proxy-test-server")), + } + var allowTarget func(hostPort string) error + if *tailscaleOnly { + allowTarget = func(hostPort string) error { + host, port, err := net.SplitHostPort(hostPort) + if err != nil { + return fmt.Errorf("invalid target %q: %v", hostPort, err) + } + if port != "443" { + return fmt.Errorf("target %q must use port 443", hostPort) + } + for allowed := range strings.SplitSeq(*extraAllowedHosts, ",") { + if host == allowed { + return nil // explicitly allowed target + } + } + if !strings.HasSuffix(host, ".tailscale.com") { + return fmt.Errorf("target %q is not a Tailscale host", hostPort) + } + return nil // valid Tailscale target + } + } + + go func() { + if err := http.ListenAndServe(":http", am.HTTPHandler(nil)); err != nil { + log.Fatalf("autocert HTTP server failed: %v", err) + } + }() + hs := &http.Server{ + Addr: *listen, + Handler: &connectproxy.Handler{ + Check: allowTarget, + Logf: log.Printf, + }, + TLSConfig: &tls.Config{ + GetCertificate: am.GetCertificate, + NextProtos: []string{ + "http/1.1", // enable HTTP/2 + acme.ALPNProto, // enable tls-alpn ACME challenges + }, + }, + } + log.Printf("Starting proxy-test-server on %s (hostname: %q)\n", *listen, *hostname) + log.Fatal(hs.ListenAndServeTLS("", "")) // cert and key are provided by autocert +} diff --git a/control/controlclient/controlclient_test.go b/control/controlclient/controlclient_test.go index f8882a4e796ca..1107f76a46a78 100644 --- a/control/controlclient/controlclient_test.go +++ b/control/controlclient/controlclient_test.go @@ -4,13 +4,35 @@ package controlclient import ( + "context" + "crypto/tls" "errors" + "flag" "fmt" "io" + "net" + "net/http" + "net/netip" + "net/url" "reflect" "slices" + "sync/atomic" "testing" + "time" + "tailscale.com/control/controlknobs" + "tailscale.com/health" + "tailscale.com/net/bakedroots" + "tailscale.com/net/connectproxy" + "tailscale.com/net/netmon" + "tailscale.com/net/tsdial" + "tailscale.com/tailcfg" + "tailscale.com/tstest" + "tailscale.com/tstest/integration/testcontrol" + "tailscale.com/tstest/tlstest" + "tailscale.com/tstime" + "tailscale.com/types/key" + "tailscale.com/types/logger" "tailscale.com/types/netmap" "tailscale.com/types/persist" ) @@ -188,3 +210,206 @@ func isRetryableErrorForTest(err error) bool { } return false } + +var liveNetworkTest = flag.Bool("live-network-test", false, "run live network tests") + +func TestDirectProxyManual(t *testing.T) { + if !*liveNetworkTest { + t.Skip("skipping without --live-network-test") + } + + dialer := &tsdial.Dialer{} + dialer.SetNetMon(netmon.NewStatic()) + + opts := Options{ + Persist: persist.Persist{}, + GetMachinePrivateKey: func() (key.MachinePrivate, error) { + return key.NewMachine(), nil + }, + ServerURL: "https://controlplane.tailscale.com", + Clock: tstime.StdClock{}, + Hostinfo: &tailcfg.Hostinfo{ + BackendLogID: "test-backend-log-id", + }, + DiscoPublicKey: key.NewDisco().Public(), + Logf: t.Logf, + HealthTracker: &health.Tracker{}, + PopBrowserURL: func(url string) { + t.Logf("PopBrowserURL: %q", url) + }, + Dialer: dialer, + ControlKnobs: &controlknobs.Knobs{}, + } + d, err := NewDirect(opts) + if err != nil { + t.Fatalf("NewDirect: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + url, err := d.TryLogin(ctx, LoginEphemeral) + if err != nil { + t.Fatalf("TryLogin: %v", err) + } + t.Logf("URL: %q", url) +} + +func TestHTTPSNoProxy(t *testing.T) { testHTTPS(t, false) } + +// TestTLSWithProxy verifies we can connect to the control plane via +// an HTTPS proxy. +func TestHTTPSWithProxy(t *testing.T) { testHTTPS(t, true) } + +func testHTTPS(t *testing.T, withProxy bool) { + bakedroots.ResetForTest(t, tlstest.TestRootCA()) + + controlLn, err := tls.Listen("tcp", "127.0.0.1:0", tlstest.ControlPlaneKeyPair.ServerTLSConfig()) + if err != nil { + t.Fatal(err) + } + defer controlLn.Close() + + proxyLn, err := tls.Listen("tcp", "127.0.0.1:0", tlstest.ProxyServerKeyPair.ServerTLSConfig()) + if err != nil { + t.Fatal(err) + } + defer proxyLn.Close() + + const requiredAuthKey = "hunter2" + const someUsername = "testuser" + const somePassword = "testpass" + + testControl := &testcontrol.Server{ + Logf: tstest.WhileTestRunningLogger(t), + RequireAuthKey: requiredAuthKey, + } + controlSrv := &http.Server{ + Handler: testControl, + ErrorLog: logger.StdLogger(t.Logf), + } + go controlSrv.Serve(controlLn) + + const fakeControlIP = "1.2.3.4" + const fakeProxyIP = "5.6.7.8" + + dialer := &tsdial.Dialer{} + dialer.SetNetMon(netmon.NewStatic()) + dialer.SetSystemDialerForTest(func(ctx context.Context, network, addr string) (net.Conn, error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("SplitHostPort(%q): %v", addr, err) + } + var d net.Dialer + if host == fakeControlIP { + return d.DialContext(ctx, network, controlLn.Addr().String()) + } + if host == fakeProxyIP { + return d.DialContext(ctx, network, proxyLn.Addr().String()) + } + return nil, fmt.Errorf("unexpected dial to %q", addr) + }) + + opts := Options{ + Persist: persist.Persist{}, + GetMachinePrivateKey: func() (key.MachinePrivate, error) { + return key.NewMachine(), nil + }, + AuthKey: requiredAuthKey, + ServerURL: "https://controlplane.tstest", + Clock: tstime.StdClock{}, + Hostinfo: &tailcfg.Hostinfo{ + BackendLogID: "test-backend-log-id", + }, + DiscoPublicKey: key.NewDisco().Public(), + Logf: t.Logf, + HealthTracker: &health.Tracker{}, + PopBrowserURL: func(url string) { + t.Logf("PopBrowserURL: %q", url) + }, + Dialer: dialer, + } + d, err := NewDirect(opts) + if err != nil { + t.Fatalf("NewDirect: %v", err) + } + + d.dnsCache.LookupIPForTest = func(ctx context.Context, host string) ([]netip.Addr, error) { + switch host { + case "controlplane.tstest": + return []netip.Addr{netip.MustParseAddr(fakeControlIP)}, nil + case "proxy.tstest": + if !withProxy { + t.Errorf("unexpected DNS lookup for %q with proxy disabled", host) + return nil, fmt.Errorf("unexpected DNS lookup for %q", host) + } + return []netip.Addr{netip.MustParseAddr(fakeProxyIP)}, nil + } + t.Errorf("unexpected DNS query for %q", host) + return []netip.Addr{}, nil + } + + var proxyReqs atomic.Int64 + if withProxy { + d.httpc.Transport.(*http.Transport).Proxy = func(req *http.Request) (*url.URL, error) { + t.Logf("using proxy for %q", req.URL) + u := &url.URL{ + Scheme: "https", + Host: "proxy.tstest:443", + User: url.UserPassword(someUsername, somePassword), + } + return u, nil + } + + connectProxy := &http.Server{ + Handler: connectProxyTo(t, "controlplane.tstest:443", controlLn.Addr().String(), &proxyReqs), + } + go connectProxy.Serve(proxyLn) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + url, err := d.TryLogin(ctx, LoginEphemeral) + if err != nil { + t.Fatalf("TryLogin: %v", err) + } + if url != "" { + t.Errorf("got URL %q, want empty", url) + } + + if withProxy { + if got, want := proxyReqs.Load(), int64(1); got != want { + t.Errorf("proxy CONNECT requests = %d; want %d", got, want) + } + } +} + +func connectProxyTo(t testing.TB, target, backendAddrPort string, reqs *atomic.Int64) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI != target { + t.Errorf("invalid CONNECT request to %q; want %q", r.RequestURI, target) + http.Error(w, "bad target", http.StatusBadRequest) + return + } + + r.Header.Set("Authorization", r.Header.Get("Proxy-Authorization")) // for the BasicAuth method. kinda trashy. + user, pass, ok := r.BasicAuth() + if !ok || user != "testuser" || pass != "testpass" { + t.Errorf("invalid CONNECT auth %q:%q; want %q:%q", user, pass, "testuser", "testpass") + http.Error(w, "bad auth", http.StatusUnauthorized) + return + } + + (&connectproxy.Handler{ + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + var d net.Dialer + c, err := d.DialContext(ctx, network, backendAddrPort) + if err == nil { + reqs.Add(1) + } + return c, err + }, + Logf: t.Logf, + }).ServeHTTP(w, r) + }) +} diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index 2d6dc6e36c299..4c9b04ce9b114 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -16,7 +16,6 @@ import ( "net" "net/http" "net/netip" - "net/url" "os" "reflect" "runtime" @@ -240,10 +239,6 @@ func NewDirect(opts Options) (*Direct, error) { opts.ControlKnobs = &controlknobs.Knobs{} } opts.ServerURL = strings.TrimRight(opts.ServerURL, "/") - serverURL, err := url.Parse(opts.ServerURL) - if err != nil { - return nil, err - } if opts.Clock == nil { opts.Clock = tstime.StdClock{} } @@ -273,7 +268,7 @@ func NewDirect(opts Options) (*Direct, error) { tr := http.DefaultTransport.(*http.Transport).Clone() tr.Proxy = tshttpproxy.ProxyFromEnvironment tshttpproxy.SetTransportGetProxyConnectHeader(tr) - tr.TLSClientConfig = tlsdial.Config(serverURL.Hostname(), opts.HealthTracker, tr.TLSClientConfig) + tr.TLSClientConfig = tlsdial.Config(opts.HealthTracker, tr.TLSClientConfig) var dialFunc netx.DialFunc dialFunc, interceptedDial = makeScreenTimeDetectingDialFunc(opts.Dialer.SystemDial) tr.DialContext = dnscache.Dialer(dialFunc, dnsCache) diff --git a/control/controlhttp/client.go b/control/controlhttp/client.go index 869bcb599c9f3..1bb60d672980d 100644 --- a/control/controlhttp/client.go +++ b/control/controlhttp/client.go @@ -534,7 +534,7 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, optAddr netip.Ad // Disable HTTP2, since h2 can't do protocol switching. tr.TLSClientConfig.NextProtos = []string{} tr.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} - tr.TLSClientConfig = tlsdial.Config(a.Hostname, a.HealthTracker, tr.TLSClientConfig) + tr.TLSClientConfig = tlsdial.Config(a.HealthTracker, tr.TLSClientConfig) if !tr.TLSClientConfig.InsecureSkipVerify { panic("unexpected") // should be set by tlsdial.Config } diff --git a/derp/derphttp/derphttp_client.go b/derp/derphttp/derphttp_client.go index 8c42e9070252e..7385f0ad1b46f 100644 --- a/derp/derphttp/derphttp_client.go +++ b/derp/derphttp/derphttp_client.go @@ -647,12 +647,13 @@ func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.C } func (c *Client) tlsClient(nc net.Conn, node *tailcfg.DERPNode) *tls.Conn { - tlsConf := tlsdial.Config(c.tlsServerName(node), c.HealthTracker, c.TLSConfig) + tlsConf := tlsdial.Config(c.HealthTracker, c.TLSConfig) if node != nil { if node.InsecureForTests { tlsConf.InsecureSkipVerify = true tlsConf.VerifyConnection = nil } + tlsConf.ServerName = c.tlsServerName(node) if node.CertName != "" { if suf, ok := strings.CutPrefix(node.CertName, "sha256-raw:"); ok { tlsdial.SetConfigExpectedCertHash(tlsConf, suf) diff --git a/derp/derphttp/derphttp_test.go b/derp/derphttp/derphttp_test.go index 25254966084ee..7f0a7e3334abf 100644 --- a/derp/derphttp/derphttp_test.go +++ b/derp/derphttp/derphttp_test.go @@ -7,10 +7,14 @@ import ( "bytes" "context" "crypto/tls" + "encoding/json" + "flag" "fmt" + "maps" "net" "net/http" "net/http/httptest" + "slices" "strings" "sync" "testing" @@ -19,6 +23,7 @@ import ( "tailscale.com/derp" "tailscale.com/net/netmon" "tailscale.com/net/netx" + "tailscale.com/tailcfg" "tailscale.com/types/key" ) @@ -556,3 +561,32 @@ func TestNotifyError(t *testing.T) { t.Fatalf("context done before receiving error: %v", ctx.Err()) } } + +var liveNetworkTest = flag.Bool("live-net-tests", false, "run live network tests") + +func TestManualDial(t *testing.T) { + if !*liveNetworkTest { + t.Skip("skipping live network test without --live-net-tests") + } + dm := &tailcfg.DERPMap{} + res, err := http.Get("https://controlplane.tailscale.com/derpmap/default") + if err != nil { + t.Fatalf("fetching DERPMap: %v", err) + } + defer res.Body.Close() + if err := json.NewDecoder(res.Body).Decode(dm); err != nil { + t.Fatalf("decoding DERPMap: %v", err) + } + + region := slices.Sorted(maps.Keys(dm.Regions))[0] + + netMon := netmon.NewStatic() + rc := NewRegionClient(key.NewNode(), t.Logf, netMon, func() *tailcfg.DERPRegion { + return dm.Regions[region] + }) + defer rc.Close() + + if err := rc.Connect(context.Background()); err != nil { + t.Fatalf("rc.Connect: %v", err) + } +} diff --git a/logpolicy/logpolicy.go b/logpolicy/logpolicy.go index fc259a417197b..b84528d7b76bd 100644 --- a/logpolicy/logpolicy.go +++ b/logpolicy/logpolicy.go @@ -9,7 +9,6 @@ package logpolicy import ( "bufio" "bytes" - "cmp" "context" "crypto/tls" "encoding/json" @@ -911,8 +910,7 @@ func (opts TransportOptions) New() http.RoundTripper { tr.TLSNextProto = map[string]func(authority string, c *tls.Conn) http.RoundTripper{} } - host := cmp.Or(opts.Host, logtail.DefaultHost) - tr.TLSClientConfig = tlsdial.Config(host, opts.Health, tr.TLSClientConfig) + tr.TLSClientConfig = tlsdial.Config(opts.Health, tr.TLSClientConfig) // Force TLS 1.3 since we know log.tailscale.com supports it. tr.TLSClientConfig.MinVersion = tls.VersionTLS13 diff --git a/net/bakedroots/bakedroots.go b/net/bakedroots/bakedroots.go index 42e70c0dd2abb..8787b4a6d200b 100644 --- a/net/bakedroots/bakedroots.go +++ b/net/bakedroots/bakedroots.go @@ -7,6 +7,7 @@ package bakedroots import ( "crypto/x509" + "fmt" "sync" "tailscale.com/util/testenv" @@ -14,7 +15,7 @@ import ( // Get returns the baked-in roots. // -// As of 2025-01-21, this includes only the LetsEncrypt ISRG Root X1 root. +// As of 2025-01-21, this includes only the LetsEncrypt ISRG Root X1 & X2 roots. func Get() *x509.CertPool { roots.once.Do(func() { roots.parsePEM(append( @@ -56,7 +57,7 @@ type rootsOnce struct { func (r *rootsOnce) parsePEM(caPEM []byte) { p := x509.NewCertPool() if !p.AppendCertsFromPEM(caPEM) { - panic("bogus PEM") + panic(fmt.Sprintf("bogus PEM: %q", caPEM)) } r.p = p } diff --git a/net/connectproxy/connectproxy.go b/net/connectproxy/connectproxy.go new file mode 100644 index 0000000000000..4bf6875029554 --- /dev/null +++ b/net/connectproxy/connectproxy.go @@ -0,0 +1,93 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package connectproxy contains some CONNECT proxy code. +package connectproxy + +import ( + "context" + "io" + "log" + "net" + "net/http" + "time" + + "tailscale.com/net/netx" + "tailscale.com/types/logger" +) + +// Handler is an HTTP CONNECT proxy handler. +type Handler struct { + // Dial, if non-nil, is an alternate dialer to use + // instead of the default dialer. + Dial netx.DialFunc + + // Logf, if non-nil, is an alterate logger to + // use instead of log.Printf. + Logf logger.Logf + + // Check, if non-nil, validates the CONNECT target. + Check func(hostPort string) error +} + +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + if r.Method != "CONNECT" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + dial := h.Dial + if dial == nil { + var d net.Dialer + dial = d.DialContext + } + logf := h.Logf + if logf == nil { + logf = log.Printf + } + + hostPort := r.RequestURI + if h.Check != nil { + if err := h.Check(hostPort); err != nil { + logf("CONNECT target %q not allowed: %v", hostPort, err) + http.Error(w, "Invalid CONNECT target", http.StatusForbidden) + return + } + } + + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + back, err := dial(ctx, "tcp", hostPort) + if err != nil { + logf("error CONNECT dialing %v: %v", hostPort, err) + http.Error(w, "Connect failure", http.StatusBadGateway) + return + } + defer back.Close() + + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "CONNECT hijack unavailable", http.StatusInternalServerError) + return + } + c, br, err := hj.Hijack() + if err != nil { + logf("CONNECT hijack: %v", err) + return + } + defer c.Close() + + io.WriteString(c, "HTTP/1.1 200 OK\r\n\r\n") + + errc := make(chan error, 2) + go func() { + _, err := io.Copy(c, back) + errc <- err + }() + go func() { + _, err := io.Copy(back, br) + errc <- err + }() + <-errc +} diff --git a/net/dnscache/dnscache.go b/net/dnscache/dnscache.go index 96550cbb17fca..d60e92f0b8bbc 100644 --- a/net/dnscache/dnscache.go +++ b/net/dnscache/dnscache.go @@ -24,6 +24,7 @@ import ( "tailscale.com/util/cloudenv" "tailscale.com/util/singleflight" "tailscale.com/util/slicesx" + "tailscale.com/util/testenv" ) var zaddr netip.Addr @@ -63,6 +64,10 @@ type Resolver struct { // If nil, net.DefaultResolver is used. Forward *net.Resolver + // LookupIPForTest, if non-nil and in tests, handles requests instead + // of the usual mechanisms. + LookupIPForTest func(ctx context.Context, host string) ([]netip.Addr, error) + // LookupIPFallback optionally provides a backup DNS mechanism // to use if Forward returns an error or no results. LookupIPFallback func(ctx context.Context, host string) ([]netip.Addr, error) @@ -284,7 +289,13 @@ func (r *Resolver) lookupIP(ctx context.Context, host string) (ip, ip6 netip.Add lookupCtx, lookupCancel := context.WithTimeout(ctx, r.lookupTimeoutForHost(host)) defer lookupCancel() - ips, err := r.fwd().LookupNetIP(lookupCtx, "ip", host) + + var ips []netip.Addr + if r.LookupIPForTest != nil && testenv.InTest() { + ips, err = r.LookupIPForTest(ctx, host) + } else { + ips, err = r.fwd().LookupNetIP(lookupCtx, "ip", host) + } if err != nil || len(ips) == 0 { if resolver, ok := r.cloudHostResolver(); ok { r.dlogf("resolving %q via cloud resolver", host) diff --git a/net/dnsfallback/dnsfallback.go b/net/dnsfallback/dnsfallback.go index 4c5d5fa2f2743..8e53c3b293cb4 100644 --- a/net/dnsfallback/dnsfallback.go +++ b/net/dnsfallback/dnsfallback.go @@ -286,7 +286,7 @@ func bootstrapDNSMap(ctx context.Context, serverName string, serverIP netip.Addr tr.DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) { return dialer.DialContext(ctx, "tcp", net.JoinHostPort(serverIP.String(), "443")) } - tr.TLSClientConfig = tlsdial.Config(serverName, ht, tr.TLSClientConfig) + tr.TLSClientConfig = tlsdial.Config(ht, tr.TLSClientConfig) c := &http.Client{Transport: tr} req, err := http.NewRequestWithContext(ctx, "GET", "https://"+serverName+"/bootstrap-dns?q="+url.QueryEscape(queryName), nil) if err != nil { diff --git a/net/tlsdial/tlsdial.go b/net/tlsdial/tlsdial.go index 1bd2450aa3c5d..80f3bfc06c4e8 100644 --- a/net/tlsdial/tlsdial.go +++ b/net/tlsdial/tlsdial.go @@ -59,18 +59,26 @@ var mitmBlockWarnable = health.Register(&health.Warnable{ ImpactsConnectivity: true, }) -// Config returns a tls.Config for connecting to a server. +// Config returns a tls.Config for connecting to a server that +// uses system roots for validation but, if those fail, also tries +// the baked-in LetsEncrypt roots as a fallback validation method. +// // If base is non-nil, it's cloned as the base config before // being configured and returned. // If ht is non-nil, it's used to report health errors. -func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { +func Config(ht *health.Tracker, base *tls.Config) *tls.Config { var conf *tls.Config if base == nil { conf = new(tls.Config) } else { conf = base.Clone() } - conf.ServerName = host + + // Note: we do NOT set conf.ServerName here (as we accidentally did + // previously), as this path is also used when dialing an HTTPS proxy server + // (through which we'll send a CONNECT request to get a TCP connection to do + // the real TCP connection) because host is the ultimate hostname, but this + // tls.Config is used for both the proxy and the ultimate target. if n := sslKeyLogFile; n != "" { f, err := os.OpenFile(n, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) @@ -93,7 +101,9 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { // (with the baked-in fallback root) in the VerifyConnection hook. conf.InsecureSkipVerify = true conf.VerifyConnection = func(cs tls.ConnectionState) (retErr error) { - if host == "log.tailscale.com" && hostinfo.IsNATLabGuestVM() { + dialedHost := cs.ServerName + + if dialedHost == "log.tailscale.com" && hostinfo.IsNATLabGuestVM() { // Allow log.tailscale.com TLS MITM for integration tests when // the client's running within a NATLab VM. return nil @@ -116,7 +126,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { // Show a dedicated warning. m, ok := blockblame.VerifyCertificate(cert) if ok { - log.Printf("tlsdial: server cert for %q looks like %q equipment (could be blocking Tailscale)", host, m.Name) + log.Printf("tlsdial: server cert seen while dialing %q looks like %q equipment (could be blocking Tailscale)", dialedHost, m.Name) ht.SetUnhealthy(mitmBlockWarnable, health.Args{"manufacturer": m.Name}) } else { ht.SetHealthy(mitmBlockWarnable) @@ -135,7 +145,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { ht.SetTLSConnectionError(cs.ServerName, nil) if selfSignedIssuer != "" { // Log the self-signed issuer, but don't treat it as an error. - log.Printf("tlsdial: warning: server cert for %q passed x509 validation but is self-signed by %q", host, selfSignedIssuer) + log.Printf("tlsdial: warning: server cert for %q passed x509 validation but is self-signed by %q", dialedHost, selfSignedIssuer) } } }() @@ -144,7 +154,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { // First try doing x509 verification with the system's // root CA pool. opts := x509.VerifyOptions{ - DNSName: cs.ServerName, + DNSName: dialedHost, Intermediates: x509.NewCertPool(), } for _, cert := range cs.PeerCertificates[1:] { @@ -152,7 +162,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { } _, errSys := cs.PeerCertificates[0].Verify(opts) if debug() { - log.Printf("tlsdial(sys %q): %v", host, errSys) + log.Printf("tlsdial(sys %q): %v", dialedHost, errSys) } // Always verify with our baked-in Let's Encrypt certificate, @@ -161,13 +171,11 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { opts.Roots = bakedroots.Get() _, bakedErr := cs.PeerCertificates[0].Verify(opts) if debug() { - log.Printf("tlsdial(bake %q): %v", host, bakedErr) + log.Printf("tlsdial(bake %q): %v", dialedHost, bakedErr) } else if bakedErr != nil { - if _, loaded := tlsdialWarningPrinted.LoadOrStore(host, true); !loaded { - if errSys == nil { - log.Printf("tlsdial: warning: server cert for %q is not a Let's Encrypt cert", host) - } else { - log.Printf("tlsdial: error: server cert for %q failed to verify and is not a Let's Encrypt cert", host) + if _, loaded := tlsdialWarningPrinted.LoadOrStore(dialedHost, true); !loaded { + if errSys != nil { + log.Printf("tlsdial: error: server cert for %q failed both system roots & Let's Encrypt root validation", dialedHost) } } } @@ -202,9 +210,6 @@ func SetConfigExpectedCert(c *tls.Config, certDNSName string) { c.ServerName = certDNSName return } - if c.VerifyPeerCertificate != nil { - panic("refusing to override tls.Config.VerifyPeerCertificate") - } // Set InsecureSkipVerify to prevent crypto/tls from doing its // own cert verification, but do the same work that it'd do // (but using certDNSName) in the VerifyPeerCertificate hook. @@ -257,29 +262,30 @@ func SetConfigExpectedCertHash(c *tls.Config, wantFullCertSHA256Hex string) { if c.VerifyPeerCertificate != nil { panic("refusing to override tls.Config.VerifyPeerCertificate") } + // Set InsecureSkipVerify to prevent crypto/tls from doing its // own cert verification, but do the same work that it'd do - // (but using certDNSName) in the VerifyPeerCertificate hook. + // (but using certDNSName) in the VerifyConnection hook. c.InsecureSkipVerify = true - c.VerifyConnection = nil - c.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + + c.VerifyConnection = func(cs tls.ConnectionState) error { + dialedHost := cs.ServerName var sawGoodCert bool - for _, rawCert := range rawCerts { - cert, err := x509.ParseCertificate(rawCert) - if err != nil { - return fmt.Errorf("ParseCertificate: %w", err) - } + + for _, cert := range cs.PeerCertificates { if strings.HasPrefix(cert.Subject.CommonName, derpconst.MetaCertCommonNamePrefix) { continue } if sawGoodCert { return errors.New("unexpected multiple certs presented") } - if fmt.Sprintf("%02x", sha256.Sum256(rawCert)) != wantFullCertSHA256Hex { + if fmt.Sprintf("%02x", sha256.Sum256(cert.Raw)) != wantFullCertSHA256Hex { return fmt.Errorf("cert hash does not match expected cert hash") } - if err := cert.VerifyHostname(c.ServerName); err != nil { - return fmt.Errorf("cert does not match server name %q: %w", c.ServerName, err) + if dialedHost != "" { // it's empty when dialing a derper by IP with no hostname + if err := cert.VerifyHostname(dialedHost); err != nil { + return fmt.Errorf("cert does not match server name %q: %w", dialedHost, err) + } } now := time.Now() if now.After(cert.NotAfter) { @@ -302,12 +308,8 @@ func SetConfigExpectedCertHash(c *tls.Config, wantFullCertSHA256Hex string) { func NewTransport() *http.Transport { return &http.Transport{ DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - host, _, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } var d tls.Dialer - d.Config = Config(host, nil, nil) + d.Config = Config(nil, nil) return d.DialContext(ctx, network, addr) }, } diff --git a/net/tlsdial/tlsdial_test.go b/net/tlsdial/tlsdial_test.go index 6723b82e0d1c9..e2c4cdd4f51cb 100644 --- a/net/tlsdial/tlsdial_test.go +++ b/net/tlsdial/tlsdial_test.go @@ -86,7 +86,7 @@ func TestFallbackRootWorks(t *testing.T) { DisableKeepAlives: true, // for test cleanup ease } ht := new(health.Tracker) - tr.TLSClientConfig = Config("tlsdial.test", ht, tr.TLSClientConfig) + tr.TLSClientConfig = Config(ht, tr.TLSClientConfig) c := &http.Client{Transport: tr} ctr0 := atomic.LoadInt32(&counterFallbackOK) diff --git a/tstest/tlstest/testdata/controlplane.tstest.key b/tstest/tlstest/testdata/controlplane.tstest.key new file mode 100644 index 0000000000000..dbe5ede348570 --- /dev/null +++ b/tstest/tlstest/testdata/controlplane.tstest.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIHcxOQNVyqvBSSlu7c93QW6OsyccjL+R1evW4acd32MWoAoGCCqGSM49 +AwEHoUQDQgAEIOY5/CQ8CMuKYPLf+r6OEneqfzQ5RfgPnLdkL22qhm8xb69ZCXxz +UecawU0KEDfHLYbUYXSuhAFxxuPh9I3x5Q== +-----END EC PRIVATE KEY----- diff --git a/tstest/tlstest/testdata/proxy.tstest.key b/tstest/tlstest/testdata/proxy.tstest.key new file mode 100644 index 0000000000000..067279089435b --- /dev/null +++ b/tstest/tlstest/testdata/proxy.tstest.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEING1XBDWFXQjqBmLjhp20hXOf2rk/I0N6W7muv9RVvk3oAoGCCqGSM49 +AwEHoUQDQgAE8lxnEEeLqYikwmXbXSsIQSw20R0oLA831s960KQZEgt0P9SbWcJc +QTk98rdfYT/QDdHn157Oh4FPcDtxmdQ4vw== +-----END EC PRIVATE KEY----- diff --git a/tstest/tlstest/testdata/root-ca.key b/tstest/tlstest/testdata/root-ca.key new file mode 100644 index 0000000000000..ece23ddf99b4a --- /dev/null +++ b/tstest/tlstest/testdata/root-ca.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIMl3xjqt1dnXBpYJSEqevirAcnSJ79I2tucdRazlrDG9oAoGCCqGSM49 +AwEHoUQDQgAEQ/+Jme+16hgO7TtPSIFHVV0Yt969ltVlARVcNUZmWc0upQaq7uiJ +Aur5KtzwxU3YI4bhNK0593OK2TLvEEWIdw== +-----END EC PRIVATE KEY----- diff --git a/tstest/tlstest/tlstest.go b/tstest/tlstest/tlstest.go new file mode 100644 index 0000000000000..f65c261e8eb15 --- /dev/null +++ b/tstest/tlstest/tlstest.go @@ -0,0 +1,167 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package tlstest contains code to help test Tailscale's client proxy support. +package tlstest + +import ( + "bytes" + "crypto/ecdsa" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + _ "embed" + "encoding/pem" + "fmt" + "math/big" + "sync" + "time" +) + +// Some baked-in ECDSA keys to speed up tests, not having to burn CPU to +// generate them each time. We only make the certs (which have expiry times) +// at runtime. +// +// They were made with: +// +// openssl ecparam -name prime256v1 -genkey -noout -out root-ca.key +var ( + //go:embed testdata/root-ca.key + rootCAKeyPEM []byte + + // TestProxyServerKey is the PEM private key for [TestProxyServerCert]. + // + //go:embed testdata/proxy.tstest.key + TestProxyServerKey []byte + + // TestControlPlaneKey is the PEM private key for [TestControlPlaneCert]. + // + //go:embed testdata/controlplane.tstest.key + TestControlPlaneKey []byte +) + +// TestRootCA returns a self-signed ECDSA root CA certificate (as PEM) for +// testing purposes. +func TestRootCA() []byte { + return bytes.Clone(testRootCAOncer()) +} + +var testRootCAOncer = sync.OnceValue(func() []byte { + key := rootCAKey() + now := time.Now().Add(-time.Hour) + tpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "Tailscale Unit Test ECDSA Root", + Organization: []string{"Tailscale Test Org"}, + }, + NotBefore: now, + NotAfter: now.AddDate(5, 0, 0), + + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + SubjectKeyId: mustSKID(&key.PublicKey), + } + + der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key) + if err != nil { + panic(err) + } + return pemCert(der) +}) + +func pemCert(der []byte) []byte { + var buf bytes.Buffer + if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil { + panic(fmt.Sprintf("failed to encode PEM: %v", err)) + } + return buf.Bytes() +} + +var rootCAKey = sync.OnceValue(func() *ecdsa.PrivateKey { + return mustParsePEM(rootCAKeyPEM, x509.ParseECPrivateKey) +}) + +func mustParsePEM[T any](pemBytes []byte, parse func([]byte) (T, error)) T { + block, rest := pem.Decode(pemBytes) + if block == nil || len(rest) > 0 { + panic("invalid PEM") + } + v, err := parse(block.Bytes) + if err != nil { + panic(fmt.Sprintf("invalid PEM: %v", err)) + } + return v +} + +// KeyPair is a simple struct to hold a certificate and its private key. +type KeyPair struct { + Domain string + KeyPEM []byte // PEM-encoded private key +} + +// ServerTLSConfig returns a TLS configuration suitable for a server +// using the KeyPair's certificate and private key. +func (p KeyPair) ServerTLSConfig() *tls.Config { + cert, err := tls.X509KeyPair(p.CertPEM(), p.KeyPEM) + if err != nil { + panic("invalid TLS key pair: " + err.Error()) + } + return &tls.Config{ + Certificates: []tls.Certificate{cert}, + } +} + +// ProxyServerKeyPair is a KeyPair for a test control plane server +// with domain name "proxy.tstest". +var ProxyServerKeyPair = KeyPair{ + Domain: "proxy.tstest", + KeyPEM: TestProxyServerKey, +} + +// ControlPlaneKeyPair is a KeyPair for a test control plane server +// with domain name "controlplane.tstest". +var ControlPlaneKeyPair = KeyPair{ + Domain: "controlplane.tstest", + KeyPEM: TestControlPlaneKey, +} + +func (p KeyPair) CertPEM() []byte { + caCert := mustParsePEM(TestRootCA(), x509.ParseCertificate) + caPriv := mustParsePEM(rootCAKeyPEM, x509.ParseECPrivateKey) + leafKey := mustParsePEM(p.KeyPEM, x509.ParseECPrivateKey) + + serial, err := rand.Int(rand.Reader, big.NewInt(0).Lsh(big.NewInt(1), 128)) + if err != nil { + panic(err) + } + + now := time.Now().Add(-time.Hour) + tpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: p.Domain}, + NotBefore: now, + NotAfter: now.AddDate(2, 0, 0), + + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: []string{p.Domain}, + } + + der, err := x509.CreateCertificate(rand.Reader, tpl, caCert, &leafKey.PublicKey, caPriv) + if err != nil { + panic(err) + } + return pemCert(der) +} + +func mustSKID(pub *ecdsa.PublicKey) []byte { + skid, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + panic(err) + } + return skid[:20] // same as x509 library +} From 583f740c0b583081b0c1a39f92e349c49c0c4a41 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 19 Jun 2025 09:47:06 -0700 Subject: [PATCH 090/263] Revert "types/netmap,wgengine/magicsock: propagate CapVer to magicsock.endpoint (#16244)" (#16322) This reverts commit 6a93b17c8cafc1d8e1c52e133511e52ed9086355. The reverted commit added more complexity than it was worth at the current stage. Handling delta CapVer changes requires extensive changes to relayManager datastructures in order to also support delta updates of relay servers. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- types/netmap/nodemut.go | 13 ------------- types/netmap/nodemut_test.go | 9 --------- wgengine/magicsock/magicsock.go | 4 ---- 3 files changed, 26 deletions(-) diff --git a/types/netmap/nodemut.go b/types/netmap/nodemut.go index ab30ef1e6a286..f4de1bf0b8f02 100644 --- a/types/netmap/nodemut.go +++ b/types/netmap/nodemut.go @@ -69,17 +69,6 @@ func (m NodeMutationLastSeen) Apply(n *tailcfg.Node) { n.LastSeen = ptr.To(m.LastSeen) } -// NodeMutationCap is a NodeMutation that says a node's -// [tailcfg.CapabilityVersion] value has changed. -type NodeMutationCap struct { - mutatingNodeID - Cap tailcfg.CapabilityVersion -} - -func (m NodeMutationCap) Apply(n *tailcfg.Node) { - n.Cap = m.Cap -} - var peerChangeFields = sync.OnceValue(func() []reflect.StructField { var fields []reflect.StructField rt := reflect.TypeFor[tailcfg.PeerChange]() @@ -116,8 +105,6 @@ func NodeMutationsFromPatch(p *tailcfg.PeerChange) (_ []NodeMutation, ok bool) { ret = append(ret, NodeMutationOnline{mutatingNodeID(p.NodeID), *p.Online}) case "LastSeen": ret = append(ret, NodeMutationLastSeen{mutatingNodeID(p.NodeID), *p.LastSeen}) - case "Cap": - ret = append(ret, NodeMutationCap{mutatingNodeID(p.NodeID), p.Cap}) } } return ret, true diff --git a/types/netmap/nodemut_test.go b/types/netmap/nodemut_test.go index 0f1cac6b247bc..374f8623ad564 100644 --- a/types/netmap/nodemut_test.go +++ b/types/netmap/nodemut_test.go @@ -177,14 +177,6 @@ func TestMutationsFromMapResponse(t *testing.T) { }, want: nil, }, - { - name: "patch-cap", - mr: fromChanges(&tailcfg.PeerChange{ - NodeID: 1, - Cap: 2, - }), - want: muts(NodeMutationCap{1, 2}), - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -203,7 +195,6 @@ func TestMutationsFromMapResponse(t *testing.T) { NodeMutationDERPHome{}, NodeMutationOnline{}, NodeMutationLastSeen{}, - NodeMutationCap{}, )); diff != "" { t.Errorf("wrong result (-want +got):\n%s", diff) } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index a6c6a3fb62206..bfc7afba95149 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3263,10 +3263,6 @@ func (c *Conn) onNodeMutationsUpdate(update NodeMutationsUpdate) { ep.mu.Lock() ep.setEndpointsLocked(views.SliceOf(m.Endpoints)) ep.mu.Unlock() - case netmap.NodeMutationCap: - ep.mu.Lock() - ep.relayCapable = capVerIsRelayCapable(m.Cap) - ep.mu.Unlock() } } } From a64ca7a5b4efed0437a1d4eace3815b4de7f6eaf Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 19 Jun 2025 07:58:19 -0700 Subject: [PATCH 091/263] tstest/tlstest: simplify, don't even bake in any keys I earlier thought this saved a second of CPU even on a fast machine, but I think when I was previously measuring, I still had a 4096 bit RSA key being generated in the code I was measuring. Measuring again for this, it's plenty fast. Prep for using this package more, for derp, etc. Updates #16315 Change-Id: I4c9008efa9aa88a3d65409d6ffd7b3807f4d75e9 Signed-off-by: Brad Fitzpatrick --- control/controlclient/controlclient_test.go | 4 +- net/bakedroots/bakedroots.go | 13 +- .../tlstest/testdata/controlplane.tstest.key | 5 - tstest/tlstest/testdata/proxy.tstest.key | 5 - tstest/tlstest/testdata/root-ca.key | 5 - tstest/tlstest/tlstest.go | 114 ++++++++++-------- tstest/tlstest/tlstest_test.go | 21 ++++ 7 files changed, 95 insertions(+), 72 deletions(-) delete mode 100644 tstest/tlstest/testdata/controlplane.tstest.key delete mode 100644 tstest/tlstest/testdata/proxy.tstest.key delete mode 100644 tstest/tlstest/testdata/root-ca.key create mode 100644 tstest/tlstest/tlstest_test.go diff --git a/control/controlclient/controlclient_test.go b/control/controlclient/controlclient_test.go index 1107f76a46a78..792c26955e5d1 100644 --- a/control/controlclient/controlclient_test.go +++ b/control/controlclient/controlclient_test.go @@ -263,13 +263,13 @@ func TestHTTPSWithProxy(t *testing.T) { testHTTPS(t, true) } func testHTTPS(t *testing.T, withProxy bool) { bakedroots.ResetForTest(t, tlstest.TestRootCA()) - controlLn, err := tls.Listen("tcp", "127.0.0.1:0", tlstest.ControlPlaneKeyPair.ServerTLSConfig()) + controlLn, err := tls.Listen("tcp", "127.0.0.1:0", tlstest.ControlPlane.ServerTLSConfig()) if err != nil { t.Fatal(err) } defer controlLn.Close() - proxyLn, err := tls.Listen("tcp", "127.0.0.1:0", tlstest.ProxyServerKeyPair.ServerTLSConfig()) + proxyLn, err := tls.Listen("tcp", "127.0.0.1:0", tlstest.ProxyServer.ServerTLSConfig()) if err != nil { t.Fatal(err) } diff --git a/net/bakedroots/bakedroots.go b/net/bakedroots/bakedroots.go index 8787b4a6d200b..b268b1546caac 100644 --- a/net/bakedroots/bakedroots.go +++ b/net/bakedroots/bakedroots.go @@ -26,16 +26,9 @@ func Get() *x509.CertPool { return roots.p } -// testingTB is a subset of testing.TB needed -// to verify the caller isn't in a parallel test. -type testingTB interface { - // Setenv panics if it's in a parallel test. - Setenv(k, v string) -} - // ResetForTest resets the cached roots for testing, // optionally setting them to caPEM if non-nil. -func ResetForTest(tb testingTB, caPEM []byte) { +func ResetForTest(tb testenv.TB, caPEM []byte) { if !testenv.InTest() { panic("not in test") } @@ -44,6 +37,10 @@ func ResetForTest(tb testingTB, caPEM []byte) { roots = rootsOnce{} if caPEM != nil { roots.once.Do(func() { roots.parsePEM(caPEM) }) + tb.Cleanup(func() { + // Reset the roots to real roots for any following test. + roots = rootsOnce{} + }) } } diff --git a/tstest/tlstest/testdata/controlplane.tstest.key b/tstest/tlstest/testdata/controlplane.tstest.key deleted file mode 100644 index dbe5ede348570..0000000000000 --- a/tstest/tlstest/testdata/controlplane.tstest.key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIHcxOQNVyqvBSSlu7c93QW6OsyccjL+R1evW4acd32MWoAoGCCqGSM49 -AwEHoUQDQgAEIOY5/CQ8CMuKYPLf+r6OEneqfzQ5RfgPnLdkL22qhm8xb69ZCXxz -UecawU0KEDfHLYbUYXSuhAFxxuPh9I3x5Q== ------END EC PRIVATE KEY----- diff --git a/tstest/tlstest/testdata/proxy.tstest.key b/tstest/tlstest/testdata/proxy.tstest.key deleted file mode 100644 index 067279089435b..0000000000000 --- a/tstest/tlstest/testdata/proxy.tstest.key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEING1XBDWFXQjqBmLjhp20hXOf2rk/I0N6W7muv9RVvk3oAoGCCqGSM49 -AwEHoUQDQgAE8lxnEEeLqYikwmXbXSsIQSw20R0oLA831s960KQZEgt0P9SbWcJc -QTk98rdfYT/QDdHn157Oh4FPcDtxmdQ4vw== ------END EC PRIVATE KEY----- diff --git a/tstest/tlstest/testdata/root-ca.key b/tstest/tlstest/testdata/root-ca.key deleted file mode 100644 index ece23ddf99b4a..0000000000000 --- a/tstest/tlstest/testdata/root-ca.key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIMl3xjqt1dnXBpYJSEqevirAcnSJ79I2tucdRazlrDG9oAoGCCqGSM49 -AwEHoUQDQgAEQ/+Jme+16hgO7TtPSIFHVV0Yt969ltVlARVcNUZmWc0upQaq7uiJ -Aur5KtzwxU3YI4bhNK0593OK2TLvEEWIdw== ------END EC PRIVATE KEY----- diff --git a/tstest/tlstest/tlstest.go b/tstest/tlstest/tlstest.go index f65c261e8eb15..76ec0e7e2dfad 100644 --- a/tstest/tlstest/tlstest.go +++ b/tstest/tlstest/tlstest.go @@ -1,12 +1,14 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -// Package tlstest contains code to help test Tailscale's client proxy support. +// Package tlstest contains code to help test Tailscale's TLS support without +// depending on real WebPKI roots or certificates during tests. package tlstest import ( "bytes" "crypto/ecdsa" + "crypto/elliptic" "crypto/rand" "crypto/tls" "crypto/x509" @@ -19,32 +21,47 @@ import ( "time" ) -// Some baked-in ECDSA keys to speed up tests, not having to burn CPU to -// generate them each time. We only make the certs (which have expiry times) -// at runtime. +// TestRootCA returns a self-signed ECDSA root CA certificate (as PEM) for +// testing purposes. // -// They were made with: +// Typical use in a test is like: // -// openssl ecparam -name prime256v1 -genkey -noout -out root-ca.key +// bakedroots.ResetForTest(t, tlstest.TestRootCA()) +func TestRootCA() []byte { + return bytes.Clone(testRootCAOncer()) +} + +// cache for [privateKey], so it always returns the same key for a given domain. var ( - //go:embed testdata/root-ca.key - rootCAKeyPEM []byte - - // TestProxyServerKey is the PEM private key for [TestProxyServerCert]. - // - //go:embed testdata/proxy.tstest.key - TestProxyServerKey []byte - - // TestControlPlaneKey is the PEM private key for [TestControlPlaneCert]. - // - //go:embed testdata/controlplane.tstest.key - TestControlPlaneKey []byte + mu sync.Mutex + privateKeys = make(map[string][]byte) // domain -> private key PEM ) -// TestRootCA returns a self-signed ECDSA root CA certificate (as PEM) for -// testing purposes. -func TestRootCA() []byte { - return bytes.Clone(testRootCAOncer()) +// caDomain is a fake domain name to repreesnt the private key for the root CA. +const caDomain = "_root" + +// privateKey returns a PEM-encoded test ECDSA private key for the given domain. +func privateKey(domain string) (pemBytes []byte) { + mu.Lock() + defer mu.Unlock() + if pemBytes, ok := privateKeys[domain]; ok { + return bytes.Clone(pemBytes) + } + defer func() { privateKeys[domain] = bytes.Clone(pemBytes) }() + + k, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + panic(fmt.Sprintf("failed to generate ECDSA key for %q: %v", domain, err)) + } + der, err := x509.MarshalECPrivateKey(k) + if err != nil { + panic(fmt.Sprintf("failed to marshal ECDSA key for %q: %v", domain, err)) + } + var buf bytes.Buffer + if err := pem.Encode(&buf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: der}); err != nil { + panic(fmt.Sprintf("failed to encode PEM: %v", err)) + } + return buf.Bytes() } var testRootCAOncer = sync.OnceValue(func() []byte { @@ -81,7 +98,7 @@ func pemCert(der []byte) []byte { } var rootCAKey = sync.OnceValue(func() *ecdsa.PrivateKey { - return mustParsePEM(rootCAKeyPEM, x509.ParseECPrivateKey) + return mustParsePEM(privateKey(caDomain), x509.ParseECPrivateKey) }) func mustParsePEM[T any](pemBytes []byte, parse func([]byte) (T, error)) T { @@ -96,16 +113,27 @@ func mustParsePEM[T any](pemBytes []byte, parse func([]byte) (T, error)) T { return v } -// KeyPair is a simple struct to hold a certificate and its private key. -type KeyPair struct { - Domain string - KeyPEM []byte // PEM-encoded private key -} +// Domain is a fake domain name used in TLS tests. +// +// They don't have real DNS records. Tests are expected to fake DNS +// lookups and dials for these domains. +type Domain string + +// ProxyServer is a domain name for a hypothetical proxy server. +const ( + ProxyServer = Domain("proxy.tstest") + + // ControlPlane is a domain name for a test control plane server. + ControlPlane = Domain("controlplane.tstest") + + // Derper is a domain name for a test DERP server. + Derper = Domain("derp.tstest") +) // ServerTLSConfig returns a TLS configuration suitable for a server // using the KeyPair's certificate and private key. -func (p KeyPair) ServerTLSConfig() *tls.Config { - cert, err := tls.X509KeyPair(p.CertPEM(), p.KeyPEM) +func (d Domain) ServerTLSConfig() *tls.Config { + cert, err := tls.X509KeyPair(d.CertPEM(), privateKey(string(d))) if err != nil { panic("invalid TLS key pair: " + err.Error()) } @@ -114,24 +142,16 @@ func (p KeyPair) ServerTLSConfig() *tls.Config { } } -// ProxyServerKeyPair is a KeyPair for a test control plane server -// with domain name "proxy.tstest". -var ProxyServerKeyPair = KeyPair{ - Domain: "proxy.tstest", - KeyPEM: TestProxyServerKey, -} - -// ControlPlaneKeyPair is a KeyPair for a test control plane server -// with domain name "controlplane.tstest". -var ControlPlaneKeyPair = KeyPair{ - Domain: "controlplane.tstest", - KeyPEM: TestControlPlaneKey, +// KeyPEM returns a PEM-encoded private key for the domain. +func (d Domain) KeyPEM() []byte { + return privateKey(string(d)) } -func (p KeyPair) CertPEM() []byte { +// CertPEM returns a PEM-encoded certificate for the domain. +func (d Domain) CertPEM() []byte { caCert := mustParsePEM(TestRootCA(), x509.ParseCertificate) - caPriv := mustParsePEM(rootCAKeyPEM, x509.ParseECPrivateKey) - leafKey := mustParsePEM(p.KeyPEM, x509.ParseECPrivateKey) + caPriv := mustParsePEM(privateKey(caDomain), x509.ParseECPrivateKey) + leafKey := mustParsePEM(d.KeyPEM(), x509.ParseECPrivateKey) serial, err := rand.Int(rand.Reader, big.NewInt(0).Lsh(big.NewInt(1), 128)) if err != nil { @@ -141,14 +161,14 @@ func (p KeyPair) CertPEM() []byte { now := time.Now().Add(-time.Hour) tpl := &x509.Certificate{ SerialNumber: serial, - Subject: pkix.Name{CommonName: p.Domain}, + Subject: pkix.Name{CommonName: string(d)}, NotBefore: now, NotAfter: now.AddDate(2, 0, 0), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true, - DNSNames: []string{p.Domain}, + DNSNames: []string{string(d)}, } der, err := x509.CreateCertificate(rand.Reader, tpl, caCert, &leafKey.PublicKey, caPriv) diff --git a/tstest/tlstest/tlstest_test.go b/tstest/tlstest/tlstest_test.go new file mode 100644 index 0000000000000..8497b872ec7c5 --- /dev/null +++ b/tstest/tlstest/tlstest_test.go @@ -0,0 +1,21 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package tlstest + +import ( + "testing" +) + +func TestPrivateKey(t *testing.T) { + a := privateKey("a.tstest") + a2 := privateKey("a.tstest") + b := privateKey("b.tstest") + + if string(a) != string(a2) { + t.Errorf("a and a2 should be equal") + } + if string(a) == string(b) { + t.Errorf("a and b should not be equal") + } +} From 253d0b026dbd55f38787d8e7334261b044b8c703 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 20 Jun 2025 10:34:47 +0100 Subject: [PATCH 092/263] cmd/k8s-operator: remove conffile hashing mechanism (#16335) Proxies know how to reload configfile on changes since 1.80, which is going to be the earliest supported proxy version with 1.84 operator, so remove the mechanism that was updating configfile hash to force proxy Pod restarts on config changes. Updates #13032 Signed-off-by: Irbe Krumina --- cmd/k8s-operator/connector_test.go | 24 ++++---- cmd/k8s-operator/ingress_test.go | 16 ++--- cmd/k8s-operator/operator_test.go | 42 ++++++------- cmd/k8s-operator/proxygroup.go | 91 +++-------------------------- cmd/k8s-operator/proxygroup_test.go | 35 +++-------- cmd/k8s-operator/sts.go | 85 +++++---------------------- cmd/k8s-operator/testutils_test.go | 19 ------ 7 files changed, 74 insertions(+), 238 deletions(-) diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index f32fe3282020c..d5829c37fe596 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -80,7 +80,7 @@ func TestConnector(t *testing.T) { app: kubetypes.AppConnector, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Connector status should get updated with the IP/hostname info when available. const hostname = "foo.tailnetxyz.ts.net" @@ -106,7 +106,7 @@ func TestConnector(t *testing.T) { opts.subnetRoutes = "10.40.0.0/14,10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Remove a route. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -114,7 +114,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Remove the subnet router. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -122,7 +122,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Re-add the subnet router. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -132,7 +132,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { @@ -176,7 +176,7 @@ func TestConnector(t *testing.T) { app: kubetypes.AppConnector, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Add an exit node. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -184,7 +184,7 @@ func TestConnector(t *testing.T) { }) opts.isExitNode = true expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { @@ -262,7 +262,7 @@ func TestConnectorWithProxyClass(t *testing.T) { app: kubetypes.AppConnector, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. Update Connector to specify a ProxyClass. ProxyClass is not yet // ready, so its configuration is NOT applied to the Connector @@ -271,7 +271,7 @@ func TestConnectorWithProxyClass(t *testing.T) { conn.Spec.ProxyClass = "custom-metadata" }) expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 3. ProxyClass is set to Ready by proxy-class reconciler. Connector // get reconciled and configuration from the ProxyClass is applied to @@ -286,7 +286,7 @@ func TestConnectorWithProxyClass(t *testing.T) { }) opts.proxyClass = pc.Name expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 4. Connector.spec.proxyClass field is unset, Connector gets // reconciled and configuration from the ProxyClass is removed from the @@ -296,7 +296,7 @@ func TestConnectorWithProxyClass(t *testing.T) { }) opts.proxyClass = "" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) } func TestConnectorWithAppConnector(t *testing.T) { @@ -352,7 +352,7 @@ func TestConnectorWithAppConnector(t *testing.T) { isAppConnector: true, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Connector's ready condition should be set to true cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer") diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index dbd6961d7d7ff..aacf27d8e6600 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -71,7 +71,7 @@ func TestTailscaleIngress(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 2. Ingress status gets updated with ingress proxy's MagicDNS name // once that becomes available. @@ -98,7 +98,7 @@ func TestTailscaleIngress(t *testing.T) { }) opts.shouldEnableForwardingClusterTrafficViaIngress = true expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 4. Resources get cleaned up when Ingress class is unset mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { @@ -162,7 +162,7 @@ func TestTailscaleIngressHostname(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 2. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint set mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) { @@ -280,7 +280,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 2. Ingress is updated to specify a ProxyClass, ProxyClass is not yet // ready, so proxy resource configuration does not change. @@ -288,7 +288,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { mak.Set(&ing.ObjectMeta.Labels, LabelProxyClass, "custom-metadata") }) expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 3. ProxyClass is set to Ready by proxy-class reconciler. Ingress get // reconciled and configuration from the ProxyClass is applied to the @@ -303,7 +303,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = pc.Name - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 4. tailscale.com/proxy-class label is removed from the Ingress, the // Ingress gets reconciled and the custom ProxyClass configuration is @@ -313,7 +313,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = "" - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) } func TestTailscaleIngressWithServiceMonitor(t *testing.T) { @@ -608,7 +608,7 @@ func TestEmptyPath(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) expectEvents(t, fr, tt.expectedEvents) }) diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index 33bf23e844d9a..ff6ba4f952749 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -130,7 +130,7 @@ func TestLoadBalancerClass(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) want.Annotations = nil want.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"} @@ -268,7 +268,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -291,7 +291,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Change the tailscale-target-fqdn annotation which should update the // StatefulSet @@ -380,7 +380,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -403,7 +403,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Change the tailscale-target-ip annotation which should update the // StatefulSet @@ -631,7 +631,7 @@ func TestAnnotations(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -737,7 +737,7 @@ func TestAnnotationIntoLB(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, since it would have normally happened at @@ -781,7 +781,7 @@ func TestAnnotationIntoLB(t *testing.T) { expectReconciled(t, sr, "default", "test") // None of the proxy machinery should have changed... expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // ... but the service should have a LoadBalancer status. want = &corev1.Service{ @@ -867,7 +867,7 @@ func TestLBIntoAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, then verify reconcile again and verify @@ -927,7 +927,7 @@ func TestLBIntoAnnotation(t *testing.T) { expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want = &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -1007,7 +1007,7 @@ func TestCustomHostname(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -1118,7 +1118,7 @@ func TestCustomPriorityClassName(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) } func TestProxyClassForService(t *testing.T) { @@ -1188,7 +1188,7 @@ func TestProxyClassForService(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. The Service gets updated with tailscale.com/proxy-class label // pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not @@ -1197,7 +1197,7 @@ func TestProxyClassForService(t *testing.T) { mak.Set(&svc.Labels, LabelProxyClass, "custom-metadata") }) expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) expectEqual(t, fc, expectedSecret(t, fc, opts)) // 3. ProxyClass is set to Ready, the Service gets reconciled by the @@ -1213,7 +1213,7 @@ func TestProxyClassForService(t *testing.T) { }) opts.proxyClass = pc.Name expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t)) // 4. tailscale.com/proxy-class label is removed from the Service, the @@ -1224,7 +1224,7 @@ func TestProxyClassForService(t *testing.T) { }) opts.proxyClass = "" expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) } func TestDefaultLoadBalancer(t *testing.T) { @@ -1280,7 +1280,7 @@ func TestDefaultLoadBalancer(t *testing.T) { clusterTargetIP: "10.20.30.40", app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) } func TestProxyFirewallMode(t *testing.T) { @@ -1336,7 +1336,7 @@ func TestProxyFirewallMode(t *testing.T) { clusterTargetIP: "10.20.30.40", app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) } func Test_isMagicDNSName(t *testing.T) { @@ -1617,7 +1617,7 @@ func Test_authKeyRemoval(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. Apply update to the Secret that imitates the proxy setting device_id. s := expectedSecret(t, fc, opts) @@ -1691,7 +1691,7 @@ func Test_externalNameService(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. Change the ExternalName and verify that changes get propagated. mustUpdate(t, sr, "default", "test", func(s *corev1.Service) { @@ -1699,7 +1699,7 @@ func Test_externalNameService(t *testing.T) { }) expectReconciled(t, sr, "default", "test") opts.clusterTargetDNS = "bar.com" - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) } func Test_metricsResourceCreation(t *testing.T) { diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index e7c0590b0dbb9..0d5eff551e8de 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -7,7 +7,6 @@ package main import ( "context" - "crypto/sha256" "encoding/json" "errors" "fmt" @@ -237,8 +236,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.ensureAddedToGaugeForProxyGroup(pg) r.mu.Unlock() - cfgHash, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass) - if err != nil { + if err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass); err != nil { return fmt.Errorf("error provisioning config Secrets: %w", err) } // State secrets are precreated so we can use the ProxyGroup CR as their owner ref. @@ -306,33 +304,10 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro proxyType: string(pg.Spec.Type), } ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger) - capver, err := r.capVerForPG(ctx, pg, logger) - if err != nil { - return fmt.Errorf("error getting device info: %w", err) - } updateSS := func(s *appsv1.StatefulSet) { - // This is a temporary workaround to ensure that egress ProxyGroup proxies with capver older than 110 - // are restarted when tailscaled configfile contents have changed. - // This workaround ensures that: - // 1. The hash mechanism is used to trigger pod restarts for proxies below capver 110. - // 2. Proxies above capver are not unnecessarily restarted when the configfile contents change. - // 3. If the hash has alreay been set, but the capver is above 110, the old hash is preserved to avoid - // unnecessary pod restarts that could result in an update loop where capver cannot be determined for a - // restarting Pod and the hash is re-added again. - // Note that this workaround is only applied to egress ProxyGroups, because ingress ProxyGroup was added after capver 110. - // Note also that the hash annotation is only set on updates, not creation, because if the StatefulSet is - // being created, there is no need for a restart. - // TODO(irbekrm): remove this in 1.84. - hash := cfgHash - if capver >= 110 { - hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash] - } s.Spec = ss.Spec - if hash != "" && pg.Spec.Type == tsapi.ProxyGroupTypeEgress { - mak.Set(&s.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, hash) - } s.ObjectMeta.Labels = ss.ObjectMeta.Labels s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations @@ -449,9 +424,8 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc return nil } -func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (hash string, err error) { +func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (err error) { logger := r.logger(pg.Name) - var configSHA256Sum string for i := range pgReplicas(pg) { cfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -467,7 +441,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName()) existingCfgSecret = cfgSecret.DeepCopy() } else if !apierrors.IsNotFound(err) { - return "", err + return err } var authKey string @@ -479,65 +453,39 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p } authKey, err = newAuthKey(ctx, r.tsClient, tags) if err != nil { - return "", err + return err } } configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret) if err != nil { - return "", fmt.Errorf("error creating tailscaled config: %w", err) + return fmt.Errorf("error creating tailscaled config: %w", err) } for cap, cfg := range configs { cfgJSON, err := json.Marshal(cfg) if err != nil { - return "", fmt.Errorf("error marshalling tailscaled config: %w", err) + return fmt.Errorf("error marshalling tailscaled config: %w", err) } mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON) } - // The config sha256 sum is a value for a hash annotation used to trigger - // pod restarts when tailscaled config changes. Any config changes apply - // to all replicas, so it is sufficient to only hash the config for the - // first replica. - // - // In future, we're aiming to eliminate restarts altogether and have - // pods dynamically reload their config when it changes. - if i == 0 { - sum := sha256.New() - for _, cfg := range configs { - // Zero out the auth key so it doesn't affect the sha256 hash when we - // remove it from the config after the pods have all authed. Otherwise - // all the pods will need to restart immediately after authing. - cfg.AuthKey = nil - b, err := json.Marshal(cfg) - if err != nil { - return "", err - } - if _, err := sum.Write(b); err != nil { - return "", err - } - } - - configSHA256Sum = fmt.Sprintf("%x", sum.Sum(nil)) - } - if existingCfgSecret != nil { if !apiequality.Semantic.DeepEqual(existingCfgSecret, cfgSecret) { logger.Debugf("Updating the existing ProxyGroup config Secret %s", cfgSecret.Name) if err := r.Update(ctx, cfgSecret); err != nil { - return "", err + return err } } } else { logger.Debugf("Creating a new config Secret %s for the ProxyGroup", cfgSecret.Name) if err := r.Create(ctx, cfgSecret); err != nil { - return "", err + return err } } } - return configSHA256Sum, nil + return nil } // ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup @@ -707,24 +655,3 @@ type nodeMetadata struct { tsID tailcfg.StableNodeID dnsName string } - -// capVerForPG returns best effort capability version for the given ProxyGroup. It attempts to find it by looking at the -// Secret + Pod for the replica with ordinal 0. Returns -1 if it is not possible to determine the capability version -// (i.e there is no Pod yet). -func (r *ProxyGroupReconciler) capVerForPG(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (tailcfg.CapabilityVersion, error) { - metas, err := r.getNodeMetadata(ctx, pg) - if err != nil { - return -1, fmt.Errorf("error getting node metadata: %w", err) - } - if len(metas) == 0 { - return -1, nil - } - dev, err := deviceInfo(metas[0].stateSecret, metas[0].podUID, logger) - if err != nil { - return -1, fmt.Errorf("error getting device info: %w", err) - } - if dev == nil { - return -1, nil - } - return dev.capver, nil -} diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index f3f87aaacf663..c556ae94a0de4 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -30,7 +30,6 @@ import ( "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/types/ptr" - "tailscale.com/util/mak" ) const testProxyImage = "tailscale/tailscale:test" @@ -40,7 +39,6 @@ var defaultProxyClassAnnotations = map[string]string{ } func TestProxyGroup(t *testing.T) { - const initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196" pc := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{ @@ -98,7 +96,7 @@ func TestProxyGroup(t *testing.T) { tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, false, "", pc) + expectProxyGroupResources(t, fc, pg, false, pc) }) t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) { @@ -119,11 +117,11 @@ func TestProxyGroup(t *testing.T) { tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, "", pc) + expectProxyGroupResources(t, fc, pg, true, pc) if expected := 1; reconciler.egressProxyGroups.Len() != expected { t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) } - expectProxyGroupResources(t, fc, pg, true, "", pc) + expectProxyGroupResources(t, fc, pg, true, pc) keyReq := tailscale.KeyCapabilities{ Devices: tailscale.KeyDeviceCapabilities{ Create: tailscale.KeyDeviceCreateCapabilities{ @@ -155,7 +153,7 @@ func TestProxyGroup(t *testing.T) { } tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) + expectProxyGroupResources(t, fc, pg, true, pc) }) t.Run("scale_up_to_3", func(t *testing.T) { @@ -166,7 +164,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) + expectProxyGroupResources(t, fc, pg, true, pc) addNodeIDToStateSecrets(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) @@ -176,7 +174,7 @@ func TestProxyGroup(t *testing.T) { TailnetIPs: []string{"1.2.3.4", "::1"}, }) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) + expectProxyGroupResources(t, fc, pg, true, pc) }) t.Run("scale_down_to_1", func(t *testing.T) { @@ -189,21 +187,7 @@ func TestProxyGroup(t *testing.T) { pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) - }) - - t.Run("trigger_config_change_and_observe_new_config_hash", func(t *testing.T) { - pc.Spec.TailscaleConfig = &tsapi.TailscaleConfig{ - AcceptRoutes: true, - } - mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { - p.Spec = pc.Spec - }) - - expectReconciled(t, reconciler, "", pg.Name) - - expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74", pc) + expectProxyGroupResources(t, fc, pg, true, pc) }) t.Run("enable_metrics", func(t *testing.T) { @@ -608,7 +592,7 @@ func verifyEnvVarNotPresent(t *testing.T, sts *appsv1.StatefulSet, name string) } } -func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string, proxyClass *tsapi.ProxyClass) { +func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, proxyClass *tsapi.ProxyClass) { t.Helper() role := pgRole(pg, tsNamespace) @@ -619,9 +603,6 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox t.Fatal(err) } statefulSet.Annotations = defaultProxyClassAnnotations - if cfgHash != "" { - mak.Set(&statefulSet.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, cfgHash) - } if shouldExist { expectEqual(t, fc, role) diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 70b25f2d28784..4c7c3ac6741a2 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -7,7 +7,6 @@ package main import ( "context" - "crypto/sha256" _ "embed" "encoding/json" "errors" @@ -91,8 +90,6 @@ const ( podAnnotationLastSetClusterDNSName = "tailscale.com/operator-last-set-cluster-dns-name" podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip" podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn" - // podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents. - podAnnotationLastSetConfigFileHash = "tailscale.com/operator-last-set-config-file-hash" proxyTypeEgress = "egress_service" proxyTypeIngressService = "ingress_service" @@ -110,7 +107,7 @@ var ( // tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods. tailscaleManagedLabels = []string{kubetypes.LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"} // tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods. - tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash} + tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN} ) type tailscaleSTSConfig struct { @@ -201,11 +198,11 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga } sts.ProxyClass = proxyClass - secretName, tsConfigHash, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc) + secretName, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc) if err != nil { return nil, fmt.Errorf("failed to create or get API key secret: %w", err) } - _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash) + _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName) if err != nil { return nil, fmt.Errorf("failed to reconcile statefulset: %w", err) } @@ -335,7 +332,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec }) } -func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaledConfigs, _ error) { +func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName string, configs tailscaledConfigs, _ error) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ // Hardcode a -0 suffix so that in future, if we support @@ -351,7 +348,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName()) orig = secret.DeepCopy() } else if !apierrors.IsNotFound(err) { - return "", "", nil, err + return "", nil, err } var authKey string @@ -361,13 +358,13 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * // ACME account key. sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels) if err != nil { - return "", "", nil, err + return "", nil, err } if sts != nil { // StatefulSet exists, so we have already created the secret. // If the secret is missing, they should delete the StatefulSet. logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName()) - return "", "", nil, nil + return "", nil, nil } // Create API Key secret which is going to be used by the statefulset // to authenticate with Tailscale. @@ -378,25 +375,20 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * } authKey, err = newAuthKey(ctx, a.tsClient, tags) if err != nil { - return "", "", nil, err + return "", nil, err } } configs, err := tailscaledConfig(stsC, authKey, orig) if err != nil { - return "", "", nil, fmt.Errorf("error creating tailscaled config: %w", err) + return "", nil, fmt.Errorf("error creating tailscaled config: %w", err) } - hash, err = tailscaledConfigHash(configs) - if err != nil { - return "", "", nil, fmt.Errorf("error calculating hash of tailscaled configs: %w", err) - } - latest := tailcfg.CapabilityVersion(-1) var latestConfig ipn.ConfigVAlpha for key, val := range configs { fn := tsoperator.TailscaledConfigFileName(key) b, err := json.Marshal(val) if err != nil { - return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err) + return "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err) } mak.Set(&secret.StringData, fn, string(b)) if key > latest { @@ -408,7 +400,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * if stsC.ServeConfig != nil { j, err := json.Marshal(stsC.ServeConfig) if err != nil { - return "", "", nil, err + return "", nil, err } mak.Set(&secret.StringData, "serve-config", string(j)) } @@ -416,15 +408,15 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * if orig != nil { logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig)) if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil { - return "", "", nil, err + return "", nil, err } } else { logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig)) if err := a.Create(ctx, secret); err != nil { - return "", "", nil, err + return "", nil, err } } - return secret.Name, hash, configs, nil + return secret.Name, configs, nil } // sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted @@ -535,7 +527,7 @@ var proxyYaml []byte //go:embed deploy/manifests/userspace-proxy.yaml var userspaceProxyYaml []byte -func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) { +func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret string) (*appsv1.StatefulSet, error) { ss := new(appsv1.StatefulSet) if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil { @@ -662,11 +654,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S }) } - dev, err := a.DeviceInfo(ctx, sts.ChildResourceLabels, logger) - if err != nil { - return nil, fmt.Errorf("failed to get device info: %w", err) - } - app, err := appInfoForProxy(sts) if err != nil { // No need to error out if now or in future we end up in a @@ -685,25 +672,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S ss = applyProxyClassToStatefulSet(sts.ProxyClass, ss, sts, logger) } updateSS := func(s *appsv1.StatefulSet) { - // This is a temporary workaround to ensure that proxies with capver older than 110 - // are restarted when tailscaled configfile contents have changed. - // This workaround ensures that: - // 1. The hash mechanism is used to trigger pod restarts for proxies below capver 110. - // 2. Proxies above capver are not unnecessarily restarted when the configfile contents change. - // 3. If the hash has alreay been set, but the capver is above 110, the old hash is preserved to avoid - // unnecessary pod restarts that could result in an update loop where capver cannot be determined for a - // restarting Pod and the hash is re-added again. - // Note that the hash annotation is only set on updates not creation, because if the StatefulSet is - // being created, there is no need for a restart. - // TODO(irbekrm): remove this in 1.84. - hash := tsConfigHash - if dev == nil || dev.capver >= 110 { - hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash] - } s.Spec = ss.Spec - if hash != "" { - mak.Set(&s.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, hash) - } s.ObjectMeta.Labels = ss.Labels s.ObjectMeta.Annotations = ss.Annotations } @@ -937,8 +906,7 @@ func readAuthKey(secret *corev1.Secret, key string) (*string, error) { } // tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy -// state and auth key and returns tailscaled config files for currently supported proxy versions and a hash of that -// configuration. +// state and auth key and returns tailscaled config files for currently supported proxy versions. func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { conf := &ipn.ConfigVAlpha{ Version: "alpha0", @@ -1031,27 +999,6 @@ type ptrObject[T any] interface { type tailscaledConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha -// hashBytes produces a hash for the provided tailscaled config that is the same across -// different invocations of this code. We do not use the -// tailscale.com/deephash.Hash here because that produces a different hash for -// the same value in different tailscale builds. The hash we are producing here -// is used to determine if the container running the Connector Tailscale node -// needs to be restarted. The container does not need restarting when the only -// thing that changed is operator version (the hash is also exposed to users via -// an annotation and might be confusing if it changes without the config having -// changed). -func tailscaledConfigHash(c tailscaledConfigs) (string, error) { - b, err := json.Marshal(c) - if err != nil { - return "", fmt.Errorf("error marshalling tailscaled configs: %w", err) - } - h := sha256.New() - if _, err = h.Write(b); err != nil { - return "", fmt.Errorf("error calculating hash: %w", err) - } - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - // createOrMaybeUpdate adds obj to the k8s cluster, unless the object already exists, // in which case update is called to make changes to it. If update is nil or returns // an error, the object is returned unmodified. diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 619aecc56816e..56542700d951c 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -62,7 +62,6 @@ type configOpts struct { subnetRoutes string isExitNode bool isAppConnector bool - confFileHash string serveConfig *ipn.ServeConfig shouldEnableForwardingClusterTrafficViaIngress bool proxyClass string // configuration from the named ProxyClass should be applied to proxy resources @@ -120,9 +119,6 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef ReadOnly: true, MountPath: "/etc/tsconfig", }} - if opts.confFileHash != "" { - mak.Set(&annots, "tailscale.com/operator-last-set-config-file-hash", opts.confFileHash) - } if opts.firewallMode != "" { tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ Name: "TS_DEBUG_FIREWALL_MODE", @@ -358,10 +354,6 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps }, }, } - ss.Spec.Template.Annotations = map[string]string{} - if opts.confFileHash != "" { - ss.Spec.Template.Annotations["tailscale.com/operator-last-set-config-file-hash"] = opts.confFileHash - } // If opts.proxyClass is set, retrieve the ProxyClass and apply // configuration from that to the StatefulSet. if opts.proxyClass != "" { @@ -842,17 +834,6 @@ func (c *fakeTSClient) Deleted() []string { return c.deleted } -// removeHashAnnotation can be used to remove declarative tailscaled config hash -// annotation from proxy StatefulSets to make the tests more maintainable (so -// that we don't have to change the annotation in each test case after any -// change to the configfile contents). -func removeHashAnnotation(sts *appsv1.StatefulSet) { - delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash) - if len(sts.Spec.Template.Annotations) == 0 { - sts.Spec.Template.Annotations = nil - } -} - func removeResourceReqs(sts *appsv1.StatefulSet) { if sts != nil { sts.Spec.Template.Spec.Resources = nil From 5a52f80c4cb4fc231faec2790a088c8cb856397f Mon Sep 17 00:00:00 2001 From: okunamayanad Date: Tue, 17 Jun 2025 04:50:01 +0300 Subject: [PATCH 093/263] docs: fix typo in commit-messages.md Updates: #cleanup Signed-off-by: okunamayanad --- docs/commit-messages.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commit-messages.md b/docs/commit-messages.md index b3881eaeb9fbb..79b16e4c6f6f2 100644 --- a/docs/commit-messages.md +++ b/docs/commit-messages.md @@ -65,7 +65,7 @@ Notably, for the subject (the first line of description): | `foo/bar:fix memory leak` | BAD: no space after colon | | `foo/bar : fix memory leak` | BAD: space before colon | | `foo/bar: fix memory leak Fixes #123` | BAD: the "Fixes" shouldn't be part of the title | - | `!fixup reviewer feedback` | BAD: we don't check in fixup commits; the history should always bissect to a clean, working tree | + | `!fixup reviewer feedback` | BAD: we don't check in fixup commits; the history should always bisect to a clean, working tree | For the body (the rest of the description): From 9af42f425ca48ca2e0dee9b3524ea586675069c6 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 19 Jun 2025 10:56:15 -0700 Subject: [PATCH 094/263] .github/workflows: shard the Windows builder It's one of the slower ones, so split it up into chunks. Updates tailscale/corp#28679 Change-Id: I16a5ba667678bf238c84417a51dda61baefbecf7 Signed-off-by: Brad Fitzpatrick --- .github/workflows/test.yml | 21 +++++++++++++++++---- cmd/testwrapper/testwrapper.go | 10 ++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d8ab863ce422..722a73f93ce33 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -210,6 +210,17 @@ jobs: windows: runs-on: windows-2022 needs: gomod-cache + name: Windows (${{ matrix.name || matrix.shard}}) + strategy: + fail-fast: false # don't abort the entire matrix if one element fails + matrix: + include: + - key: "win-bench" + name: "benchmarks" + - key: "win-shard-1-2" + shard: "1/2" + - key: "win-shard-2-2" + shard: "2/2" steps: - name: checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -237,14 +248,16 @@ jobs: ~\AppData\Local\go-build # The -2- here should be incremented when the scheme of data to be # cached changes (e.g. path above changes). - key: ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} + key: ${{ github.job }}-${{ matrix.key }}-go-2-${{ hashFiles('**/go.sum') }}-${{ github.run_id }} restore-keys: | - ${{ github.job }}-${{ runner.os }}-go-2-${{ hashFiles('**/go.sum') }} - ${{ github.job }}-${{ runner.os }}-go-2- + ${{ github.job }}-${{ matrix.key }}-go-2-${{ hashFiles('**/go.sum') }} + ${{ github.job }}-${{ matrix.key }}-go-2- - name: test + if: matrix.key != 'win-bench' # skip on bench builder working-directory: src - run: go run ./cmd/testwrapper ./... + run: go run ./cmd/testwrapper sharded:${{ matrix.shard }} - name: bench all + if: matrix.key == 'win-bench' working-directory: src # Don't use -bench=. -benchtime=1x. # Somewhere in the layers (powershell?) diff --git a/cmd/testwrapper/testwrapper.go b/cmd/testwrapper/testwrapper.go index 53c1b1d05f7ca..173edee733f04 100644 --- a/cmd/testwrapper/testwrapper.go +++ b/cmd/testwrapper/testwrapper.go @@ -213,6 +213,16 @@ func main() { return } + // As a special case, if the packages looks like "sharded:1/2" then shell out to + // ./tool/listpkgs to cut up the package list pieces for each sharded builder. + if nOfM, ok := strings.CutPrefix(packages[0], "sharded:"); ok && len(packages) == 1 { + out, err := exec.Command("go", "run", "tailscale.com/tool/listpkgs", "-shard", nOfM, "./...").Output() + if err != nil { + log.Fatalf("failed to list packages for sharded test: %v", err) + } + packages = strings.Split(strings.TrimSpace(string(out)), "\n") + } + ctx := context.Background() type nextRun struct { tests []*packageTests From ca06d944c5622e89ce1ae8e507149af2f858d2a0 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 19 Jun 2025 18:35:49 -0700 Subject: [PATCH 095/263] .github/workflows: try running Windows jobs on bigger VMs Updates tailscale/corp#28679 Change-Id: Iee3f3820d2d8308fff3494e300ad3939e3ed2598 Signed-off-by: Brad Fitzpatrick --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 722a73f93ce33..2ebb82a8582c8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -208,7 +208,10 @@ jobs: find $(go env GOCACHE) -type f -mmin +90 -delete windows: - runs-on: windows-2022 + # windows-8vpu is a 2022 GitHub-managed runner in our + # org with 8 cores and 32 GB of RAM: + # https://github.com/organizations/tailscale/settings/actions/github-hosted-runners/1 + runs-on: windows-8vcpu needs: gomod-cache name: Windows (${{ matrix.name || matrix.shard}}) strategy: From bb085cfa3e434a5a8da2d27eca6e94c49bebc036 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 19 Jun 2025 20:48:50 -0700 Subject: [PATCH 096/263] tool: add go toolchain wrapper for Windows go.cmd lets you run just "./tool/go" on Windows the same as Linux/Darwin. The batch script (go.md) then just invokes PowerShell which is more powerful than batch. I wanted this while debugging Windows CI performance by reproducing slow tests on my local Windows laptop. Updates tailscale/corp#28679 Change-Id: I6e520968da3cef3032091c1c4f4237f663cefcab Signed-off-by: Brad Fitzpatrick --- .github/workflows/test.yml | 16 +++++++++- tool/go.cmd | 2 ++ tool/go.ps1 | 64 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tool/go.cmd create mode 100644 tool/go.ps1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ebb82a8582c8..2e80b44dcc4d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -220,6 +220,8 @@ jobs: include: - key: "win-bench" name: "benchmarks" + - key: "win-tool-go" + name: "./tool/go" - key: "win-shard-1-2" shard: "1/2" - key: "win-shard-2-2" @@ -231,12 +233,14 @@ jobs: path: src - name: Install Go + if: matrix.key != 'win-tool-go' uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: go-version-file: src/go.mod cache: false - name: Restore Go module cache + if: matrix.key != 'win-tool-go' uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: gomodcache @@ -244,6 +248,7 @@ jobs: enableCrossOsArchive: true - name: Restore Cache + if: matrix.key != 'win-tool-go' uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: | @@ -255,10 +260,17 @@ jobs: restore-keys: | ${{ github.job }}-${{ matrix.key }}-go-2-${{ hashFiles('**/go.sum') }} ${{ github.job }}-${{ matrix.key }}-go-2- + + - name: test-tool-go + if: matrix.key == 'win-tool-go' + working-directory: src + run: ./tool/go version + - name: test - if: matrix.key != 'win-bench' # skip on bench builder + if: matrix.key != 'win-bench' && matrix.key != 'win-tool-go' # skip on bench builder working-directory: src run: go run ./cmd/testwrapper sharded:${{ matrix.shard }} + - name: bench all if: matrix.key == 'win-bench' working-directory: src @@ -266,7 +278,9 @@ jobs: # Somewhere in the layers (powershell?) # the equals signs cause great confusion. run: go test ./... -bench . -benchtime 1x -run "^$" + - name: Tidy cache + if: matrix.key != 'win-tool-go' working-directory: src shell: bash run: | diff --git a/tool/go.cmd b/tool/go.cmd new file mode 100644 index 0000000000000..51bace110d59b --- /dev/null +++ b/tool/go.cmd @@ -0,0 +1,2 @@ +@echo off +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0go.ps1" %* diff --git a/tool/go.ps1 b/tool/go.ps1 new file mode 100644 index 0000000000000..49313ffbabee9 --- /dev/null +++ b/tool/go.ps1 @@ -0,0 +1,64 @@ +<# + go.ps1 – Tailscale Go toolchain fetching wrapper for Windows/PowerShell + • Reads go.toolchain.rev one dir above this script + • If the requested commit hash isn't cached, downloads and unpacks + https://github.com/tailscale/go/releases/download/build-${REV}/${OS}-${ARCH}.tar.gz + • Finally execs the toolchain's "go" binary, forwarding all args & exit-code +#> + +param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]] $Args +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if ($env:CI -eq 'true' -and $env:NODEBUG -ne 'true') { + $VerbosePreference = 'Continue' +} + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..') +$REV = (Get-Content (Join-Path $repoRoot 'go.toolchain.rev') -Raw).Trim() + +if ([IO.Path]::IsPathRooted($REV)) { + $toolchain = $REV +} else { + if (-not [string]::IsNullOrWhiteSpace($env:TSGO_CACHE_ROOT)) { + $cacheRoot = $env:TSGO_CACHE_ROOT + } else { + $cacheRoot = Join-Path $env:USERPROFILE '.cache\tsgo' + } + + $toolchain = Join-Path $cacheRoot $REV + $marker = "$toolchain.extracted" + + if (-not (Test-Path $marker)) { + Write-Host "# Downloading Go toolchain $REV" -ForegroundColor Cyan + if (Test-Path $toolchain) { Remove-Item -Recurse -Force $toolchain } + + # Removing the marker file again (even though it shouldn't still exist) + # because the equivalent Bash script also does so (to guard against + # concurrent cache fills?). + # TODO(bradfitz): remove this and add some proper locking instead? + if (Test-Path $marker ) { Remove-Item -Force $marker } + + New-Item -ItemType Directory -Path $cacheRoot -Force | Out-Null + + $url = "https://github.com/tailscale/go/releases/download/build-$REV/windows-amd64.tar.gz" + $tgz = "$toolchain.tar.gz" + Invoke-WebRequest -Uri $url -OutFile $tgz -UseBasicParsing -ErrorAction Stop + + New-Item -ItemType Directory -Path $toolchain -Force | Out-Null + tar --strip-components=1 -xzf $tgz -C $toolchain + Remove-Item $tgz + Set-Content -Path $marker -Value $REV + } +} + +$goExe = Join-Path $toolchain 'bin\go.exe' +if (-not (Test-Path $goExe)) { throw "go executable not found at $goExe" } + +& $goExe @Args +exit $LASTEXITCODE + From 12e92b1b085b72e900e001d2bd5c827ed395bd57 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 20 Jun 2025 10:25:42 -0700 Subject: [PATCH 097/263] tsconsensus: skipping slow non-applicable tests on Windows for now Updates #16340 Change-Id: I61b0186295c095f99c5be81dc4dced5853025d35 Signed-off-by: Brad Fitzpatrick --- tsconsensus/tsconsensus_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tsconsensus/tsconsensus_test.go b/tsconsensus/tsconsensus_test.go index d1b92f8a489f7..bfb6b3e0688cc 100644 --- a/tsconsensus/tsconsensus_test.go +++ b/tsconsensus/tsconsensus_test.go @@ -17,6 +17,7 @@ import ( "net/netip" "os" "path/filepath" + "runtime" "strings" "sync" "testing" @@ -37,6 +38,7 @@ import ( "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/types/views" + "tailscale.com/util/cibuild" "tailscale.com/util/racebuild" ) @@ -113,6 +115,9 @@ func (f *fsm) Restore(rc io.ReadCloser) error { } func testConfig(t *testing.T) { + if runtime.GOOS == "windows" && cibuild.On() { + t.Skip("cmd/natc isn't supported on Windows, so skipping tsconsensus tests on CI for now; see https://github.com/tailscale/tailscale/issues/16340") + } // -race AND Parallel makes things start to take too long. if !racebuild.On { t.Parallel() From d3bb34c628b01953c1f064d75d01c0a41e4d41ab Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Fri, 20 Jun 2025 15:00:28 -0700 Subject: [PATCH 098/263] wgengine/magicsock: generate relay server set from tailnet policy (#16331) Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/magicsock.go | 173 +++++++++++++++++--- wgengine/magicsock/magicsock_test.go | 202 +++++++++++++++++++++++- wgengine/magicsock/relaymanager.go | 29 ++++ wgengine/magicsock/relaymanager_test.go | 6 + 4 files changed, 386 insertions(+), 24 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index bfc7afba95149..0679a4ebd049e 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -14,6 +14,7 @@ import ( "expvar" "fmt" "io" + "math" "net" "net/netip" "reflect" @@ -348,17 +349,19 @@ type Conn struct { // magicsock could do with any complexity reduction it can get. netInfoLast *tailcfg.NetInfo - derpMap *tailcfg.DERPMap // nil (or zero regions/nodes) means DERP is disabled - peers views.Slice[tailcfg.NodeView] // from last onNodeViewsUpdate update - lastFlags debugFlags // at time of last onNodeViewsUpdate - firstAddrForTest netip.Addr // from last onNodeViewsUpdate update; for tests only - privateKey key.NodePrivate // WireGuard private key for this node - everHadKey bool // whether we ever had a non-zero private key - myDerp int // nearest DERP region ID; 0 means none/unknown - homeless bool // if true, don't try to find & stay conneted to a DERP home (myDerp will stay 0) - derpStarted chan struct{} // closed on first connection to DERP; for tests & cleaner Close - activeDerp map[int]activeDerp // DERP regionID -> connection to a node in that region - prevDerp map[int]*syncs.WaitGroupChan + derpMap *tailcfg.DERPMap // nil (or zero regions/nodes) means DERP is disabled + self tailcfg.NodeView // from last onNodeViewsUpdate + peers views.Slice[tailcfg.NodeView] // from last onNodeViewsUpdate, sorted by Node.ID; Note: [netmap.NodeMutation]'s rx'd in onNodeMutationsUpdate are never applied + filt *filter.Filter // from last onFilterUpdate + relayClientEnabled bool // whether we can allocate UDP relay endpoints on UDP relay servers + lastFlags debugFlags // at time of last onNodeViewsUpdate + privateKey key.NodePrivate // WireGuard private key for this node + everHadKey bool // whether we ever had a non-zero private key + myDerp int // nearest DERP region ID; 0 means none/unknown + homeless bool // if true, don't try to find & stay conneted to a DERP home (myDerp will stay 0) + derpStarted chan struct{} // closed on first connection to DERP; for tests & cleaner Close + activeDerp map[int]activeDerp // DERP regionID -> connection to a node in that region + prevDerp map[int]*syncs.WaitGroupChan // derpRoute contains optional alternate routes to use as an // optimization instead of contacting a peer via their home @@ -516,7 +519,7 @@ func (o *Options) derpActiveFunc() func() { // this type out of magicsock. type NodeViewsUpdate struct { SelfNode tailcfg.NodeView - Peers []tailcfg.NodeView + Peers []tailcfg.NodeView // sorted by Node.ID } // NodeMutationsUpdate represents an update event of one or more @@ -2555,38 +2558,160 @@ func (c *Conn) SetProbeUDPLifetime(v bool) { func capVerIsRelayCapable(version tailcfg.CapabilityVersion) bool { // TODO(jwhited): implement once capVer is bumped - return false + return version == math.MinInt32 +} + +func capVerIsRelayServerCapable(version tailcfg.CapabilityVersion) bool { + // TODO(jwhited): implement once capVer is bumped + return version == math.MinInt32 } +// onFilterUpdate is called when a [FilterUpdate] is received over the +// [eventbus.Bus]. func (c *Conn) onFilterUpdate(f FilterUpdate) { - // TODO(jwhited): implement + c.mu.Lock() + c.filt = f.Filter + self := c.self + peers := c.peers + relayClientEnabled := c.relayClientEnabled + c.mu.Unlock() // release c.mu before potentially calling c.updateRelayServersSet which is O(m * n) + + if !relayClientEnabled { + // Early return if we cannot operate as a relay client. + return + } + + // The filter has changed, and we are operating as a relay server client. + // Re-evaluate it in order to produce an updated relay server set. + c.updateRelayServersSet(f.Filter, self, peers) +} + +// updateRelayServersSet iterates all peers, evaluating filt for each one in +// order to determine which peers are relay server candidates. filt, self, and +// peers are passed as args (vs c.mu-guarded fields) to enable callers to +// release c.mu before calling as this is O(m * n) (we iterate all cap rules 'm' +// in filt for every peer 'n'). +// TODO: Optimize this so that it's not O(m * n). This might involve: +// 1. Changes to [filter.Filter], e.g. adding a CapsWithValues() to check for +// a given capability instead of building and returning a map of all of +// them. +// 2. Moving this work upstream into [nodeBackend] or similar, and publishing +// the computed result over the eventbus instead. +func (c *Conn) updateRelayServersSet(filt *filter.Filter, self tailcfg.NodeView, peers views.Slice[tailcfg.NodeView]) { + relayServers := make(set.Set[netip.AddrPort]) + for _, peer := range peers.All() { + peerAPI := peerAPIIfCandidateRelayServer(filt, self, peer) + if peerAPI.IsValid() { + relayServers.Add(peerAPI) + } + } + c.relayManager.handleRelayServersSet(relayServers) +} + +// peerAPIIfCandidateRelayServer returns the peer API address of peer if it +// is considered to be a candidate relay server upon evaluation against filt and +// self, otherwise it returns a zero value. +func peerAPIIfCandidateRelayServer(filt *filter.Filter, self, peer tailcfg.NodeView) netip.AddrPort { + if filt == nil || + !self.Valid() || + !peer.Valid() || + !capVerIsRelayServerCapable(peer.Cap()) || + !peer.Hostinfo().Valid() { + return netip.AddrPort{} + } + for _, peerPrefix := range peer.Addresses().All() { + if !peerPrefix.IsSingleIP() { + continue + } + peerAddr := peerPrefix.Addr() + for _, selfPrefix := range self.Addresses().All() { + if !selfPrefix.IsSingleIP() { + continue + } + selfAddr := selfPrefix.Addr() + if selfAddr.BitLen() == peerAddr.BitLen() { // same address family + if filt.CapsWithValues(peerAddr, selfAddr).HasCapability(tailcfg.PeerCapabilityRelayTarget) { + for _, s := range peer.Hostinfo().Services().All() { + if peerAddr.Is4() && s.Proto == tailcfg.PeerAPI4 || + peerAddr.Is6() && s.Proto == tailcfg.PeerAPI6 { + return netip.AddrPortFrom(peerAddr, s.Port) + } + } + return netip.AddrPort{} // no peerAPI + } else { + // [nodeBackend.peerCapsLocked] only returns/considers the + // [tailcfg.PeerCapMap] between the passed src and the + // _first_ host (/32 or /128) address for self. We are + // consistent with that behavior here. If self and peer + // host addresses are of the same address family they either + // have the capability or not. We do not check against + // additional host addresses of the same address family. + return netip.AddrPort{} + } + } + } + } + return netip.AddrPort{} } // onNodeViewsUpdate is called when a [NodeViewsUpdate] is received over the // [eventbus.Bus]. func (c *Conn) onNodeViewsUpdate(update NodeViewsUpdate) { + peersChanged := c.updateNodes(update) + + relayClientEnabled := update.SelfNode.Valid() && + update.SelfNode.HasCap(tailcfg.NodeAttrRelayClient) && + envknob.UseWIPCode() + + c.mu.Lock() + relayClientChanged := c.relayClientEnabled != relayClientEnabled + c.relayClientEnabled = relayClientEnabled + filt := c.filt + self := c.self + peers := c.peers + c.mu.Unlock() // release c.mu before potentially calling c.updateRelayServersSet which is O(m * n) + + if peersChanged || relayClientChanged { + if !relayClientEnabled { + c.relayManager.handleRelayServersSet(nil) + } else { + c.updateRelayServersSet(filt, self, peers) + } + } +} + +// updateNodes updates [Conn] to reflect the [tailcfg.NodeView]'s contained +// in update. It returns true if update.Peers was unequal to c.peers, otherwise +// false. +func (c *Conn) updateNodes(update NodeViewsUpdate) (peersChanged bool) { c.mu.Lock() defer c.mu.Unlock() if c.closed { - return + return false } priorPeers := c.peers metricNumPeers.Set(int64(len(update.Peers))) - // Update c.netMap regardless, before the following early return. + // Update c.self & c.peers regardless, before the following early return. + c.self = update.SelfNode curPeers := views.SliceOf(update.Peers) c.peers = curPeers + // [debugFlags] are mutable in [Conn.SetSilentDisco] & + // [Conn.SetProbeUDPLifetime]. These setters are passed [controlknobs.Knobs] + // values by [ipnlocal.LocalBackend] around netmap reception. + // [controlknobs.Knobs] are simply self [tailcfg.NodeCapability]'s. They are + // useful as a global view of notable feature toggles, but the magicsock + // setters are completely unnecessary as we have the same values right here + // (update.SelfNode.Capabilities) at a time they are considered most + // up-to-date. + // TODO: mutate [debugFlags] here instead of in various [Conn] setters. flags := c.debugFlagsLocked() - if update.SelfNode.Valid() && update.SelfNode.Addresses().Len() > 0 { - c.firstAddrForTest = update.SelfNode.Addresses().At(0).Addr() - } else { - c.firstAddrForTest = netip.Addr{} - } - if nodesEqual(priorPeers, curPeers) && c.lastFlags == flags { + peersChanged = !nodesEqual(priorPeers, curPeers) + if !peersChanged && c.lastFlags == flags { // The rest of this function is all adjusting state for peers that have // changed. But if the set of peers is equal and the debug flags (for // silent disco and probe UDP lifetime) haven't changed, there is no @@ -2728,6 +2853,8 @@ func (c *Conn) onNodeViewsUpdate(update NodeViewsUpdate) { delete(c.discoInfo, dk) } } + + return peersChanged } func devPanicf(format string, a ...any) { @@ -3245,7 +3372,7 @@ func simpleDur(d time.Duration) time.Duration { } // onNodeMutationsUpdate is called when a [NodeMutationsUpdate] is received over -// the [eventbus.Bus]. +// the [eventbus.Bus]. Note: It does not apply these mutations to c.peers. func (c *Conn) onNodeMutationsUpdate(update NodeMutationsUpdate) { c.mu.Lock() defer c.mu.Unlock() diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 7fa062fa87df2..8aa9a09d2c15a 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -19,6 +19,7 @@ import ( "net/http/httptest" "net/netip" "os" + "reflect" "runtime" "strconv" "strings" @@ -71,6 +72,7 @@ import ( "tailscale.com/util/slicesx" "tailscale.com/util/usermetric" "tailscale.com/wgengine/filter" + "tailscale.com/wgengine/filter/filtertype" "tailscale.com/wgengine/wgcfg" "tailscale.com/wgengine/wgcfg/nmcfg" "tailscale.com/wgengine/wglog" @@ -275,7 +277,10 @@ func (s *magicStack) Status() *ipnstate.Status { func (s *magicStack) IP() netip.Addr { for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { s.conn.mu.Lock() - addr := s.conn.firstAddrForTest + var addr netip.Addr + if s.conn.self.Valid() && s.conn.self.Addresses().Len() > 0 { + addr = s.conn.self.Addresses().At(0).Addr() + } s.conn.mu.Unlock() if addr.IsValid() { return addr @@ -3378,3 +3383,198 @@ func Test_virtualNetworkID(t *testing.T) { }) } } + +func Test_peerAPIIfCandidateRelayServer(t *testing.T) { + selfOnlyIPv4 := &tailcfg.Node{ + Cap: math.MinInt32, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("1.1.1.1/32"), + }, + } + selfOnlyIPv6 := selfOnlyIPv4.Clone() + selfOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::1/128") + + peerHostinfo := &tailcfg.Hostinfo{ + Services: []tailcfg.Service{ + { + Proto: tailcfg.PeerAPI4, + Port: 4, + }, + { + Proto: tailcfg.PeerAPI6, + Port: 6, + }, + }, + } + peerOnlyIPv4 := &tailcfg.Node{ + Cap: math.MinInt32, + CapMap: map[tailcfg.NodeCapability][]tailcfg.RawMessage{ + tailcfg.NodeAttrRelayServer: nil, + }, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("2.2.2.2/32"), + }, + Hostinfo: peerHostinfo.View(), + } + + peerOnlyIPv6 := peerOnlyIPv4.Clone() + peerOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::2/128") + + peerOnlyIPv4ZeroCapVer := peerOnlyIPv4.Clone() + peerOnlyIPv4ZeroCapVer.Cap = 0 + + peerOnlyIPv4NilHostinfo := peerOnlyIPv4.Clone() + peerOnlyIPv4NilHostinfo.Hostinfo = tailcfg.HostinfoView{} + + tests := []struct { + name string + filt *filter.Filter + self tailcfg.NodeView + peer tailcfg.NodeView + want netip.AddrPort + }{ + { + name: "match v4", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("1.1.1.1/32"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfOnlyIPv4.View(), + peer: peerOnlyIPv4.View(), + want: netip.MustParseAddrPort("2.2.2.2:4"), + }, + { + name: "match v6", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("::2/128")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("::1/128"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfOnlyIPv6.View(), + peer: peerOnlyIPv6.View(), + want: netip.MustParseAddrPort("[::2]:6"), + }, + { + name: "no match dst", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("::2/128")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("::3/128"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfOnlyIPv6.View(), + peer: peerOnlyIPv6.View(), + }, + { + name: "no match peer cap", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("::2/128")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("::1/128"), + Cap: tailcfg.PeerCapabilityIngress, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfOnlyIPv6.View(), + peer: peerOnlyIPv6.View(), + }, + { + name: "cap ver not relay capable", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("1.1.1.1/32"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: peerOnlyIPv4.View(), + peer: peerOnlyIPv4ZeroCapVer.View(), + }, + { + name: "nil filt", + filt: nil, + self: selfOnlyIPv4.View(), + peer: peerOnlyIPv4.View(), + }, + { + name: "nil self", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("1.1.1.1/32"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: tailcfg.NodeView{}, + peer: peerOnlyIPv4.View(), + }, + { + name: "nil peer", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("1.1.1.1/32"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfOnlyIPv4.View(), + peer: tailcfg.NodeView{}, + }, + { + name: "nil peer hostinfo", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("1.1.1.1/32"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfOnlyIPv4.View(), + peer: peerOnlyIPv4NilHostinfo.View(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := peerAPIIfCandidateRelayServer(tt.filt, tt.self, tt.peer); !reflect.DeepEqual(got, tt.want) { + t.Errorf("peerAPIIfCandidateRelayServer() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 2b636dc5758d1..3c8ceb2de8625 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -51,6 +51,7 @@ type relayManager struct { cancelWorkCh chan *endpoint newServerEndpointCh chan newRelayServerEndpointEvent rxHandshakeDiscoMsgCh chan relayHandshakeDiscoMsgEvent + serversCh chan set.Set[netip.AddrPort] discoInfoMu sync.Mutex // guards the following field discoInfoByServerDisco map[key.DiscoPublic]*relayHandshakeDiscoInfo @@ -174,7 +175,29 @@ func (r *relayManager) runLoop() { if !r.hasActiveWorkRunLoop() { return } + case serversUpdate := <-r.serversCh: + r.handleServersUpdateRunLoop(serversUpdate) + if !r.hasActiveWorkRunLoop() { + return + } + } + } +} + +func (r *relayManager) handleServersUpdateRunLoop(update set.Set[netip.AddrPort]) { + for k, v := range r.serversByAddrPort { + if !update.Contains(k) { + delete(r.serversByAddrPort, k) + delete(r.serversByDisco, v) + } + } + for _, v := range update.Slice() { + _, ok := r.serversByAddrPort[v] + if ok { + // don't zero known disco keys + continue } + r.serversByAddrPort[v] = key.DiscoPublic{} } } @@ -215,6 +238,7 @@ func (r *relayManager) init() { r.cancelWorkCh = make(chan *endpoint) r.newServerEndpointCh = make(chan newRelayServerEndpointEvent) r.rxHandshakeDiscoMsgCh = make(chan relayHandshakeDiscoMsgEvent) + r.serversCh = make(chan set.Set[netip.AddrPort]) r.runLoopStoppedCh = make(chan struct{}, 1) r.runLoopStoppedCh <- struct{}{} }) @@ -299,6 +323,11 @@ func (r *relayManager) handleGeneveEncapDiscoMsgNotBestAddr(dm disco.Message, di relayManagerInputEvent(r, nil, &r.rxHandshakeDiscoMsgCh, relayHandshakeDiscoMsgEvent{msg: dm, disco: di.discoKey, from: src.ap, vni: src.vni.get(), at: time.Now()}) } +// handleRelayServersSet handles an update of the complete relay server set. +func (r *relayManager) handleRelayServersSet(servers set.Set[netip.AddrPort]) { + relayManagerInputEvent(r, nil, &r.serversCh, servers) +} + // relayManagerInputEvent initializes [relayManager] if necessary, starts // relayManager.runLoop() if it is not running, and writes 'event' on 'eventCh'. // diff --git a/wgengine/magicsock/relaymanager_test.go b/wgengine/magicsock/relaymanager_test.go index be0582669c964..6055c2d72b4ef 100644 --- a/wgengine/magicsock/relaymanager_test.go +++ b/wgengine/magicsock/relaymanager_test.go @@ -4,10 +4,12 @@ package magicsock import ( + "net/netip" "testing" "tailscale.com/disco" "tailscale.com/types/key" + "tailscale.com/util/set" ) func TestRelayManagerInitAndIdle(t *testing.T) { @@ -26,4 +28,8 @@ func TestRelayManagerInitAndIdle(t *testing.T) { rm = relayManager{} rm.handleGeneveEncapDiscoMsgNotBestAddr(&disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, epAddr{}) <-rm.runLoopStoppedCh + + rm = relayManager{} + rm.handleRelayServersSet(make(set.Set[netip.AddrPort])) + <-rm.runLoopStoppedCh } From cd9b9a8cadfd03b9e304ca8a2ff0900d016387fc Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Fri, 20 Jun 2025 19:23:52 -0700 Subject: [PATCH 099/263] wgengine/magicsock: fix relay endpoint allocation URL (#16344) Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/relaymanager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 3c8ceb2de8625..81a71b20e22f1 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -737,7 +737,7 @@ func (r *relayManager) allocateSingleServer(ctx context.Context, wg *sync.WaitGr const reqTimeout = time.Second * 10 reqCtx, cancel := context.WithTimeout(ctx, reqTimeout) defer cancel() - req, err := http.NewRequestWithContext(reqCtx, httpm.POST, "http://"+server.String()+"/relay/endpoint", &b) + req, err := http.NewRequestWithContext(reqCtx, httpm.POST, "http://"+server.String()+"/v0/relay/endpoint", &b) if err != nil { return } From e935a28a196f4ccb212ed44c23b62f4e40a7f243 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Sat, 21 Jun 2025 19:09:19 -0700 Subject: [PATCH 100/263] wgengine/magicsock: set rxDiscoMsgCh field in relayHandshakeWork (#16349) Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/relaymanager.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 81a71b20e22f1..3e72ff0f08ae3 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -567,11 +567,12 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay // We're ready to start a new handshake. ctx, cancel := context.WithCancel(context.Background()) work := &relayHandshakeWork{ - ep: newServerEndpoint.ep, - se: newServerEndpoint.se, - doneCh: make(chan relayEndpointHandshakeWorkDoneEvent, 1), - ctx: ctx, - cancel: cancel, + ep: newServerEndpoint.ep, + se: newServerEndpoint.se, + rxDiscoMsgCh: make(chan relayHandshakeDiscoMsgEvent), + doneCh: make(chan relayEndpointHandshakeWorkDoneEvent, 1), + ctx: ctx, + cancel: cancel, } if byServerDisco == nil { byServerDisco = make(map[key.DiscoPublic]*relayHandshakeWork) From 61958f531c5c6a004415b46eb341f2dc289288cd Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Sat, 21 Jun 2025 19:09:36 -0700 Subject: [PATCH 101/263] wgengine/magicsock: set conn field in relayHandshakeDiscoMsgEvent (#16348) Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 2 +- wgengine/magicsock/magicsock.go | 6 +++--- wgengine/magicsock/relaymanager.go | 4 ++-- wgengine/magicsock/relaymanager_test.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 23316dcb454cf..fb5a28c2832fd 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1601,7 +1601,7 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd if src.vni.isSet() && src != de.bestAddr.epAddr { // "src" is not our bestAddr, but [relayManager] might be in the // middle of probing it, awaiting pong reception. Make it aware. - de.c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(m, di, src) + de.c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(de.c, m, di, src) return false } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 0679a4ebd049e..a96eaf3d800b9 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2053,7 +2053,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake c.logf("[unexpected] %T packets should not come from a relay server with Geneve control bit set", dm) return } - c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(challenge, di, src) + c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(c, challenge, di, src) return } @@ -2075,7 +2075,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake return true }) if !knownTxID && src.vni.isSet() { - c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(dm, di, src) + c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(c, dm, di, src) } case *disco.CallMeMaybe, *disco.CallMeMaybeVia: var via *disco.CallMeMaybeVia @@ -2221,7 +2221,7 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src epAddr, di *discoInfo, derpN // using it as a bestAddr. [relayManager] might be in the middle of // probing it or attempting to set it as best via // [endpoint.relayEndpointReady()]. Make [relayManager] aware. - c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(dm, di, src) + c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(c, dm, di, src) return } default: // no VNI diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 3e72ff0f08ae3..e655ec99230a3 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -319,8 +319,8 @@ func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, dm *disco.CallMeMaybeV // handleGeneveEncapDiscoMsgNotBestAddr handles reception of Geneve-encapsulated // disco messages if they are not associated with any known // [*endpoint.bestAddr]. -func (r *relayManager) handleGeneveEncapDiscoMsgNotBestAddr(dm disco.Message, di *discoInfo, src epAddr) { - relayManagerInputEvent(r, nil, &r.rxHandshakeDiscoMsgCh, relayHandshakeDiscoMsgEvent{msg: dm, disco: di.discoKey, from: src.ap, vni: src.vni.get(), at: time.Now()}) +func (r *relayManager) handleGeneveEncapDiscoMsgNotBestAddr(conn *Conn, dm disco.Message, di *discoInfo, src epAddr) { + relayManagerInputEvent(r, nil, &r.rxHandshakeDiscoMsgCh, relayHandshakeDiscoMsgEvent{conn: conn, msg: dm, disco: di.discoKey, from: src.ap, vni: src.vni.get(), at: time.Now()}) } // handleRelayServersSet handles an update of the complete relay server set. diff --git a/wgengine/magicsock/relaymanager_test.go b/wgengine/magicsock/relaymanager_test.go index 6055c2d72b4ef..de282b4990637 100644 --- a/wgengine/magicsock/relaymanager_test.go +++ b/wgengine/magicsock/relaymanager_test.go @@ -26,7 +26,7 @@ func TestRelayManagerInitAndIdle(t *testing.T) { <-rm.runLoopStoppedCh rm = relayManager{} - rm.handleGeneveEncapDiscoMsgNotBestAddr(&disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, epAddr{}) + rm.handleGeneveEncapDiscoMsgNotBestAddr(&Conn{discoPrivate: key.NewDisco()}, &disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, epAddr{}) <-rm.runLoopStoppedCh rm = relayManager{} From 0905936c45b6380d65d347e3cb9037f64991b8f4 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Sat, 21 Jun 2025 21:14:42 -0700 Subject: [PATCH 102/263] wgengine/magicsock: set Geneve header protocol for WireGuard (#16350) Otherwise receives interpret as naked WireGuard. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/batching_conn_linux.go | 2 ++ wgengine/magicsock/rebinding_conn.go | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/wgengine/magicsock/batching_conn_linux.go b/wgengine/magicsock/batching_conn_linux.go index c9aaff168b5b6..a0607c624445c 100644 --- a/wgengine/magicsock/batching_conn_linux.go +++ b/wgengine/magicsock/batching_conn_linux.go @@ -114,6 +114,7 @@ func (c *linuxBatchingConn) coalesceMessages(addr *net.UDPAddr, vni virtualNetwo vniIsSet := vni.isSet() var gh packet.GeneveHeader if vniIsSet { + gh.Protocol = packet.GeneveProtocolWireGuard gh.VNI = vni.get() } for i, buff := range buffs { @@ -202,6 +203,7 @@ retry: vniIsSet := addr.vni.isSet() var gh packet.GeneveHeader if vniIsSet { + gh.Protocol = packet.GeneveProtocolWireGuard gh.VNI = addr.vni.get() offset -= packet.GeneveFixedHeaderLength } diff --git a/wgengine/magicsock/rebinding_conn.go b/wgengine/magicsock/rebinding_conn.go index 51e97c8ccae2e..8b9ad4bb0bead 100644 --- a/wgengine/magicsock/rebinding_conn.go +++ b/wgengine/magicsock/rebinding_conn.go @@ -85,7 +85,8 @@ func (c *RebindingUDPConn) WriteBatchTo(buffs [][]byte, addr epAddr, offset int) var gh packet.GeneveHeader if vniIsSet { gh = packet.GeneveHeader{ - VNI: addr.vni.get(), + Protocol: packet.GeneveProtocolWireGuard, + VNI: addr.vni.get(), } } for _, buf := range buffs { From b3e74367d84650600b25162510d8beaf8a460240 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 22 Jun 2025 21:15:20 -0700 Subject: [PATCH 103/263] tool: rename go.ps1 to go-win.ps1 for cmd.exe+Powershell compat This tweaks the just-added ./tool/go.{cmd,ps1} port of ./tool/go for Windows. Otherwise in Windows Terminal in Powershell, running just ".\tool\go" picks up go.ps1 before go.cmd, which means execution gets denied without the cmd script's -ExecutionPolicy Bypass part letting it work. This makes it work in both cmd.exe and in Powershell. Updates tailscale/corp#28679 Change-Id: Iaf628a9fd6cb95670633b2dbdb635dfb8afaa006 Signed-off-by: Brad Fitzpatrick --- tool/{go.ps1 => go-win.ps1} | 0 tool/go.cmd | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename tool/{go.ps1 => go-win.ps1} (100%) diff --git a/tool/go.ps1 b/tool/go-win.ps1 similarity index 100% rename from tool/go.ps1 rename to tool/go-win.ps1 diff --git a/tool/go.cmd b/tool/go.cmd index 51bace110d59b..04172a28d5b25 100644 --- a/tool/go.cmd +++ b/tool/go.cmd @@ -1,2 +1,2 @@ @echo off -powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0go.ps1" %* +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0go-win.ps1" %* From 9309760263e7c7c34522871752cf1da08b82b72a Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 19 Jun 2025 11:31:47 +0200 Subject: [PATCH 104/263] util/prompt: make yes/no prompt reusable Updates #19445 Signed-off-by: Kristoffer Dalby --- cmd/tailscale/cli/serve_v2.go | 3 ++- cmd/tailscale/cli/update.go | 18 ++---------------- cmd/tailscale/depaware.txt | 1 + util/prompt/prompt.go | 24 ++++++++++++++++++++++++ 4 files changed, 29 insertions(+), 17 deletions(-) create mode 100644 util/prompt/prompt.go diff --git a/cmd/tailscale/cli/serve_v2.go b/cmd/tailscale/cli/serve_v2.go index 3e173ce28d8c1..bb51fb7d0e131 100644 --- a/cmd/tailscale/cli/serve_v2.go +++ b/cmd/tailscale/cli/serve_v2.go @@ -28,6 +28,7 @@ import ( "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" "tailscale.com/util/mak" + "tailscale.com/util/prompt" "tailscale.com/util/slicesx" "tailscale.com/version" ) @@ -757,7 +758,7 @@ func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, dnsName string, srvPort u if len(mounts) > 1 { msg := fmt.Sprintf("Are you sure you want to delete %d handlers under port %s?", len(mounts), portStr) - if !e.yes && !promptYesNo(msg) { + if !e.yes && !prompt.YesNo(msg) { return nil } } diff --git a/cmd/tailscale/cli/update.go b/cmd/tailscale/cli/update.go index 69d1aa97b43f7..7c0269f6a7687 100644 --- a/cmd/tailscale/cli/update.go +++ b/cmd/tailscale/cli/update.go @@ -9,10 +9,10 @@ import ( "flag" "fmt" "runtime" - "strings" "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/clientupdate" + "tailscale.com/util/prompt" "tailscale.com/version" "tailscale.com/version/distro" ) @@ -87,19 +87,5 @@ func confirmUpdate(ver string) bool { } msg := fmt.Sprintf("This will update Tailscale from %v to %v. Continue?", version.Short(), ver) - return promptYesNo(msg) -} - -// PromptYesNo takes a question and prompts the user to answer the -// question with a yes or no. It appends a [y/n] to the message. -func promptYesNo(msg string) bool { - fmt.Print(msg + " [y/n] ") - var resp string - fmt.Scanln(&resp) - resp = strings.ToLower(resp) - switch resp { - case "y", "yes", "sure": - return true - } - return false + return prompt.YesNo(msg) } diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 69d054ea42fb6..e44e20e8c92b2 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -172,6 +172,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/util/multierr from tailscale.com/control/controlhttp+ tailscale.com/util/must from tailscale.com/clientupdate/distsign+ tailscale.com/util/nocasemaps from tailscale.com/types/ipproto + tailscale.com/util/prompt from tailscale.com/cmd/tailscale/cli tailscale.com/util/quarantine from tailscale.com/cmd/tailscale/cli tailscale.com/util/rands from tailscale.com/tsweb tailscale.com/util/set from tailscale.com/derp+ diff --git a/util/prompt/prompt.go b/util/prompt/prompt.go new file mode 100644 index 0000000000000..4e589ceb32b52 --- /dev/null +++ b/util/prompt/prompt.go @@ -0,0 +1,24 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package prompt provides a simple way to prompt the user for input. +package prompt + +import ( + "fmt" + "strings" +) + +// YesNo takes a question and prompts the user to answer the +// question with a yes or no. It appends a [y/n] to the message. +func YesNo(msg string) bool { + fmt.Print(msg + " [y/n] ") + var resp string + fmt.Scanln(&resp) + resp = strings.ToLower(resp) + switch resp { + case "y", "yes", "sure": + return true + } + return false +} From 01982552663848378ba6cd6ac27013fe4d65f84b Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Thu, 19 Jun 2025 11:32:54 +0200 Subject: [PATCH 105/263] cmd/tailscale: warn user about nllock key removal without resigning Fixes #19445 Signed-off-by: Kristoffer Dalby --- cmd/tailscale/cli/network-lock.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmd/tailscale/cli/network-lock.go b/cmd/tailscale/cli/network-lock.go index ae1e90bbfaea9..871a931b54ba5 100644 --- a/cmd/tailscale/cli/network-lock.go +++ b/cmd/tailscale/cli/network-lock.go @@ -17,12 +17,14 @@ import ( "strings" "time" + "github.com/mattn/go-isatty" "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/ipn/ipnstate" "tailscale.com/tka" "tailscale.com/tsconst" "tailscale.com/types/key" "tailscale.com/types/tkatype" + "tailscale.com/util/prompt" ) var netlockCmd = &ffcli.Command{ @@ -369,6 +371,18 @@ func runNetworkLockRemove(ctx context.Context, args []string) error { } } } + } else { + if isatty.IsTerminal(os.Stdout.Fd()) { + fmt.Printf(`Warning +Removal of a signing key(s) without resigning nodes (--re-sign=false) +will cause any nodes signed by the the given key(s) to be locked out +of the Tailscale network. Proceed with caution. +`) + if !prompt.YesNo("Are you sure you want to remove the signing key(s)?") { + fmt.Printf("aborting removal of signing key(s)\n") + os.Exit(0) + } + } } return localClient.NetworkLockModify(ctx, nil, removeKeys) From 9288efe592d45c2578278c61ac0bddd4db57e901 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 23 Jun 2025 08:53:29 -0700 Subject: [PATCH 106/263] wgengine/magicsock: remove premature return in handshakeServerEndpoint (#16351) Any return underneath this select case must belong to a type switch case. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/relaymanager.go | 1 - 1 file changed, 1 deletion(-) diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index e655ec99230a3..4ccfbb501ed94 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -691,7 +691,6 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { // unexpected message type, silently discard continue } - return case <-timer.C: // The handshake timed out. return From a589863d61725bf027bb03a1389c7900dce611b8 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 23 Jun 2025 15:50:43 -0700 Subject: [PATCH 107/263] feature/relayserver,net/udprelay,wgengine/magicsock: implement retry (#16347) udprelay.Server is lazily initialized when the first request is received over peerAPI. These early requests have a high chance of failure until the first address discovery cycle has completed. Return an ErrServerNotReady error until the first address discovery cycle has completed, and plumb retry handling for this error all the way back to the client in relayManager. relayManager can now retry after a few seconds instead of waiting for the next path discovery cycle, which could take another minute or longer. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- feature/relayserver/relayserver.go | 8 +++ net/udprelay/server.go | 37 +++++++++---- wgengine/magicsock/relaymanager.go | 85 ++++++++++++++++++++++-------- 3 files changed, 96 insertions(+), 34 deletions(-) diff --git a/feature/relayserver/relayserver.go b/feature/relayserver/relayserver.go index a38587aa37b3a..4634f3ac27151 100644 --- a/feature/relayserver/relayserver.go +++ b/feature/relayserver/relayserver.go @@ -8,9 +8,11 @@ package relayserver import ( "encoding/json" "errors" + "fmt" "io" "net/http" "sync" + "time" "tailscale.com/envknob" "tailscale.com/feature" @@ -184,6 +186,12 @@ func handlePeerAPIRelayAllocateEndpoint(h ipnlocal.PeerAPIHandler, w http.Respon } ep, err := rs.AllocateEndpoint(allocateEndpointReq.DiscoKeys[0], allocateEndpointReq.DiscoKeys[1]) if err != nil { + var notReady udprelay.ErrServerNotReady + if errors.As(err, ¬Ready) { + w.Header().Set("Retry-After", fmt.Sprintf("%d", notReady.RetryAfter.Round(time.Second)/time.Second)) + httpErrAndLog(err.Error(), http.StatusServiceUnavailable) + return + } httpErrAndLog(err.Error(), http.StatusInternalServerError) return } diff --git a/net/udprelay/server.go b/net/udprelay/server.go index f7f5868c06f21..8b9e95fb1e728 100644 --- a/net/udprelay/server.go +++ b/net/udprelay/server.go @@ -63,13 +63,14 @@ type Server struct { closeCh chan struct{} netChecker *netcheck.Client - mu sync.Mutex // guards the following fields - addrPorts []netip.AddrPort // the ip:port pairs returned as candidate endpoints - closed bool - lamportID uint64 - vniPool []uint32 // the pool of available VNIs - byVNI map[uint32]*serverEndpoint - byDisco map[pairOfDiscoPubKeys]*serverEndpoint + mu sync.Mutex // guards the following fields + addrDiscoveryOnce bool // addrDiscovery completed once (successfully or unsuccessfully) + addrPorts []netip.AddrPort // the ip:port pairs returned as candidate endpoints + closed bool + lamportID uint64 + vniPool []uint32 // the pool of available VNIs + byVNI map[uint32]*serverEndpoint + byDisco map[pairOfDiscoPubKeys]*serverEndpoint } // pairOfDiscoPubKeys is a pair of key.DiscoPublic. It must be constructed via @@ -321,8 +322,7 @@ func NewServer(logf logger.Logf, port int, overrideAddrs []netip.Addr) (s *Serve s.wg.Add(1) go s.endpointGCLoop() if len(overrideAddrs) > 0 { - var addrPorts set.Set[netip.AddrPort] - addrPorts.Make() + addrPorts := make(set.Set[netip.AddrPort], len(overrideAddrs)) for _, addr := range overrideAddrs { if addr.IsValid() { addrPorts.Add(netip.AddrPortFrom(addr, boundPort)) @@ -401,12 +401,12 @@ func (s *Server) addrDiscoveryLoop() { } s.mu.Lock() s.addrPorts = addrPorts + s.addrDiscoveryOnce = true s.mu.Unlock() case <-s.closeCh: return } } - } func (s *Server) listenOn(port int) (uint16, error) { @@ -521,10 +521,22 @@ func (s *Server) packetReadLoop() { var ErrServerClosed = errors.New("server closed") +// ErrServerNotReady indicates the server is not ready. Allocation should be +// requested after waiting for at least RetryAfter duration. +type ErrServerNotReady struct { + RetryAfter time.Duration +} + +func (e ErrServerNotReady) Error() string { + return fmt.Sprintf("server not ready, retry after %v", e.RetryAfter) +} + // AllocateEndpoint allocates an [endpoint.ServerEndpoint] for the provided pair // of [key.DiscoPublic]'s. If an allocation already exists for discoA and discoB // it is returned without modification/reallocation. AllocateEndpoint returns -// [ErrServerClosed] if the server has been closed. +// the following notable errors: +// 1. [ErrServerClosed] if the server has been closed. +// 2. [ErrServerNotReady] if the server is not ready. func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.ServerEndpoint, error) { s.mu.Lock() defer s.mu.Unlock() @@ -533,6 +545,9 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv } if len(s.addrPorts) == 0 { + if !s.addrDiscoveryOnce { + return endpoint.ServerEndpoint{}, ErrServerNotReady{RetryAfter: 3 * time.Second} + } return endpoint.ServerEndpoint{}, errors.New("server addrPorts are not yet known") } diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 4ccfbb501ed94..d149d0c595e70 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -7,9 +7,12 @@ import ( "bytes" "context" "encoding/json" + "errors" + "fmt" "io" "net/http" "net/netip" + "strconv" "sync" "time" @@ -716,46 +719,82 @@ func (r *relayManager) allocateAllServersRunLoop(ep *endpoint) { }() } -func (r *relayManager) allocateSingleServer(ctx context.Context, wg *sync.WaitGroup, server netip.AddrPort, ep *endpoint) { - // TODO(jwhited): introduce client metrics counters for notable failures - defer wg.Done() - var b bytes.Buffer - remoteDisco := ep.disco.Load() - if remoteDisco == nil { - return - } +type errNotReady struct{ retryAfter time.Duration } + +func (e errNotReady) Error() string { + return fmt.Sprintf("server not ready, retry after %v", e.retryAfter) +} + +const reqTimeout = time.Second * 10 + +func doAllocate(ctx context.Context, server netip.AddrPort, discoKeys [2]key.DiscoPublic) (udprelay.ServerEndpoint, error) { + var reqBody bytes.Buffer type allocateRelayEndpointReq struct { DiscoKeys []key.DiscoPublic } a := &allocateRelayEndpointReq{ - DiscoKeys: []key.DiscoPublic{ep.c.discoPublic, remoteDisco.key}, + DiscoKeys: []key.DiscoPublic{discoKeys[0], discoKeys[1]}, } - err := json.NewEncoder(&b).Encode(a) + err := json.NewEncoder(&reqBody).Encode(a) if err != nil { - return + return udprelay.ServerEndpoint{}, err } - const reqTimeout = time.Second * 10 reqCtx, cancel := context.WithTimeout(ctx, reqTimeout) defer cancel() - req, err := http.NewRequestWithContext(reqCtx, httpm.POST, "http://"+server.String()+"/v0/relay/endpoint", &b) + req, err := http.NewRequestWithContext(reqCtx, httpm.POST, "http://"+server.String()+"/v0/relay/endpoint", &reqBody) if err != nil { - return + return udprelay.ServerEndpoint{}, err } resp, err := http.DefaultClient.Do(req) if err != nil { - return + return udprelay.ServerEndpoint{}, err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { + switch resp.StatusCode { + case http.StatusOK: + var se udprelay.ServerEndpoint + err = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&se) + return se, err + case http.StatusServiceUnavailable: + raHeader := resp.Header.Get("Retry-After") + raSeconds, err := strconv.ParseUint(raHeader, 10, 32) + if err == nil { + return udprelay.ServerEndpoint{}, errNotReady{retryAfter: time.Second * time.Duration(raSeconds)} + } + fallthrough + default: + return udprelay.ServerEndpoint{}, fmt.Errorf("non-200 status: %d", resp.StatusCode) + } +} + +func (r *relayManager) allocateSingleServer(ctx context.Context, wg *sync.WaitGroup, server netip.AddrPort, ep *endpoint) { + // TODO(jwhited): introduce client metrics counters for notable failures + defer wg.Done() + remoteDisco := ep.disco.Load() + if remoteDisco == nil { return } - var se udprelay.ServerEndpoint - err = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&se) - if err != nil { + firstTry := true + for { + se, err := doAllocate(ctx, server, [2]key.DiscoPublic{ep.c.discoPublic, remoteDisco.key}) + if err == nil { + relayManagerInputEvent(r, ctx, &r.newServerEndpointCh, newRelayServerEndpointEvent{ + ep: ep, + se: se, + }) + return + } + ep.c.logf("[v1] magicsock: relayManager: error allocating endpoint on %v for %v: %v", server, ep.discoShort(), err) + var notReady errNotReady + if firstTry && errors.As(err, ¬Ready) { + select { + case <-ctx.Done(): + return + case <-time.After(min(notReady.retryAfter, reqTimeout)): + firstTry = false + continue + } + } return } - relayManagerInputEvent(r, ctx, &r.newServerEndpointCh, newRelayServerEndpointEvent{ - ep: ep, - se: se, - }) } From 31eebdb0f8b42d40f0360a835e25d4d35c1cf420 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 23 Jun 2025 16:13:58 -0700 Subject: [PATCH 108/263] wgengine/magicsock: send CallMeMaybeVia for relay endpoints (#16360) If we acted as the allocator we are responsible for signaling it to the remote peer in a CallMeMaybeVia message over DERP. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/relaymanager.go | 38 ++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index d149d0c595e70..f22e281e6944b 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -30,8 +30,9 @@ import ( // // [relayManager] methods can be called by [Conn] and [endpoint] while their .mu // mutexes are held. Therefore, in order to avoid deadlocks, [relayManager] must -// never attempt to acquire those mutexes, including synchronous calls back -// towards [Conn] or [endpoint] methods that acquire them. +// never attempt to acquire those mutexes synchronously from its runLoop(), +// including synchronous calls back towards [Conn] or [endpoint] methods that +// acquire them. type relayManager struct { initOnce sync.Once @@ -584,9 +585,37 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay byServerDisco[newServerEndpoint.se.ServerDisco] = work r.handshakeWorkByServerDiscoVNI[sdv] = work + if newServerEndpoint.server.IsValid() { + // Send CallMeMaybeVia to the remote peer if we allocated this endpoint. + go r.sendCallMeMaybeVia(work.ep, work.se) + } + go r.handshakeServerEndpoint(work) } +// sendCallMeMaybeVia sends a [disco.CallMeMaybeVia] to ep over DERP. It must be +// called as part of a goroutine independent from runLoop(), for 2 reasons: +// 1. it acquires ep.mu (refer to [relayManager] docs for reasoning) +// 2. it makes a networking syscall, which can introduce unwanted backpressure +func (r *relayManager) sendCallMeMaybeVia(ep *endpoint, se udprelay.ServerEndpoint) { + ep.mu.Lock() + derpAddr := ep.derpAddr + ep.mu.Unlock() + epDisco := ep.disco.Load() + if epDisco == nil || !derpAddr.IsValid() { + return + } + callMeMaybeVia := &disco.CallMeMaybeVia{ + ServerDisco: se.ServerDisco, + LamportID: se.LamportID, + VNI: se.VNI, + BindLifetime: se.BindLifetime.Duration, + SteadyStateLifetime: se.SteadyStateLifetime.Duration, + AddrPorts: se.AddrPorts, + } + ep.c.sendDiscoMessage(epAddr{ap: derpAddr}, ep.publicKey, epDisco.key, callMeMaybeVia, discoVerboseLog) +} + func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { done := relayEndpointHandshakeWorkDoneEvent{work: work} r.ensureDiscoInfoFor(work) @@ -779,8 +808,9 @@ func (r *relayManager) allocateSingleServer(ctx context.Context, wg *sync.WaitGr se, err := doAllocate(ctx, server, [2]key.DiscoPublic{ep.c.discoPublic, remoteDisco.key}) if err == nil { relayManagerInputEvent(r, ctx, &r.newServerEndpointCh, newRelayServerEndpointEvent{ - ep: ep, - se: se, + ep: ep, + se: se, + server: server, // we allocated this endpoint (vs CallMeMaybeVia reception), mark it as such }) return } From 4a1fc378d1a8fa4d7f5beef318830d8354f76d1c Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Mon, 23 Jun 2025 17:55:23 -0500 Subject: [PATCH 109/263] release/dist: switch back to Ubuntu 20.04 for building QNAP packages After the switch to 24.04, unsigned packages did not build correctly (came out as only a few KBs). Fixes tailscale/tailscale-qpkg#148 Signed-off-by: Percy Wegmann --- release/dist/qnap/files/scripts/Dockerfile.qpkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/dist/qnap/files/scripts/Dockerfile.qpkg b/release/dist/qnap/files/scripts/Dockerfile.qpkg index 1f4c2406d7642..542eb95e1a7cc 100644 --- a/release/dist/qnap/files/scripts/Dockerfile.qpkg +++ b/release/dist/qnap/files/scripts/Dockerfile.qpkg @@ -1,4 +1,4 @@ -FROM ubuntu:24.04 +FROM ubuntu:20.04 RUN apt-get update -y && \ apt-get install -y --no-install-recommends \ From 9e28bfc69c0127a21fbce6beeaee2d763fe78d2a Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Tue, 24 Jun 2025 13:39:29 -0500 Subject: [PATCH 110/263] ipn/ipnlocal,wgengine/magicsock: wait for magicsock to process pending events on authReconfig Updates #16369 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 5 ++++ ipn/ipnlocal/local_test.go | 6 ++++ ipn/ipnlocal/state_test.go | 51 ++++++++++++++++++++++++++++++++- wgengine/magicsock/magicsock.go | 34 ++++++++++++++++++++++ 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 908418d4aad2c..5467088f7c91a 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -4853,6 +4853,11 @@ func (b *LocalBackend) readvertiseAppConnectorRoutes() { // updates are not currently blocked, based on the cached netmap and // user prefs. func (b *LocalBackend) authReconfig() { + // Wait for magicsock to process pending [eventbus] events, + // such as netmap updates. This should be completed before + // wireguard-go is reconfigured. See tailscale/tailscale#16369. + b.MagicConn().Synchronize() + b.mu.Lock() blocked := b.blocked prefs := b.pm.CurrentPrefs() diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 6e24f43006bd4..6e62786883d1a 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -85,6 +85,12 @@ func makeNodeKeyFromID(nodeID tailcfg.NodeID) key.NodePublic { return key.NodePublicFromRaw32(memro.B(raw)) } +func makeDiscoKeyFromID(nodeID tailcfg.NodeID) (ret key.DiscoPublic) { + raw := make([]byte, 32) + binary.BigEndian.PutUint64(raw[24:], uint64(nodeID)) + return key.DiscoPublicFromRaw32(memro.B(raw)) +} + func TestShrinkDefaultRoute(t *testing.T) { tests := []struct { route string diff --git a/ipn/ipnlocal/state_test.go b/ipn/ipnlocal/state_test.go index 5d9e8b169f0a5..2921de2032913 100644 --- a/ipn/ipnlocal/state_test.go +++ b/ipn/ipnlocal/state_test.go @@ -1114,6 +1114,8 @@ func TestEngineReconfigOnStateChange(t *testing.T) { disconnect := &ipn.MaskedPrefs{Prefs: ipn.Prefs{WantRunning: false}, WantRunningSet: true} node1 := testNetmapForNode(1, "node-1", []netip.Prefix{netip.MustParsePrefix("100.64.1.1/32")}) node2 := testNetmapForNode(2, "node-2", []netip.Prefix{netip.MustParsePrefix("100.64.1.2/32")}) + node3 := testNetmapForNode(3, "node-3", []netip.Prefix{netip.MustParsePrefix("100.64.1.3/32")}) + node3.Peers = []tailcfg.NodeView{node1.SelfNode, node2.SelfNode} routesWithQuad100 := func(extra ...netip.Prefix) []netip.Prefix { return append(extra, netip.MustParsePrefix("100.100.100.100/32")) } @@ -1308,6 +1310,40 @@ func TestEngineReconfigOnStateChange(t *testing.T) { Hosts: hostsFor(node1), }, }, + { + name: "Start/Connect/Login/WithPeers", + steps: func(t *testing.T, lb *LocalBackend, cc func() *mockControl) { + mustDo(t)(lb.Start(ipn.Options{})) + mustDo2(t)(lb.EditPrefs(connect)) + cc().authenticated(node3) + }, + wantState: ipn.Starting, + wantCfg: &wgcfg.Config{ + Name: "tailscale", + NodeID: node3.SelfNode.StableID(), + Peers: []wgcfg.Peer{ + { + PublicKey: node1.SelfNode.Key(), + DiscoKey: node1.SelfNode.DiscoKey(), + }, + { + PublicKey: node2.SelfNode.Key(), + DiscoKey: node2.SelfNode.DiscoKey(), + }, + }, + Addresses: node3.SelfNode.Addresses().AsSlice(), + }, + wantRouterCfg: &router.Config{ + SNATSubnetRoutes: true, + NetfilterMode: preftype.NetfilterOn, + LocalAddrs: node3.SelfNode.Addresses().AsSlice(), + Routes: routesWithQuad100(), + }, + wantDNSCfg: &dns.Config{ + Routes: map[dnsname.FQDN][]*dnstype.Resolver{}, + Hosts: hostsFor(node3), + }, + }, } for _, tt := range tests { @@ -1322,8 +1358,18 @@ func TestEngineReconfigOnStateChange(t *testing.T) { t.Errorf("State: got %v; want %v", gotState, tt.wantState) } + if engine.Config() != nil { + for _, p := range engine.Config().Peers { + pKey := p.PublicKey.UntypedHexString() + _, err := lb.MagicConn().ParseEndpoint(pKey) + if err != nil { + t.Errorf("ParseEndpoint(%q) failed: %v", pKey, err) + } + } + } + opts := []cmp.Option{ - cmpopts.EquateComparable(key.NodePublic{}, netip.Addr{}, netip.Prefix{}), + cmpopts.EquateComparable(key.NodePublic{}, key.DiscoPublic{}, netip.Addr{}, netip.Prefix{}), } if diff := cmp.Diff(tt.wantCfg, engine.Config(), opts...); diff != "" { t.Errorf("wgcfg.Config(+got -want): %v", diff) @@ -1356,6 +1402,8 @@ func testNetmapForNode(userID tailcfg.UserID, name string, addresses []netip.Pre Addresses: addresses, MachineAuthorized: true, } + self.Key = makeNodeKeyFromID(self.ID) + self.DiscoKey = makeDiscoKeyFromID(self.ID) return &netmap.NetworkMap{ SelfNode: self.View(), Name: self.Name, @@ -1403,6 +1451,7 @@ func newLocalBackendWithMockEngineAndControl(t *testing.T, enableLogging bool) ( magicConn, err := magicsock.NewConn(magicsock.Options{ Logf: logf, + EventBus: sys.Bus.Get(), NetMon: dialer.NetMon(), Metrics: sys.UserMetricsRegistry(), HealthTracker: sys.HealthTracker(), diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index a96eaf3d800b9..d7b52269984da 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -167,6 +167,8 @@ type Conn struct { filterSub *eventbus.Subscriber[FilterUpdate] nodeViewsSub *eventbus.Subscriber[NodeViewsUpdate] nodeMutsSub *eventbus.Subscriber[NodeMutationsUpdate] + syncSub *eventbus.Subscriber[syncPoint] + syncPub *eventbus.Publisher[syncPoint] subsDoneCh chan struct{} // closed when consumeEventbusTopics returns // pconn4 and pconn6 are the underlying UDP sockets used to @@ -538,6 +540,21 @@ type FilterUpdate struct { *filter.Filter } +// syncPoint is an event published over an [eventbus.Bus] by [Conn.Synchronize]. +// It serves as a synchronization point, allowing to wait until magicsock +// has processed all pending events. +type syncPoint chan struct{} + +// Wait blocks until [syncPoint.Signal] is called. +func (s syncPoint) Wait() { + <-s +} + +// Signal signals the sync point, unblocking the [syncPoint.Wait] call. +func (s syncPoint) Signal() { + close(s) +} + // newConn is the error-free, network-listening-side-effect-free based // of NewConn. Mostly for tests. func newConn(logf logger.Logf) *Conn { @@ -593,10 +610,25 @@ func (c *Conn) consumeEventbusTopics() { c.onNodeViewsUpdate(nodeViews) case nodeMuts := <-c.nodeMutsSub.Events(): c.onNodeMutationsUpdate(nodeMuts) + case syncPoint := <-c.syncSub.Events(): + c.dlogf("magicsock: received sync point after reconfig") + syncPoint.Signal() } } } +// Synchronize waits for all [eventbus] events published +// prior to this call to be processed by the receiver. +func (c *Conn) Synchronize() { + if c.syncPub == nil { + // Eventbus is not used; no need to synchronize (in certain tests). + return + } + sp := syncPoint(make(chan struct{})) + c.syncPub.Publish(sp) + sp.Wait() +} + // NewConn creates a magic Conn listening on opts.Port. // As the set of possible endpoints for a Conn changes, the // callback opts.EndpointsFunc is called. @@ -624,6 +656,8 @@ func NewConn(opts Options) (*Conn, error) { c.filterSub = eventbus.Subscribe[FilterUpdate](c.eventClient) c.nodeViewsSub = eventbus.Subscribe[NodeViewsUpdate](c.eventClient) c.nodeMutsSub = eventbus.Subscribe[NodeMutationsUpdate](c.eventClient) + c.syncSub = eventbus.Subscribe[syncPoint](c.eventClient) + c.syncPub = eventbus.Publish[syncPoint](c.eventClient) c.subsDoneCh = make(chan struct{}) go c.consumeEventbusTopics() } From 83cd446b5d2cd136e87023187949bbd45710be7a Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Tue, 24 Jun 2025 16:56:28 -0500 Subject: [PATCH 111/263] release/dist/qnap: upgrade to Ubuntu 24.04 Docker image 20.04 is no longer supported. This pulls in changes to the QDK package that were required to make build succeed on 24.04. Updates https://github.com/tailscale/corp/issues/29849 Signed-off-by: Percy Wegmann --- release/dist/qnap/files/scripts/Dockerfile.qpkg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/dist/qnap/files/scripts/Dockerfile.qpkg b/release/dist/qnap/files/scripts/Dockerfile.qpkg index 542eb95e1a7cc..dbcaac11668f0 100644 --- a/release/dist/qnap/files/scripts/Dockerfile.qpkg +++ b/release/dist/qnap/files/scripts/Dockerfile.qpkg @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 RUN apt-get update -y && \ apt-get install -y --no-install-recommends \ @@ -10,7 +10,7 @@ RUN apt-get update -y && \ patch # Install QNAP QDK (force a specific version to pick up updates) -RUN git clone https://github.com/tailscale/QDK.git && cd /QDK && git reset --hard 9a31a67387c583d19a81a378dcf7c25e2abe231d +RUN git clone https://github.com/tailscale/QDK.git && cd /QDK && git reset --hard 6aba74f6b4c8ea0c30b8aec9f3476f428f6a58a1 RUN cd /QDK && ./InstallToUbuntu.sh install ENV PATH="/usr/share/QDK/bin:${PATH}" From f2f1236ad4174ca46402f26139cca71dd1c94c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus=20Lensb=C3=B8l?= Date: Wed, 25 Jun 2025 09:00:34 -0400 Subject: [PATCH 112/263] util/eventbus: add test helpers to simplify testing events (#16294) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of every module having to come up with a set of test methods for the event bus, this handful of test helpers hides a lot of the needed setup for the testing of the event bus. The tests in portmapper is also ported over to the new helpers. Updates #15160 Signed-off-by: Claus Lensbøl --- net/portmapper/portmapper.go | 2 +- net/portmapper/portmapper_test.go | 17 +- util/eventbus/doc.go | 10 + util/eventbus/eventbustest/doc.go | 45 +++ util/eventbus/eventbustest/eventbustest.go | 203 ++++++++++ .../eventbustest/eventbustest_test.go | 366 ++++++++++++++++++ util/eventbus/eventbustest/examples_test.go | 201 ++++++++++ 7 files changed, 831 insertions(+), 13 deletions(-) create mode 100644 util/eventbus/eventbustest/doc.go create mode 100644 util/eventbus/eventbustest/eventbustest.go create mode 100644 util/eventbus/eventbustest/eventbustest_test.go create mode 100644 util/eventbus/eventbustest/examples_test.go diff --git a/net/portmapper/portmapper.go b/net/portmapper/portmapper.go index 59f88e96604a5..1c6c7634bf34a 100644 --- a/net/portmapper/portmapper.go +++ b/net/portmapper/portmapper.go @@ -515,7 +515,7 @@ func (c *Client) createMapping() { GoodUntil: mapping.GoodUntil(), }) } - if c.onChange != nil { + if c.onChange != nil && c.pubClient == nil { go c.onChange() } } diff --git a/net/portmapper/portmapper_test.go b/net/portmapper/portmapper_test.go index 515a0c28c993f..e66d3c159eccb 100644 --- a/net/portmapper/portmapper_test.go +++ b/net/portmapper/portmapper_test.go @@ -12,7 +12,7 @@ import ( "time" "tailscale.com/control/controlknobs" - "tailscale.com/util/eventbus" + "tailscale.com/util/eventbus/eventbustest" ) func TestCreateOrGetMapping(t *testing.T) { @@ -142,22 +142,15 @@ func TestUpdateEvent(t *testing.T) { t.Fatalf("Create test gateway: %v", err) } - bus := eventbus.New() - defer bus.Close() + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) - sub := eventbus.Subscribe[Mapping](bus.Client("TestUpdateEvent")) c := newTestClient(t, igd, bus) if _, err := c.Probe(t.Context()); err != nil { t.Fatalf("Probe failed: %v", err) } c.GetCachedMappingOrStartCreatingOne() - - select { - case evt := <-sub.Events(): - t.Logf("Received portmap update: %+v", evt) - case <-sub.Done(): - t.Error("Subscriber closed prematurely") - case <-time.After(5 * time.Second): - t.Error("Timed out waiting for an update event") + if err := eventbustest.Expect(tw, eventbustest.Type[Mapping]()); err != nil { + t.Error(err.Error()) } } diff --git a/util/eventbus/doc.go b/util/eventbus/doc.go index 964a686eae109..f95f9398c8de9 100644 --- a/util/eventbus/doc.go +++ b/util/eventbus/doc.go @@ -89,4 +89,14 @@ // The [Debugger], obtained through [Bus.Debugger], provides // introspection facilities to monitor events flowing through the bus, // and inspect publisher and subscriber state. +// +// Additionally, a debug command exists for monitoring the eventbus: +// +// tailscale debug daemon-bus-events +// +// # Testing facilities +// +// Helpers for testing code with the eventbus can be found in: +// +// eventbus/eventbustest package eventbus diff --git a/util/eventbus/eventbustest/doc.go b/util/eventbus/eventbustest/doc.go new file mode 100644 index 0000000000000..9e39504a83521 --- /dev/null +++ b/util/eventbus/eventbustest/doc.go @@ -0,0 +1,45 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package eventbustest provides helper methods for testing an [eventbus.Bus]. +// +// # Usage +// +// A [Watcher] presents a set of generic helpers for testing events. +// +// To test code that generates events, create a [Watcher] from the [eventbus.Bus] +// used by the code under test, run the code to generate events, then use the watcher +// to verify that the expected events were produced. In outline: +// +// bus := eventbustest.NewBus(t) +// tw := eventbustest.NewWatcher(t, bus) +// somethingThatEmitsSomeEvent() +// if err := eventbustest.Expect(tw, eventbustest.Type[EventFoo]()); err != nil { +// t.Error(err.Error()) +// } +// +// As shown, [Expect] checks that at least one event of the given type occurs +// in the stream generated by the code under test. +// +// The following functions all take an any parameter representing a function. +// This function will take an argument of the expected type and is used to test +// for the events on the eventbus being of the given type. The function can +// take the shape described in [Expect]. +// +// [Type] is a helper for only testing event type. +// +// To check for specific properties of an event, use [Expect], and pass a function +// as the second argument that tests for those properties. +// +// To test for multiple events, use [Expect], which checks that the stream +// contains the given events in the given order, possibly with other events +// interspersed. +// +// To test the complete contents of the stream, use [ExpectExactly], which +// checks that the stream contains exactly the given events in the given order, +// and no others. +// +// See the [usage examples]. +// +// [usage examples]: https://github.com/tailscale/tailscale/blob/main/util/eventbus/eventbustest/examples_test.go +package eventbustest diff --git a/util/eventbus/eventbustest/eventbustest.go b/util/eventbus/eventbustest/eventbustest.go new file mode 100644 index 0000000000000..75d430d53683e --- /dev/null +++ b/util/eventbus/eventbustest/eventbustest.go @@ -0,0 +1,203 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package eventbustest + +import ( + "errors" + "fmt" + "reflect" + "testing" + "time" + + "tailscale.com/util/eventbus" +) + +// NewBus constructs an [eventbus.Bus] that will be shut automatically when +// its controlling test ends. +func NewBus(t *testing.T) *eventbus.Bus { + bus := eventbus.New() + t.Cleanup(bus.Close) + return bus +} + +// NewTestWatcher constructs a [Watcher] that can be used to check the stream of +// events generated by code under test. After construction the caller may use +// [Expect] and [ExpectExactly], to verify that the desired events were captured. +func NewWatcher(t *testing.T, bus *eventbus.Bus) *Watcher { + tw := &Watcher{ + mon: bus.Debugger().WatchBus(), + TimeOut: 5 * time.Second, + chDone: make(chan bool, 1), + events: make(chan any, 100), + } + if deadline, ok := t.Deadline(); ok { + tw.TimeOut = deadline.Sub(time.Now()) + } + t.Cleanup(tw.done) + go tw.watch() + return tw +} + +// Watcher monitors and holds events for test expectations. +type Watcher struct { + mon *eventbus.Subscriber[eventbus.RoutedEvent] + events chan any + chDone chan bool + // TimeOut defines when the Expect* functions should stop looking for events + // coming from the Watcher. The value is set by [NewWatcher] and defaults to + // the deadline passed in by [testing.T]. If looking to verify the absence + // of an event, the TimeOut can be set to a lower value after creating the + // Watcher. + TimeOut time.Duration +} + +// Type is a helper representing the expectation to see an event of type T, without +// caring about the content of the event. +// It makes it possible to use helpers like: +// +// eventbustest.ExpectFilter(tw, eventbustest.Type[EventFoo]()) +func Type[T any]() func(T) { return func(T) {} } + +// Expect verifies that the given events are a subsequence of the events +// observed by tw. That is, tw must contain at least one event matching the type +// of each argument in the given order, other event types are allowed to occur in +// between without error. The given events are represented by a function +// that must have one of the following forms: +// +// // Tests for the event type only +// func(e ExpectedType) +// +// // Tests for event type and whatever is defined in the body. +// // If return is false, the test will look for other events of that type +// // If return is true, the test will look for the next given event +// // if a list is given +// func(e ExpectedType) bool +// +// // Tests for event type and whatever is defined in the body. +// // The boolean return works as above. +// // The if error != nil, the test helper will return that error immediately. +// func(e ExpectedType) (bool, error) +// +// If the list of events must match exactly with no extra events, +// use [ExpectExactly]. +func Expect(tw *Watcher, filters ...any) error { + if len(filters) == 0 { + return errors.New("no event filters were provided") + } + eventCount := 0 + head := 0 + for head < len(filters) { + eventFunc := eventFilter(filters[head]) + select { + case event := <-tw.events: + eventCount++ + if ok, err := eventFunc(event); err != nil { + return err + } else if ok { + head++ + } + case <-time.After(tw.TimeOut): + return fmt.Errorf( + "timed out waiting for event, saw %d events, %d was expected", + eventCount, head) + case <-tw.chDone: + return errors.New("watcher closed while waiting for events") + } + } + return nil +} + +// ExpectExactly checks for some number of events showing up on the event bus +// in a given order, returning an error if the events does not match the given list +// exactly. The given events are represented by a function as described in +// [Expect]. Use [Expect] if other events are allowed. +func ExpectExactly(tw *Watcher, filters ...any) error { + if len(filters) == 0 { + return errors.New("no event filters were provided") + } + eventCount := 0 + for pos, next := range filters { + eventFunc := eventFilter(next) + fnType := reflect.TypeOf(next) + argType := fnType.In(0) + select { + case event := <-tw.events: + eventCount++ + typeEvent := reflect.TypeOf(event) + if typeEvent != argType { + return fmt.Errorf( + "expected event type %s, saw %s, at index %d", + argType, typeEvent, pos) + } else if ok, err := eventFunc(event); err != nil { + return err + } else if !ok { + return fmt.Errorf( + "expected test ok for type %s, at index %d", argType, pos) + } + case <-time.After(tw.TimeOut): + return fmt.Errorf( + "timed out waiting for event, saw %d events, %d was expected", + eventCount, pos) + case <-tw.chDone: + return errors.New("watcher closed while waiting for events") + } + } + return nil +} + +func (tw *Watcher) watch() { + for { + select { + case event := <-tw.mon.Events(): + tw.events <- event.Event + case <-tw.chDone: + tw.mon.Close() + return + } + } +} + +// done tells the watcher to stop monitoring for new events. +func (tw *Watcher) done() { + close(tw.chDone) +} + +type filter = func(any) (bool, error) + +func eventFilter(f any) filter { + ft := reflect.TypeOf(f) + if ft.Kind() != reflect.Func { + panic("filter is not a function") + } else if ft.NumIn() != 1 { + panic(fmt.Sprintf("function takes %d arguments, want 1", ft.NumIn())) + } + var fixup func([]reflect.Value) []reflect.Value + switch ft.NumOut() { + case 0: + fixup = func([]reflect.Value) []reflect.Value { + return []reflect.Value{reflect.ValueOf(true), reflect.Zero(reflect.TypeFor[error]())} + } + case 1: + if ft.Out(0) != reflect.TypeFor[bool]() { + panic(fmt.Sprintf("result is %T, want bool", ft.Out(0))) + } + fixup = func(vals []reflect.Value) []reflect.Value { + return append(vals, reflect.Zero(reflect.TypeFor[error]())) + } + case 2: + if ft.Out(0) != reflect.TypeFor[bool]() || ft.Out(1) != reflect.TypeFor[error]() { + panic(fmt.Sprintf("results are %T, %T; want bool, error", ft.Out(0), ft.Out(1))) + } + fixup = func(vals []reflect.Value) []reflect.Value { return vals } + default: + panic(fmt.Sprintf("function returns %d values", ft.NumOut())) + } + fv := reflect.ValueOf(f) + return reflect.MakeFunc(reflect.TypeFor[filter](), func(args []reflect.Value) []reflect.Value { + if !args[0].IsValid() || args[0].Elem().Type() != ft.In(0) { + return []reflect.Value{reflect.ValueOf(false), reflect.Zero(reflect.TypeFor[error]())} + } + return fixup(fv.Call([]reflect.Value{args[0].Elem()})) + }).Interface().(filter) +} diff --git a/util/eventbus/eventbustest/eventbustest_test.go b/util/eventbus/eventbustest/eventbustest_test.go new file mode 100644 index 0000000000000..fd95973e5538d --- /dev/null +++ b/util/eventbus/eventbustest/eventbustest_test.go @@ -0,0 +1,366 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package eventbustest_test + +import ( + "fmt" + "testing" + "time" + + "tailscale.com/util/eventbus" + "tailscale.com/util/eventbus/eventbustest" +) + +type EventFoo struct { + Value int +} + +type EventBar struct { + Value string +} + +type EventBaz struct { + Value []float64 +} + +func TestExpectFilter(t *testing.T) { + tests := []struct { + name string + events []int + expectFunc any + wantErr bool + }{ + { + name: "single event", + events: []int{42}, + expectFunc: eventbustest.Type[EventFoo](), + wantErr: false, + }, + { + name: "multiple events, single expectation", + events: []int{42, 1, 2, 3, 4, 5}, + expectFunc: eventbustest.Type[EventFoo](), + wantErr: false, + }, + { + name: "filter on event with function", + events: []int{24, 42}, + expectFunc: func(event EventFoo) (bool, error) { + if event.Value == 42 { + return true, nil + } + return false, nil + }, + wantErr: false, + }, + { + name: "first event has to be func", + events: []int{24, 42}, + expectFunc: func(event EventFoo) (bool, error) { + if event.Value != 42 { + return false, fmt.Errorf("expected 42, got %d", event.Value) + } + return false, nil + }, + wantErr: true, + }, + { + name: "no events", + events: []int{}, + expectFunc: func(event EventFoo) (bool, error) { + return true, nil + }, + wantErr: true, + }, + } + + bus := eventbustest.NewBus(t) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tw := eventbustest.NewWatcher(t, bus) + // TODO(cmol): When synctest is out of experimental, use that instead: + // https://go.dev/blog/synctest + tw.TimeOut = 10 * time.Millisecond + + client := bus.Client("testClient") + defer client.Close() + updater := eventbus.Publish[EventFoo](client) + + for _, i := range tt.events { + updater.Publish(EventFoo{i}) + } + + if err := eventbustest.Expect(tw, tt.expectFunc); (err != nil) != tt.wantErr { + t.Errorf("ExpectFilter[EventFoo]: error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestExpectEvents(t *testing.T) { + tests := []struct { + name string + events []any + expectEvents []any + wantErr bool + }{ + { + name: "No expectations", + events: []any{EventFoo{}}, + expectEvents: []any{}, + wantErr: true, + }, + { + name: "One event", + events: []any{EventFoo{}}, + expectEvents: []any{eventbustest.Type[EventFoo]()}, + wantErr: false, + }, + { + name: "Two events", + events: []any{EventFoo{}, EventBar{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: false, + }, + { + name: "Two expected events with another in the middle", + events: []any{EventFoo{}, EventBaz{}, EventBar{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: false, + }, + { + name: "Missing event", + events: []any{EventFoo{}, EventBaz{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: true, + }, + { + name: "One event with specific value", + events: []any{EventFoo{42}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: false, + }, + { + name: "Two event with one specific value", + events: []any{EventFoo{43}, EventFoo{42}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: false, + }, + { + name: "One event with wrong value", + events: []any{EventFoo{43}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: true, + }, + { + name: "Two events with specific values", + events: []any{EventFoo{42}, EventFoo{42}, EventBar{"42"}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + func(ev EventBar) (bool, error) { + if ev.Value == "42" { + return true, nil + } + return false, nil + }, + }, + wantErr: false, + }, + } + + bus := eventbustest.NewBus(t) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tw := eventbustest.NewWatcher(t, bus) + // TODO(cmol): When synctest is out of experimental, use that instead: + // https://go.dev/blog/synctest + tw.TimeOut = 10 * time.Millisecond + + client := bus.Client("testClient") + defer client.Close() + updaterFoo := eventbus.Publish[EventFoo](client) + updaterBar := eventbus.Publish[EventBar](client) + updaterBaz := eventbus.Publish[EventBaz](client) + + for _, ev := range tt.events { + switch ev.(type) { + case EventFoo: + evCast := ev.(EventFoo) + updaterFoo.Publish(evCast) + case EventBar: + evCast := ev.(EventBar) + updaterBar.Publish(evCast) + case EventBaz: + evCast := ev.(EventBaz) + updaterBaz.Publish(evCast) + } + } + + if err := eventbustest.Expect(tw, tt.expectEvents...); (err != nil) != tt.wantErr { + t.Errorf("ExpectEvents: error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestExpectExactlyEventsFilter(t *testing.T) { + tests := []struct { + name string + events []any + expectEvents []any + wantErr bool + }{ + { + name: "No expectations", + events: []any{EventFoo{}}, + expectEvents: []any{}, + wantErr: true, + }, + { + name: "One event", + events: []any{EventFoo{}}, + expectEvents: []any{eventbustest.Type[EventFoo]()}, + wantErr: false, + }, + { + name: "Two events", + events: []any{EventFoo{}, EventBar{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: false, + }, + { + name: "Two expected events with another in the middle", + events: []any{EventFoo{}, EventBaz{}, EventBar{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: true, + }, + { + name: "Missing event", + events: []any{EventFoo{}, EventBaz{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: true, + }, + { + name: "One event with value", + events: []any{EventFoo{42}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: false, + }, + { + name: "Two event with one specific value", + events: []any{EventFoo{43}, EventFoo{42}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: true, + }, + { + name: "One event with wrong value", + events: []any{EventFoo{43}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: true, + }, + { + name: "Two events with specific values", + events: []any{EventFoo{42}, EventFoo{42}, EventBar{"42"}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + func(ev EventBar) (bool, error) { + if ev.Value == "42" { + return true, nil + } + return false, nil + }, + }, + wantErr: true, + }, + } + + bus := eventbustest.NewBus(t) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tw := eventbustest.NewWatcher(t, bus) + // TODO(cmol): When synctest is out of experimental, use that instead: + // https://go.dev/blog/synctest + tw.TimeOut = 10 * time.Millisecond + + client := bus.Client("testClient") + defer client.Close() + updaterFoo := eventbus.Publish[EventFoo](client) + updaterBar := eventbus.Publish[EventBar](client) + updaterBaz := eventbus.Publish[EventBaz](client) + + for _, ev := range tt.events { + switch ev.(type) { + case EventFoo: + evCast := ev.(EventFoo) + updaterFoo.Publish(evCast) + case EventBar: + evCast := ev.(EventBar) + updaterBar.Publish(evCast) + case EventBaz: + evCast := ev.(EventBaz) + updaterBaz.Publish(evCast) + } + } + + if err := eventbustest.ExpectExactly(tw, tt.expectEvents...); (err != nil) != tt.wantErr { + t.Errorf("ExpectEvents: error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/util/eventbus/eventbustest/examples_test.go b/util/eventbus/eventbustest/examples_test.go new file mode 100644 index 0000000000000..914e29933b2a2 --- /dev/null +++ b/util/eventbus/eventbustest/examples_test.go @@ -0,0 +1,201 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package eventbustest_test + +import ( + "testing" + + "tailscale.com/util/eventbus" + "tailscale.com/util/eventbus/eventbustest" +) + +func TestExample_Expect(t *testing.T) { + type eventOfInterest struct{} + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updater := eventbus.Publish[eventOfInterest](client) + updater.Publish(eventOfInterest{}) + + if err := eventbustest.Expect(tw, eventbustest.Type[eventOfInterest]()); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } + // Output: + // OK +} + +func TestExample_Expect_WithFunction(t *testing.T) { + type eventOfInterest struct { + value int + } + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updater := eventbus.Publish[eventOfInterest](client) + updater.Publish(eventOfInterest{43}) + updater.Publish(eventOfInterest{42}) + + // Look for an event of eventOfInterest with a specific value + if err := eventbustest.Expect(tw, func(event eventOfInterest) (bool, error) { + if event.value != 42 { + return false, nil // Look for another event with the expected value. + // You could alternatively return an error here to ensure that the + // first seen eventOfInterest matches the value: + // return false, fmt.Errorf("expected 42, got %d", event.value) + } + return true, nil + }); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } + // Output: + // OK +} + +func TestExample_Expect_MultipleEvents(t *testing.T) { + type eventOfInterest struct{} + type eventOfNoConcern struct{} + type eventOfCuriosity struct{} + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updaterInterest := eventbus.Publish[eventOfInterest](client) + updaterConcern := eventbus.Publish[eventOfNoConcern](client) + updaterCuriosity := eventbus.Publish[eventOfCuriosity](client) + updaterInterest.Publish(eventOfInterest{}) + updaterConcern.Publish(eventOfNoConcern{}) + updaterCuriosity.Publish(eventOfCuriosity{}) + + // Even though three events was published, we just care about the two + if err := eventbustest.Expect(tw, + eventbustest.Type[eventOfInterest](), + eventbustest.Type[eventOfCuriosity]()); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } + // Output: + // OK +} + +func TestExample_ExpectExactly_MultipleEvents(t *testing.T) { + type eventOfInterest struct{} + type eventOfNoConcern struct{} + type eventOfCuriosity struct{} + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updaterInterest := eventbus.Publish[eventOfInterest](client) + updaterConcern := eventbus.Publish[eventOfNoConcern](client) + updaterCuriosity := eventbus.Publish[eventOfCuriosity](client) + updaterInterest.Publish(eventOfInterest{}) + updaterConcern.Publish(eventOfNoConcern{}) + updaterCuriosity.Publish(eventOfCuriosity{}) + + // Will fail as more events than the two expected comes in + if err := eventbustest.ExpectExactly(tw, + eventbustest.Type[eventOfInterest](), + eventbustest.Type[eventOfCuriosity]()); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } +} + +func TestExample_Expect_WithMultipleFunctions(t *testing.T) { + type eventOfInterest struct { + value int + } + type eventOfNoConcern struct{} + type eventOfCuriosity struct { + value string + } + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updaterInterest := eventbus.Publish[eventOfInterest](client) + updaterConcern := eventbus.Publish[eventOfNoConcern](client) + updaterCuriosity := eventbus.Publish[eventOfCuriosity](client) + updaterInterest.Publish(eventOfInterest{42}) + updaterConcern.Publish(eventOfNoConcern{}) + updaterCuriosity.Publish(eventOfCuriosity{"42"}) + + interest := func(event eventOfInterest) (bool, error) { + if event.value == 42 { + return true, nil + } + return false, nil + } + curiosity := func(event eventOfCuriosity) (bool, error) { + if event.value == "42" { + return true, nil + } + return false, nil + } + + // Will fail as more events than the two expected comes in + if err := eventbustest.Expect(tw, interest, curiosity); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } + // Output: + // OK +} + +func TestExample_ExpectExactly_WithMultipleFuncions(t *testing.T) { + type eventOfInterest struct { + value int + } + type eventOfNoConcern struct{} + type eventOfCuriosity struct { + value string + } + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updaterInterest := eventbus.Publish[eventOfInterest](client) + updaterConcern := eventbus.Publish[eventOfNoConcern](client) + updaterCuriosity := eventbus.Publish[eventOfCuriosity](client) + updaterInterest.Publish(eventOfInterest{42}) + updaterConcern.Publish(eventOfNoConcern{}) + updaterCuriosity.Publish(eventOfCuriosity{"42"}) + + interest := func(event eventOfInterest) (bool, error) { + if event.value == 42 { + return true, nil + } + return false, nil + } + curiosity := func(event eventOfCuriosity) (bool, error) { + if event.value == "42" { + return true, nil + } + return false, nil + } + + // Will fail as more events than the two expected comes in + if err := eventbustest.ExpectExactly(tw, interest, curiosity); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } + // Output: + // expected event type eventbustest.eventOfCuriosity, saw eventbustest.eventOfNoConcern, at index 1 +} From b75fe9eeca13d7ff651c83d8202d22cce466dc08 Mon Sep 17 00:00:00 2001 From: David Bond Date: Wed, 25 Jun 2025 14:14:17 +0100 Subject: [PATCH 113/263] cmd/k8s-operator: Add NOTES.txt to Helm chart (#16364) This commit adds a NOTES.txt to the operator helm chart that will be written to the terminal upon successful installation of the operator. It includes a small list of knowledgebase articles with possible next steps for the actor that installed the operator to the cluster. It also provides possible commands to use for explaining the custom resources. Fixes #13427 Signed-off-by: David Bond --- .gitignore | 3 +++ .../deploy/chart/templates/NOTES.txt | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 cmd/k8s-operator/deploy/chart/templates/NOTES.txt diff --git a/.gitignore b/.gitignore index 47d2bbe959ae1..3941fd06ef6d5 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,6 @@ client/web/build/assets *.xcworkspacedata /tstest/tailmac/bin /tstest/tailmac/build + +# Ignore personal IntelliJ settings +.idea/ diff --git a/cmd/k8s-operator/deploy/chart/templates/NOTES.txt b/cmd/k8s-operator/deploy/chart/templates/NOTES.txt new file mode 100644 index 0000000000000..5678e597a6824 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/templates/NOTES.txt @@ -0,0 +1,25 @@ +You have successfully installed the Tailscale Kubernetes Operator! + +Once connected, the operator should appear as a device within the Tailscale admin console: +https://login.tailscale.com/admin/machines + +If you have not used the Tailscale operator before, here are some examples to try out: + +* Private Kubernetes API access and authorization using the API server proxy + https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy + +* Private access to cluster Services using an ingress proxy + https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress + +* Private access to the cluster's available subnets using a subnet router + https://tailscale.com/kb/1441/kubernetes-operator-connector + +You can also explore the CRDs, operator, and associated resources within the {{ .Release.Namespace }} namespace: + +$ kubectl explain connector +$ kubectl explain proxygroup +$ kubectl explain proxyclass +$ kubectl explain recorder +$ kubectl explain dnsconfig + +$ kubectl --namespace={{ .Release.Namespace }} get pods From 35b11e7be55088e282b5e240b9473968eebeb002 Mon Sep 17 00:00:00 2001 From: Laszlo Magyar Date: Wed, 25 Jun 2025 20:26:11 +0300 Subject: [PATCH 114/263] envknob/featureknob: restore SSH and exit-node capability for Home Assistant (#16263) SSH was disabled in #10538 Exit node was disabled in #13726 This enables ssh and exit-node options in case of Home Assistant. Fixes #15552 Signed-off-by: Laszlo Magyar --- envknob/featureknob/featureknob.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/envknob/featureknob/featureknob.go b/envknob/featureknob/featureknob.go index e9b871f74a8c0..5a54a1c42978d 100644 --- a/envknob/featureknob/featureknob.go +++ b/envknob/featureknob/featureknob.go @@ -10,7 +10,6 @@ import ( "runtime" "tailscale.com/envknob" - "tailscale.com/hostinfo" "tailscale.com/version" "tailscale.com/version/distro" ) @@ -26,14 +25,6 @@ func CanRunTailscaleSSH() error { if distro.Get() == distro.QNAP && !envknob.UseWIPCode() { return errors.New("The Tailscale SSH server does not run on QNAP.") } - - // Setting SSH on Home Assistant causes trouble on startup - // (since the flag is not being passed to `tailscale up`). - // Although Tailscale SSH does work here, - // it's not terribly useful since it's running in a separate container. - if hostinfo.GetEnvType() == hostinfo.HomeAssistantAddOn { - return errors.New("The Tailscale SSH server does not run on HomeAssistant.") - } // otherwise okay case "darwin": // okay only in tailscaled mode for now. @@ -58,10 +49,5 @@ func CanUseExitNode() error { distro.QNAP: return errors.New("Tailscale exit nodes cannot be used on " + string(dist)) } - - if hostinfo.GetEnvType() == hostinfo.HomeAssistantAddOn { - return errors.New("Tailscale exit nodes cannot be used on HomeAssistant.") - } - return nil } From 37eca1785c280311b16133e6bd455fa062df29e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus=20Lensb=C3=B8l?= Date: Wed, 25 Jun 2025 14:44:01 -0400 Subject: [PATCH 115/263] net/netmon: add tests for the events over the eventbus (#16382) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates #15160 Signed-off-by: Claus Lensbøl --- net/netmon/netmon_test.go | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/net/netmon/netmon_test.go b/net/netmon/netmon_test.go index a9af8fb004af3..b8ec1b75f97ec 100644 --- a/net/netmon/netmon_test.go +++ b/net/netmon/netmon_test.go @@ -12,6 +12,7 @@ import ( "time" "tailscale.com/util/eventbus" + "tailscale.com/util/eventbus/eventbustest" "tailscale.com/util/mak" ) @@ -68,6 +69,23 @@ func TestMonitorInjectEvent(t *testing.T) { } } +func TestMonitorInjectEventOnBus(t *testing.T) { + bus := eventbustest.NewBus(t) + + mon, err := New(bus, t.Logf) + if err != nil { + t.Fatal(err) + } + defer mon.Close() + tw := eventbustest.NewWatcher(t, bus) + + mon.Start() + mon.InjectEvent() + if err := eventbustest.Expect(tw, eventbustest.Type[*ChangeDelta]()); err != nil { + t.Error(err) + } +} + var ( monitor = flag.String("monitor", "", `go into monitor mode like 'route monitor'; test never terminates. Value can be either "raw" or "callback"`) monitorDuration = flag.Duration("monitor-duration", 0, "if non-zero, how long to run TestMonitorMode. Zero means forever.") @@ -77,13 +95,13 @@ func TestMonitorMode(t *testing.T) { switch *monitor { case "": t.Skip("skipping non-test without --monitor") - case "raw", "callback": + case "raw", "callback", "eventbus": default: - t.Skipf(`invalid --monitor value: must be "raw" or "callback"`) + t.Skipf(`invalid --monitor value: must be "raw", "callback" or "eventbus"`) } - bus := eventbus.New() - defer bus.Close() + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) mon, err := New(bus, t.Logf) if err != nil { @@ -124,6 +142,16 @@ func TestMonitorMode(t *testing.T) { mon.Start() <-done t.Logf("%v callbacks", n) + case "eventbus": + tw.TimeOut = *monitorDuration + n := 0 + mon.Start() + eventbustest.Expect(tw, func(event *ChangeDelta) (bool, error) { + n++ + t.Logf("cb: changed=%v, ifSt=%v", event.Major, event.New) + return false, nil // Return false, indicating we wanna look for more events + }) + t.Logf("%v events", n) } } From 51d00e135b6c5775f60f77ecd2a94e327aabd1f6 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 25 Jun 2025 19:13:02 -0700 Subject: [PATCH 116/263] wgengine/magicsock: fix relayManager alloc work cleanup (#16387) Premature cancellation was preventing the work from ever being cleaned up in runLoop(). Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/relaymanager.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index f22e281e6944b..7b378838a145c 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -743,8 +743,11 @@ func (r *relayManager) allocateAllServersRunLoop(ep *endpoint) { r.allocWorkByEndpoint[ep] = started go func() { started.wg.Wait() - started.cancel() relayManagerInputEvent(r, ctx, &r.allocateWorkDoneCh, relayEndpointAllocWorkDoneEvent{work: started}) + // cleanup context cancellation must come after the + // relayManagerInputEvent call, otherwise it returns early without + // writing the event to runLoop(). + started.cancel() }() } From aa106c92a4ed6d66b26d455dc4bff23516514af1 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Wed, 25 Jun 2025 21:30:44 -0700 Subject: [PATCH 117/263] .github/workflows: request @tailscale/dataplane review DERP changes (#16372) For any changes that involve DERP, automatically add the @tailscale/dataplane team as a reviewer. Updates #cleanup Signed-off-by: Simon Law --- .../workflows/request-dataplane-review.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/request-dataplane-review.yml diff --git a/.github/workflows/request-dataplane-review.yml b/.github/workflows/request-dataplane-review.yml new file mode 100644 index 0000000000000..836fef6fbce7c --- /dev/null +++ b/.github/workflows/request-dataplane-review.yml @@ -0,0 +1,31 @@ +name: request-dataplane-review + +on: + pull_request: + branches: + - "*" + paths: + - ".github/workflows/request-dataplane-review.yml" + - "**/*derp*" + - "**/derp*/**" + +jobs: + request-dataplane-review: + name: Request Dataplane Review + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Get access token + uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 + id: generate-token + with: + # Get token for app: https://github.com/apps/change-visibility-bot + app-id: ${{ secrets.VISIBILITY_BOT_APP_ID }} + private-key: ${{ secrets.VISIBILITY_BOT_APP_PRIVATE_KEY }} + - name: Add reviewers + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + url: ${{ github.event.pull_request.html_url }} + run: | + gh pr edit "$url" --add-reviewer tailscale/dataplane From 47dff33eac2441003a1d8ca4e98d56660f8119d4 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Wed, 25 Jun 2025 18:07:49 -0700 Subject: [PATCH 118/263] tool/gocross: remove GOROOT to ensure correct toolchain use go(1) repsects GOROOT if set, but tool/go / gocross-wrapper.sh are explicitly intending to use our toolchain. We don't need to set GOROOT, just unset it, and then go(1) handles the rest. Updates tailscale/corp#26717 Signed-off-by: James Tucker --- tool/gocross/gocross-wrapper.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tool/gocross/gocross-wrapper.sh b/tool/gocross/gocross-wrapper.sh index e9fca2aea71b5..90485d31b95af 100755 --- a/tool/gocross/gocross-wrapper.sh +++ b/tool/gocross/gocross-wrapper.sh @@ -133,6 +133,12 @@ fi repo_root="${BASH_SOURCE%/*}/../.." +# Some scripts/package systems set GOROOT even though they should only be +# setting $PATH. Stop them from breaking builds - go(1) respects GOROOT and +# so if it is left on here, compilation units depending on our Go fork will +# fail (such as those which depend on our net/ patches). +unset GOROOT + # gocross is opt-in as of 2025-06-16. See tailscale/corp#26717 # and comment above in this file. if [ "${TS_USE_GOCROSS:-}" != "1" ]; then From 99aaa6e92cda572896503538e1716851358e42d6 Mon Sep 17 00:00:00 2001 From: JerryYan Date: Fri, 27 Jun 2025 00:43:48 +0800 Subject: [PATCH 119/263] ipn/ipnlocal: update PeerByID to return SelfNode and rename it to NodeByID (#16096) Like NodeByKey, add an if stmt for checking the NodeId is SelfNode. Updates #16052 Signed-off-by: Jerry Yan <792602257@qq.com> --- ipn/ipnlocal/drive.go | 2 +- ipn/ipnlocal/local.go | 14 +++++--------- ipn/ipnlocal/local_test.go | 2 +- ipn/ipnlocal/node_backend.go | 7 ++++++- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ipn/ipnlocal/drive.go b/ipn/ipnlocal/drive.go index a06ea5e8c41ba..6a6f9bcd2b24a 100644 --- a/ipn/ipnlocal/drive.go +++ b/ipn/ipnlocal/drive.go @@ -318,7 +318,7 @@ func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Rem // - They are online // - They are allowed to share at least one folder with us cn := b.currentNode() - peer, ok := cn.PeerByID(peerID) + peer, ok := cn.NodeByID(peerID) if !ok { return false } diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 5467088f7c91a..9cec088f1f28b 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1391,7 +1391,7 @@ func profileFromView(v tailcfg.UserProfileView) tailcfg.UserProfile { func (b *LocalBackend) WhoIsNodeKey(k key.NodePublic) (n tailcfg.NodeView, u tailcfg.UserProfile, ok bool) { cn := b.currentNode() if nid, ok := cn.NodeByKey(k); ok { - if n, ok := cn.PeerByID(nid); ok { + if n, ok := cn.NodeByID(nid); ok { up, ok := cn.NetMap().UserProfiles[n.User()] u = profileFromView(up) return n, u, ok @@ -1457,13 +1457,9 @@ func (b *LocalBackend) WhoIs(proto string, ipp netip.AddrPort) (n tailcfg.NodeVi if nm == nil { return failf("no netmap") } - n, ok = cn.PeerByID(nid) + n, ok = cn.NodeByID(nid) if !ok { - // Check if this the self-node, which would not appear in peers. - if !nm.SelfNode.Valid() || nid != nm.SelfNode.ID() { - return zero, u, false - } - n = nm.SelfNode + return zero, u, false } up, ok := cn.UserByID(n.User()) if !ok { @@ -1968,7 +1964,7 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo if !ok || mo.Online { continue } - n, ok := cn.PeerByID(m.NodeIDBeingMutated()) + n, ok := cn.NodeByID(m.NodeIDBeingMutated()) if !ok || n.StableID() != exitNodeID { continue } @@ -7724,7 +7720,7 @@ func (b *LocalBackend) srcIPHasCapForFilter(srcIP netip.Addr, cap tailcfg.NodeCa if !ok { return false } - n, ok := cn.PeerByID(nodeID) + n, ok := cn.NodeByID(nodeID) if !ok { return false } diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 6e62786883d1a..16dbef62a4190 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -1005,7 +1005,7 @@ func TestUpdateNetmapDelta(t *testing.T) { }, } for _, want := range wants { - gotv, ok := b.currentNode().PeerByID(want.ID) + gotv, ok := b.currentNode().NodeByID(want.ID) if !ok { t.Errorf("netmap.Peer %v missing from b.profile.Peers", want.ID) continue diff --git a/ipn/ipnlocal/node_backend.go b/ipn/ipnlocal/node_backend.go index 05389a677d4f5..ec503f1300ca5 100644 --- a/ipn/ipnlocal/node_backend.go +++ b/ipn/ipnlocal/node_backend.go @@ -206,9 +206,14 @@ func (nb *nodeBackend) NodeByKey(k key.NodePublic) (_ tailcfg.NodeID, ok bool) { return 0, false } -func (nb *nodeBackend) PeerByID(id tailcfg.NodeID) (_ tailcfg.NodeView, ok bool) { +func (nb *nodeBackend) NodeByID(id tailcfg.NodeID) (_ tailcfg.NodeView, ok bool) { nb.mu.Lock() defer nb.mu.Unlock() + if nb.netMap != nil { + if self := nb.netMap.SelfNode; self.Valid() && self.ID() == id { + return self, true + } + } n, ok := nb.peers[id] return n, ok } From d2c1ed22c39096f11cfd7920449ff746b865a025 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Thu, 26 Jun 2025 13:37:21 -0700 Subject: [PATCH 120/263] .github/workflows: replace tibdex with official GitHub Action (#16385) GitHub used to recommend the tibdex/github-app-token GitHub Action until they wrote their own actions/create-github-app-token. This patch replaces the use of the third-party action with the official one. Updates #cleanup Signed-off-by: Simon Law --- .github/workflows/update-flake.yml | 9 ++++----- .github/workflows/update-webclient-prebuilt.yml | 11 ++++------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/update-flake.yml b/.github/workflows/update-flake.yml index af7bdff1ee66d..61a09cea1c990 100644 --- a/.github/workflows/update-flake.yml +++ b/.github/workflows/update-flake.yml @@ -27,13 +27,12 @@ jobs: run: ./update-flake.sh - name: Get access token - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 id: generate-token with: - app_id: ${{ secrets.LICENSING_APP_ID }} - installation_retrieval_mode: "id" - installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }} - private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }} + # Get token for app: https://github.com/apps/tailscale-code-updater + app-id: ${{ secrets.CODE_UPDATER_APP_ID }} + private-key: ${{ secrets.CODE_UPDATER_APP_PRIVATE_KEY }} - name: Send pull request uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8 diff --git a/.github/workflows/update-webclient-prebuilt.yml b/.github/workflows/update-webclient-prebuilt.yml index f1c2b0c3b9368..5565b8c86c4bf 100644 --- a/.github/workflows/update-webclient-prebuilt.yml +++ b/.github/workflows/update-webclient-prebuilt.yml @@ -23,15 +23,12 @@ jobs: ./tool/go mod tidy - name: Get access token - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 id: generate-token with: - # TODO(will): this should use the code updater app rather than licensing. - # It has the same permissions, so not a big deal, but still. - app_id: ${{ secrets.LICENSING_APP_ID }} - installation_retrieval_mode: "id" - installation_retrieval_payload: ${{ secrets.LICENSING_APP_INSTALLATION_ID }} - private_key: ${{ secrets.LICENSING_APP_PRIVATE_KEY }} + # Get token for app: https://github.com/apps/tailscale-code-updater + app-id: ${{ secrets.CODE_UPDATER_APP_ID }} + private-key: ${{ secrets.CODE_UPDATER_APP_PRIVATE_KEY }} - name: Send pull request id: pull-request From 6feb3c35cb851d54b613236c31f2dd3c03dbd6b7 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Thu, 26 Jun 2025 17:09:13 -0700 Subject: [PATCH 121/263] ipn/store: automatically migrate between plaintext and encrypted state (#16318) Add a new `--encrypt-state` flag to `cmd/tailscaled`. Based on that flag, migrate the existing state file to/from encrypted format if needed. Updates #15830 Signed-off-by: Andrew Lytvynov --- atomicfile/atomicfile.go | 6 +- cmd/tailscaled/tailscaled.go | 49 +++++- cmd/tsconnect/src/lib/js-state-store.ts | 3 + cmd/tsconnect/src/types/wasm_js.d.ts | 1 + cmd/tsconnect/wasm/wasm_js.go | 24 +++ docs/windows/policy/en-US/tailscale.adml | 11 +- docs/windows/policy/tailscale.admx | 14 ++ feature/tpm/tpm.go | 20 ++- feature/tpm/tpm_test.go | 165 +++++++++++++++++- ipn/ipnlocal/state_test.go | 8 +- ipn/store.go | 6 + ipn/store/awsstore/store_aws.go | 5 + ipn/store/kubestore/store_kube.go | 5 + ipn/store/mem/store_mem.go | 14 ++ ipn/store/stores.go | 140 +++++++++++++++ ipn/store_test.go | 14 ++ tailcfg/tailcfg.go | 3 + tstest/integration/integration.go | 16 +- tstest/integration/integration_test.go | 59 +++++++ .../tailscaled_deps_test_darwin.go | 1 + .../tailscaled_deps_test_freebsd.go | 1 + .../integration/tailscaled_deps_test_linux.go | 1 + .../tailscaled_deps_test_openbsd.go | 1 + util/syspolicy/policy_keys.go | 5 + 24 files changed, 546 insertions(+), 26 deletions(-) diff --git a/atomicfile/atomicfile.go b/atomicfile/atomicfile.go index b3c8c93da2af9..9cae9bb750fa8 100644 --- a/atomicfile/atomicfile.go +++ b/atomicfile/atomicfile.go @@ -48,5 +48,9 @@ func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { if err := f.Close(); err != nil { return err } - return rename(tmpName, filename) + return Rename(tmpName, filename) } + +// Rename srcFile to dstFile, similar to [os.Rename] but preserving file +// attributes and ACLs on Windows. +func Rename(srcFile, dstFile string) error { return rename(srcFile, dstFile) } diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index 61b811c129454..3987b0c26927f 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -64,6 +64,7 @@ import ( "tailscale.com/util/clientmetric" "tailscale.com/util/multierr" "tailscale.com/util/osshare" + "tailscale.com/util/syspolicy" "tailscale.com/version" "tailscale.com/version/distro" "tailscale.com/wgengine" @@ -126,6 +127,7 @@ var args struct { debug string port uint16 statepath string + encryptState bool statedir string socketpath string birdSocketPath string @@ -193,6 +195,7 @@ func main() { flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`) flag.Var(flagtype.PortValue(&args.port, defaultPort()), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select") flag.StringVar(&args.statepath, "state", "", "absolute path of state file; use 'kube:' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM; use 'mem:' to not store state and register as an ephemeral node. If empty and --statedir is provided, the default is /tailscaled.state. Default: "+paths.DefaultTailscaledStateFile()) + flag.BoolVar(&args.encryptState, "encrypt-state", defaultEncryptState(), "encrypt the state file on disk; uses TPM on Linux and Windows, on all other platforms this flag is not supported") flag.StringVar(&args.statedir, "statedir", "", "path to directory for storage of config state, TLS certs, temporary incoming Taildrop files, etc. If empty, it's derived from --state when possible.") flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket") flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket") @@ -268,6 +271,28 @@ func main() { } } + if args.encryptState { + if runtime.GOOS != "linux" && runtime.GOOS != "windows" { + log.SetFlags(0) + log.Fatalf("--encrypt-state is not supported on %s", runtime.GOOS) + } + // Check if we have TPM support in this build. + if !store.HasKnownProviderPrefix(store.TPMPrefix + "/") { + log.SetFlags(0) + log.Fatal("--encrypt-state is not supported in this build of tailscaled") + } + // Check if we have TPM access. + if !hostinfo.New().TPM.Present() { + log.SetFlags(0) + log.Fatal("--encrypt-state is not supported on this device or a TPM is not accessible") + } + // Check for conflicting prefix in --state, like arn: or kube:. + if args.statepath != "" && store.HasKnownProviderPrefix(args.statepath) { + log.SetFlags(0) + log.Fatal("--encrypt-state can only be used with --state set to a local file path") + } + } + if args.disableLogs { envknob.SetNoLogsNoSupport() } @@ -315,13 +340,17 @@ func trySynologyMigration(p string) error { } func statePathOrDefault() string { + var path string if args.statepath != "" { - return args.statepath + path = args.statepath } - if args.statedir != "" { - return filepath.Join(args.statedir, "tailscaled.state") + if path == "" && args.statedir != "" { + path = filepath.Join(args.statedir, "tailscaled.state") } - return "" + if path != "" && !store.HasKnownProviderPrefix(path) && args.encryptState { + path = store.TPMPrefix + path + } + return path } // serverOptions is the configuration of the Tailscale node agent. @@ -974,3 +1003,15 @@ func applyIntegrationTestEnvKnob() { } } } + +func defaultEncryptState() bool { + if runtime.GOOS != "windows" && runtime.GOOS != "linux" { + // TPM encryption is only configurable on Windows and Linux. Other + // platforms either use system APIs and are not configurable + // (Android/Apple), or don't support any form of encryption yet + // (plan9/FreeBSD/etc). + return false + } + v, _ := syspolicy.GetBoolean(syspolicy.EncryptState, false) + return v +} diff --git a/cmd/tsconnect/src/lib/js-state-store.ts b/cmd/tsconnect/src/lib/js-state-store.ts index e57dfd98efabd..7f2fc8087e768 100644 --- a/cmd/tsconnect/src/lib/js-state-store.ts +++ b/cmd/tsconnect/src/lib/js-state-store.ts @@ -10,4 +10,7 @@ export const sessionStateStorage: IPNStateStorage = { getState(id) { return window.sessionStorage[`ipn-state-${id}`] || "" }, + all() { + return JSON.stringify(window.sessionStorage) + }, } diff --git a/cmd/tsconnect/src/types/wasm_js.d.ts b/cmd/tsconnect/src/types/wasm_js.d.ts index 492197ccb1a9b..f47a972b03fba 100644 --- a/cmd/tsconnect/src/types/wasm_js.d.ts +++ b/cmd/tsconnect/src/types/wasm_js.d.ts @@ -44,6 +44,7 @@ declare global { interface IPNStateStorage { setState(id: string, value: string): void getState(id: string): string + all(): string } type IPNConfig = { diff --git a/cmd/tsconnect/wasm/wasm_js.go b/cmd/tsconnect/wasm/wasm_js.go index 779a87e49dec9..c5ff56120f492 100644 --- a/cmd/tsconnect/wasm/wasm_js.go +++ b/cmd/tsconnect/wasm/wasm_js.go @@ -15,6 +15,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "iter" "log" "math/rand/v2" "net" @@ -579,6 +580,29 @@ func (s *jsStateStore) WriteState(id ipn.StateKey, bs []byte) error { return nil } +func (s *jsStateStore) All() iter.Seq2[ipn.StateKey, []byte] { + return func(yield func(ipn.StateKey, []byte) bool) { + jsValue := s.jsStateStorage.Call("all") + if jsValue.String() == "" { + return + } + buf, err := hex.DecodeString(jsValue.String()) + if err != nil { + return + } + var state map[string][]byte + if err := json.Unmarshal(buf, &state); err != nil { + return + } + + for k, v := range state { + if !yield(ipn.StateKey(k), v) { + break + } + } + } +} + func mapSlice[T any, M any](a []T, f func(T) M) []M { n := make([]M, len(a)) for i, e := range a { diff --git a/docs/windows/policy/en-US/tailscale.adml b/docs/windows/policy/en-US/tailscale.adml index 62ff94da7096d..c09d847bc7c0d 100644 --- a/docs/windows/policy/en-US/tailscale.adml +++ b/docs/windows/policy/en-US/tailscale.adml @@ -19,6 +19,7 @@ Tailscale version 1.80.0 and later Tailscale version 1.82.0 and later Tailscale version 1.84.0 and later + Tailscale version 1.86.0 and later Tailscale UI customization Settings @@ -67,7 +68,7 @@ If you disable or do not configure this policy setting, an interactive user logi See https://tailscale.com/kb/1315/mdm-keys#set-an-auth-key for more details.]]> Require using a specific Exit Node + Encrypt client state file stored on disk + diff --git a/docs/windows/policy/tailscale.admx b/docs/windows/policy/tailscale.admx index d97b24c36b5df..0a8aa1a75eb50 100644 --- a/docs/windows/policy/tailscale.admx +++ b/docs/windows/policy/tailscale.admx @@ -66,6 +66,10 @@ displayName="$(string.SINCE_V1_84)"> + + + @@ -365,5 +369,15 @@ + + + + + + + + + + diff --git a/feature/tpm/tpm.go b/feature/tpm/tpm.go index 6feac85e35ecb..64656d412a2e7 100644 --- a/feature/tpm/tpm.go +++ b/feature/tpm/tpm.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "iter" "log" "os" "path/filepath" @@ -37,7 +38,7 @@ func init() { hostinfo.RegisterHostinfoNewHook(func(hi *tailcfg.Hostinfo) { hi.TPM = infoOnce() }) - store.Register(storePrefix, newStore) + store.Register(store.TPMPrefix, newStore) } func info() *tailcfg.TPMInfo { @@ -103,10 +104,8 @@ func propToString(v uint32) string { return string(slices.DeleteFunc(chars, func(b byte) bool { return b < ' ' || b > '~' })) } -const storePrefix = "tpmseal:" - func newStore(logf logger.Logf, path string) (ipn.StateStore, error) { - path = strings.TrimPrefix(path, storePrefix) + path = strings.TrimPrefix(path, store.TPMPrefix) if err := paths.MkStateDir(filepath.Dir(path)); err != nil { return nil, fmt.Errorf("creating state directory: %w", err) } @@ -205,6 +204,19 @@ func (s *tpmStore) writeSealed() error { return atomicfile.WriteFile(s.path, buf, 0600) } +func (s *tpmStore) All() iter.Seq2[ipn.StateKey, []byte] { + return func(yield func(ipn.StateKey, []byte) bool) { + s.mu.Lock() + defer s.mu.Unlock() + + for k, v := range s.cache { + if !yield(k, v) { + break + } + } + } +} + // The nested levels of encoding and encryption are confusing, so here's what's // going on in plain English. // diff --git a/feature/tpm/tpm_test.go b/feature/tpm/tpm_test.go index a022b69b2bf04..b08681354a1e4 100644 --- a/feature/tpm/tpm_test.go +++ b/feature/tpm/tpm_test.go @@ -6,13 +6,22 @@ package tpm import ( "bytes" "crypto/rand" + "encoding/json" "errors" + "fmt" + "maps" + "os" "path/filepath" + "slices" "strconv" + "strings" "testing" + "github.com/google/go-cmp/cmp" "tailscale.com/ipn" "tailscale.com/ipn/store" + "tailscale.com/ipn/store/mem" + "tailscale.com/types/logger" ) func TestPropToString(t *testing.T) { @@ -29,11 +38,9 @@ func TestPropToString(t *testing.T) { } func skipWithoutTPM(t testing.TB) { - tpm, err := open() - if err != nil { + if !tpmSupported() { t.Skip("TPM not available") } - tpm.Close() } func TestSealUnseal(t *testing.T) { @@ -67,7 +74,7 @@ func TestSealUnseal(t *testing.T) { func TestStore(t *testing.T) { skipWithoutTPM(t) - path := storePrefix + filepath.Join(t.TempDir(), "state") + path := store.TPMPrefix + filepath.Join(t.TempDir(), "state") store, err := newStore(t.Logf, path) if err != nil { t.Fatal(err) @@ -180,3 +187,153 @@ func BenchmarkStore(b *testing.B) { }) } } + +func TestMigrateStateToTPM(t *testing.T) { + if !tpmSupported() { + t.Logf("using mock tpmseal provider") + store.RegisterForTest(t, store.TPMPrefix, newMockTPMSeal) + } + + storePath := filepath.Join(t.TempDir(), "store") + // Make sure migration doesn't cause a failure when no state file exists. + if _, err := store.New(t.Logf, store.TPMPrefix+storePath); err != nil { + t.Fatalf("store.New failed for new tpmseal store: %v", err) + } + os.Remove(storePath) + + initial, err := store.New(t.Logf, storePath) + if err != nil { + t.Fatalf("store.New failed for new file store: %v", err) + } + + // Populate initial state file. + content := map[ipn.StateKey][]byte{ + "foo": []byte("bar"), + "baz": []byte("qux"), + } + for k, v := range content { + if err := initial.WriteState(k, v); err != nil { + t.Fatal(err) + } + } + // Expected file keys for plaintext and sealed versions of state. + keysPlaintext := []string{"foo", "baz"} + keysTPMSeal := []string{"key", "nonce", "data"} + + for _, tt := range []struct { + desc string + path string + wantKeys []string + }{ + { + desc: "plaintext-to-plaintext", + path: storePath, + wantKeys: keysPlaintext, + }, + { + desc: "plaintext-to-tpmseal", + path: store.TPMPrefix + storePath, + wantKeys: keysTPMSeal, + }, + { + desc: "tpmseal-to-tpmseal", + path: store.TPMPrefix + storePath, + wantKeys: keysTPMSeal, + }, + { + desc: "tpmseal-to-plaintext", + path: storePath, + wantKeys: keysPlaintext, + }, + } { + t.Run(tt.desc, func(t *testing.T) { + s, err := store.New(t.Logf, tt.path) + if err != nil { + t.Fatalf("migration failed: %v", err) + } + gotContent := maps.Collect(s.All()) + if diff := cmp.Diff(content, gotContent); diff != "" { + t.Errorf("unexpected content after migration, diff:\n%s", diff) + } + + buf, err := os.ReadFile(storePath) + if err != nil { + t.Fatal(err) + } + var data map[string]any + if err := json.Unmarshal(buf, &data); err != nil { + t.Fatal(err) + } + gotKeys := slices.Collect(maps.Keys(data)) + slices.Sort(gotKeys) + slices.Sort(tt.wantKeys) + if diff := cmp.Diff(gotKeys, tt.wantKeys); diff != "" { + t.Errorf("unexpected content keys after migration, diff:\n%s", diff) + } + }) + } +} + +func tpmSupported() bool { + tpm, err := open() + if err != nil { + return false + } + tpm.Close() + return true +} + +type mockTPMSealProvider struct { + path string + mem.Store +} + +func newMockTPMSeal(logf logger.Logf, path string) (ipn.StateStore, error) { + path, ok := strings.CutPrefix(path, store.TPMPrefix) + if !ok { + return nil, fmt.Errorf("%q missing tpmseal: prefix", path) + } + s := &mockTPMSealProvider{path: path, Store: mem.Store{}} + buf, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return s, s.flushState() + } + if err != nil { + return nil, err + } + var data struct { + Key string + Nonce string + Data map[ipn.StateKey][]byte + } + if err := json.Unmarshal(buf, &data); err != nil { + return nil, err + } + if data.Key == "" || data.Nonce == "" { + return nil, fmt.Errorf("%q missing key or nonce", path) + } + for k, v := range data.Data { + s.Store.WriteState(k, v) + } + return s, nil +} + +func (p *mockTPMSealProvider) WriteState(k ipn.StateKey, v []byte) error { + if err := p.Store.WriteState(k, v); err != nil { + return err + } + return p.flushState() +} + +func (p *mockTPMSealProvider) flushState() error { + data := map[string]any{ + "key": "foo", + "nonce": "bar", + "data": maps.Collect(p.Store.All()), + } + buf, err := json.Marshal(data) + if err != nil { + return err + } + return os.WriteFile(p.path, buf, 0600) +} diff --git a/ipn/ipnlocal/state_test.go b/ipn/ipnlocal/state_test.go index 2921de2032913..eb36643856f82 100644 --- a/ipn/ipnlocal/state_test.go +++ b/ipn/ipnlocal/state_test.go @@ -1013,17 +1013,13 @@ func TestEditPrefsHasNoKeys(t *testing.T) { } type testStateStorage struct { - mem mem.Store + mem.Store written atomic.Bool } -func (s *testStateStorage) ReadState(id ipn.StateKey) ([]byte, error) { - return s.mem.ReadState(id) -} - func (s *testStateStorage) WriteState(id ipn.StateKey, bs []byte) error { s.written.Store(true) - return s.mem.WriteState(id, bs) + return s.Store.WriteState(id, bs) } // awaitWrite clears the "I've seen writes" bit, in prep for a future diff --git a/ipn/store.go b/ipn/store.go index 550aa8cba819a..e176e48421216 100644 --- a/ipn/store.go +++ b/ipn/store.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "iter" "net" "strconv" ) @@ -83,6 +84,11 @@ type StateStore interface { // instead, which only writes if the value is different from what's // already in the store. WriteState(id StateKey, bs []byte) error + // All returns an iterator over all StateStore keys. Using ReadState or + // WriteState is not safe while iterating and can lead to a deadlock. + // The order of keys in the iterator is not specified and may change + // between runs. + All() iter.Seq2[StateKey, []byte] } // WriteState is a wrapper around store.WriteState that only writes if diff --git a/ipn/store/awsstore/store_aws.go b/ipn/store/awsstore/store_aws.go index 40bbbf0370822..523d1657b109d 100644 --- a/ipn/store/awsstore/store_aws.go +++ b/ipn/store/awsstore/store_aws.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "iter" "net/url" "regexp" "strings" @@ -253,3 +254,7 @@ func (s *awsStore) persistState() error { _, err = s.ssmClient.PutParameter(context.TODO(), in) return err } + +func (s *awsStore) All() iter.Seq2[ipn.StateKey, []byte] { + return s.memory.All() +} diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go index 14025bbb4150a..f6bedbf0b8054 100644 --- a/ipn/store/kubestore/store_kube.go +++ b/ipn/store/kubestore/store_kube.go @@ -7,6 +7,7 @@ package kubestore import ( "context" "fmt" + "iter" "log" "net" "os" @@ -428,3 +429,7 @@ func sanitizeKey[T ~string](k T) string { return '_' }, string(k)) } + +func (s *Store) All() iter.Seq2[ipn.StateKey, []byte] { + return s.memory.All() +} diff --git a/ipn/store/mem/store_mem.go b/ipn/store/mem/store_mem.go index 6f474ce993b43..6c22aefd547f8 100644 --- a/ipn/store/mem/store_mem.go +++ b/ipn/store/mem/store_mem.go @@ -7,6 +7,7 @@ package mem import ( "bytes" "encoding/json" + "iter" "sync" xmaps "golang.org/x/exp/maps" @@ -85,3 +86,16 @@ func (s *Store) ExportToJSON() ([]byte, error) { } return json.MarshalIndent(s.cache, "", " ") } + +func (s *Store) All() iter.Seq2[ipn.StateKey, []byte] { + return func(yield func(ipn.StateKey, []byte) bool) { + s.mu.Lock() + defer s.mu.Unlock() + + for k, v := range s.cache { + if !yield(k, v) { + break + } + } + } +} diff --git a/ipn/store/stores.go b/ipn/store/stores.go index 1a98574c91cf9..43c79639934b8 100644 --- a/ipn/store/stores.go +++ b/ipn/store/stores.go @@ -7,10 +7,14 @@ package store import ( "bytes" "encoding/json" + "errors" "fmt" + "iter" + "maps" "os" "path/filepath" "runtime" + "slices" "strings" "sync" @@ -20,6 +24,7 @@ import ( "tailscale.com/paths" "tailscale.com/types/logger" "tailscale.com/util/mak" + "tailscale.com/util/testenv" ) // Provider returns a StateStore for the provided path. @@ -32,6 +37,9 @@ func init() { var knownStores map[string]Provider +// TPMPrefix is the path prefix used for TPM-encrypted StateStore. +const TPMPrefix = "tpmseal:" + // New returns a StateStore based on the provided arg // and registered stores. // The arg is of the form "prefix:rest", where prefix was previously @@ -53,12 +61,23 @@ func New(logf logger.Logf, path string) (ipn.StateStore, error) { if strings.HasPrefix(path, prefix) { // We can't strip the prefix here as some NewStoreFunc (like arn:) // expect the prefix. + if prefix == TPMPrefix { + if runtime.GOOS == "windows" { + path = TPMPrefix + TryWindowsAppDataMigration(logf, strings.TrimPrefix(path, TPMPrefix)) + } + if err := maybeMigrateLocalStateFile(logf, path); err != nil { + return nil, fmt.Errorf("failed to migrate existing state file to TPM-sealed format: %w", err) + } + } return sf(logf, path) } } if runtime.GOOS == "windows" { path = TryWindowsAppDataMigration(logf, path) } + if err := maybeMigrateLocalStateFile(logf, path); err != nil { + return nil, fmt.Errorf("failed to migrate existing TPM-sealed state file to plaintext format: %w", err) + } return NewFileStore(logf, path) } @@ -77,6 +96,29 @@ func Register(prefix string, fn Provider) { mak.Set(&knownStores, prefix, fn) } +// RegisterForTest registers a prefix to be used for NewStore in tests. An +// existing registered prefix will be replaced. +func RegisterForTest(t testenv.TB, prefix string, fn Provider) { + if len(prefix) == 0 { + panic("prefix is empty") + } + old := maps.Clone(knownStores) + t.Cleanup(func() { knownStores = old }) + + mak.Set(&knownStores, prefix, fn) +} + +// HasKnownProviderPrefix reports whether path uses one of the registered +// Provider prefixes. +func HasKnownProviderPrefix(path string) bool { + for prefix := range knownStores { + if strings.HasPrefix(path, prefix) { + return true + } + } + return false +} + // TryWindowsAppDataMigration attempts to copy the Windows state file // from its old location to the new location. (Issue 2856) // @@ -179,3 +221,101 @@ func (s *FileStore) WriteState(id ipn.StateKey, bs []byte) error { } return atomicfile.WriteFile(s.path, bs, 0600) } + +func (s *FileStore) All() iter.Seq2[ipn.StateKey, []byte] { + return func(yield func(ipn.StateKey, []byte) bool) { + s.mu.Lock() + defer s.mu.Unlock() + + for k, v := range s.cache { + if !yield(k, v) { + break + } + } + } +} + +func maybeMigrateLocalStateFile(logf logger.Logf, path string) error { + path, toTPM := strings.CutPrefix(path, TPMPrefix) + + // Extract JSON keys from the file on disk and guess what kind it is. + bs, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + var content map[string]any + if err := json.Unmarshal(bs, &content); err != nil { + return fmt.Errorf("failed to unmarshal %q: %w", path, err) + } + keys := slices.Sorted(maps.Keys(content)) + tpmKeys := []string{"key", "nonce", "data"} + slices.Sort(tpmKeys) + // TPM-sealed files will have exactly these keys. + existingFileSealed := slices.Equal(keys, tpmKeys) + // Plaintext files for nodes that registered at least once will have this + // key, plus other dynamic ones. + _, existingFilePlaintext := content["_machinekey"] + isTPM := existingFileSealed && !existingFilePlaintext + + if isTPM == toTPM { + // No migration needed. + return nil + } + + newTPMStore, ok := knownStores[TPMPrefix] + if !ok { + return errors.New("this build does not support TPM integration") + } + + // Open from (old format) and to (new format) stores for migration. The + // "to" store will be at tmpPath. + var from, to ipn.StateStore + tmpPath := path + ".tmp" + if toTPM { + // Migrate plaintext file to be TPM-sealed. + from, err = NewFileStore(logf, path) + if err != nil { + return fmt.Errorf("NewFileStore(%q): %w", path, err) + } + to, err = newTPMStore(logf, TPMPrefix+tmpPath) + if err != nil { + return fmt.Errorf("newTPMStore(%q): %w", tmpPath, err) + } + } else { + // Migrate TPM-selaed file to plaintext. + from, err = newTPMStore(logf, TPMPrefix+path) + if err != nil { + return fmt.Errorf("newTPMStore(%q): %w", path, err) + } + to, err = NewFileStore(logf, tmpPath) + if err != nil { + return fmt.Errorf("NewFileStore(%q): %w", tmpPath, err) + } + } + defer os.Remove(tmpPath) + + // Copy all the items. This is pretty inefficient, because both stores + // write the file to disk for each WriteState, but that's ok for a one-time + // migration. + for k, v := range from.All() { + if err := to.WriteState(k, v); err != nil { + return err + } + } + + // Finally, overwrite the state file with the new one we created at + // tmpPath. + if err := atomicfile.Rename(tmpPath, path); err != nil { + return err + } + + if toTPM { + logf("migrated %q from plaintext to TPM-sealed format", path) + } else { + logf("migrated %q from TPM-sealed to plaintext format", path) + } + return nil +} diff --git a/ipn/store_test.go b/ipn/store_test.go index fcc082d8a8a87..4dd7321b9048d 100644 --- a/ipn/store_test.go +++ b/ipn/store_test.go @@ -5,6 +5,7 @@ package ipn import ( "bytes" + "iter" "sync" "testing" @@ -31,6 +32,19 @@ func (s *memStore) WriteState(k StateKey, v []byte) error { return nil } +func (s *memStore) All() iter.Seq2[StateKey, []byte] { + return func(yield func(StateKey, []byte) bool) { + s.mu.Lock() + defer s.mu.Unlock() + + for k, v := range s.m { + if !yield(k, v) { + break + } + } + } +} + func TestWriteState(t *testing.T) { var ss StateStore = new(memStore) WriteState(ss, "foo", []byte("bar")) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 4679609f3e9d4..23f3cc49b152b 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -908,6 +908,9 @@ type TPMInfo struct { SpecRevision int `json:",omitempty"` } +// Present reports whether a TPM device is present on this machine. +func (t *TPMInfo) Present() bool { return t != nil } + // ServiceName is the name of a service, of the form `svc:dns-label`. Services // represent some kind of application provided for users of the tailnet with a // MagicDNS name and possibly dedicated IP addresses. Currently (2024-01-21), diff --git a/tstest/integration/integration.go b/tstest/integration/integration.go index d64bfbbd9d755..987bb569a4f66 100644 --- a/tstest/integration/integration.go +++ b/tstest/integration/integration.go @@ -569,11 +569,12 @@ type TestNode struct { env *TestEnv tailscaledParser *nodeOutputParser - dir string // temp dir for sock & state - configFile string // or empty for none - sockFile string - stateFile string - upFlagGOOS string // if non-empty, sets TS_DEBUG_UP_FLAG_GOOS for cmd/tailscale CLI + dir string // temp dir for sock & state + configFile string // or empty for none + sockFile string + stateFile string + upFlagGOOS string // if non-empty, sets TS_DEBUG_UP_FLAG_GOOS for cmd/tailscale CLI + encryptState bool mu sync.Mutex onLogLine []func([]byte) @@ -640,7 +641,7 @@ func (n *TestNode) diskPrefs() *ipn.Prefs { if _, err := os.ReadFile(n.stateFile); err != nil { t.Fatalf("reading prefs: %v", err) } - fs, err := store.NewFileStore(nil, n.stateFile) + fs, err := store.New(nil, n.stateFile) if err != nil { t.Fatalf("reading prefs, NewFileStore: %v", err) } @@ -822,6 +823,9 @@ func (n *TestNode) StartDaemonAsIPNGOOS(ipnGOOS string) *Daemon { if n.configFile != "" { cmd.Args = append(cmd.Args, "--config="+n.configFile) } + if n.encryptState { + cmd.Args = append(cmd.Args, "--encrypt-state") + } cmd.Env = append(os.Environ(), "TS_DEBUG_PERMIT_HTTP_C2N=1", "TS_LOG_TARGET="+n.env.LogCatcherServer.URL, diff --git a/tstest/integration/integration_test.go b/tstest/integration/integration_test.go index 90cc7e443b5d3..7cb251f31c344 100644 --- a/tstest/integration/integration_test.go +++ b/tstest/integration/integration_test.go @@ -21,6 +21,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "strconv" "sync/atomic" "testing" @@ -32,6 +33,7 @@ import ( "tailscale.com/client/tailscale" "tailscale.com/clientupdate" "tailscale.com/cmd/testwrapper/flakytest" + "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/net/tsaddr" "tailscale.com/net/tstun" @@ -1470,3 +1472,60 @@ func TestNetstackUDPLoopback(t *testing.T) { d1.MustCleanShutdown(t) } + +func TestEncryptStateMigration(t *testing.T) { + if !hostinfo.New().TPM.Present() { + t.Skip("TPM not available") + } + if runtime.GOOS != "linux" && runtime.GOOS != "windows" { + t.Skip("--encrypt-state for tailscaled state not supported on this platform") + } + tstest.Shard(t) + tstest.Parallel(t) + env := NewTestEnv(t) + n := NewTestNode(t, env) + + runNode := func(t *testing.T, wantStateKeys []string) { + t.Helper() + + // Run the node. + d := n.StartDaemon() + n.AwaitResponding() + n.MustUp() + n.AwaitRunning() + + // Check the contents of the state file. + buf, err := os.ReadFile(n.stateFile) + if err != nil { + t.Fatalf("reading %q: %v", n.stateFile, err) + } + t.Logf("state file content:\n%s", buf) + var content map[string]any + if err := json.Unmarshal(buf, &content); err != nil { + t.Fatalf("parsing %q: %v", n.stateFile, err) + } + for _, k := range wantStateKeys { + if _, ok := content[k]; !ok { + t.Errorf("state file is missing key %q", k) + } + } + + // Stop the node. + d.MustCleanShutdown(t) + } + + wantPlaintextStateKeys := []string{"_machinekey", "_current-profile", "_profiles"} + wantEncryptedStateKeys := []string{"key", "nonce", "data"} + t.Run("regular-state", func(t *testing.T) { + n.encryptState = false + runNode(t, wantPlaintextStateKeys) + }) + t.Run("migrate-to-encrypted", func(t *testing.T) { + n.encryptState = true + runNode(t, wantEncryptedStateKeys) + }) + t.Run("migrate-to-plaintext", func(t *testing.T) { + n.encryptState = false + runNode(t, wantPlaintextStateKeys) + }) +} diff --git a/tstest/integration/tailscaled_deps_test_darwin.go b/tstest/integration/tailscaled_deps_test_darwin.go index 321ba25668c1f..a73c6ebf649f2 100644 --- a/tstest/integration/tailscaled_deps_test_darwin.go +++ b/tstest/integration/tailscaled_deps_test_darwin.go @@ -51,6 +51,7 @@ import ( _ "tailscale.com/util/eventbus" _ "tailscale.com/util/multierr" _ "tailscale.com/util/osshare" + _ "tailscale.com/util/syspolicy" _ "tailscale.com/version" _ "tailscale.com/version/distro" _ "tailscale.com/wgengine" diff --git a/tstest/integration/tailscaled_deps_test_freebsd.go b/tstest/integration/tailscaled_deps_test_freebsd.go index 321ba25668c1f..a73c6ebf649f2 100644 --- a/tstest/integration/tailscaled_deps_test_freebsd.go +++ b/tstest/integration/tailscaled_deps_test_freebsd.go @@ -51,6 +51,7 @@ import ( _ "tailscale.com/util/eventbus" _ "tailscale.com/util/multierr" _ "tailscale.com/util/osshare" + _ "tailscale.com/util/syspolicy" _ "tailscale.com/version" _ "tailscale.com/version/distro" _ "tailscale.com/wgengine" diff --git a/tstest/integration/tailscaled_deps_test_linux.go b/tstest/integration/tailscaled_deps_test_linux.go index 321ba25668c1f..a73c6ebf649f2 100644 --- a/tstest/integration/tailscaled_deps_test_linux.go +++ b/tstest/integration/tailscaled_deps_test_linux.go @@ -51,6 +51,7 @@ import ( _ "tailscale.com/util/eventbus" _ "tailscale.com/util/multierr" _ "tailscale.com/util/osshare" + _ "tailscale.com/util/syspolicy" _ "tailscale.com/version" _ "tailscale.com/version/distro" _ "tailscale.com/wgengine" diff --git a/tstest/integration/tailscaled_deps_test_openbsd.go b/tstest/integration/tailscaled_deps_test_openbsd.go index 321ba25668c1f..a73c6ebf649f2 100644 --- a/tstest/integration/tailscaled_deps_test_openbsd.go +++ b/tstest/integration/tailscaled_deps_test_openbsd.go @@ -51,6 +51,7 @@ import ( _ "tailscale.com/util/eventbus" _ "tailscale.com/util/multierr" _ "tailscale.com/util/osshare" + _ "tailscale.com/util/syspolicy" _ "tailscale.com/version" _ "tailscale.com/version/distro" _ "tailscale.com/wgengine" diff --git a/util/syspolicy/policy_keys.go b/util/syspolicy/policy_keys.go index ed00d0004abb1..b19a3e7fec61f 100644 --- a/util/syspolicy/policy_keys.go +++ b/util/syspolicy/policy_keys.go @@ -120,6 +120,10 @@ const ( LogSCMInteractions Key = "LogSCMInteractions" FlushDNSOnSessionUnlock Key = "FlushDNSOnSessionUnlock" + // EncryptState is a boolean setting that specifies whether to encrypt the + // tailscaled state file with a TPM device. + EncryptState Key = "EncryptState" + // PostureChecking indicates if posture checking is enabled and the client shall gather // posture data. // Key is a string value that specifies an option: "always", "never", "user-decides". @@ -186,6 +190,7 @@ var implicitDefinitions = []*setting.Definition{ setting.NewDefinition(ExitNodeID, setting.DeviceSetting, setting.StringValue), setting.NewDefinition(ExitNodeIP, setting.DeviceSetting, setting.StringValue), setting.NewDefinition(FlushDNSOnSessionUnlock, setting.DeviceSetting, setting.BooleanValue), + setting.NewDefinition(EncryptState, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(Hostname, setting.DeviceSetting, setting.StringValue), setting.NewDefinition(LogSCMInteractions, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(LogTarget, setting.DeviceSetting, setting.StringValue), From b2bf7e988e110e2a7245ac67792f666e1cd114f1 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 26 Jun 2025 18:39:47 -0700 Subject: [PATCH 122/263] wgengine/magicsock: add envknob to toggle UDP relay feature (#16396) Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/debugknobs.go | 6 ++++++ wgengine/magicsock/debugknobs_stubs.go | 1 + wgengine/magicsock/magicsock.go | 6 +++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/wgengine/magicsock/debugknobs.go b/wgengine/magicsock/debugknobs.go index f8fd9f0407d44..0558953887ae0 100644 --- a/wgengine/magicsock/debugknobs.go +++ b/wgengine/magicsock/debugknobs.go @@ -62,6 +62,12 @@ var ( // //lint:ignore U1000 used on Linux/Darwin only debugPMTUD = envknob.RegisterBool("TS_DEBUG_PMTUD") + // debugAssumeUDPRelayCapable forces magicsock to assume that all peers are + // UDP relay capable clients and servers. This will eventually be replaced + // by a [tailcfg.CapabilityVersion] comparison. It enables early testing of + // the UDP relay feature before we have established related + // [tailcfg.CapabilityVersion]'s. + debugAssumeUDPRelayCapable = envknob.RegisterBool("TS_DEBUG_ASSUME_UDP_RELAY_CAPABLE") // Hey you! Adding a new debugknob? Make sure to stub it out in the // debugknobs_stubs.go file too. ) diff --git a/wgengine/magicsock/debugknobs_stubs.go b/wgengine/magicsock/debugknobs_stubs.go index 336d7baa19645..3d23b1f8e8f01 100644 --- a/wgengine/magicsock/debugknobs_stubs.go +++ b/wgengine/magicsock/debugknobs_stubs.go @@ -31,3 +31,4 @@ func debugRingBufferMaxSizeBytes() int { return 0 } func inTest() bool { return false } func debugPeerMap() bool { return false } func pretendpoints() []netip.AddrPort { return []netip.AddrPort{} } +func debugAssumeUDPRelayCapable() bool { return false } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index d7b52269984da..e76d0054f04c2 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2592,12 +2592,12 @@ func (c *Conn) SetProbeUDPLifetime(v bool) { func capVerIsRelayCapable(version tailcfg.CapabilityVersion) bool { // TODO(jwhited): implement once capVer is bumped - return version == math.MinInt32 + return version == math.MinInt32 || debugAssumeUDPRelayCapable() } func capVerIsRelayServerCapable(version tailcfg.CapabilityVersion) bool { - // TODO(jwhited): implement once capVer is bumped - return version == math.MinInt32 + // TODO(jwhited): implement once capVer is bumped & update Test_peerAPIIfCandidateRelayServer + return version == math.MinInt32 || debugAssumeUDPRelayCapable() } // onFilterUpdate is called when a [FilterUpdate] is received over the From b32a01b2dc44986ce83d2dd091f53c31e9a25391 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 26 Jun 2025 19:30:14 -0700 Subject: [PATCH 123/263] disco,net/udprelay,wgengine/magicsock: support relay re-binding (#16388) Relay handshakes may now occur multiple times over the lifetime of a relay server endpoint. Handshake messages now include a handshake generation, which is client specified, as a means to trigger safe challenge reset server-side. Relay servers continue to enforce challenge values as single use. They will only send a given value once, in reply to the first arriving bind message for a handshake generation. VNI has been added to the handshake messages, and we expect the outer Geneve header value to match the sealed value upon reception. Remote peer disco pub key is now also included in handshake messages, and it must match the receiver's expectation for the remote, participating party. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- disco/disco.go | 110 ++++++++++++++++------ disco/disco_test.go | 27 ++++-- net/udprelay/server.go | 146 +++++++++++++++-------------- net/udprelay/server_test.go | 80 +++++++++++++--- wgengine/magicsock/relaymanager.go | 46 +++++++-- 5 files changed, 276 insertions(+), 133 deletions(-) diff --git a/disco/disco.go b/disco/disco.go index 0854eb4c0af5a..d4623c119dbcb 100644 --- a/disco/disco.go +++ b/disco/disco.go @@ -321,79 +321,131 @@ const ( BindUDPRelayHandshakeStateAnswerReceived ) -// bindUDPRelayEndpointLen is the length of a marshalled BindUDPRelayEndpoint -// message, without the message header. -const bindUDPRelayEndpointLen = BindUDPRelayEndpointChallengeLen +// bindUDPRelayEndpointCommonLen is the length of a marshalled +// [BindUDPRelayEndpointCommon], without the message header. +const bindUDPRelayEndpointCommonLen = 72 + +// BindUDPRelayChallengeLen is the length of the Challenge field carried in +// [BindUDPRelayEndpointChallenge] & [BindUDPRelayEndpointAnswer] messages. +const BindUDPRelayChallengeLen = 32 + +// BindUDPRelayEndpointCommon contains fields that are common across all 3 +// UDP relay handshake message types. All 4 field values are expected to be +// consistent for the lifetime of a handshake besides Challenge, which is +// irrelevant in a [BindUDPRelayEndpoint] message. +type BindUDPRelayEndpointCommon struct { + // VNI is the Geneve header Virtual Network Identifier field value, which + // must match this disco-sealed value upon reception. If they are + // non-matching it indicates the cleartext Geneve header was tampered with + // and/or mangled. + VNI uint32 + // Generation represents the handshake generation. Clients must set a new, + // nonzero value at the start of every handshake. + Generation uint32 + // RemoteKey is the disco key of the remote peer participating over this + // relay endpoint. + RemoteKey key.DiscoPublic + // Challenge is set by the server in a [BindUDPRelayEndpointChallenge] + // message, and expected to be echoed back by the client in a + // [BindUDPRelayEndpointAnswer] message. Its value is irrelevant in a + // [BindUDPRelayEndpoint] message, where it simply serves a padding purpose + // ensuring all handshake messages are equal in size. + Challenge [BindUDPRelayChallengeLen]byte +} + +// encode encodes m in b. b must be at least bindUDPRelayEndpointCommonLen bytes +// long. +func (m *BindUDPRelayEndpointCommon) encode(b []byte) { + binary.BigEndian.PutUint32(b, m.VNI) + b = b[4:] + binary.BigEndian.PutUint32(b, m.Generation) + b = b[4:] + m.RemoteKey.AppendTo(b[:0]) + b = b[key.DiscoPublicRawLen:] + copy(b, m.Challenge[:]) +} + +// decode decodes m from b. +func (m *BindUDPRelayEndpointCommon) decode(b []byte) error { + if len(b) < bindUDPRelayEndpointCommonLen { + return errShort + } + m.VNI = binary.BigEndian.Uint32(b) + b = b[4:] + m.Generation = binary.BigEndian.Uint32(b) + b = b[4:] + m.RemoteKey = key.DiscoPublicFromRaw32(mem.B(b[:key.DiscoPublicRawLen])) + b = b[key.DiscoPublicRawLen:] + copy(m.Challenge[:], b[:BindUDPRelayChallengeLen]) + return nil +} // BindUDPRelayEndpoint is the first messaged transmitted from UDP relay client -// towards UDP relay server as part of the 3-way bind handshake. It is padded to -// match the length of BindUDPRelayEndpointChallenge. This message type is -// currently considered experimental and is not yet tied to a +// towards UDP relay server as part of the 3-way bind handshake. This message +// type is currently considered experimental and is not yet tied to a // tailcfg.CapabilityVersion. type BindUDPRelayEndpoint struct { + BindUDPRelayEndpointCommon } func (m *BindUDPRelayEndpoint) AppendMarshal(b []byte) []byte { - ret, _ := appendMsgHeader(b, TypeBindUDPRelayEndpoint, v0, bindUDPRelayEndpointLen) + ret, d := appendMsgHeader(b, TypeBindUDPRelayEndpoint, v0, bindUDPRelayEndpointCommonLen) + m.BindUDPRelayEndpointCommon.encode(d) return ret } func parseBindUDPRelayEndpoint(ver uint8, p []byte) (m *BindUDPRelayEndpoint, err error) { m = new(BindUDPRelayEndpoint) + err = m.BindUDPRelayEndpointCommon.decode(p) + if err != nil { + return nil, err + } return m, nil } -// BindUDPRelayEndpointChallengeLen is the length of a marshalled -// BindUDPRelayEndpointChallenge message, without the message header. -const BindUDPRelayEndpointChallengeLen = 32 - // BindUDPRelayEndpointChallenge is transmitted from UDP relay server towards // UDP relay client in response to a BindUDPRelayEndpoint message as part of the // 3-way bind handshake. This message type is currently considered experimental // and is not yet tied to a tailcfg.CapabilityVersion. type BindUDPRelayEndpointChallenge struct { - Challenge [BindUDPRelayEndpointChallengeLen]byte + BindUDPRelayEndpointCommon } func (m *BindUDPRelayEndpointChallenge) AppendMarshal(b []byte) []byte { - ret, d := appendMsgHeader(b, TypeBindUDPRelayEndpointChallenge, v0, BindUDPRelayEndpointChallengeLen) - copy(d, m.Challenge[:]) + ret, d := appendMsgHeader(b, TypeBindUDPRelayEndpointChallenge, v0, bindUDPRelayEndpointCommonLen) + m.BindUDPRelayEndpointCommon.encode(d) return ret } func parseBindUDPRelayEndpointChallenge(ver uint8, p []byte) (m *BindUDPRelayEndpointChallenge, err error) { - if len(p) < BindUDPRelayEndpointChallengeLen { - return nil, errShort - } m = new(BindUDPRelayEndpointChallenge) - copy(m.Challenge[:], p[:]) + err = m.BindUDPRelayEndpointCommon.decode(p) + if err != nil { + return nil, err + } return m, nil } -// bindUDPRelayEndpointAnswerLen is the length of a marshalled -// BindUDPRelayEndpointAnswer message, without the message header. -const bindUDPRelayEndpointAnswerLen = BindUDPRelayEndpointChallengeLen - // BindUDPRelayEndpointAnswer is transmitted from UDP relay client to UDP relay // server in response to a BindUDPRelayEndpointChallenge message. This message // type is currently considered experimental and is not yet tied to a // tailcfg.CapabilityVersion. type BindUDPRelayEndpointAnswer struct { - Answer [bindUDPRelayEndpointAnswerLen]byte + BindUDPRelayEndpointCommon } func (m *BindUDPRelayEndpointAnswer) AppendMarshal(b []byte) []byte { - ret, d := appendMsgHeader(b, TypeBindUDPRelayEndpointAnswer, v0, bindUDPRelayEndpointAnswerLen) - copy(d, m.Answer[:]) + ret, d := appendMsgHeader(b, TypeBindUDPRelayEndpointAnswer, v0, bindUDPRelayEndpointCommonLen) + m.BindUDPRelayEndpointCommon.encode(d) return ret } func parseBindUDPRelayEndpointAnswer(ver uint8, p []byte) (m *BindUDPRelayEndpointAnswer, err error) { - if len(p) < bindUDPRelayEndpointAnswerLen { - return nil, errShort - } m = new(BindUDPRelayEndpointAnswer) - copy(m.Answer[:], p[:]) + err = m.BindUDPRelayEndpointCommon.decode(p) + if err != nil { + return nil, err + } return m, nil } diff --git a/disco/disco_test.go b/disco/disco_test.go index f2a29a744992f..9fb71ff83b73b 100644 --- a/disco/disco_test.go +++ b/disco/disco_test.go @@ -16,6 +16,15 @@ import ( ) func TestMarshalAndParse(t *testing.T) { + relayHandshakeCommon := BindUDPRelayEndpointCommon{ + VNI: 1, + Generation: 2, + RemoteKey: key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})), + Challenge: [BindUDPRelayChallengeLen]byte{ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + }, + } + tests := []struct { name string want string @@ -86,26 +95,24 @@ func TestMarshalAndParse(t *testing.T) { }, { name: "bind_udp_relay_endpoint", - m: &BindUDPRelayEndpoint{}, - want: "04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + m: &BindUDPRelayEndpoint{ + relayHandshakeCommon, + }, + want: "04 00 00 00 00 01 00 00 00 02 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f", }, { name: "bind_udp_relay_endpoint_challenge", m: &BindUDPRelayEndpointChallenge{ - Challenge: [BindUDPRelayEndpointChallengeLen]byte{ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - }, + relayHandshakeCommon, }, - want: "05 00 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f", + want: "05 00 00 00 00 01 00 00 00 02 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f", }, { name: "bind_udp_relay_endpoint_answer", m: &BindUDPRelayEndpointAnswer{ - Answer: [bindUDPRelayEndpointAnswerLen]byte{ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - }, + relayHandshakeCommon, }, - want: "06 00 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f", + want: "06 00 00 00 00 01 00 00 00 02 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f", }, { name: "call_me_maybe_via", diff --git a/net/udprelay/server.go b/net/udprelay/server.go index 8b9e95fb1e728..e32f8917c520c 100644 --- a/net/udprelay/server.go +++ b/net/udprelay/server.go @@ -96,12 +96,13 @@ type serverEndpoint struct { // indexing of this array aligns with the following fields, e.g. // discoSharedSecrets[0] is the shared secret to use when sealing // Disco protocol messages for transmission towards discoPubKeys[0]. - discoPubKeys pairOfDiscoPubKeys - discoSharedSecrets [2]key.DiscoShared - handshakeState [2]disco.BindUDPRelayHandshakeState - addrPorts [2]netip.AddrPort - lastSeen [2]time.Time // TODO(jwhited): consider using mono.Time - challenge [2][disco.BindUDPRelayEndpointChallengeLen]byte + discoPubKeys pairOfDiscoPubKeys + discoSharedSecrets [2]key.DiscoShared + handshakeGeneration [2]uint32 // or zero if a handshake has never started for that relay leg + handshakeAddrPorts [2]netip.AddrPort // or zero value if a handshake has never started for that relay leg + boundAddrPorts [2]netip.AddrPort // or zero value if a handshake has never completed for that relay leg + lastSeen [2]time.Time // TODO(jwhited): consider using mono.Time + challenge [2][disco.BindUDPRelayChallengeLen]byte lamportID uint64 vni uint32 @@ -112,69 +113,77 @@ func (e *serverEndpoint) handleDiscoControlMsg(from netip.AddrPort, senderIndex if senderIndex != 0 && senderIndex != 1 { return } - handshakeState := e.handshakeState[senderIndex] - if handshakeState == disco.BindUDPRelayHandshakeStateAnswerReceived { - // this sender is already bound - return + + otherSender := 0 + if senderIndex == 0 { + otherSender = 1 } + + validateVNIAndRemoteKey := func(common disco.BindUDPRelayEndpointCommon) error { + if common.VNI != e.vni { + return errors.New("mismatching VNI") + } + if common.RemoteKey.Compare(e.discoPubKeys[otherSender]) != 0 { + return errors.New("mismatching RemoteKey") + } + return nil + } + switch discoMsg := discoMsg.(type) { case *disco.BindUDPRelayEndpoint: - switch handshakeState { - case disco.BindUDPRelayHandshakeStateInit: - // set sender addr - e.addrPorts[senderIndex] = from - fallthrough - case disco.BindUDPRelayHandshakeStateChallengeSent: - if from != e.addrPorts[senderIndex] { - // this is a later arriving bind from a different source, or - // a retransmit and the sender's source has changed, discard - return - } - m := new(disco.BindUDPRelayEndpointChallenge) - copy(m.Challenge[:], e.challenge[senderIndex][:]) - reply := make([]byte, packet.GeneveFixedHeaderLength, 512) - gh := packet.GeneveHeader{Control: true, VNI: e.vni, Protocol: packet.GeneveProtocolDisco} - err := gh.Encode(reply) - if err != nil { - return - } - reply = append(reply, disco.Magic...) - reply = serverDisco.AppendTo(reply) - box := e.discoSharedSecrets[senderIndex].Seal(m.AppendMarshal(nil)) - reply = append(reply, box...) - uw.WriteMsgUDPAddrPort(reply, nil, from) - // set new state - e.handshakeState[senderIndex] = disco.BindUDPRelayHandshakeStateChallengeSent + err := validateVNIAndRemoteKey(discoMsg.BindUDPRelayEndpointCommon) + if err != nil { + // silently drop return - default: - // disco.BindUDPRelayEndpoint is unexpected in all other handshake states + } + if discoMsg.Generation == 0 { + // Generation must be nonzero, silently drop + return + } + if e.handshakeGeneration[senderIndex] == discoMsg.Generation { + // we've seen this generation before, silently drop + return + } + e.handshakeGeneration[senderIndex] = discoMsg.Generation + e.handshakeAddrPorts[senderIndex] = from + m := new(disco.BindUDPRelayEndpointChallenge) + m.VNI = e.vni + m.Generation = discoMsg.Generation + m.RemoteKey = e.discoPubKeys[otherSender] + rand.Read(e.challenge[senderIndex][:]) + copy(m.Challenge[:], e.challenge[senderIndex][:]) + reply := make([]byte, packet.GeneveFixedHeaderLength, 512) + gh := packet.GeneveHeader{Control: true, VNI: e.vni, Protocol: packet.GeneveProtocolDisco} + err = gh.Encode(reply) + if err != nil { return } + reply = append(reply, disco.Magic...) + reply = serverDisco.AppendTo(reply) + box := e.discoSharedSecrets[senderIndex].Seal(m.AppendMarshal(nil)) + reply = append(reply, box...) + uw.WriteMsgUDPAddrPort(reply, nil, from) + return case *disco.BindUDPRelayEndpointAnswer: - switch handshakeState { - case disco.BindUDPRelayHandshakeStateChallengeSent: - if from != e.addrPorts[senderIndex] { - // sender source has changed - return - } - if !bytes.Equal(discoMsg.Answer[:], e.challenge[senderIndex][:]) { - // bad answer - return - } - // sender is now bound - // TODO: Consider installing a fast path via netfilter or similar to - // relay (NAT) data packets for this serverEndpoint. - e.handshakeState[senderIndex] = disco.BindUDPRelayHandshakeStateAnswerReceived - // record last seen as bound time - e.lastSeen[senderIndex] = time.Now() + err := validateVNIAndRemoteKey(discoMsg.BindUDPRelayEndpointCommon) + if err != nil { + // silently drop return - default: - // disco.BindUDPRelayEndpointAnswer is unexpected in all other handshake - // states, or we've already handled it + } + generation := e.handshakeGeneration[senderIndex] + if generation == 0 || // we have no active handshake + generation != discoMsg.Generation || // mismatching generation for the active handshake + e.handshakeAddrPorts[senderIndex] != from || // mismatching source for the active handshake + !bytes.Equal(e.challenge[senderIndex][:], discoMsg.Challenge[:]) { // mismatching answer for the active handshake + // silently drop return } + // Handshake complete. Update the binding for this sender. + e.boundAddrPorts[senderIndex] = from + e.lastSeen[senderIndex] = time.Now() // record last seen as bound time + return default: - // unexpected Disco message type + // unexpected message types, silently drop return } } @@ -225,12 +234,12 @@ func (e *serverEndpoint) handlePacket(from netip.AddrPort, gh packet.GeneveHeade } var to netip.AddrPort switch { - case from == e.addrPorts[0]: + case from == e.boundAddrPorts[0]: e.lastSeen[0] = time.Now() - to = e.addrPorts[1] - case from == e.addrPorts[1]: + to = e.boundAddrPorts[1] + case from == e.boundAddrPorts[1]: e.lastSeen[1] = time.Now() - to = e.addrPorts[0] + to = e.boundAddrPorts[0] default: // unrecognized source return @@ -240,11 +249,6 @@ func (e *serverEndpoint) handlePacket(from netip.AddrPort, gh packet.GeneveHeade return } - if e.isBound() { - // control packet, but serverEndpoint is already bound - return - } - if gh.Protocol != packet.GeneveProtocolDisco { // control packet, but not Disco return @@ -267,11 +271,11 @@ func (e *serverEndpoint) isExpired(now time.Time, bindLifetime, steadyStateLifet return false } -// isBound returns true if both clients have completed their 3-way handshake, +// isBound returns true if both clients have completed a 3-way handshake, // otherwise false. func (e *serverEndpoint) isBound() bool { - return e.handshakeState[0] == disco.BindUDPRelayHandshakeStateAnswerReceived && - e.handshakeState[1] == disco.BindUDPRelayHandshakeStateAnswerReceived + return e.boundAddrPorts[0].IsValid() && + e.boundAddrPorts[1].IsValid() } // NewServer constructs a [Server] listening on 0.0.0.0:'port'. IPv6 is not yet @@ -591,8 +595,6 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv e.discoSharedSecrets[0] = s.disco.Shared(e.discoPubKeys[0]) e.discoSharedSecrets[1] = s.disco.Shared(e.discoPubKeys[1]) e.vni, s.vniPool = s.vniPool[0], s.vniPool[1:] - rand.Read(e.challenge[0][:]) - rand.Read(e.challenge[1][:]) s.byDisco[pair] = e s.byVNI[e.vni] = e diff --git a/net/udprelay/server_test.go b/net/udprelay/server_test.go index a4e5ca451af2e..3fcb9b8b198c2 100644 --- a/net/udprelay/server_test.go +++ b/net/udprelay/server_test.go @@ -19,23 +19,27 @@ import ( ) type testClient struct { - vni uint32 - local key.DiscoPrivate - server key.DiscoPublic - uc *net.UDPConn + vni uint32 + handshakeGeneration uint32 + local key.DiscoPrivate + remote key.DiscoPublic + server key.DiscoPublic + uc *net.UDPConn } -func newTestClient(t *testing.T, vni uint32, serverEndpoint netip.AddrPort, local key.DiscoPrivate, server key.DiscoPublic) *testClient { +func newTestClient(t *testing.T, vni uint32, serverEndpoint netip.AddrPort, local key.DiscoPrivate, remote, server key.DiscoPublic) *testClient { rAddr := &net.UDPAddr{IP: serverEndpoint.Addr().AsSlice(), Port: int(serverEndpoint.Port())} uc, err := net.DialUDP("udp4", nil, rAddr) if err != nil { t.Fatal(err) } return &testClient{ - vni: vni, - local: local, - server: server, - uc: uc, + vni: vni, + handshakeGeneration: 1, + local: local, + remote: remote, + server: server, + uc: uc, } } @@ -137,13 +141,35 @@ func (c *testClient) readControlDiscoMsg(t *testing.T) disco.Message { } func (c *testClient) handshake(t *testing.T) { - c.writeControlDiscoMsg(t, &disco.BindUDPRelayEndpoint{}) + generation := c.handshakeGeneration + c.handshakeGeneration++ + common := disco.BindUDPRelayEndpointCommon{ + VNI: c.vni, + Generation: generation, + RemoteKey: c.remote, + } + c.writeControlDiscoMsg(t, &disco.BindUDPRelayEndpoint{ + BindUDPRelayEndpointCommon: common, + }) msg := c.readControlDiscoMsg(t) challenge, ok := msg.(*disco.BindUDPRelayEndpointChallenge) if !ok { - t.Fatal("unexepcted disco message type") + t.Fatal("unexpected disco message type") + } + if challenge.Generation != common.Generation { + t.Fatalf("rx'd challenge.Generation (%d) != %d", challenge.Generation, common.Generation) + } + if challenge.VNI != common.VNI { + t.Fatalf("rx'd challenge.VNI (%d) != %d", challenge.VNI, common.VNI) + } + if challenge.RemoteKey != common.RemoteKey { + t.Fatalf("rx'd challenge.RemoteKey (%v) != %v", challenge.RemoteKey, common.RemoteKey) } - c.writeControlDiscoMsg(t, &disco.BindUDPRelayEndpointAnswer{Answer: challenge.Challenge}) + answer := &disco.BindUDPRelayEndpointAnswer{ + BindUDPRelayEndpointCommon: common, + } + answer.Challenge = challenge.Challenge + c.writeControlDiscoMsg(t, answer) } func (c *testClient) close() { @@ -179,9 +205,9 @@ func TestServer(t *testing.T) { if len(endpoint.AddrPorts) != 1 { t.Fatalf("unexpected endpoint.AddrPorts: %v", endpoint.AddrPorts) } - tcA := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoA, endpoint.ServerDisco) + tcA := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoA, discoB.Public(), endpoint.ServerDisco) defer tcA.close() - tcB := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoB, endpoint.ServerDisco) + tcB := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoB, discoA.Public(), endpoint.ServerDisco) defer tcB.close() tcA.handshake(t) @@ -209,4 +235,30 @@ func TestServer(t *testing.T) { if !bytes.Equal(txToA, rxFromB) { t.Fatal("unexpected msg B->A") } + + tcAOnNewPort := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoA, discoB.Public(), endpoint.ServerDisco) + tcAOnNewPort.handshakeGeneration = tcA.handshakeGeneration + 1 + defer tcAOnNewPort.close() + + // Handshake client A on a new source IP:port, verify we receive packets on the new binding + tcAOnNewPort.handshake(t) + txToAOnNewPort := []byte{7, 8, 9} + tcB.writeDataPkt(t, txToAOnNewPort) + rxFromB = tcAOnNewPort.readDataPkt(t) + if !bytes.Equal(txToAOnNewPort, rxFromB) { + t.Fatal("unexpected msg B->A") + } + + tcBOnNewPort := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoB, discoA.Public(), endpoint.ServerDisco) + tcBOnNewPort.handshakeGeneration = tcB.handshakeGeneration + 1 + defer tcBOnNewPort.close() + + // Handshake client B on a new source IP:port, verify we receive packets on the new binding + tcBOnNewPort.handshake(t) + txToBOnNewPort := []byte{7, 8, 9} + tcAOnNewPort.writeDataPkt(t, txToBOnNewPort) + rxFromA = tcBOnNewPort.readDataPkt(t) + if !bytes.Equal(txToBOnNewPort, rxFromA) { + t.Fatal("unexpected msg A->B") + } } diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 7b378838a145c..6418a43641200 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -45,6 +45,7 @@ type relayManager struct { handshakeWorkByServerDiscoVNI map[serverDiscoVNI]*relayHandshakeWork handshakeWorkAwaitingPong map[*relayHandshakeWork]addrPortVNI addrPortVNIToHandshakeWork map[addrPortVNI]*relayHandshakeWork + handshakeGeneration uint32 // =================================================================== // The following chan fields serve event inputs to a single goroutine, @@ -590,7 +591,12 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay go r.sendCallMeMaybeVia(work.ep, work.se) } - go r.handshakeServerEndpoint(work) + r.handshakeGeneration++ + if r.handshakeGeneration == 0 { // generation must be nonzero + r.handshakeGeneration++ + } + + go r.handshakeServerEndpoint(work, r.handshakeGeneration) } // sendCallMeMaybeVia sends a [disco.CallMeMaybeVia] to ep over DERP. It must be @@ -616,7 +622,7 @@ func (r *relayManager) sendCallMeMaybeVia(ep *endpoint, se udprelay.ServerEndpoi ep.c.sendDiscoMessage(epAddr{ap: derpAddr}, ep.publicKey, epDisco.key, callMeMaybeVia, discoVerboseLog) } -func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { +func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork, generation uint32) { done := relayEndpointHandshakeWorkDoneEvent{work: work} r.ensureDiscoInfoFor(work) @@ -627,8 +633,21 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { work.cancel() }() + epDisco := work.ep.disco.Load() + if epDisco == nil { + return + } + + common := disco.BindUDPRelayEndpointCommon{ + VNI: work.se.VNI, + Generation: generation, + RemoteKey: epDisco.key, + } + sentBindAny := false - bind := &disco.BindUDPRelayEndpoint{} + bind := &disco.BindUDPRelayEndpoint{ + BindUDPRelayEndpointCommon: common, + } vni := virtualNetworkID{} vni.set(work.se.VNI) for _, addrPort := range work.se.AddrPorts { @@ -661,10 +680,6 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { if len(sentPingAt) == limitPings { return } - epDisco := work.ep.disco.Load() - if epDisco == nil { - return - } txid := stun.NewTxID() sentPingAt[txid] = time.Now() ping := &disco.Ping{ @@ -673,13 +688,24 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { } go func() { if withAnswer != nil { - answer := &disco.BindUDPRelayEndpointAnswer{Answer: *withAnswer} + answer := &disco.BindUDPRelayEndpointAnswer{BindUDPRelayEndpointCommon: common} + answer.Challenge = *withAnswer work.ep.c.sendDiscoMessage(epAddr{ap: to, vni: vni}, key.NodePublic{}, work.se.ServerDisco, answer, discoVerboseLog) } work.ep.c.sendDiscoMessage(epAddr{ap: to, vni: vni}, key.NodePublic{}, epDisco.key, ping, discoVerboseLog) }() } + validateVNIAndRemoteKey := func(common disco.BindUDPRelayEndpointCommon) error { + if common.VNI != work.se.VNI { + return errors.New("mismatching VNI") + } + if common.RemoteKey.Compare(epDisco.key) != 0 { + return errors.New("mismatching RemoteKey") + } + return nil + } + // This for{select{}} is responsible for handshaking and tx'ing ping/pong // when the handshake is complete. for { @@ -689,6 +715,10 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { case msgEvent := <-work.rxDiscoMsgCh: switch msg := msgEvent.msg.(type) { case *disco.BindUDPRelayEndpointChallenge: + err := validateVNIAndRemoteKey(msg.BindUDPRelayEndpointCommon) + if err != nil { + continue + } if handshakeState >= disco.BindUDPRelayHandshakeStateAnswerSent { continue } From 4a7b8afabfe71cde16445b416e7c93274ff657b8 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 20 Jun 2025 13:21:31 +0200 Subject: [PATCH 124/263] cmd/tailscale: add tlpub: prefix to lock log output Updates tailscale/corp#23258 Signed-off-by: Kristoffer Dalby --- cmd/tailscale/cli/network-lock.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/tailscale/cli/network-lock.go b/cmd/tailscale/cli/network-lock.go index 871a931b54ba5..9ab2b11b0bf01 100644 --- a/cmd/tailscale/cli/network-lock.go +++ b/cmd/tailscale/cli/network-lock.go @@ -623,7 +623,7 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er printKey := func(key *tka.Key, prefix string) { fmt.Fprintf(&stanza, "%sType: %s\n", prefix, key.Kind.String()) if keyID, err := key.ID(); err == nil { - fmt.Fprintf(&stanza, "%sKeyID: %x\n", prefix, keyID) + fmt.Fprintf(&stanza, "%sKeyID: tlpub:%x\n", prefix, keyID) } else { // Older versions of the client shouldn't explode when they encounter an // unknown key type. @@ -645,10 +645,10 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er case tka.AUMAddKey.String(): printKey(aum.Key, "") case tka.AUMRemoveKey.String(): - fmt.Fprintf(&stanza, "KeyID: %x\n", aum.KeyID) + fmt.Fprintf(&stanza, "KeyID: tlpub:%x\n", aum.KeyID) case tka.AUMUpdateKey.String(): - fmt.Fprintf(&stanza, "KeyID: %x\n", aum.KeyID) + fmt.Fprintf(&stanza, "KeyID: tlpub:%x\n", aum.KeyID) if aum.Votes != nil { fmt.Fprintf(&stanza, "Votes: %d\n", aum.Votes) } From df786be14d40a9aadd82c5900bcf4d79b3e6af4f Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 27 Jun 2025 11:54:01 +0200 Subject: [PATCH 125/263] cmd/tailscale: use text format for TKA head Updates tailscale/corp#23258 Signed-off-by: Kristoffer Dalby --- cmd/tailscale/cli/network-lock.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/tailscale/cli/network-lock.go b/cmd/tailscale/cli/network-lock.go index 9ab2b11b0bf01..d19909576c090 100644 --- a/cmd/tailscale/cli/network-lock.go +++ b/cmd/tailscale/cli/network-lock.go @@ -639,7 +639,11 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er return "", fmt.Errorf("decoding: %w", err) } - fmt.Fprintf(&stanza, "%supdate %x (%s)%s\n", terminalYellow, update.Hash, update.Change, terminalClear) + tkaHead, err := aum.Hash().MarshalText() + if err != nil { + return "", fmt.Errorf("decoding AUM hash: %w", err) + } + fmt.Fprintf(&stanza, "%supdate %s (%s)%s\n", terminalYellow, string(tkaHead), update.Change, terminalClear) switch update.Change { case tka.AUMAddKey.String(): From 53f67c43961a322048bc2c858181b331d2f35695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus=20Lensb=C3=B8l?= Date: Fri, 27 Jun 2025 10:03:56 -0400 Subject: [PATCH 126/263] util/eventbus: fix docstrings (#16401) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates #15160 Signed-off-by: Claus Lensbøl --- util/eventbus/bus.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/eventbus/bus.go b/util/eventbus/bus.go index 45d12da2f3736..e5bf7329a67ee 100644 --- a/util/eventbus/bus.go +++ b/util/eventbus/bus.go @@ -40,8 +40,8 @@ type Bus struct { clients set.Set[*Client] } -// New returns a new bus. Use [PublisherOf] to make event publishers, -// and [Bus.Queue] and [Subscribe] to make event subscribers. +// New returns a new bus. Use [Publish] to make event publishers, +// and [Subscribe] and [SubscribeFunc] to make event subscribers. func New() *Bus { ret := &Bus{ write: make(chan PublishedEvent), From f81baa2d56795267df835f770d0779d414aed283 Mon Sep 17 00:00:00 2001 From: Tom Meadows Date: Fri, 27 Jun 2025 17:12:14 +0100 Subject: [PATCH 127/263] cmd/k8s-operator, k8s-operator: support Static Endpoints on ProxyGroups (#16115) updates: #14674 Signed-off-by: chaosinthecrd --- .../deploy/chart/templates/operator-rbac.yaml | 3 + .../crds/tailscale.com_proxyclasses.yaml | 45 + .../crds/tailscale.com_proxygroups.yaml | 5 + .../deploy/manifests/operator.yaml | 58 ++ cmd/k8s-operator/nodeport-service-ports.go | 203 +++++ .../nodeport-services-ports_test.go | 277 +++++++ cmd/k8s-operator/operator.go | 88 +- cmd/k8s-operator/proxyclass.go | 33 +- cmd/k8s-operator/proxyclass_test.go | 118 ++- cmd/k8s-operator/proxygroup.go | 389 ++++++++- cmd/k8s-operator/proxygroup_specs.go | 45 +- cmd/k8s-operator/proxygroup_test.go | 771 +++++++++++++++++- k8s-operator/api.md | 55 ++ .../apis/v1alpha1/types_proxyclass.go | 122 +++ .../apis/v1alpha1/types_proxygroup.go | 4 + .../apis/v1alpha1/zz_generated.deepcopy.go | 91 +++ 16 files changed, 2244 insertions(+), 63 deletions(-) create mode 100644 cmd/k8s-operator/nodeport-service-ports.go create mode 100644 cmd/k8s-operator/nodeport-services-ports_test.go diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 00d8318acdce4..5eb920a6f41c4 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -16,6 +16,9 @@ kind: ClusterRole metadata: name: tailscale-operator rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events", "services", "services/status"] verbs: ["create","delete","deletecollection","get","list","patch","update","watch"] diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index 1541234755029..fcf1b27aaf318 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -2203,6 +2203,51 @@ spec: won't make it *more* imbalanced. It's a required field. type: string + staticEndpoints: + description: |- + Configuration for 'static endpoints' on proxies in order to facilitate + direct connections from other devices on the tailnet. + See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. + type: object + required: + - nodePort + properties: + nodePort: + description: The configuration for static endpoints using NodePort Services. + type: object + required: + - ports + properties: + ports: + description: |- + The port ranges from which the operator will select NodePorts for the Services. + You must ensure that firewall rules allow UDP ingress traffic for these ports + to the node's external IPs. + The ports must be in the range of service node ports for the cluster (default `30000-32767`). + See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. + type: array + minItems: 1 + items: + type: object + required: + - port + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be used. This field cannot be defined if the port field is not defined. + The endPort must be either unset, or equal or greater than port. + type: integer + port: + description: port represents a port selected to be used. This is a required field. + type: integer + selector: + description: |- + A selector which will be used to select the node's that will have their `ExternalIP`'s advertised + by the ProxyGroup as Static Endpoints. + type: object + additionalProperties: + type: string tailscale: description: |- TailscaleConfig contains options to configure the tailscale-specific diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index 4b9149e23e55b..f695e989d7b85 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -196,6 +196,11 @@ spec: If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the node. type: string + staticEndpoints: + description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. + type: array + items: + type: string tailnetIPs: description: |- TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 1d910cf92c6c6..fa18a5debeaa9 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -2679,6 +2679,51 @@ spec: type: array type: object type: object + staticEndpoints: + description: |- + Configuration for 'static endpoints' on proxies in order to facilitate + direct connections from other devices on the tailnet. + See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. + properties: + nodePort: + description: The configuration for static endpoints using NodePort Services. + properties: + ports: + description: |- + The port ranges from which the operator will select NodePorts for the Services. + You must ensure that firewall rules allow UDP ingress traffic for these ports + to the node's external IPs. + The ports must be in the range of service node ports for the cluster (default `30000-32767`). + See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. + items: + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be used. This field cannot be defined if the port field is not defined. + The endPort must be either unset, or equal or greater than port. + type: integer + port: + description: port represents a port selected to be used. This is a required field. + type: integer + required: + - port + type: object + minItems: 1 + type: array + selector: + additionalProperties: + type: string + description: |- + A selector which will be used to select the node's that will have their `ExternalIP`'s advertised + by the ProxyGroup as Static Endpoints. + type: object + required: + - ports + type: object + required: + - nodePort + type: object tailscale: description: |- TailscaleConfig contains options to configure the tailscale-specific @@ -2976,6 +3021,11 @@ spec: If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the node. type: string + staticEndpoints: + description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. + items: + type: string + type: array tailnetIPs: description: |- TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) @@ -4791,6 +4841,14 @@ kind: ClusterRole metadata: name: tailscale-operator rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - "" resources: diff --git a/cmd/k8s-operator/nodeport-service-ports.go b/cmd/k8s-operator/nodeport-service-ports.go new file mode 100644 index 0000000000000..a9504e3e94f88 --- /dev/null +++ b/cmd/k8s-operator/nodeport-service-ports.go @@ -0,0 +1,203 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "context" + "fmt" + "math/rand/v2" + "regexp" + "sort" + "strconv" + "strings" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + k8soperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubetypes" +) + +const ( + tailscaledPortMax = 65535 + tailscaledPortMin = 1024 + testSvcName = "test-node-port-range" + + invalidSvcNodePort = 777777 +) + +// getServicesNodePortRange is a hacky function that attempts to determine Service NodePort range by +// creating a deliberately invalid Service with a NodePort that is too large and parsing the returned +// validation error. Returns nil if unable to determine port range. +// https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport +func getServicesNodePortRange(ctx context.Context, c client.Client, tsNamespace string, logger *zap.SugaredLogger) *tsapi.PortRange { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: testSvcName, + Namespace: tsNamespace, + Labels: map[string]string{ + kubetypes.LabelManaged: "true", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + { + Name: testSvcName, + Port: 8080, + TargetPort: intstr.FromInt32(8080), + Protocol: corev1.ProtocolUDP, + NodePort: invalidSvcNodePort, + }, + }, + }, + } + + // NOTE(ChaosInTheCRD): ideally this would be a server side dry-run but could not get it working + err := c.Create(ctx, svc) + if err == nil { + return nil + } + + if validPorts := getServicesNodePortRangeFromErr(err.Error()); validPorts != "" { + pr, err := parseServicesNodePortRange(validPorts) + if err != nil { + logger.Debugf("failed to parse NodePort range set for Kubernetes Cluster: %w", err) + return nil + } + + return pr + } + + return nil +} + +func getServicesNodePortRangeFromErr(err string) string { + reg := regexp.MustCompile(`\d{1,5}-\d{1,5}`) + matches := reg.FindAllString(err, -1) + if len(matches) != 1 { + return "" + } + + return matches[0] +} + +// parseServicesNodePortRange converts the `ValidPorts` string field in the Kubernetes PortAllocator error and converts it to +// PortRange +func parseServicesNodePortRange(p string) (*tsapi.PortRange, error) { + parts := strings.Split(p, "-") + s, err := strconv.ParseUint(parts[0], 10, 16) + if err != nil { + return nil, fmt.Errorf("failed to parse string as uint16: %w", err) + } + + var e uint64 + switch len(parts) { + case 1: + e = uint64(s) + case 2: + e, err = strconv.ParseUint(parts[1], 10, 16) + if err != nil { + return nil, fmt.Errorf("failed to parse string as uint16: %w", err) + } + default: + return nil, fmt.Errorf("failed to parse port range %q", p) + } + + portRange := &tsapi.PortRange{Port: uint16(s), EndPort: uint16(e)} + if !portRange.IsValid() { + return nil, fmt.Errorf("port range %q is not valid", portRange.String()) + } + + return portRange, nil +} + +// validateNodePortRanges checks that the port range specified is valid. It also ensures that the specified ranges +// lie within the NodePort Service port range specified for the Kubernetes API Server. +func validateNodePortRanges(ctx context.Context, c client.Client, kubeRange *tsapi.PortRange, pc *tsapi.ProxyClass) error { + if pc.Spec.StaticEndpoints == nil { + return nil + } + + portRanges := pc.Spec.StaticEndpoints.NodePort.Ports + + if kubeRange != nil { + for _, pr := range portRanges { + if !kubeRange.Contains(pr.Port) || (pr.EndPort != 0 && !kubeRange.Contains(pr.EndPort)) { + return fmt.Errorf("range %q is not within Cluster configured range %q", pr.String(), kubeRange.String()) + } + } + } + + for _, r := range portRanges { + if !r.IsValid() { + return fmt.Errorf("port range %q is invalid", r.String()) + } + } + + // TODO(ChaosInTheCRD): if a ProxyClass that made another invalid (due to port range clash) is deleted, + // the invalid ProxyClass doesn't get reconciled on, and therefore will not go valid. We should fix this. + proxyClassRanges, err := getPortsForProxyClasses(ctx, c) + if err != nil { + return fmt.Errorf("failed to get port ranges for ProxyClasses: %w", err) + } + + for _, r := range portRanges { + for pcName, pcr := range proxyClassRanges { + if pcName == pc.Name { + continue + } + if pcr.ClashesWith(r) { + return fmt.Errorf("port ranges for ProxyClass %q clash with existing ProxyClass %q", pc.Name, pcName) + } + } + } + + if len(portRanges) == 1 { + return nil + } + + sort.Slice(portRanges, func(i, j int) bool { + return portRanges[i].Port < portRanges[j].Port + }) + + for i := 1; i < len(portRanges); i++ { + prev := portRanges[i-1] + curr := portRanges[i] + if curr.Port <= prev.Port || curr.Port <= prev.EndPort { + return fmt.Errorf("overlapping ranges: %q and %q", prev.String(), curr.String()) + } + } + + return nil +} + +// getPortsForProxyClasses gets the port ranges for all the other existing ProxyClasses +func getPortsForProxyClasses(ctx context.Context, c client.Client) (map[string]tsapi.PortRanges, error) { + pcs := new(tsapi.ProxyClassList) + + err := c.List(ctx, pcs) + if err != nil { + return nil, fmt.Errorf("failed to list ProxyClasses: %w", err) + } + + portRanges := make(map[string]tsapi.PortRanges) + for _, i := range pcs.Items { + if !k8soperator.ProxyClassIsReady(&i) { + continue + } + if se := i.Spec.StaticEndpoints; se != nil && se.NodePort != nil { + portRanges[i.Name] = se.NodePort.Ports + } + } + + return portRanges, nil +} + +func getRandomPort() uint16 { + return uint16(rand.IntN(tailscaledPortMax-tailscaledPortMin+1) + tailscaledPortMin) +} diff --git a/cmd/k8s-operator/nodeport-services-ports_test.go b/cmd/k8s-operator/nodeport-services-ports_test.go new file mode 100644 index 0000000000000..9418bb8446bd8 --- /dev/null +++ b/cmd/k8s-operator/nodeport-services-ports_test.go @@ -0,0 +1,277 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstest" +) + +func TestGetServicesNodePortRangeFromErr(t *testing.T) { + tests := []struct { + name string + errStr string + want string + }{ + { + name: "valid_error_string", + errStr: "NodePort 777777 is not in the allowed range 30000-32767", + want: "30000-32767", + }, + { + name: "error_string_with_different_message", + errStr: "some other error without a port range", + want: "", + }, + { + name: "error_string_with_multiple_port_ranges", + errStr: "range 1000-2000 and another range 3000-4000", + want: "", + }, + { + name: "empty_error_string", + errStr: "", + want: "", + }, + { + name: "error_string_with_range_at_start", + errStr: "30000-32767 is the range", + want: "30000-32767", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getServicesNodePortRangeFromErr(tt.errStr); got != tt.want { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseServicesNodePortRange(t *testing.T) { + tests := []struct { + name string + p string + want *tsapi.PortRange + wantErr bool + }{ + { + name: "valid_range", + p: "30000-32767", + want: &tsapi.PortRange{Port: 30000, EndPort: 32767}, + wantErr: false, + }, + { + name: "single_port_range", + p: "30000", + want: &tsapi.PortRange{Port: 30000, EndPort: 30000}, + wantErr: false, + }, + { + name: "invalid_format_non_numeric_end", + p: "30000-abc", + want: nil, + wantErr: true, + }, + { + name: "invalid_format_non_numeric_start", + p: "abc-32767", + want: nil, + wantErr: true, + }, + { + name: "empty_string", + p: "", + want: nil, + wantErr: true, + }, + { + name: "too_many_parts", + p: "1-2-3", + want: nil, + wantErr: true, + }, + { + name: "port_too_large_start", + p: "65536-65537", + want: nil, + wantErr: true, + }, + { + name: "port_too_large_end", + p: "30000-65536", + want: nil, + wantErr: true, + }, + { + name: "inverted_range", + p: "32767-30000", + want: nil, + wantErr: true, // IsValid() will fail + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + portRange, err := parseServicesNodePortRange(tt.p) + if (err != nil) != tt.wantErr { + t.Errorf("error = %v, wantErr %v", err, tt.wantErr) + return + } + + if tt.wantErr { + return + } + + if portRange == nil { + t.Fatalf("got nil port range, expected %v", tt.want) + } + + if portRange.Port != tt.want.Port || portRange.EndPort != tt.want.EndPort { + t.Errorf("got = %v, want %v", portRange, tt.want) + } + }) + } +} + +func TestValidateNodePortRanges(t *testing.T) { + tests := []struct { + name string + portRanges []tsapi.PortRange + wantErr bool + }{ + { + name: "valid_ranges_with_unknown_kube_range", + portRanges: []tsapi.PortRange{ + {Port: 30003, EndPort: 30005}, + {Port: 30006, EndPort: 30007}, + }, + wantErr: false, + }, + { + name: "overlapping_ranges", + portRanges: []tsapi.PortRange{ + {Port: 30000, EndPort: 30010}, + {Port: 30005, EndPort: 30015}, + }, + wantErr: true, + }, + { + name: "adjacent_ranges_no_overlap", + portRanges: []tsapi.PortRange{ + {Port: 30010, EndPort: 30020}, + {Port: 30021, EndPort: 30022}, + }, + wantErr: false, + }, + { + name: "identical_ranges_are_overlapping", + portRanges: []tsapi.PortRange{ + {Port: 30005, EndPort: 30010}, + {Port: 30005, EndPort: 30010}, + }, + wantErr: true, + }, + { + name: "range_clashes_with_existing_proxyclass", + portRanges: []tsapi.PortRange{ + {Port: 31005, EndPort: 32070}, + }, + wantErr: true, + }, + } + + // as part of this test, we want to create an adjacent ProxyClass in order to ensure that if it clashes with the one created in this test + // that we get an error + cl := tstest.NewClock(tstest.ClockOpts{}) + opc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Annotations: defaultProxyClassAnnotations, + }, + StaticEndpoints: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 31000}, {Port: 32000}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + }, + Status: tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + Reason: reasonProxyClassValid, + Message: reasonProxyClassValid, + LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }}, + }, + } + + fc := fake.NewClientBuilder(). + WithObjects(opc). + WithStatusSubresource(opc). + WithScheme(tsapi.GlobalScheme). + Build() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Annotations: defaultProxyClassAnnotations, + }, + StaticEndpoints: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: tt.portRanges, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + }, + Status: tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + Reason: reasonProxyClassValid, + Message: reasonProxyClassValid, + LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }}, + }, + } + err := validateNodePortRanges(context.Background(), fc, &tsapi.PortRange{Port: 30000, EndPort: 32767}, pc) + if (err != nil) != tt.wantErr { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +func TestGetRandomPort(t *testing.T) { + for range 100 { + port := getRandomPort() + if port < tailscaledPortMin || port > tailscaledPortMax { + t.Errorf("generated port %d which is out of range [%d, %d]", port, tailscaledPortMin, tailscaledPortMax) + } + } +} diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index a08dd4da8c52f..cd1ae8158d01c 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -26,7 +26,9 @@ import ( networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + klabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" @@ -39,6 +41,7 @@ import ( kzap "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "tailscale.com/client/local" "tailscale.com/client/tailscale" @@ -228,6 +231,17 @@ waitOnline: return s, tsc } +// predicate function for filtering to ensure we *don't* reconcile on tailscale managed Kubernetes Services +func serviceManagedResourceFilterPredicate() predicate.Predicate { + return predicate.NewPredicateFuncs(func(object client.Object) bool { + if svc, ok := object.(*corev1.Service); !ok { + return false + } else { + return !isManagedResource(svc) + } + }) +} + // runReconcilers starts the controller-runtime manager and registers the // ServiceReconciler. It blocks forever. func runReconcilers(opts reconcilerOpts) { @@ -374,7 +388,7 @@ func runReconcilers(opts reconcilerOpts) { ingressSvcFromEpsFilter := handler.EnqueueRequestsFromMapFunc(ingressSvcFromEps(mgr.GetClient(), opts.log.Named("service-pg-reconciler"))) err = builder. ControllerManagedBy(mgr). - For(&corev1.Service{}). + For(&corev1.Service{}, builder.WithPredicates(serviceManagedResourceFilterPredicate())). Named("service-pg-reconciler"). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(HAServicesFromSecret(mgr.GetClient(), startlog))). Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter). @@ -519,16 +533,19 @@ func runReconcilers(opts reconcilerOpts) { // ProxyClass reconciler gets triggered on ServiceMonitor CRD changes to ensure that any ProxyClasses, that // define that a ServiceMonitor should be created, were set to invalid because the CRD did not exist get // reconciled if the CRD is applied at a later point. + kPortRange := getServicesNodePortRange(context.Background(), mgr.GetClient(), opts.tailscaleNamespace, startlog) serviceMonitorFilter := handler.EnqueueRequestsFromMapFunc(proxyClassesWithServiceMonitor(mgr.GetClient(), opts.log)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyClass{}). Named("proxyclass-reconciler"). Watches(&apiextensionsv1.CustomResourceDefinition{}, serviceMonitorFilter). Complete(&ProxyClassReconciler{ - Client: mgr.GetClient(), - recorder: eventRecorder, - logger: opts.log.Named("proxyclass-reconciler"), - clock: tstime.DefaultClock{}, + Client: mgr.GetClient(), + nodePortRange: kPortRange, + recorder: eventRecorder, + tsNamespace: opts.tailscaleNamespace, + logger: opts.log.Named("proxyclass-reconciler"), + clock: tstime.DefaultClock{}, }) if err != nil { startlog.Fatal("could not create proxyclass reconciler: %v", err) @@ -587,9 +604,11 @@ func runReconcilers(opts reconcilerOpts) { // ProxyGroup reconciler. ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{}) proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog)) + nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyGroup{}). Named("proxygroup-reconciler"). + Watches(&corev1.Service{}, ownedByProxyGroupFilter). Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter). Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter). Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter). @@ -597,6 +616,7 @@ func runReconcilers(opts reconcilerOpts) { Watches(&rbacv1.Role{}, ownedByProxyGroupFilter). Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter). Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup). + Watches(&corev1.Node{}, nodeFilterForProxyGroup). Complete(&ProxyGroupReconciler{ recorder: eventRecorder, Client: mgr.GetClient(), @@ -840,6 +860,64 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger) } } +// nodeHandlerForProxyGroup returns a handler that, for a given Node, returns a +// list of reconcile requests for ProxyGroups that should be reconciled for the +// Node event. ProxyGroups need to be reconciled for Node events if they are +// configured to expose tailscaled static endpoints to tailnet using NodePort +// Services. +func nodeHandlerForProxyGroup(cl client.Client, defaultProxyClass string, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + pgList := new(tsapi.ProxyGroupList) + if err := cl.List(ctx, pgList); err != nil { + logger.Debugf("error listing ProxyGroups for ProxyClass: %v", err) + return nil + } + + reqs := make([]reconcile.Request, 0) + for _, pg := range pgList.Items { + if pg.Spec.ProxyClass == "" && defaultProxyClass == "" { + continue + } + + pc := defaultProxyClass + if pc == "" { + pc = pg.Spec.ProxyClass + } + + proxyClass := &tsapi.ProxyClass{} + if err := cl.Get(ctx, types.NamespacedName{Name: pc}, proxyClass); err != nil { + logger.Debugf("error getting ProxyClass %q: %v", pg.Spec.ProxyClass, err) + return nil + } + + stat := proxyClass.Spec.StaticEndpoints + if stat == nil { + continue + } + + // If the selector is empty, all nodes match. + // TODO(ChaosInTheCRD): think about how this must be handled if we want to limit the number of nodes used + if len(stat.NodePort.Selector) == 0 { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + continue + } + + selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: stat.NodePort.Selector, + }) + if err != nil { + logger.Debugf("error converting `spec.staticEndpoints.nodePort.selector` to Selector: %v", err) + return nil + } + + if selector.Matches(klabels.Set(o.GetLabels())) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + } + } + return reqs + } +} + // proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass, // returns a list of reconcile requests for all Connectors that have // .spec.proxyClass set. diff --git a/cmd/k8s-operator/proxyclass.go b/cmd/k8s-operator/proxyclass.go index 5ec9897d0a8b7..2d51b351d3907 100644 --- a/cmd/k8s-operator/proxyclass.go +++ b/cmd/k8s-operator/proxyclass.go @@ -44,22 +44,24 @@ const ( type ProxyClassReconciler struct { client.Client - recorder record.EventRecorder - logger *zap.SugaredLogger - clock tstime.Clock + recorder record.EventRecorder + logger *zap.SugaredLogger + clock tstime.Clock + tsNamespace string mu sync.Mutex // protects following // managedProxyClasses is a set of all ProxyClass resources that we're currently // managing. This is only used for metrics. managedProxyClasses set.Slice[types.UID] + // nodePortRange is the NodePort range set for the Kubernetes Cluster. This is used + // when validating port ranges configured by users for spec.StaticEndpoints + nodePortRange *tsapi.PortRange } -var ( - // gaugeProxyClassResources tracks the number of ProxyClass resources - // that we're currently managing. - gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources") -) +// gaugeProxyClassResources tracks the number of ProxyClass resources +// that we're currently managing. +var gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources") func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { logger := pcr.logger.With("ProxyClass", req.Name) @@ -96,7 +98,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re pcr.mu.Unlock() oldPCStatus := pc.Status.DeepCopy() - if errs := pcr.validate(ctx, pc); errs != nil { + if errs := pcr.validate(ctx, pc, logger); errs != nil { msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error()) pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg) tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger) @@ -112,7 +114,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re return reconcile.Result{}, nil } -func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass) (violations field.ErrorList) { +func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) (violations field.ErrorList) { if sts := pc.Spec.StatefulSet; sts != nil { if len(sts.Labels) > 0 { if errs := metavalidation.ValidateLabels(sts.Labels.Parse(), field.NewPath(".spec.statefulSet.labels")); errs != nil { @@ -183,6 +185,17 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl violations = append(violations, errs...) } } + + if stat := pc.Spec.StaticEndpoints; stat != nil { + if err := validateNodePortRanges(ctx, pcr.Client, pcr.nodePortRange, pc); err != nil { + var prs tsapi.PortRanges = stat.NodePort.Ports + violations = append(violations, field.TypeInvalid(field.NewPath("spec", "staticEndpoints", "nodePort", "ports"), prs.String(), err.Error())) + } + + if len(stat.NodePort.Selector) < 1 { + logger.Debug("no Selectors specified on `spec.staticEndpoints.nodePort.selectors` field") + } + } // We do not validate embedded fields (security context, resource // requirements etc) as we inherit upstream validation for those fields. // Invalid values would get rejected by upstream validations at apply diff --git a/cmd/k8s-operator/proxyclass_test.go b/cmd/k8s-operator/proxyclass_test.go index 48290eea782b5..ae0f63d99ea4d 100644 --- a/cmd/k8s-operator/proxyclass_test.go +++ b/cmd/k8s-operator/proxyclass_test.go @@ -131,9 +131,11 @@ func TestProxyClass(t *testing.T) { proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Env = []tsapi.Env{{Name: "TS_USERSPACE", Value: "true"}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH"}, {Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS"}} }) - expectedEvents := []string{"Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", + expectedEvents := []string{ + "Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", "Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_TS_CONFIGFILE_PATH env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", - "Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future."} + "Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", + } expectReconciled(t, pcr, "", "test") expectEvents(t, fr, expectedEvents) @@ -176,6 +178,110 @@ func TestProxyClass(t *testing.T) { expectEqual(t, fc, pc) } +func TestValidateProxyClassStaticEndpoints(t *testing.T) { + for name, tc := range map[string]struct { + staticEndpointConfig *tsapi.StaticEndpointsConfig + valid bool + }{ + "no_static_endpoints": { + staticEndpointConfig: nil, + valid: true, + }, + "valid_specific_ports": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: true, + }, + "valid_port_ranges": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3000, EndPort: 3002}, + {Port: 3005, EndPort: 3007}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: true, + }, + "overlapping_port_ranges": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 1000, EndPort: 2000}, + {Port: 1500, EndPort: 1800}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: false, + }, + "clashing_port_and_range": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3005}, + {Port: 3001, EndPort: 3010}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: false, + }, + "malformed_port_range": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001, EndPort: 3000}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: false, + }, + "empty_selector": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{{Port: 3000}}, + Selector: map[string]string{}, + }, + }, + valid: true, + }, + } { + t.Run(name, func(t *testing.T) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + Build() + zl, _ := zap.NewDevelopment() + pcr := &ProxyClassReconciler{ + logger: zl.Sugar(), + Client: fc, + } + + pc := &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + StaticEndpoints: tc.staticEndpointConfig, + }, + } + + logger := pcr.logger.With("ProxyClass", pc) + err := pcr.validate(context.Background(), pc, logger) + valid := err == nil + if valid != tc.valid { + t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err) + } + }) + } +} + func TestValidateProxyClass(t *testing.T) { for name, tc := range map[string]struct { pc *tsapi.ProxyClass @@ -219,8 +325,12 @@ func TestValidateProxyClass(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - pcr := &ProxyClassReconciler{} - err := pcr.validate(context.Background(), tc.pc) + zl, _ := zap.NewDevelopment() + pcr := &ProxyClassReconciler{ + logger: zl.Sugar(), + } + logger := pcr.logger.With("ProxyClass", tc.pc) + err := pcr.validate(context.Background(), tc.pc, logger) valid := err == nil if valid != tc.valid { t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 0d5eff551e8de..328262031b85c 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "net/http" + "net/netip" "slices" "strings" "sync" @@ -24,6 +25,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -48,7 +50,8 @@ const ( reasonProxyGroupInvalid = "ProxyGroupInvalid" // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c - optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" + optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" + staticEndpointsMaxAddrs = 2 ) var ( @@ -174,7 +177,8 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } } - if err = r.maybeProvision(ctx, pg, proxyClass); err != nil { + isProvisioned, err := r.maybeProvision(ctx, pg, proxyClass) + if err != nil { reason := reasonProxyGroupCreationFailed msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", err) if strings.Contains(err.Error(), optimisticLockErrorMsg) { @@ -185,9 +189,20 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } else { r.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg) } + return setStatusReady(pg, metav1.ConditionFalse, reason, msg) } + if !isProvisioned { + if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) { + // An error encountered here should get returned by the Reconcile function. + if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { + return reconcile.Result{}, errors.Join(err, updateErr) + } + } + return + } + desiredReplicas := int(pgReplicas(pg)) if len(pg.Status.Devices) < desiredReplicas { message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas) @@ -230,15 +245,42 @@ func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc } } -func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error { +func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (isProvisioned bool, err error) { logger := r.logger(pg.Name) r.mu.Lock() r.ensureAddedToGaugeForProxyGroup(pg) r.mu.Unlock() - if err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass); err != nil { - return fmt.Errorf("error provisioning config Secrets: %w", err) + svcToNodePorts := make(map[string]uint16) + var tailscaledPort *uint16 + if proxyClass != nil && proxyClass.Spec.StaticEndpoints != nil { + svcToNodePorts, tailscaledPort, err = r.ensureNodePortServiceCreated(ctx, pg, proxyClass) + if err != nil { + wrappedErr := fmt.Errorf("error provisioning NodePort Services for static endpoints: %w", err) + var allocatePortErr *allocatePortsErr + if errors.As(err, &allocatePortErr) { + reason := reasonProxyGroupCreationFailed + msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", wrappedErr) + r.setStatusReady(pg, metav1.ConditionFalse, reason, msg, logger) + return false, nil + } + return false, wrappedErr + } } + + staticEndpoints, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass, svcToNodePorts) + if err != nil { + wrappedErr := fmt.Errorf("error provisioning config Secrets: %w", err) + var selectorErr *FindStaticEndpointErr + if errors.As(err, &selectorErr) { + reason := reasonProxyGroupCreationFailed + msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", wrappedErr) + r.setStatusReady(pg, metav1.ConditionFalse, reason, msg, logger) + return false, nil + } + return false, wrappedErr + } + // State secrets are precreated so we can use the ProxyGroup CR as their owner ref. stateSecrets := pgStateSecrets(pg, r.tsNamespace) for _, sec := range stateSecrets { @@ -247,7 +289,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences }); err != nil { - return fmt.Errorf("error provisioning state Secrets: %w", err) + return false, fmt.Errorf("error provisioning state Secrets: %w", err) } } sa := pgServiceAccount(pg, r.tsNamespace) @@ -256,7 +298,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences }); err != nil { - return fmt.Errorf("error provisioning ServiceAccount: %w", err) + return false, fmt.Errorf("error provisioning ServiceAccount: %w", err) } role := pgRole(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { @@ -265,7 +307,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences r.Rules = role.Rules }); err != nil { - return fmt.Errorf("error provisioning Role: %w", err) + return false, fmt.Errorf("error provisioning Role: %w", err) } roleBinding := pgRoleBinding(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { @@ -275,7 +317,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.RoleRef = roleBinding.RoleRef r.Subjects = roleBinding.Subjects }); err != nil { - return fmt.Errorf("error provisioning RoleBinding: %w", err) + return false, fmt.Errorf("error provisioning RoleBinding: %w", err) } if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { cm, hp := pgEgressCM(pg, r.tsNamespace) @@ -284,7 +326,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp) }); err != nil { - return fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err) + return false, fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err) } } if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { @@ -293,12 +335,12 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro existing.ObjectMeta.Labels = cm.ObjectMeta.Labels existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences }); err != nil { - return fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err) + return false, fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err) } } - ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, proxyClass) + ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, tailscaledPort, proxyClass) if err != nil { - return fmt.Errorf("error generating StatefulSet spec: %w", err) + return false, fmt.Errorf("error generating StatefulSet spec: %w", err) } cfg := &tailscaleSTSConfig{ proxyType: string(pg.Spec.Type), @@ -306,7 +348,6 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger) updateSS := func(s *appsv1.StatefulSet) { - s.Spec = ss.Spec s.ObjectMeta.Labels = ss.ObjectMeta.Labels @@ -314,7 +355,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences } if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, updateSS); err != nil { - return fmt.Errorf("error provisioning StatefulSet: %w", err) + return false, fmt.Errorf("error provisioning StatefulSet: %w", err) } mo := &metricsOpts{ tsNamespace: r.tsNamespace, @@ -323,26 +364,150 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro proxyType: "proxygroup", } if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil { - return fmt.Errorf("error reconciling metrics resources: %w", err) + return false, fmt.Errorf("error reconciling metrics resources: %w", err) } - if err := r.cleanupDanglingResources(ctx, pg); err != nil { - return fmt.Errorf("error cleaning up dangling resources: %w", err) + if err := r.cleanupDanglingResources(ctx, pg, proxyClass); err != nil { + return false, fmt.Errorf("error cleaning up dangling resources: %w", err) } - devices, err := r.getDeviceInfo(ctx, pg) + devices, err := r.getDeviceInfo(ctx, staticEndpoints, pg) if err != nil { - return fmt.Errorf("failed to get device info: %w", err) + return false, fmt.Errorf("failed to get device info: %w", err) } pg.Status.Devices = devices - return nil + return true, nil +} + +// getServicePortsForProxyGroups returns a map of ProxyGroup Service names to their NodePorts, +// and a set of all allocated NodePorts for quick occupancy checking. +func getServicePortsForProxyGroups(ctx context.Context, c client.Client, namespace string, portRanges tsapi.PortRanges) (map[string]uint16, set.Set[uint16], error) { + svcs := new(corev1.ServiceList) + matchingLabels := client.MatchingLabels(map[string]string{ + LabelParentType: "proxygroup", + }) + + err := c.List(ctx, svcs, matchingLabels, client.InNamespace(namespace)) + if err != nil { + return nil, nil, fmt.Errorf("failed to list ProxyGroup Services: %w", err) + } + + svcToNodePorts := map[string]uint16{} + usedPorts := set.Set[uint16]{} + for _, svc := range svcs.Items { + if len(svc.Spec.Ports) == 1 && svc.Spec.Ports[0].NodePort != 0 { + p := uint16(svc.Spec.Ports[0].NodePort) + if portRanges.Contains(p) { + svcToNodePorts[svc.Name] = p + usedPorts.Add(p) + } + } + } + + return svcToNodePorts, usedPorts, nil +} + +type allocatePortsErr struct { + msg string +} + +func (e *allocatePortsErr) Error() string { + return e.msg +} + +func (r *ProxyGroupReconciler) allocatePorts(ctx context.Context, pg *tsapi.ProxyGroup, proxyClassName string, portRanges tsapi.PortRanges) (map[string]uint16, error) { + replicaCount := int(pgReplicas(pg)) + svcToNodePorts, usedPorts, err := getServicePortsForProxyGroups(ctx, r.Client, r.tsNamespace, portRanges) + if err != nil { + return nil, &allocatePortsErr{msg: fmt.Sprintf("failed to find ports for existing ProxyGroup NodePort Services: %s", err.Error())} + } + + replicasAllocated := 0 + for i := range pgReplicas(pg) { + if _, ok := svcToNodePorts[pgNodePortServiceName(pg.Name, i)]; !ok { + svcToNodePorts[pgNodePortServiceName(pg.Name, i)] = 0 + } else { + replicasAllocated++ + } + } + + for replica, port := range svcToNodePorts { + if port == 0 { + for p := range portRanges.All() { + if !usedPorts.Contains(p) { + svcToNodePorts[replica] = p + usedPorts.Add(p) + replicasAllocated++ + break + } + } + } + } + + if replicasAllocated < replicaCount { + return nil, &allocatePortsErr{msg: fmt.Sprintf("not enough available ports to allocate all replicas (needed %d, got %d). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass %q must have bigger range allocated", replicaCount, usedPorts.Len(), proxyClassName)} + } + + return svcToNodePorts, nil +} + +func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) (map[string]uint16, *uint16, error) { + // NOTE: (ChaosInTheCRD) we want the same TargetPort for every static endpoint NodePort Service for the ProxyGroup + tailscaledPort := getRandomPort() + svcs := []*corev1.Service{} + for i := range pgReplicas(pg) { + replicaName := pgNodePortServiceName(pg.Name, i) + + svc := &corev1.Service{} + err := r.Get(ctx, types.NamespacedName{Name: replicaName, Namespace: r.tsNamespace}, svc) + if err != nil && !apierrors.IsNotFound(err) { + return nil, nil, fmt.Errorf("error getting Kubernetes Service %q: %w", replicaName, err) + } + if apierrors.IsNotFound(err) { + svcs = append(svcs, pgNodePortService(pg, replicaName, r.tsNamespace)) + } else { + // NOTE: if we can we want to recover the random port used for tailscaled, + // as well as the NodePort previously used for that Service + if len(svc.Spec.Ports) == 1 { + if svc.Spec.Ports[0].Port != 0 { + tailscaledPort = uint16(svc.Spec.Ports[0].Port) + } + } + svcs = append(svcs, svc) + } + } + + svcToNodePorts, err := r.allocatePorts(ctx, pg, pc.Name, pc.Spec.StaticEndpoints.NodePort.Ports) + if err != nil { + return nil, nil, fmt.Errorf("failed to allocate NodePorts to ProxyGroup Services: %w", err) + } + + for _, svc := range svcs { + // NOTE: we know that every service is going to have 1 port here + svc.Spec.Ports[0].Port = int32(tailscaledPort) + svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(tailscaledPort)) + svc.Spec.Ports[0].NodePort = int32(svcToNodePorts[svc.Name]) + + _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, svc, func(s *corev1.Service) { + s.ObjectMeta.Labels = svc.ObjectMeta.Labels + s.ObjectMeta.Annotations = svc.ObjectMeta.Annotations + s.ObjectMeta.OwnerReferences = svc.ObjectMeta.OwnerReferences + s.Spec.Selector = svc.Spec.Selector + s.Spec.Ports = svc.Spec.Ports + }) + if err != nil { + return nil, nil, fmt.Errorf("error creating/updating Kubernetes NodePort Service %q: %w", svc.Name, err) + } + } + + return svcToNodePorts, ptr.To(tailscaledPort), nil } // cleanupDanglingResources ensures we don't leak config secrets, state secrets, and // tailnet devices when the number of replicas specified is reduced. -func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup) error { +func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) error { logger := r.logger(pg.Name) metadata, err := r.getNodeMetadata(ctx, pg) if err != nil { @@ -371,6 +536,30 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg return fmt.Errorf("error deleting config Secret %s: %w", configSecret.Name, err) } } + // NOTE(ChaosInTheCRD): we shouldn't need to get the service first, checking for a not found error should be enough + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-nodeport", m.stateSecret.Name), + Namespace: m.stateSecret.Namespace, + }, + } + if err := r.Delete(ctx, svc); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting static endpoints Kubernetes Service %q: %w", svc.Name, err) + } + } + } + + // If the ProxyClass has its StaticEndpoints config removed, we want to remove all of the NodePort Services + if pc != nil && pc.Spec.StaticEndpoints == nil { + labels := map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentType: proxyTypeProxyGroup, + LabelParentName: pg.Name, + } + if err := r.DeleteAllOf(ctx, &corev1.Service{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + return fmt.Errorf("error deleting Kubernetes Services for static endpoints: %w", err) + } } return nil @@ -396,7 +585,8 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy mo := &metricsOpts{ proxyLabels: pgLabels(pg.Name, nil), tsNamespace: r.tsNamespace, - proxyType: "proxygroup"} + proxyType: "proxygroup", + } if err := maybeCleanupMetricsResources(ctx, mo, r.Client); err != nil { return false, fmt.Errorf("error cleaning up metrics resources: %w", err) } @@ -424,8 +614,9 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc return nil } -func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (err error) { +func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass, svcToNodePorts map[string]uint16) (endpoints map[string][]netip.AddrPort, err error) { logger := r.logger(pg.Name) + endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg)) for i := range pgReplicas(pg) { cfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -441,7 +632,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName()) existingCfgSecret = cfgSecret.DeepCopy() } else if !apierrors.IsNotFound(err) { - return err + return nil, err } var authKey string @@ -453,19 +644,32 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p } authKey, err = newAuthKey(ctx, r.tsClient, tags) if err != nil { - return err + return nil, err + } + } + + replicaName := pgNodePortServiceName(pg.Name, i) + if len(svcToNodePorts) > 0 { + port, ok := svcToNodePorts[replicaName] + if !ok { + return nil, fmt.Errorf("could not find configured NodePort for ProxyGroup replica %q", replicaName) + } + + endpoints[replicaName], err = r.findStaticEndpoints(ctx, existingCfgSecret, proxyClass, port, logger) + if err != nil { + return nil, fmt.Errorf("could not find static endpoints for replica %q: %w", replicaName, err) } } - configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret) + configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret, endpoints[replicaName]) if err != nil { - return fmt.Errorf("error creating tailscaled config: %w", err) + return nil, fmt.Errorf("error creating tailscaled config: %w", err) } for cap, cfg := range configs { cfgJSON, err := json.Marshal(cfg) if err != nil { - return fmt.Errorf("error marshalling tailscaled config: %w", err) + return nil, fmt.Errorf("error marshalling tailscaled config: %w", err) } mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON) } @@ -474,18 +678,111 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p if !apiequality.Semantic.DeepEqual(existingCfgSecret, cfgSecret) { logger.Debugf("Updating the existing ProxyGroup config Secret %s", cfgSecret.Name) if err := r.Update(ctx, cfgSecret); err != nil { - return err + return nil, err } } } else { logger.Debugf("Creating a new config Secret %s for the ProxyGroup", cfgSecret.Name) if err := r.Create(ctx, cfgSecret); err != nil { - return err + return nil, err } } } - return nil + return endpoints, nil +} + +type FindStaticEndpointErr struct { + msg string +} + +func (e *FindStaticEndpointErr) Error() string { + return e.msg +} + +// findStaticEndpoints returns up to two `netip.AddrPort` entries, derived from the ExternalIPs of Nodes that +// match the `proxyClass`'s selector within the StaticEndpoints configuration. The port is set to the replica's NodePort Service Port. +func (r *ProxyGroupReconciler) findStaticEndpoints(ctx context.Context, existingCfgSecret *corev1.Secret, proxyClass *tsapi.ProxyClass, port uint16, logger *zap.SugaredLogger) ([]netip.AddrPort, error) { + var currAddrs []netip.AddrPort + if existingCfgSecret != nil { + oldConfB := existingCfgSecret.Data[tsoperator.TailscaledConfigFileName(106)] + if len(oldConfB) > 0 { + var oldConf ipn.ConfigVAlpha + if err := json.Unmarshal(oldConfB, &oldConf); err == nil { + currAddrs = oldConf.StaticEndpoints + } else { + logger.Debugf("failed to unmarshal tailscaled config from secret %q: %v", existingCfgSecret.Name, err) + } + } else { + logger.Debugf("failed to get tailscaled config from secret %q: empty data", existingCfgSecret.Name) + } + } + + nodes := new(corev1.NodeList) + selectors := client.MatchingLabels(proxyClass.Spec.StaticEndpoints.NodePort.Selector) + + err := r.List(ctx, nodes, selectors) + if err != nil { + return nil, fmt.Errorf("failed to list nodes: %w", err) + } + + if len(nodes.Items) == 0 { + return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass %q", proxyClass.Name)} + } + + endpoints := []netip.AddrPort{} + + // NOTE(ChaosInTheCRD): Setting a hard limit of two static endpoints. + newAddrs := []netip.AddrPort{} + for _, n := range nodes.Items { + for _, a := range n.Status.Addresses { + if a.Type == corev1.NodeExternalIP { + addr := getStaticEndpointAddress(&a, port) + if addr == nil { + logger.Debugf("failed to parse %q address on node %q: %q", corev1.NodeExternalIP, n.Name, a.Address) + continue + } + + // we want to add the currently used IPs first before + // adding new ones. + if currAddrs != nil && slices.Contains(currAddrs, *addr) { + endpoints = append(endpoints, *addr) + } else { + newAddrs = append(newAddrs, *addr) + } + } + + if len(endpoints) == 2 { + break + } + } + } + + // if the 2 endpoints limit hasn't been reached, we + // can start adding newIPs. + if len(endpoints) < 2 { + for _, a := range newAddrs { + endpoints = append(endpoints, a) + if len(endpoints) == 2 { + break + } + } + } + + if len(endpoints) == 0 { + return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to find any `status.addresses` of type %q on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass %q", corev1.NodeExternalIP, proxyClass.Name)} + } + + return endpoints, nil +} + +func getStaticEndpointAddress(a *corev1.NodeAddress, port uint16) *netip.AddrPort { + addr, err := netip.ParseAddr(a.Address) + if err != nil { + return nil + } + + return ptr.To(netip.AddrPortFrom(addr, port)) } // ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup @@ -514,7 +811,7 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) } -func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { +func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret, staticEndpoints []netip.AddrPort) (tailscaledConfigs, error) { conf := &ipn.ConfigVAlpha{ Version: "alpha0", AcceptDNS: "false", @@ -531,6 +828,10 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32 conf.AcceptRoutes = "true" } + if len(staticEndpoints) > 0 { + conf.StaticEndpoints = staticEndpoints + } + deviceAuthed := false for _, d := range pg.Status.Devices { if d.Hostname == *conf.Hostname { @@ -624,7 +925,7 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr return metadata, nil } -func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) { +func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, staticEndpoints map[string][]netip.AddrPort, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) { metadata, err := r.getNodeMetadata(ctx, pg) if err != nil { return nil, err @@ -638,10 +939,21 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.Prox if !ok { continue } - devices = append(devices, tsapi.TailnetDevice{ + + dev := tsapi.TailnetDevice{ Hostname: device.Hostname, TailnetIPs: device.TailnetIPs, - }) + } + + if ep, ok := staticEndpoints[device.Hostname]; ok && len(ep) > 0 { + eps := make([]string, 0, len(ep)) + for _, e := range ep { + eps = append(eps, e.String()) + } + dev.StaticEndpoints = eps + } + + devices = append(devices, dev) } return devices, nil @@ -655,3 +967,8 @@ type nodeMetadata struct { tsID tailcfg.StableNodeID dnsName string } + +func (pr *ProxyGroupReconciler) setStatusReady(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason string, msg string, logger *zap.SugaredLogger) { + pr.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, msg, pg.Generation, pr.clock, logger) +} diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 1d12c39e0241e..20e797f0c07cd 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -9,6 +9,7 @@ import ( "fmt" "slices" "strconv" + "strings" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -23,12 +24,43 @@ import ( "tailscale.com/types/ptr" ) -// deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully. -const deletionGracePeriodSeconds int64 = 360 +const ( + // deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully. + deletionGracePeriodSeconds int64 = 360 + staticEndpointPortName = "static-endpoint-port" +) + +func pgNodePortServiceName(proxyGroupName string, replica int32) string { + return fmt.Sprintf("%s-%d-nodeport", proxyGroupName, replica) +} + +func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + // NOTE(ChaosInTheCRD): we set the ports once we've iterated over every svc and found any old configuration we want to persist. + { + Name: staticEndpointPortName, + Protocol: corev1.ProtocolUDP, + }, + }, + Selector: map[string]string{ + appsv1.StatefulSetPodNameLabel: strings.TrimSuffix(name, "-nodeport"), + }, + }, + } +} // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be // applied over the top after. -func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { +func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { ss := new(appsv1.StatefulSet) if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) @@ -144,6 +176,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string }, } + if port != nil { + envs = append(envs, corev1.EnvVar{ + Name: "PORT", + Value: strconv.Itoa(int(*port)), + }) + } + if tsFirewallMode != "" { envs = append(envs, corev1.EnvVar{ Name: "TS_DEBUG_FIREWALL_MODE", diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index c556ae94a0de4..8ffce2c0c68ac 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -9,6 +9,8 @@ import ( "context" "encoding/json" "fmt" + "net/netip" + "slices" "testing" "time" @@ -18,6 +20,7 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" @@ -32,14 +35,772 @@ import ( "tailscale.com/types/ptr" ) -const testProxyImage = "tailscale/tailscale:test" +const ( + testProxyImage = "tailscale/tailscale:test" + initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196" +) + +var ( + defaultProxyClassAnnotations = map[string]string{ + "some-annotation": "from-the-proxy-class", + } + + defaultReplicas = ptr.To(int32(2)) + defaultStaticEndpointConfig = &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 30001}, {Port: 30002}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + } +) + +func TestProxyGroupWithStaticEndpoints(t *testing.T) { + type testNodeAddr struct { + ip string + addrType corev1.NodeAddressType + } -var defaultProxyClassAnnotations = map[string]string{ - "some-annotation": "from-the-proxy-class", + type testNode struct { + name string + addresses []testNodeAddr + labels map[string]string + } + + type reconcile struct { + staticEndpointConfig *tsapi.StaticEndpointsConfig + replicas *int32 + nodes []testNode + expectedIPs []netip.Addr + expectedEvents []string + expectedErr string + expectStatefulSet bool + } + + testCases := []struct { + name string + description string + reconciles []reconcile + }{ + { + // the reconciler should manage to create static endpoints when Nodes have IPv6 addresses. + name: "IPv6", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + {Port: 3009}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: ptr.To(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "2001:0db8::1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "2001:0db8::2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "2001:0db8::3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("2001:0db8::1"), netip.MustParseAddr("2001:0db8::2"), netip.MustParseAddr("2001:0db8::3")}, + expectedEvents: []string{}, + expectedErr: "", + expectStatefulSet: true, + }, + }, + }, + { + // declaring specific ports (with no `endPort`s) in the `spec.staticEndpoints.nodePort` should work. + name: "SpecificPorts", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + {Port: 3009}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: ptr.To(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("192.168.0.1"), netip.MustParseAddr("192.168.0.2"), netip.MustParseAddr("192.168.0.3")}, + expectedEvents: []string{}, + expectedErr: "", + expectStatefulSet: true, + }, + }, + }, + { + // if too narrow a range of `spec.staticEndpoints.nodePort.Ports` on the proxyClass should result in no StatefulSet being created. + name: "NotEnoughPorts", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: ptr.To(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning NodePort Services for static endpoints: failed to allocate NodePorts to ProxyGroup Services: not enough available ports to allocate all replicas (needed 4, got 3). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass \"default-pc\" must have bigger range allocated"}, + expectedErr: "", + expectStatefulSet: false, + }, + }, + }, + { + // when supplying a variety of ranges that are not clashing, the reconciler should manage to create a StatefulSet. + name: "NonClashingRanges", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3000, EndPort: 3002}, + {Port: 3003, EndPort: 3005}, + {Port: 3006}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: ptr.To(int32(3)), + nodes: []testNode{ + {name: "node1", addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}}, + {name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}}, + {name: "node3", addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}}, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.0.0.3")}, + expectedEvents: []string{}, + expectedErr: "", + expectStatefulSet: true, + }, + }, + }, + { + // when there isn't a node that matches the selector, the ProxyGroup enters a failed state as there are no valid Static Endpoints. + // while it does create an event on the resource, It does not return an error + name: "NoMatchingNodes", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3000, EndPort: 3005}, + }, + Selector: map[string]string{ + "zone": "us-west", + }, + }, + }, + replicas: defaultReplicas, + nodes: []testNode{ + {name: "node1", addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"zone": "eu-central"}}, + {name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}}, labels: map[string]string{"zone": "eu-central"}}, + }, + expectedIPs: []netip.Addr{}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""}, + expectedErr: "", + expectStatefulSet: false, + }, + }, + }, + { + // when all the nodes have only have addresses of type InternalIP populated in their status, the ProxyGroup enters a failed state as there are no valid Static Endpoints. + // while it does create an event on the resource, It does not return an error + name: "AllInternalIPAddresses", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + {Port: 3009}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: ptr.To(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to find any `status.addresses` of type \"ExternalIP\" on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass \"default-pc\""}, + expectedErr: "", + expectStatefulSet: false, + }, + }, + }, + { + // When the node's (and some of their addresses) change between reconciles, the reconciler should first pick addresses that + // have been used previously (provided that they are still populated on a node that matches the selector) + name: "NodeIPChangesAndPersists", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.10", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectStatefulSet: true, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + }, + }, + }, + { + // given a new node being created with a new IP, and a node previously used for Static Endpoints being removed, the Static Endpoints should be updated + // correctly + name: "NodeIPChangesWithNewNode", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.3")}, + expectStatefulSet: true, + }, + }, + }, + { + // when all the node IPs change, they should all update + name: "AllNodeIPsChange", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.100", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.200", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.100"), netip.MustParseAddr("10.0.0.200")}, + expectStatefulSet: true, + }, + }, + }, + { + // if there are less ExternalIPs after changes to the nodes between reconciles, the reconciler should complete without issues + name: "LessExternalIPsAfterChange", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1")}, + expectStatefulSet: true, + }, + }, + }, + { + // if node address parsing fails (given an invalid address), the reconciler should continue without failure and find other + // valid addresses + name: "NodeAddressParsingFails", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "invalid-ip", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "invalid-ip", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + }, + }, + { + // if the node's become unlabeled, the ProxyGroup should enter a ProxyGroupInvalid state, but the reconciler should not fail + name: "NodesBecomeUnlabeled", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{}, + }, + { + name: "node4", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""}, + expectStatefulSet: true, + }, + }, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + tsClient := &fakeTSClient{} + zl, _ := zap.NewDevelopment() + fr := record.NewFakeRecorder(10) + cl := tstest.NewClock(tstest.ClockOpts{}) + + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Annotations: defaultProxyClassAnnotations, + }, + }, + Status: tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + Reason: reasonProxyClassValid, + Message: reasonProxyClassValid, + LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }}, + }, + } + + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeEgress, + ProxyClass: pc.Name, + }, + } + + fc := fake.NewClientBuilder(). + WithObjects(pc, pg). + WithStatusSubresource(pc, pg). + WithScheme(tsapi.GlobalScheme). + Build() + + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + proxyImage: testProxyImage, + defaultTags: []string{"tag:test-tag"}, + tsFirewallMode: "auto", + defaultProxyClass: "default-pc", + + Client: fc, + tsClient: tsClient, + recorder: fr, + clock: cl, + } + + for i, r := range tt.reconciles { + createdNodes := []corev1.Node{} + t.Run(tt.name, func(t *testing.T) { + for _, n := range r.nodes { + no := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: n.name, + Labels: n.labels, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{}, + }, + } + for _, addr := range n.addresses { + no.Status.Addresses = append(no.Status.Addresses, corev1.NodeAddress{ + Type: addr.addrType, + Address: addr.ip, + }) + } + if err := fc.Create(context.Background(), no); err != nil { + t.Fatalf("failed to create node %q: %v", n.name, err) + } + createdNodes = append(createdNodes, *no) + t.Logf("created node %q with data", n.name) + } + + reconciler.l = zl.Sugar().With("TestName", tt.name).With("Reconcile", i) + pg.Spec.Replicas = r.replicas + pc.Spec.StaticEndpoints = r.staticEndpointConfig + + createOrUpdate(context.Background(), fc, "", pg, func(o *tsapi.ProxyGroup) { + o.Spec.Replicas = pg.Spec.Replicas + }) + + createOrUpdate(context.Background(), fc, "", pc, func(o *tsapi.ProxyClass) { + o.Spec.StaticEndpoints = pc.Spec.StaticEndpoints + }) + + if r.expectedErr != "" { + expectError(t, reconciler, "", pg.Name) + } else { + expectReconciled(t, reconciler, "", pg.Name) + } + expectEvents(t, fr, r.expectedEvents) + + sts := &appsv1.StatefulSet{} + err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts) + if r.expectStatefulSet { + if err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + for j := range 2 { + sec := &corev1.Secret{} + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: fmt.Sprintf("%s-%d-config", pg.Name, j)}, sec); err != nil { + t.Fatalf("failed to get state Secret for replica %d: %v", j, err) + } + + config := &ipn.ConfigVAlpha{} + foundConfig := false + for _, d := range sec.Data { + if err := json.Unmarshal(d, config); err == nil { + foundConfig = true + break + } + } + if !foundConfig { + t.Fatalf("could not unmarshal config from secret data for replica %d", j) + } + + if len(config.StaticEndpoints) > staticEndpointsMaxAddrs { + t.Fatalf("expected %d StaticEndpoints in config Secret, but got %d for replica %d. Found Static Endpoints: %v", staticEndpointsMaxAddrs, len(config.StaticEndpoints), j, config.StaticEndpoints) + } + + for _, e := range config.StaticEndpoints { + if !slices.Contains(r.expectedIPs, e.Addr()) { + t.Fatalf("found unexpected static endpoint IP %q for replica %d. Expected one of %v", e.Addr().String(), j, r.expectedIPs) + } + if c := r.staticEndpointConfig; c != nil && c.NodePort.Ports != nil { + var ports tsapi.PortRanges = c.NodePort.Ports + found := false + for port := range ports.All() { + if port == e.Port() { + found = true + break + } + } + + if !found { + t.Fatalf("found unexpected static endpoint port %d for replica %d. Expected one of %v .", e.Port(), j, ports.All()) + } + } else { + if e.Port() != 3001 && e.Port() != 3002 { + t.Fatalf("found unexpected static endpoint port %d for replica %d. Expected 3001 or 3002.", e.Port(), j) + } + } + } + } + + pgroup := &tsapi.ProxyGroup{} + err = fc.Get(context.Background(), client.ObjectKey{Name: pg.Name}, pgroup) + if err != nil { + t.Fatalf("failed to get ProxyGroup %q: %v", pg.Name, err) + } + + t.Logf("getting proxygroup after reconcile") + for _, d := range pgroup.Status.Devices { + t.Logf("found device %q", d.Hostname) + for _, e := range d.StaticEndpoints { + t.Logf("found static endpoint %q", e) + } + } + } else { + if err == nil { + t.Fatal("expected error when getting Statefulset") + } + } + }) + + // node cleanup between reconciles + // we created a new set of nodes for each + for _, n := range createdNodes { + err := fc.Delete(context.Background(), &n) + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("failed to delete node: %v", err) + } + } + } + + t.Run("delete_and_cleanup", func(t *testing.T) { + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + proxyImage: testProxyImage, + defaultTags: []string{"tag:test-tag"}, + tsFirewallMode: "auto", + defaultProxyClass: "default-pc", + + Client: fc, + tsClient: tsClient, + recorder: fr, + l: zl.Sugar().With("TestName", tt.name).With("Reconcile", "cleanup"), + clock: cl, + } + + if err := fc.Delete(context.Background(), pg); err != nil { + t.Fatalf("error deleting ProxyGroup: %v", err) + } + + expectReconciled(t, reconciler, "", pg.Name) + expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name) + + if err := fc.Delete(context.Background(), pc); err != nil { + t.Fatalf("error deleting ProxyClass: %v", err) + } + expectMissing[tsapi.ProxyClass](t, fc, "", pc.Name) + }) + }) + } } func TestProxyGroup(t *testing.T) { - pc := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{ Name: "default-pc", @@ -598,7 +1359,7 @@ func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.Prox role := pgRole(pg, tsNamespace) roleBinding := pgRoleBinding(pg, tsNamespace) serviceAccount := pgServiceAccount(pg, tsNamespace) - statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", proxyClass) + statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass) if err != nil { t.Fatal(err) } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 03bb8989b9782..aba5f9e2df4b2 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -425,6 +425,23 @@ _Appears in:_ | `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
          Currently you must manually update your cluster DNS config to add
          this address as a stub nameserver for ts.net for cluster workloads to be
          able to resolve MagicDNS names associated with egress or Ingress
          proxies.
          The IP address will change if you delete and recreate the DNSConfig. | | | +#### NodePortConfig + + + + + + + +_Appears in:_ +- [StaticEndpointsConfig](#staticendpointsconfig) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `ports` _[PortRange](#portrange) array_ | The port ranges from which the operator will select NodePorts for the Services.
          You must ensure that firewall rules allow UDP ingress traffic for these ports
          to the node's external IPs.
          The ports must be in the range of service node ports for the cluster (default `30000-32767`).
          See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. | | MinItems: 1
          | +| `selector` _object (keys:string, values:string)_ | A selector which will be used to select the node's that will have their `ExternalIP`'s advertised
          by the ProxyGroup as Static Endpoints. | | | + + #### Pod @@ -451,6 +468,26 @@ _Appears in:_ | `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#topologyspreadconstraint-v1-core) array_ | Proxy Pod's topology spread constraints.
          By default Tailscale Kubernetes operator does not apply any topology spread constraints.
          https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ | | | +#### PortRange + + + + + + + +_Appears in:_ +- [NodePortConfig](#nodeportconfig) +- [PortRanges](#portranges) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `port` _integer_ | port represents a port selected to be used. This is a required field. | | | +| `endPort` _integer_ | endPort indicates that the range of ports from port to endPort if set, inclusive,
          should be used. This field cannot be defined if the port field is not defined.
          The endPort must be either unset, or equal or greater than port. | | | + + + + #### ProxyClass @@ -518,6 +555,7 @@ _Appears in:_ | `metrics` _[Metrics](#metrics)_ | Configuration for proxy metrics. Metrics are currently not supported
          for egress proxies and for Ingress proxies that have been configured
          with tailscale.com/experimental-forward-cluster-traffic-via-ingress
          annotation. Note that the metrics are currently considered unstable
          and will likely change in breaking ways in the future - we only
          recommend that you use those for debugging purposes. | | | | `tailscale` _[TailscaleConfig](#tailscaleconfig)_ | TailscaleConfig contains options to configure the tailscale-specific
          parameters of proxies. | | | | `useLetsEncryptStagingEnvironment` _boolean_ | Set UseLetsEncryptStagingEnvironment to true to issue TLS
          certificates for any HTTPS endpoints exposed to the tailnet from
          LetsEncrypt's staging environment.
          https://letsencrypt.org/docs/staging-environment/
          This setting only affects Tailscale Ingress resources.
          By default Ingress TLS certificates are issued from LetsEncrypt's
          production environment.
          Changing this setting true -> false, will result in any
          existing certs being re-issued from the production environment.
          Changing this setting false (default) -> true, when certs have already
          been provisioned from production environment will NOT result in certs
          being re-issued from the staging environment before they need to be
          renewed. | | | +| `staticEndpoints` _[StaticEndpointsConfig](#staticendpointsconfig)_ | Configuration for 'static endpoints' on proxies in order to facilitate
          direct connections from other devices on the tailnet.
          See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. | | | #### ProxyClassStatus @@ -935,6 +973,22 @@ _Appears in:_ | `pod` _[Pod](#pod)_ | Configuration for the proxy Pod. | | | +#### StaticEndpointsConfig + + + + + + + +_Appears in:_ +- [ProxyClassSpec](#proxyclassspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `nodePort` _[NodePortConfig](#nodeportconfig)_ | The configuration for static endpoints using NodePort Services. | | | + + #### Storage @@ -1015,6 +1069,7 @@ _Appears in:_ | --- | --- | --- | --- | | `hostname` _string_ | Hostname is the fully qualified domain name of the device.
          If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
          node. | | | | `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
          assigned to the device. | | | +| `staticEndpoints` _string array_ | StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. | | | #### TailscaleConfig diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 899abf096bb86..9221c60f3c870 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -6,6 +6,10 @@ package v1alpha1 import ( + "fmt" + "iter" + "strings" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -82,6 +86,124 @@ type ProxyClassSpec struct { // renewed. // +optional UseLetsEncryptStagingEnvironment bool `json:"useLetsEncryptStagingEnvironment,omitempty"` + // Configuration for 'static endpoints' on proxies in order to facilitate + // direct connections from other devices on the tailnet. + // See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. + // +optional + StaticEndpoints *StaticEndpointsConfig `json:"staticEndpoints,omitempty"` +} + +type StaticEndpointsConfig struct { + // The configuration for static endpoints using NodePort Services. + NodePort *NodePortConfig `json:"nodePort"` +} + +type NodePortConfig struct { + // The port ranges from which the operator will select NodePorts for the Services. + // You must ensure that firewall rules allow UDP ingress traffic for these ports + // to the node's external IPs. + // The ports must be in the range of service node ports for the cluster (default `30000-32767`). + // See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. + // +kubebuilder:validation:MinItems=1 + Ports []PortRange `json:"ports"` + // A selector which will be used to select the node's that will have their `ExternalIP`'s advertised + // by the ProxyGroup as Static Endpoints. + Selector map[string]string `json:"selector,omitempty"` +} + +// PortRanges is a list of PortRange(s) +type PortRanges []PortRange + +func (prs PortRanges) String() string { + var prStrings []string + + for _, pr := range prs { + prStrings = append(prStrings, pr.String()) + } + + return strings.Join(prStrings, ", ") +} + +// All allows us to iterate over all the ports in the PortRanges +func (prs PortRanges) All() iter.Seq[uint16] { + return func(yield func(uint16) bool) { + for _, pr := range prs { + end := pr.EndPort + if end == 0 { + end = pr.Port + } + + for port := pr.Port; port <= end; port++ { + if !yield(port) { + return + } + } + } + } +} + +// Contains reports whether port is in any of the PortRanges. +func (prs PortRanges) Contains(port uint16) bool { + for _, r := range prs { + if r.Contains(port) { + return true + } + } + + return false +} + +// ClashesWith reports whether the supplied PortRange clashes with any of the PortRanges. +func (prs PortRanges) ClashesWith(pr PortRange) bool { + for p := range prs.All() { + if pr.Contains(p) { + return true + } + } + + return false +} + +type PortRange struct { + // port represents a port selected to be used. This is a required field. + Port uint16 `json:"port"` + + // endPort indicates that the range of ports from port to endPort if set, inclusive, + // should be used. This field cannot be defined if the port field is not defined. + // The endPort must be either unset, or equal or greater than port. + // +optional + EndPort uint16 `json:"endPort,omitempty"` +} + +// Contains reports whether port is in pr. +func (pr PortRange) Contains(port uint16) bool { + switch pr.EndPort { + case 0: + return port == pr.Port + default: + return port >= pr.Port && port <= pr.EndPort + } +} + +// String returns the PortRange in a string form. +func (pr PortRange) String() string { + if pr.EndPort == 0 { + return fmt.Sprintf("%d", pr.Port) + } + + return fmt.Sprintf("%d-%d", pr.Port, pr.EndPort) +} + +// IsValid reports whether the port range is valid. +func (pr PortRange) IsValid() bool { + if pr.Port == 0 { + return false + } + if pr.EndPort == 0 { + return true + } + + return pr.Port <= pr.EndPort } type TailscaleConfig struct { diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index ac87cc6caf892..17b13064bb4fc 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -111,6 +111,10 @@ type TailnetDevice struct { // assigned to the device. // +optional TailnetIPs []string `json:"tailnetIPs,omitempty"` + + // StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. + // +optional + StaticEndpoints []string `json:"staticEndpoints,omitempty"` } // +kubebuilder:validation:Type=string diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index e091272075ce2..ffc04d3b9dde3 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -407,6 +407,33 @@ func (in *NameserverStatus) DeepCopy() *NameserverStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePortConfig) DeepCopyInto(out *NodePortConfig) { + *out = *in + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]PortRange, len(*in)) + copy(*out, *in) + } + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePortConfig. +func (in *NodePortConfig) DeepCopy() *NodePortConfig { + if in == nil { + return nil + } + out := new(NodePortConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pod) DeepCopyInto(out *Pod) { *out = *in @@ -482,6 +509,40 @@ func (in *Pod) DeepCopy() *Pod { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortRange) DeepCopyInto(out *PortRange) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRange. +func (in *PortRange) DeepCopy() *PortRange { + if in == nil { + return nil + } + out := new(PortRange) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in PortRanges) DeepCopyInto(out *PortRanges) { + { + in := &in + *out = make(PortRanges, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRanges. +func (in PortRanges) DeepCopy() PortRanges { + if in == nil { + return nil + } + out := new(PortRanges) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxyClass) DeepCopyInto(out *ProxyClass) { *out = *in @@ -559,6 +620,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) { *out = new(TailscaleConfig) **out = **in } + if in.StaticEndpoints != nil { + in, out := &in.StaticEndpoints, &out.StaticEndpoints + *out = new(StaticEndpointsConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec. @@ -1096,6 +1162,26 @@ func (in *StatefulSet) DeepCopy() *StatefulSet { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticEndpointsConfig) DeepCopyInto(out *StaticEndpointsConfig) { + *out = *in + if in.NodePort != nil { + in, out := &in.NodePort, &out.NodePort + *out = new(NodePortConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticEndpointsConfig. +func (in *StaticEndpointsConfig) DeepCopy() *StaticEndpointsConfig { + if in == nil { + return nil + } + out := new(StaticEndpointsConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Storage) DeepCopyInto(out *Storage) { *out = *in @@ -1163,6 +1249,11 @@ func (in *TailnetDevice) DeepCopyInto(out *TailnetDevice) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.StaticEndpoints != nil { + in, out := &in.StaticEndpoints, &out.StaticEndpoints + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetDevice. From 711698f5a985a5c93649b31c9f49ed6d22a91c42 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Fri, 27 Jun 2025 18:10:04 +0100 Subject: [PATCH 128/263] cmd/{containerboot,k8s-operator}: use state Secret for checking device auth (#16328) Previously, the operator checked the ProxyGroup status fields for information on how many of the proxies had successfully authed. Use their state Secrets instead as a more reliable source of truth. containerboot has written device_fqdn and device_ips keys to the state Secret since inception, and pod_uid since 1.78.0, so there's no need to use the API for that data. Read it from the state Secret for consistency. However, to ensure we don't read data from a previous run of containerboot, make sure we reset containerboot's state keys on startup. One other knock-on effect of that is ProxyGroups can briefly be marked not Ready while a Pod is restarting. Introduce a new ProxyGroupAvailable condition to more accurately reflect when downstream controllers can implement flows that rely on a ProxyGroup having at least 1 proxy Pod running. Fixes #16327 Change-Id: I026c18e9d23e87109a471a87b8e4fb6271716a66 Signed-off-by: Tom Proctor --- cmd/containerboot/kube.go | 42 +++-- cmd/containerboot/kube_test.go | 80 +++++++++ cmd/containerboot/main.go | 19 +- cmd/containerboot/main_test.go | 132 +++++++------- cmd/k8s-operator/egress-services-readiness.go | 2 +- .../egress-services-readiness_test.go | 2 +- cmd/k8s-operator/egress-services.go | 2 +- cmd/k8s-operator/ingress-for-pg.go | 6 +- cmd/k8s-operator/ingress-for-pg_test.go | 10 +- cmd/k8s-operator/proxygroup.go | 167 +++++++++++------- cmd/k8s-operator/proxygroup_specs.go | 6 +- cmd/k8s-operator/proxygroup_test.go | 18 +- cmd/k8s-operator/sts.go | 36 ++-- cmd/k8s-operator/svc-for-pg.go | 2 +- cmd/k8s-operator/svc-for-pg_test.go | 4 +- cmd/k8s-operator/tsrecorder.go | 9 +- k8s-operator/apis/v1alpha1/types_connector.go | 11 +- k8s-operator/conditions.go | 10 +- kube/kubeclient/fake_client.go | 17 +- 19 files changed, 373 insertions(+), 202 deletions(-) diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go index 0a2dfa1bf342f..d4a974e6f3a24 100644 --- a/cmd/containerboot/kube.go +++ b/cmd/containerboot/kube.go @@ -18,12 +18,15 @@ import ( "time" "tailscale.com/ipn" + "tailscale.com/kube/egressservices" + "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" "tailscale.com/logtail/backoff" "tailscale.com/tailcfg" "tailscale.com/types/logger" + "tailscale.com/util/set" ) // kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use @@ -117,20 +120,39 @@ func (kc *kubeClient) deleteAuthKey(ctx context.Context) error { return nil } -// storeCapVerUID stores the current capability version of tailscale and, if provided, UID of the Pod in the tailscale -// state Secret. -// These two fields are used by the Kubernetes Operator to observe the current capability version of tailscaled running in this container. -func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error { - capVerS := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion) - d := map[string][]byte{ - kubetypes.KeyCapVer: []byte(capVerS), +// resetContainerbootState resets state from previous runs of containerboot to +// ensure the operator doesn't use stale state when a Pod is first recreated. +func (kc *kubeClient) resetContainerbootState(ctx context.Context, podUID string) error { + existingSecret, err := kc.GetSecret(ctx, kc.stateSecret) + if err != nil { + return fmt.Errorf("failed to read state Secret %q to reset state: %w", kc.stateSecret, err) + } + + s := &kubeapi.Secret{ + Data: map[string][]byte{ + kubetypes.KeyCapVer: fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion), + }, } if podUID != "" { - d[kubetypes.KeyPodUID] = []byte(podUID) + s.Data[kubetypes.KeyPodUID] = []byte(podUID) } - s := &kubeapi.Secret{ - Data: d, + + toClear := set.SetOf([]string{ + kubetypes.KeyDeviceID, + kubetypes.KeyDeviceFQDN, + kubetypes.KeyDeviceIPs, + kubetypes.KeyHTTPSEndpoint, + egressservices.KeyEgressServices, + ingressservices.IngressConfigKey, + }) + for key := range existingSecret.Data { + if toClear.Contains(key) { + // It's fine to leave the key in place as a debugging breadcrumb, + // it should get a new value soon. + s.Data[key] = nil + } } + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") } diff --git a/cmd/containerboot/kube_test.go b/cmd/containerboot/kube_test.go index 413971bc6df23..c33714ed12ace 100644 --- a/cmd/containerboot/kube_test.go +++ b/cmd/containerboot/kube_test.go @@ -8,13 +8,18 @@ package main import ( "context" "errors" + "fmt" "testing" "time" "github.com/google/go-cmp/cmp" "tailscale.com/ipn" + "tailscale.com/kube/egressservices" + "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" ) func TestSetupKube(t *testing.T) { @@ -238,3 +243,78 @@ func TestWaitForConsistentState(t *testing.T) { t.Fatalf("expected nil, got %v", err) } } + +func TestResetContainerbootState(t *testing.T) { + capver := fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion) + for name, tc := range map[string]struct { + podUID string + initial map[string][]byte + expected map[string][]byte + }{ + "empty_initial": { + podUID: "1234", + initial: map[string][]byte{}, + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: []byte("1234"), + }, + }, + "empty_initial_no_pod_uid": { + initial: map[string][]byte{}, + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + }, + }, + "only_relevant_keys_updated": { + podUID: "1234", + initial: map[string][]byte{ + kubetypes.KeyCapVer: []byte("1"), + kubetypes.KeyPodUID: []byte("5678"), + kubetypes.KeyDeviceID: []byte("device-id"), + kubetypes.KeyDeviceFQDN: []byte("device-fqdn"), + kubetypes.KeyDeviceIPs: []byte(`["192.0.2.1"]`), + kubetypes.KeyHTTPSEndpoint: []byte("https://example.com"), + egressservices.KeyEgressServices: []byte("egress-services"), + ingressservices.IngressConfigKey: []byte("ingress-config"), + "_current-profile": []byte("current-profile"), + "_machinekey": []byte("machine-key"), + "_profiles": []byte("profiles"), + "_serve_e0ce": []byte("serve-e0ce"), + "profile-e0ce": []byte("profile-e0ce"), + }, + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: []byte("1234"), + // Cleared keys. + kubetypes.KeyDeviceID: nil, + kubetypes.KeyDeviceFQDN: nil, + kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + // Tailscaled keys not included in patch. + }, + }, + } { + t.Run(name, func(t *testing.T) { + var actual map[string][]byte + kc := &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{ + GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) { + return &kubeapi.Secret{ + Data: tc.initial, + }, nil + }, + StrategicMergePatchSecretImpl: func(ctx context.Context, name string, secret *kubeapi.Secret, _ string) error { + actual = secret.Data + return nil + }, + }} + if err := kc.resetContainerbootState(context.Background(), tc.podUID); err != nil { + t.Fatalf("resetContainerbootState() error = %v", err) + } + if diff := cmp.Diff(tc.expected, actual); diff != "" { + t.Errorf("resetContainerbootState() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 9543308975b79..52b30b8375a4c 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -188,6 +188,14 @@ func run() error { if err := cfg.setupKube(bootCtx, kc); err != nil { return fmt.Errorf("error setting up for running on Kubernetes: %w", err) } + // Clear out any state from previous runs of containerboot. Check + // hasKubeStateStore because although we know we're in kube, that + // doesn't guarantee the state store is properly configured. + if hasKubeStateStore(cfg) { + if err := kc.resetContainerbootState(bootCtx, cfg.PodUID); err != nil { + return fmt.Errorf("error clearing previous state from Secret: %w", err) + } + } } client, daemonProcess, err := startTailscaled(bootCtx, cfg) @@ -367,11 +375,6 @@ authLoop: if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil { return fmt.Errorf("failed to unset serve config: %w", err) } - if hasKubeStateStore(cfg) { - if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil { - return fmt.Errorf("failed to update HTTPS endpoint in tailscale state: %w", err) - } - } } if hasKubeStateStore(cfg) && isTwoStepConfigAuthOnce(cfg) { @@ -384,12 +387,6 @@ authLoop: } } - if hasKubeStateStore(cfg) { - if err := kc.storeCapVerUID(ctx, cfg.PodUID); err != nil { - return fmt.Errorf("storing capability version and UID: %w", err) - } - } - w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState) if err != nil { return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err) diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index c7293c77a4afa..96feef682af5b 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -460,6 +460,7 @@ func TestContainerBoot(t *testing.T) { Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, + "POD_UID": "some-pod-uid", }, KubeSecret: map[string]string{ "authkey": "tskey-key", @@ -471,17 +472,20 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: "some-pod-uid", }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: "some-pod-uid", }, }, }, @@ -554,7 +558,8 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { @@ -565,7 +570,8 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { @@ -574,10 +580,10 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false", }, WantKubeSecret: map[string]string{ - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, }, }, @@ -599,17 +605,18 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, }, { @@ -624,11 +631,11 @@ func TestContainerBoot(t *testing.T) { }, }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "new-name.test.ts.net", - "device_id": "newID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "new-name.test.ts.net", + "device_id": "newID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, }, }, @@ -912,18 +919,19 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "https_endpoint": "no-https", - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + "https_endpoint": "no-https", + kubetypes.KeyCapVer: capver, }, }, }, @@ -947,7 +955,8 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, EndpointStatuses: map[string]int{ egressSvcTerminateURL(env.localAddrPort): 200, @@ -956,12 +965,12 @@ func TestContainerBoot(t *testing.T) { { Notify: runningNotify, WantKubeSecret: map[string]string{ - "egress-services": mustBase64(t, egressStatus), - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "egress-services": string(mustJSON(t, egressStatus)), + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, EndpointStatuses: map[string]int{ egressSvcTerminateURL(env.localAddrPort): 200, @@ -1002,7 +1011,8 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { @@ -1016,10 +1026,11 @@ func TestContainerBoot(t *testing.T) { // Missing "_current-profile" key. }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "_machinekey": "foo", - "_profiles": "foo", - "profile-baff": "foo", + "authkey": "tskey-key", + "_machinekey": "foo", + "_profiles": "foo", + "profile-baff": "foo", + kubetypes.KeyCapVer: capver, }, WantLog: "Waiting for tailscaled to finish writing state to Secret \"tailscale\"", }, @@ -1029,11 +1040,12 @@ func TestContainerBoot(t *testing.T) { "_current-profile": "foo", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "_machinekey": "foo", - "_profiles": "foo", - "profile-baff": "foo", - "_current-profile": "foo", + "authkey": "tskey-key", + "_machinekey": "foo", + "_profiles": "foo", + "profile-baff": "foo", + "_current-profile": "foo", + kubetypes.KeyCapVer: capver, }, WantLog: "HTTP server at [::]:9002 closed", WantExitCode: ptr.To(0), @@ -1061,7 +1073,7 @@ func TestContainerBoot(t *testing.T) { fmt.Sprintf("TS_TEST_SOCKET=%s", env.lapi.Path), fmt.Sprintf("TS_SOCKET=%s", env.runningSockPath), fmt.Sprintf("TS_TEST_ONLY_ROOT=%s", env.d), - fmt.Sprint("TS_TEST_FAKE_NETFILTER=true"), + "TS_TEST_FAKE_NETFILTER=true", } for k, v := range tc.Env { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) @@ -1489,10 +1501,7 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { } switch r.Header.Get("Content-Type") { case "application/json-patch+json": - req := []struct { - Op string `json:"op"` - Path string `json:"path"` - }{} + req := []kubeclient.JSONPatch{} if err := json.Unmarshal(bs, &req); err != nil { panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) } @@ -1503,23 +1512,20 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) } delete(k.secret, strings.TrimPrefix(op.Path, "/data/")) - case "replace": + case "add", "replace": path, ok := strings.CutPrefix(op.Path, "/data/") if !ok { panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) } - req := make([]kubeclient.JSONPatch, 0) - if err := json.Unmarshal(bs, &req); err != nil { - panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) + val, ok := op.Value.(string) + if !ok { + panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", op.Value)) } - - for _, patch := range req { - val, ok := patch.Value.(string) - if !ok { - panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", patch.Value)) - } - k.secret[path] = val + v, err := base64.StdEncoding.DecodeString(val) + if err != nil { + panic(fmt.Sprintf("json patch value %q is not base64 encoded: %v", val, err)) } + k.secret[path] = string(v) default: panic(fmt.Sprintf("unsupported json-patch op %q", op.Op)) } diff --git a/cmd/k8s-operator/egress-services-readiness.go b/cmd/k8s-operator/egress-services-readiness.go index 5e95a52790395..ecf99b63cda44 100644 --- a/cmd/k8s-operator/egress-services-readiness.go +++ b/cmd/k8s-operator/egress-services-readiness.go @@ -102,7 +102,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re msg = err.Error() return res, err } - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { l.Infof("ProxyGroup for Service is not ready, waiting...") reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady st = metav1.ConditionFalse diff --git a/cmd/k8s-operator/egress-services-readiness_test.go b/cmd/k8s-operator/egress-services-readiness_test.go index ce947329ddfb8..f80759aef927b 100644 --- a/cmd/k8s-operator/egress-services-readiness_test.go +++ b/cmd/k8s-operator/egress-services-readiness_test.go @@ -137,7 +137,7 @@ func setReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replic } func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, l *zap.SugaredLogger) { - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l) } func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) { diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 7103205ac2c3f..ca6562071eba7 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -531,7 +531,7 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) } diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index ea31dbd63befb..09417fd0c8878 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -182,7 +182,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) } - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { logger.Infof("ProxyGroup is not (yet) ready") return false, nil } @@ -666,7 +666,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki } // Validate TLS configuration - if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) { + if len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) { errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS)) } @@ -683,7 +683,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki } // Validate ProxyGroup readiness - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name)) } diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 05f4827927923..2308514f3af9c 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -305,7 +305,7 @@ func TestValidateIngress(t *testing.T) { Status: tsapi.ProxyGroupStatus{ Conditions: []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, @@ -399,7 +399,7 @@ func TestValidateIngress(t *testing.T) { Status: tsapi.ProxyGroupStatus{ Conditions: []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionFalse, ObservedGeneration: 1, }, @@ -755,7 +755,7 @@ func verifyTailscaledConfig(t *testing.T, fc client.Client, pgName string, expec Labels: pgSecretLabels(pgName, "config"), }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)), + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)), }, }) } @@ -794,13 +794,13 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) { Labels: pgSecretLabels(pgName, "config"), }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): []byte("{}"), + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte("{}"), }, } mustCreate(t, fc, pgCfgSecret) pg.Status.Conditions = []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 328262031b85c..bedf06ba0ac28 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -52,6 +52,17 @@ const ( // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" staticEndpointsMaxAddrs = 2 + + // The minimum tailcfg.CapabilityVersion that deployed clients are expected + // to support to be compatible with the current ProxyGroup controller. + // If the controller needs to depend on newer client behaviour, it should + // maintain backwards compatible logic for older capability versions for 3 + // stable releases, as per documentation on supported version drift: + // https://tailscale.com/kb/1236/kubernetes-operator#supported-versions + // + // tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was + // first introduced. + pgMinCapabilityVersion = 106 ) var ( @@ -204,14 +215,27 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } desiredReplicas := int(pgReplicas(pg)) + + // Set ProxyGroupAvailable condition. + status := metav1.ConditionFalse + reason := reasonProxyGroupCreating + message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas) + if len(pg.Status.Devices) > 0 { + status = metav1.ConditionTrue + if len(pg.Status.Devices) == desiredReplicas { + reason = reasonProxyGroupReady + } + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, pg.Generation, r.clock, logger) + + // Set ProxyGroupReady condition. if len(pg.Status.Devices) < desiredReplicas { - message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas) logger.Debug(message) return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) } if len(pg.Status.Devices) > desiredReplicas { - message := fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(pg.Status.Devices)-desiredReplicas) + message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(pg.Status.Devices)-desiredReplicas) logger.Debug(message) return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) } @@ -524,17 +548,13 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil { return err } - if err := r.Delete(ctx, m.stateSecret); err != nil { - if !apierrors.IsNotFound(err) { - return fmt.Errorf("error deleting state Secret %s: %w", m.stateSecret.Name, err) - } + if err := r.Delete(ctx, m.stateSecret); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting state Secret %q: %w", m.stateSecret.Name, err) } configSecret := m.stateSecret.DeepCopy() configSecret.Name += "-config" - if err := r.Delete(ctx, configSecret); err != nil { - if !apierrors.IsNotFound(err) { - return fmt.Errorf("error deleting config Secret %s: %w", configSecret.Name, err) - } + if err := r.Delete(ctx, configSecret); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting config Secret %q: %w", configSecret.Name, err) } // NOTE(ChaosInTheCRD): we shouldn't need to get the service first, checking for a not found error should be enough svc := &corev1.Service{ @@ -635,17 +655,38 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p return nil, err } - var authKey string + var authKey *string if existingCfgSecret == nil { logger.Debugf("Creating authkey for new ProxyGroup proxy") tags := pg.Spec.Tags.Stringify() if len(tags) == 0 { tags = r.defaultTags } - authKey, err = newAuthKey(ctx, r.tsClient, tags) + key, err := newAuthKey(ctx, r.tsClient, tags) if err != nil { return nil, err } + authKey = &key + } + + if authKey == nil { + // Get state Secret to check if it's already authed. + stateSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgStateSecretName(pg.Name, i), + Namespace: r.tsNamespace, + }, + } + if err := r.Get(ctx, client.ObjectKeyFromObject(stateSecret), stateSecret); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + + if shouldRetainAuthKey(stateSecret) && existingCfgSecret != nil { + authKey, err = authKeyFromSecret(existingCfgSecret) + if err != nil { + return nil, fmt.Errorf("error retrieving auth key from existing config Secret: %w", err) + } + } } replicaName := pgNodePortServiceName(pg.Name, i) @@ -661,7 +702,14 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p } } - configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret, endpoints[replicaName]) + // AdvertiseServices config is set by ingress-pg-reconciler, so make sure we + // don't overwrite it if already set. + existingAdvertiseServices, err := extractAdvertiseServicesConfig(existingCfgSecret) + if err != nil { + return nil, err + } + + configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[replicaName], existingAdvertiseServices) if err != nil { return nil, fmt.Errorf("error creating tailscaled config: %w", err) } @@ -811,20 +859,22 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) } -func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret, staticEndpoints []netip.AddrPort) (tailscaledConfigs, error) { +func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string) (tailscaledConfigs, error) { conf := &ipn.ConfigVAlpha{ - Version: "alpha0", - AcceptDNS: "false", - AcceptRoutes: "false", // AcceptRoutes defaults to true - Locked: "false", - Hostname: ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)), + Version: "alpha0", + AcceptDNS: "false", + AcceptRoutes: "false", // AcceptRoutes defaults to true + Locked: "false", + Hostname: ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)), + AdvertiseServices: oldAdvertiseServices, + AuthKey: authKey, } if pg.Spec.HostnamePrefix != "" { conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx)) } - if shouldAcceptRoutes(class) { + if shouldAcceptRoutes(pc) { conf.AcceptRoutes = "true" } @@ -832,51 +882,26 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32 conf.StaticEndpoints = staticEndpoints } - deviceAuthed := false - for _, d := range pg.Status.Devices { - if d.Hostname == *conf.Hostname { - deviceAuthed = true - break - } - } - - if authKey != "" { - conf.AuthKey = &authKey - } else if !deviceAuthed { - key, err := authKeyFromSecret(oldSecret) - if err != nil { - return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err) - } - conf.AuthKey = key - } - capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) - - // AdvertiseServices config is set by ingress-pg-reconciler, so make sure we - // don't overwrite it here. - if err := copyAdvertiseServicesConfig(conf, oldSecret, 106); err != nil { - return nil, err - } - capVerConfigs[106] = *conf - return capVerConfigs, nil + return map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha{ + pgMinCapabilityVersion: *conf, + }, nil } -func copyAdvertiseServicesConfig(conf *ipn.ConfigVAlpha, oldSecret *corev1.Secret, capVer tailcfg.CapabilityVersion) error { - if oldSecret == nil { - return nil +func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error) { + if cfgSecret == nil { + return nil, nil } - oldConfB := oldSecret.Data[tsoperator.TailscaledConfigFileName(capVer)] - if len(oldConfB) == 0 { - return nil + conf, err := latestConfigFromSecret(cfgSecret) + if err != nil { + return nil, err } - var oldConf ipn.ConfigVAlpha - if err := json.Unmarshal(oldConfB, &oldConf); err != nil { - return fmt.Errorf("error unmarshalling existing config: %w", err) + if conf == nil { + return nil, nil } - conf.AdvertiseServices = oldConf.AdvertiseServices - return nil + return conf.AdvertiseServices, nil } func (r *ProxyGroupReconciler) validate(_ *tsapi.ProxyGroup) error { @@ -914,7 +939,7 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr dnsName: prefs.Config.UserProfile.LoginName, } pod := &corev1.Pod{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: secret.Name}, pod); err != nil && !apierrors.IsNotFound(err) { + if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: fmt.Sprintf("%s-%d", pg.Name, ordinal)}, pod); err != nil && !apierrors.IsNotFound(err) { return nil, err } else if err == nil { nm.podUID = string(pod.UID) @@ -932,17 +957,23 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, staticEndpoint } for _, m := range metadata { - device, ok, err := getDeviceInfo(ctx, r.tsClient, m.stateSecret) - if err != nil { - return nil, err - } - if !ok { + if !strings.EqualFold(string(m.stateSecret.Data[kubetypes.KeyPodUID]), m.podUID) { + // Current Pod has not yet written its UID to the state Secret, data may + // be stale. continue } - dev := tsapi.TailnetDevice{ - Hostname: device.Hostname, - TailnetIPs: device.TailnetIPs, + device := tsapi.TailnetDevice{} + if ipsB := m.stateSecret.Data[kubetypes.KeyDeviceIPs]; len(ipsB) > 0 { + ips := []string{} + if err := json.Unmarshal(ipsB, &ips); err != nil { + return nil, fmt.Errorf("failed to extract device IPs from state Secret %q: %w", m.stateSecret.Name, err) + } + device.TailnetIPs = ips + } + + if hostname, _, ok := strings.Cut(string(m.stateSecret.Data[kubetypes.KeyDeviceFQDN]), "."); ok { + device.Hostname = hostname } if ep, ok := staticEndpoints[device.Hostname]; ok && len(ep) > 0 { @@ -950,10 +981,10 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, staticEndpoint for _, e := range ep { eps = append(eps, e.String()) } - dev.StaticEndpoints = eps + device.StaticEndpoints = eps } - devices = append(devices, dev) + devices = append(devices, device) } return devices, nil diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 20e797f0c07cd..50d9c2d5fd8f9 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -351,7 +351,7 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S for i := range pgReplicas(pg) { secrets = append(secrets, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%d", pg.Name, i), + Name: pgStateSecretName(pg.Name, i), Namespace: namespace, Labels: pgSecretLabels(pg.Name, "state"), OwnerReferences: pgOwnerReference(pg), @@ -422,6 +422,10 @@ func pgConfigSecretName(pgName string, i int32) string { return fmt.Sprintf("%s-%d-config", pgName, i) } +func pgStateSecretName(pgName string, i int32) string { + return fmt.Sprintf("%s-%d", pgName, i) +} + func pgEgressCMName(pg string) string { return fmt.Sprintf("%s-egress-config", pg) } diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 8ffce2c0c68ac..87b04a434c102 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -877,6 +877,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) if expected := 1; reconciler.egressProxyGroups.Len() != expected { @@ -913,6 +914,7 @@ func TestProxyGroup(t *testing.T) { }, } tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) }) @@ -924,12 +926,14 @@ func TestProxyGroup(t *testing.T) { }) expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) addNodeIDToStateSecrets(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar()) pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{ Hostname: "hostname-nodeid-2", TailnetIPs: []string{"1.2.3.4", "::1"}, @@ -947,6 +951,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) }) @@ -1224,7 +1229,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { Namespace: tsNamespace, }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): existingConfigBytes, + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): existingConfigBytes, }, }) @@ -1261,7 +1266,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { ResourceVersion: "2", }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): expectedConfigBytes, + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): expectedConfigBytes, }, }) } @@ -1421,8 +1426,13 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) { s.Data = map[string][]byte{ - currentProfileKey: []byte(key), - key: bytes, + currentProfileKey: []byte(key), + key: bytes, + kubetypes.KeyDeviceIPs: []byte(`["1.2.3.4", "::1"]`), + kubetypes.KeyDeviceFQDN: []byte(fmt.Sprintf("hostname-nodeid-%d.tails-scales.ts.net", i)), + // TODO(tomhjp): We have two different mechanisms to retrieve device IDs. + // Consolidate on this one. + kubetypes.KeyDeviceID: []byte(fmt.Sprintf("nodeid-%d", i)), } }) } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 4c7c3ac6741a2..3e3d2d5903a7b 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -897,14 +897,6 @@ func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { } } -func readAuthKey(secret *corev1.Secret, key string) (*string, error) { - origConf := &ipn.ConfigVAlpha{} - if err := json.Unmarshal([]byte(secret.Data[key]), origConf); err != nil { - return nil, fmt.Errorf("error unmarshaling previous tailscaled config in %q: %w", key, err) - } - return origConf.AuthKey, nil -} - // tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy // state and auth key and returns tailscaled config files for currently supported proxy versions. func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { @@ -951,7 +943,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co return capVerConfigs, nil } -func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { +// latestConfigFromSecret returns the ipn.ConfigVAlpha with the highest capver +// as found in the Secret's key names, e.g. "cap-107.hujson" has capver 107. +// If no config is found, it returns nil. +func latestConfigFromSecret(s *corev1.Secret) (*ipn.ConfigVAlpha, error) { latest := tailcfg.CapabilityVersion(-1) latestStr := "" for k, data := range s.Data { @@ -968,12 +963,31 @@ func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { latest = v } } + + var conf *ipn.ConfigVAlpha + if latestStr != "" { + conf = &ipn.ConfigVAlpha{} + if err := json.Unmarshal([]byte(s.Data[latestStr]), conf); err != nil { + return nil, fmt.Errorf("error unmarshaling tailscaled config from Secret %q in field %q: %w", s.Name, latestStr, err) + } + } + + return conf, nil +} + +func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { + conf, err := latestConfigFromSecret(s) + if err != nil { + return nil, err + } + // Allow for configs that don't contain an auth key. Perhaps // users have some mechanisms to delete them. Auth key is // normally not needed after the initial login. - if latestStr != "" { - return readAuthKey(s, latestStr) + if conf != nil { + key = conf.AuthKey } + return key, nil } diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index c9b5b8ae69a18..9846513c78d74 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -164,7 +164,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin } return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) } - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { logger.Infof("ProxyGroup is not (yet) ready") return false, nil } diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go index 5772cd5d64e7f..e08bfd80d318b 100644 --- a/cmd/k8s-operator/svc-for-pg_test.go +++ b/cmd/k8s-operator/svc-for-pg_test.go @@ -142,7 +142,7 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien Labels: pgSecretLabels("test-pg", "config"), }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): []byte(`{"Version":""}`), + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(`{"Version":""}`), }, } @@ -179,7 +179,7 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien // Set ProxyGroup status to ready pg.Status.Conditions = []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, diff --git a/cmd/k8s-operator/tsrecorder.go b/cmd/k8s-operator/tsrecorder.go index 081543cd384db..cbabc1d89e475 100644 --- a/cmd/k8s-operator/tsrecorder.go +++ b/cmd/k8s-operator/tsrecorder.go @@ -446,18 +446,15 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) return tsapi.RecorderTailnetDevice{}, false, err } - return getDeviceInfo(ctx, r.tsClient, secret) -} - -func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret) (d tsapi.RecorderTailnetDevice, ok bool, err error) { prefs, ok, err := getDevicePrefs(secret) if !ok || err != nil { return tsapi.RecorderTailnetDevice{}, false, err } // TODO(tomhjp): The profile info doesn't include addresses, which is why we - // need the API. Should we instead update the profile to include addresses? - device, err := tsClient.Device(ctx, string(prefs.Config.NodeID), nil) + // need the API. Should maybe update tsrecorder to write IPs to the state + // Secret like containerboot does. + device, err := r.tsClient.Device(ctx, string(prefs.Config.NodeID), nil) if err != nil { return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err) } diff --git a/k8s-operator/apis/v1alpha1/types_connector.go b/k8s-operator/apis/v1alpha1/types_connector.go index b8b7a935e3a3f..88fd07346cd5b 100644 --- a/k8s-operator/apis/v1alpha1/types_connector.go +++ b/k8s-operator/apis/v1alpha1/types_connector.go @@ -205,11 +205,12 @@ type ConnectorStatus struct { type ConditionType string const ( - ConnectorReady ConditionType = `ConnectorReady` - ProxyClassReady ConditionType = `ProxyClassReady` - ProxyGroupReady ConditionType = `ProxyGroupReady` - ProxyReady ConditionType = `TailscaleProxyReady` // a Tailscale-specific condition type for corev1.Service - RecorderReady ConditionType = `RecorderReady` + ConnectorReady ConditionType = `ConnectorReady` + ProxyClassReady ConditionType = `ProxyClassReady` + ProxyGroupReady ConditionType = `ProxyGroupReady` // All proxy Pods running. + ProxyGroupAvailable ConditionType = `ProxyGroupAvailable` // At least one proxy Pod running. + ProxyReady ConditionType = `TailscaleProxyReady` // a Tailscale-specific condition type for corev1.Service + RecorderReady ConditionType = `RecorderReady` // EgressSvcValid gets set on a user configured ExternalName Service that defines a tailnet target to be exposed // on a ProxyGroup. // Set to true if the user provided configuration is valid. diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go index abe8f7f9cc6fa..1d30f352c3603 100644 --- a/k8s-operator/conditions.go +++ b/k8s-operator/conditions.go @@ -137,8 +137,16 @@ func ProxyClassIsReady(pc *tsapi.ProxyClass) bool { } func ProxyGroupIsReady(pg *tsapi.ProxyGroup) bool { + return proxyGroupCondition(pg, tsapi.ProxyGroupReady) +} + +func ProxyGroupAvailable(pg *tsapi.ProxyGroup) bool { + return proxyGroupCondition(pg, tsapi.ProxyGroupAvailable) +} + +func proxyGroupCondition(pg *tsapi.ProxyGroup, condType tsapi.ConditionType) bool { idx := xslices.IndexFunc(pg.Status.Conditions, func(cond metav1.Condition) bool { - return cond.Type == string(tsapi.ProxyGroupReady) + return cond.Type == string(condType) }) if idx == -1 { return false diff --git a/kube/kubeclient/fake_client.go b/kube/kubeclient/fake_client.go index c21dc2bf89e61..15ebb5f443f2a 100644 --- a/kube/kubeclient/fake_client.go +++ b/kube/kubeclient/fake_client.go @@ -13,12 +13,13 @@ import ( var _ Client = &FakeClient{} type FakeClient struct { - GetSecretImpl func(context.Context, string) (*kubeapi.Secret, error) - CheckSecretPermissionsImpl func(ctx context.Context, name string) (bool, bool, error) - CreateSecretImpl func(context.Context, *kubeapi.Secret) error - UpdateSecretImpl func(context.Context, *kubeapi.Secret) error - JSONPatchResourceImpl func(context.Context, string, string, []JSONPatch) error - ListSecretsImpl func(context.Context, map[string]string) (*kubeapi.SecretList, error) + GetSecretImpl func(context.Context, string) (*kubeapi.Secret, error) + CheckSecretPermissionsImpl func(ctx context.Context, name string) (bool, bool, error) + CreateSecretImpl func(context.Context, *kubeapi.Secret) error + UpdateSecretImpl func(context.Context, *kubeapi.Secret) error + JSONPatchResourceImpl func(context.Context, string, string, []JSONPatch) error + ListSecretsImpl func(context.Context, map[string]string) (*kubeapi.SecretList, error) + StrategicMergePatchSecretImpl func(context.Context, string, *kubeapi.Secret, string) error } func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (bool, bool, error) { @@ -30,8 +31,8 @@ func (fc *FakeClient) GetSecret(ctx context.Context, name string) (*kubeapi.Secr func (fc *FakeClient) SetURL(_ string) {} func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) { } -func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error { - return nil +func (fc *FakeClient) StrategicMergePatchSecret(ctx context.Context, name string, s *kubeapi.Secret, fieldManager string) error { + return fc.StrategicMergePatchSecretImpl(ctx, name, s, fieldManager) } func (fc *FakeClient) Event(context.Context, string, string, string) error { return nil From 0a64e86a0df89db063211a826ba4f62eb5ec959f Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Fri, 27 Jun 2025 13:56:55 -0700 Subject: [PATCH 129/263] wgengine/magicsock: move UDP relay path discovery to heartbeat() (#16407) This was previously hooked around direct UDP path discovery / CallMeMaybe transmission, and related conditions. Now it is subject to relay-specific considerations. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 80 +++++++++++++++++++++++------ wgengine/magicsock/endpoint_test.go | 42 +++++++++++++++ wgengine/magicsock/magicsock.go | 14 +++-- 3 files changed, 118 insertions(+), 18 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index fb5a28c2832fd..29ae025f4d159 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -75,11 +75,12 @@ type endpoint struct { // mu protects all following fields. mu sync.Mutex // Lock ordering: Conn.mu, then endpoint.mu - heartBeatTimer *time.Timer // nil when idle - lastSendExt mono.Time // last time there were outgoing packets sent to this peer from an external trigger (e.g. wireguard-go or disco pingCLI) - lastSendAny mono.Time // last time there were outgoing packets sent this peer from any trigger, internal or external to magicsock - lastFullPing mono.Time // last time we pinged all disco or wireguard only endpoints - derpAddr netip.AddrPort // fallback/bootstrap path, if non-zero (non-zero for well-behaved clients) + heartBeatTimer *time.Timer // nil when idle + lastSendExt mono.Time // last time there were outgoing packets sent to this peer from an external trigger (e.g. wireguard-go or disco pingCLI) + lastSendAny mono.Time // last time there were outgoing packets sent this peer from any trigger, internal or external to magicsock + lastFullPing mono.Time // last time we pinged all disco or wireguard only endpoints + lastUDPRelayPathDiscovery mono.Time // last time we ran UDP relay path discovery + derpAddr netip.AddrPort // fallback/bootstrap path, if non-zero (non-zero for well-behaved clients) bestAddr addrQuality // best non-DERP path; zero if none; mutate via setBestAddrLocked() bestAddrAt mono.Time // time best address re-confirmed @@ -851,6 +852,10 @@ func (de *endpoint) heartbeat() { de.sendDiscoPingsLocked(now, true) } + if de.wantUDPRelayPathDiscoveryLocked(now) { + de.discoverUDPRelayPathsLocked(now) + } + de.heartBeatTimer = time.AfterFunc(heartbeatInterval, de.heartbeat) } @@ -861,6 +866,45 @@ func (de *endpoint) setHeartbeatDisabled(v bool) { de.heartbeatDisabled = v } +// discoverUDPRelayPathsLocked starts UDP relay path discovery. +func (de *endpoint) discoverUDPRelayPathsLocked(now mono.Time) { + // TODO(jwhited): return early if there are no relay servers set, otherwise + // we spin up and down relayManager.runLoop unnecessarily. + de.lastUDPRelayPathDiscovery = now + de.c.relayManager.allocateAndHandshakeAllServers(de) +} + +// wantUDPRelayPathDiscoveryLocked reports whether we should kick off UDP relay +// path discovery. +func (de *endpoint) wantUDPRelayPathDiscoveryLocked(now mono.Time) bool { + if runtime.GOOS == "js" { + return false + } + if !de.relayCapable { + return false + } + if de.bestAddr.isDirect() && now.Before(de.trustBestAddrUntil) { + return false + } + if !de.lastUDPRelayPathDiscovery.IsZero() && now.Sub(de.lastUDPRelayPathDiscovery) < discoverUDPRelayPathsInterval { + return false + } + // TODO(jwhited): consider applying 'goodEnoughLatency' suppression here, + // but not until we have a strategy for triggering CallMeMaybeVia regularly + // and/or enabling inbound packets to act as a UDP relay path discovery + // trigger, otherwise clients without relay servers may fall off a UDP + // relay path and never come back. They are dependent on the remote side + // regularly TX'ing CallMeMaybeVia, which currently only happens as part + // of full UDP relay path discovery. + if now.After(de.trustBestAddrUntil) { + return true + } + if !de.lastUDPRelayPathDiscovery.IsZero() && now.Sub(de.lastUDPRelayPathDiscovery) >= upgradeUDPRelayInterval { + return true + } + return false +} + // wantFullPingLocked reports whether we should ping to all our peers looking for // a better path. // @@ -869,7 +913,7 @@ func (de *endpoint) wantFullPingLocked(now mono.Time) bool { if runtime.GOOS == "js" { return false } - if !de.bestAddr.ap.IsValid() || de.lastFullPing.IsZero() { + if !de.bestAddr.isDirect() || de.lastFullPing.IsZero() { return true } if now.After(de.trustBestAddrUntil) { @@ -878,7 +922,7 @@ func (de *endpoint) wantFullPingLocked(now mono.Time) bool { if de.bestAddr.latency <= goodEnoughLatency { return false } - if now.Sub(de.lastFullPing) >= upgradeInterval { + if now.Sub(de.lastFullPing) >= upgradeUDPDirectInterval { return true } return false @@ -964,9 +1008,16 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { if startWGPing { de.sendWireGuardOnlyPingsLocked(now) } - } else if !udpAddr.ap.IsValid() || now.After(de.trustBestAddrUntil) { + } else if !udpAddr.isDirect() || now.After(de.trustBestAddrUntil) { de.sendDiscoPingsLocked(now, true) } + // TODO(jwhited): consider triggering UDP relay path discovery here under + // certain conditions. We currently only trigger it in heartbeat(), which + // is both good and bad. It's good because the first heartbeat() tick is 3s + // after the first packet, which gives us time to discover a UDP direct + // path and potentially avoid what would be wasted UDP relay path discovery + // work. It's bad because we might not discover a UDP direct path, and we + // incur a 3s delay before we try to discover a UDP relay path. de.noteTxActivityExtTriggerLocked(now) de.lastSendAny = now de.mu.Unlock() @@ -1275,13 +1326,6 @@ func (de *endpoint) sendDiscoPingsLocked(now mono.Time, sendCallMeMaybe bool) { // sent so our firewall ports are probably open and now // would be a good time for them to connect. go de.c.enqueueCallMeMaybe(derpAddr, de) - - // Schedule allocation of relay endpoints. We make no considerations for - // current relay endpoints or best UDP path state for now, keep it - // simple. - if de.relayCapable { - go de.c.relayManager.allocateAndHandshakeAllServers(de) - } } } @@ -1703,6 +1747,12 @@ type epAddr struct { vni virtualNetworkID // vni.isSet() indicates if this [epAddr] involves a Geneve header } +// isDirect returns true if e.ap is valid and not tailcfg.DerpMagicIPAddr, +// and a VNI is not set. +func (e epAddr) isDirect() bool { + return e.ap.IsValid() && e.ap.Addr() != tailcfg.DerpMagicIPAddr && !e.vni.isSet() +} + func (e epAddr) String() string { if !e.vni.isSet() { return e.ap.String() diff --git a/wgengine/magicsock/endpoint_test.go b/wgengine/magicsock/endpoint_test.go index b1e8cab91bcd1..3a1e55b8b9728 100644 --- a/wgengine/magicsock/endpoint_test.go +++ b/wgengine/magicsock/endpoint_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "tailscale.com/tailcfg" "tailscale.com/types/key" ) @@ -323,3 +324,44 @@ func Test_endpoint_maybeProbeUDPLifetimeLocked(t *testing.T) { }) } } + +func Test_epAddr_isDirectUDP(t *testing.T) { + vni := virtualNetworkID{} + vni.set(7) + tests := []struct { + name string + ap netip.AddrPort + vni virtualNetworkID + want bool + }{ + { + name: "true", + ap: netip.MustParseAddrPort("192.0.2.1:7"), + vni: virtualNetworkID{}, + want: true, + }, + { + name: "false derp magic addr", + ap: netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, 0), + vni: virtualNetworkID{}, + want: false, + }, + { + name: "false vni set", + ap: netip.MustParseAddrPort("192.0.2.1:7"), + vni: vni, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := epAddr{ + ap: tt.ap, + vni: tt.vni, + } + if got := e.isDirect(); got != tt.want { + t.Errorf("isDirect() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index e76d0054f04c2..553543b0f496d 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3494,9 +3494,17 @@ const ( // keep NAT mappings alive. sessionActiveTimeout = 45 * time.Second - // upgradeInterval is how often we try to upgrade to a better path - // even if we have some non-DERP route that works. - upgradeInterval = 1 * time.Minute + // upgradeUDPDirectInterval is how often we try to upgrade to a better, + // direct UDP path even if we have some direct UDP path that works. + upgradeUDPDirectInterval = 1 * time.Minute + + // upgradeUDPRelayInterval is how often we try to discover UDP relay paths + // even if we have a UDP relay path that works. + upgradeUDPRelayInterval = 1 * time.Minute + + // discoverUDPRelayPathsInterval is the minimum time between UDP relay path + // discovery. + discoverUDPRelayPathsInterval = 30 * time.Second // heartbeatInterval is how often pings to the best UDP address // are sent. From 76b9afb54d7ddf662a6d5e47ab42021a2e6dba36 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Fri, 27 Jun 2025 15:14:18 -0700 Subject: [PATCH 130/263] ipn/store: make StateStore.All optional (#16409) This method is only needed to migrate between store.FileStore and tpm.tpmStore. We can make a runtime type assertion instead of implementing an unused method for every platform. Updates #15830 Signed-off-by: Andrew Lytvynov --- cmd/tsconnect/src/lib/js-state-store.ts | 3 --- cmd/tsconnect/src/types/wasm_js.d.ts | 1 - cmd/tsconnect/wasm/wasm_js.go | 24 -------------------- feature/tpm/tpm.go | 4 ++++ feature/tpm/tpm_test.go | 29 +++++++++++++++---------- ipn/store.go | 6 ----- ipn/store/awsstore/store_aws.go | 5 ----- ipn/store/kubestore/store_kube.go | 5 ----- ipn/store/mem/store_mem.go | 14 ------------ ipn/store/stores.go | 24 +++++++++++++++++++- 10 files changed, 45 insertions(+), 70 deletions(-) diff --git a/cmd/tsconnect/src/lib/js-state-store.ts b/cmd/tsconnect/src/lib/js-state-store.ts index 7f2fc8087e768..e57dfd98efabd 100644 --- a/cmd/tsconnect/src/lib/js-state-store.ts +++ b/cmd/tsconnect/src/lib/js-state-store.ts @@ -10,7 +10,4 @@ export const sessionStateStorage: IPNStateStorage = { getState(id) { return window.sessionStorage[`ipn-state-${id}`] || "" }, - all() { - return JSON.stringify(window.sessionStorage) - }, } diff --git a/cmd/tsconnect/src/types/wasm_js.d.ts b/cmd/tsconnect/src/types/wasm_js.d.ts index f47a972b03fba..492197ccb1a9b 100644 --- a/cmd/tsconnect/src/types/wasm_js.d.ts +++ b/cmd/tsconnect/src/types/wasm_js.d.ts @@ -44,7 +44,6 @@ declare global { interface IPNStateStorage { setState(id: string, value: string): void getState(id: string): string - all(): string } type IPNConfig = { diff --git a/cmd/tsconnect/wasm/wasm_js.go b/cmd/tsconnect/wasm/wasm_js.go index c5ff56120f492..779a87e49dec9 100644 --- a/cmd/tsconnect/wasm/wasm_js.go +++ b/cmd/tsconnect/wasm/wasm_js.go @@ -15,7 +15,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "iter" "log" "math/rand/v2" "net" @@ -580,29 +579,6 @@ func (s *jsStateStore) WriteState(id ipn.StateKey, bs []byte) error { return nil } -func (s *jsStateStore) All() iter.Seq2[ipn.StateKey, []byte] { - return func(yield func(ipn.StateKey, []byte) bool) { - jsValue := s.jsStateStorage.Call("all") - if jsValue.String() == "" { - return - } - buf, err := hex.DecodeString(jsValue.String()) - if err != nil { - return - } - var state map[string][]byte - if err := json.Unmarshal(buf, &state); err != nil { - return - } - - for k, v := range state { - if !yield(ipn.StateKey(k), v) { - break - } - } - } -} - func mapSlice[T any, M any](a []T, f func(T) M) []M { n := make([]M, len(a)) for i, e := range a { diff --git a/feature/tpm/tpm.go b/feature/tpm/tpm.go index 64656d412a2e7..5ec084effa27d 100644 --- a/feature/tpm/tpm.go +++ b/feature/tpm/tpm.go @@ -217,6 +217,10 @@ func (s *tpmStore) All() iter.Seq2[ipn.StateKey, []byte] { } } +// Ensure tpmStore implements store.ExportableStore for migration to/from +// store.FileStore. +var _ store.ExportableStore = (*tpmStore)(nil) + // The nested levels of encoding and encryption are confusing, so here's what's // going on in plain English. // diff --git a/feature/tpm/tpm_test.go b/feature/tpm/tpm_test.go index b08681354a1e4..f4497f8c72732 100644 --- a/feature/tpm/tpm_test.go +++ b/feature/tpm/tpm_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "iter" "maps" "os" "path/filepath" @@ -20,8 +21,8 @@ import ( "github.com/google/go-cmp/cmp" "tailscale.com/ipn" "tailscale.com/ipn/store" - "tailscale.com/ipn/store/mem" "tailscale.com/types/logger" + "tailscale.com/util/mak" ) func TestPropToString(t *testing.T) { @@ -251,7 +252,9 @@ func TestMigrateStateToTPM(t *testing.T) { if err != nil { t.Fatalf("migration failed: %v", err) } - gotContent := maps.Collect(s.All()) + gotContent := maps.Collect(s.(interface { + All() iter.Seq2[ipn.StateKey, []byte] + }).All()) if diff := cmp.Diff(content, gotContent); diff != "" { t.Errorf("unexpected content after migration, diff:\n%s", diff) } @@ -285,7 +288,7 @@ func tpmSupported() bool { type mockTPMSealProvider struct { path string - mem.Store + data map[ipn.StateKey][]byte } func newMockTPMSeal(logf logger.Logf, path string) (ipn.StateStore, error) { @@ -293,7 +296,7 @@ func newMockTPMSeal(logf logger.Logf, path string) (ipn.StateStore, error) { if !ok { return nil, fmt.Errorf("%q missing tpmseal: prefix", path) } - s := &mockTPMSealProvider{path: path, Store: mem.Store{}} + s := &mockTPMSealProvider{path: path} buf, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return s, s.flushState() @@ -312,24 +315,28 @@ func newMockTPMSeal(logf logger.Logf, path string) (ipn.StateStore, error) { if data.Key == "" || data.Nonce == "" { return nil, fmt.Errorf("%q missing key or nonce", path) } - for k, v := range data.Data { - s.Store.WriteState(k, v) - } + s.data = data.Data return s, nil } +func (p *mockTPMSealProvider) ReadState(k ipn.StateKey) ([]byte, error) { + return p.data[k], nil +} + func (p *mockTPMSealProvider) WriteState(k ipn.StateKey, v []byte) error { - if err := p.Store.WriteState(k, v); err != nil { - return err - } + mak.Set(&p.data, k, v) return p.flushState() } +func (p *mockTPMSealProvider) All() iter.Seq2[ipn.StateKey, []byte] { + return maps.All(p.data) +} + func (p *mockTPMSealProvider) flushState() error { data := map[string]any{ "key": "foo", "nonce": "bar", - "data": maps.Collect(p.Store.All()), + "data": p.data, } buf, err := json.Marshal(data) if err != nil { diff --git a/ipn/store.go b/ipn/store.go index e176e48421216..550aa8cba819a 100644 --- a/ipn/store.go +++ b/ipn/store.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "iter" "net" "strconv" ) @@ -84,11 +83,6 @@ type StateStore interface { // instead, which only writes if the value is different from what's // already in the store. WriteState(id StateKey, bs []byte) error - // All returns an iterator over all StateStore keys. Using ReadState or - // WriteState is not safe while iterating and can lead to a deadlock. - // The order of keys in the iterator is not specified and may change - // between runs. - All() iter.Seq2[StateKey, []byte] } // WriteState is a wrapper around store.WriteState that only writes if diff --git a/ipn/store/awsstore/store_aws.go b/ipn/store/awsstore/store_aws.go index 523d1657b109d..40bbbf0370822 100644 --- a/ipn/store/awsstore/store_aws.go +++ b/ipn/store/awsstore/store_aws.go @@ -10,7 +10,6 @@ import ( "context" "errors" "fmt" - "iter" "net/url" "regexp" "strings" @@ -254,7 +253,3 @@ func (s *awsStore) persistState() error { _, err = s.ssmClient.PutParameter(context.TODO(), in) return err } - -func (s *awsStore) All() iter.Seq2[ipn.StateKey, []byte] { - return s.memory.All() -} diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go index f6bedbf0b8054..14025bbb4150a 100644 --- a/ipn/store/kubestore/store_kube.go +++ b/ipn/store/kubestore/store_kube.go @@ -7,7 +7,6 @@ package kubestore import ( "context" "fmt" - "iter" "log" "net" "os" @@ -429,7 +428,3 @@ func sanitizeKey[T ~string](k T) string { return '_' }, string(k)) } - -func (s *Store) All() iter.Seq2[ipn.StateKey, []byte] { - return s.memory.All() -} diff --git a/ipn/store/mem/store_mem.go b/ipn/store/mem/store_mem.go index 6c22aefd547f8..6f474ce993b43 100644 --- a/ipn/store/mem/store_mem.go +++ b/ipn/store/mem/store_mem.go @@ -7,7 +7,6 @@ package mem import ( "bytes" "encoding/json" - "iter" "sync" xmaps "golang.org/x/exp/maps" @@ -86,16 +85,3 @@ func (s *Store) ExportToJSON() ([]byte, error) { } return json.MarshalIndent(s.cache, "", " ") } - -func (s *Store) All() iter.Seq2[ipn.StateKey, []byte] { - return func(yield func(ipn.StateKey, []byte) bool) { - s.mu.Lock() - defer s.mu.Unlock() - - for k, v := range s.cache { - if !yield(k, v) { - break - } - } - } -} diff --git a/ipn/store/stores.go b/ipn/store/stores.go index 43c79639934b8..bf175da41d8aa 100644 --- a/ipn/store/stores.go +++ b/ipn/store/stores.go @@ -235,6 +235,23 @@ func (s *FileStore) All() iter.Seq2[ipn.StateKey, []byte] { } } +// Ensure FileStore implements ExportableStore for migration to/from +// tpm.tpmStore. +var _ ExportableStore = (*FileStore)(nil) + +// ExportableStore is an ipn.StateStore that can export all of its contents. +// This interface is optional to implement, and used for migrating the state +// between different store implementations. +type ExportableStore interface { + ipn.StateStore + + // All returns an iterator over all store keys. Using ReadState or + // WriteState is not safe while iterating and can lead to a deadlock. The + // order of keys in the iterator is not specified and may change between + // runs. + All() iter.Seq2[ipn.StateKey, []byte] +} + func maybeMigrateLocalStateFile(logf logger.Logf, path string) error { path, toTPM := strings.CutPrefix(path, TPMPrefix) @@ -297,10 +314,15 @@ func maybeMigrateLocalStateFile(logf logger.Logf, path string) error { } defer os.Remove(tmpPath) + fromExp, ok := from.(ExportableStore) + if !ok { + return fmt.Errorf("%T does not implement the exportableStore interface", from) + } + // Copy all the items. This is pretty inefficient, because both stores // write the file to disk for each WriteState, but that's ok for a one-time // migration. - for k, v := range from.All() { + for k, v := range fromExp.All() { if err := to.WriteState(k, v); err != nil { return err } From 544aee9d08e214c3fc2699916c2ed410b2fb79eb Mon Sep 17 00:00:00 2001 From: Simon Law Date: Fri, 27 Jun 2025 17:30:43 -0700 Subject: [PATCH 131/263] tsidp: update README to refer to community projects (#16411) We dropped the idea of the Experimental release stage in tailscale/tailscale-www#7697, in favour of Community Projects. Updates #cleanup Signed-off-by: Simon Law --- cmd/tsidp/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/tsidp/README.md b/cmd/tsidp/README.md index fce844e0b309c..780d9ab95b037 100644 --- a/cmd/tsidp/README.md +++ b/cmd/tsidp/README.md @@ -1,6 +1,6 @@ # `tsidp` - Tailscale OpenID Connect (OIDC) Identity Provider -[![status: experimental](https://img.shields.io/badge/status-experimental-blue)](https://tailscale.com/kb/1167/release-stages/#experimental) +[![status: community project](https://img.shields.io/badge/status-community_project-blue)](https://tailscale.com/kb/1531/community-projects) `tsidp` is an OIDC Identity Provider (IdP) server that integrates with your Tailscale network. It allows you to use Tailscale identities for authentication in applications that support OpenID Connect, enabling single sign-on (SSO) capabilities within your tailnet. @@ -89,7 +89,7 @@ The `tsidp` server supports several command-line flags: ## Support -This is an [experimental](https://tailscale.com/kb/1167/release-stages#experimental), work in progress feature. For issues or questions, file issues on the [GitHub repository](https://github.com/tailscale/tailscale) +This is an experimental, work in progress, [community project](https://tailscale.com/kb/1531/community-projects). For issues or questions, file issues on the [GitHub repository](https://github.com/tailscale/tailscale). ## License From 3dc694b4f1983dfcb1731cdf3f29aa6e4f058505 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Fri, 27 Jun 2025 19:11:59 -0700 Subject: [PATCH 132/263] wgengine/magicsock: clear UDP relay bestAddr's on disco ping timeout (#16410) Otherwise we can end up mirroring packets to them forever. We may eventually want to relax this to direct paths as well, but start with UDP relay paths, which have a higher chance of becoming untrusted and never working again, to be conservative. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 29ae025f4d159..9edc6403e6132 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1129,7 +1129,12 @@ func (de *endpoint) discoPingTimeout(txid stun.TxID) { if !ok { return } - if debugDisco() || !de.bestAddr.ap.IsValid() || mono.Now().After(de.trustBestAddrUntil) { + bestUntrusted := mono.Now().After(de.trustBestAddrUntil) + if sp.to == de.bestAddr.epAddr && sp.to.vni.isSet() && bestUntrusted { + // TODO(jwhited): consider applying this to direct UDP paths as well + de.clearBestAddrLocked() + } + if debugDisco() || !de.bestAddr.ap.IsValid() || bestUntrusted { de.c.dlogf("[v1] magicsock: disco: timeout waiting for pong %x from %v (%v, %v)", txid[:6], sp.to, de.publicKey.ShortString(), de.discoShort()) } de.removeSentDiscoPingLocked(txid, sp, discoPingTimedOut) From ee8c3560ef74613443859e1f78a0c9b9071bac76 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 23 Jun 2025 09:02:03 -0700 Subject: [PATCH 133/263] tailcfg: format integer IDs as decimal consistently The server-side code already does e.g. "nodeid:%d" instead of "%x" and as a result we have to second guess a lot of identifiers that could be hex or decimal. This stops the bleeding and means in a year and change we'll stop seeing the hex forms. Updates tailscale/corp#29827 Change-Id: Ie5785a07fc32631f7c949348d3453538ab170e6d Signed-off-by: Brad Fitzpatrick --- tailcfg/tailcfg.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 23f3cc49b152b..fb7d54c388619 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2265,10 +2265,10 @@ type Debug struct { Exit *int `json:",omitempty"` } -func (id ID) String() string { return fmt.Sprintf("id:%x", int64(id)) } -func (id UserID) String() string { return fmt.Sprintf("userid:%x", int64(id)) } -func (id LoginID) String() string { return fmt.Sprintf("loginid:%x", int64(id)) } -func (id NodeID) String() string { return fmt.Sprintf("nodeid:%x", int64(id)) } +func (id ID) String() string { return fmt.Sprintf("id:%d", int64(id)) } +func (id UserID) String() string { return fmt.Sprintf("userid:%d", int64(id)) } +func (id LoginID) String() string { return fmt.Sprintf("loginid:%d", int64(id)) } +func (id NodeID) String() string { return fmt.Sprintf("nodeid:%d", int64(id)) } // Equal reports whether n and n2 are equal. func (n *Node) Equal(n2 *Node) bool { From f85e4bcb3287a0adef9567ff79ba58d9cec4e1d2 Mon Sep 17 00:00:00 2001 From: Will Norris Date: Fri, 27 Jun 2025 13:11:59 -0400 Subject: [PATCH 134/263] client/systray: replace counter metric with gauge Replace the existing systray_start counter metrics with a systray_running gauge metrics. This also adds an IncrementGauge method to local client to parallel IncrementCounter. The LocalAPI handler supports both, we've just never added a client method for gauges. Updates #1708 Change-Id: Ia101a4a3005adb9118051b3416f5a64a4a45987d Signed-off-by: Will Norris --- client/local/local.go | 17 +++++++++++++++++ client/systray/systray.go | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/client/local/local.go b/client/local/local.go index 12bf2f7d6fef3..74c4f0b6f8a2c 100644 --- a/client/local/local.go +++ b/client/local/local.go @@ -398,6 +398,23 @@ func (lc *Client) IncrementCounter(ctx context.Context, name string, delta int) return err } +// IncrementGauge increments the value of a Tailscale daemon's gauge +// metric by the given delta. If the metric has yet to exist, a new gauge +// metric is created and initialized to delta. The delta value can be negative. +func (lc *Client) IncrementGauge(ctx context.Context, name string, delta int) error { + type metricUpdate struct { + Name string `json:"name"` + Type string `json:"type"` + Value int `json:"value"` // amount to increment by + } + _, err := lc.send(ctx, "POST", "/localapi/v0/upload-client-metrics", 200, jsonBody([]metricUpdate{{ + Name: name, + Type: "gauge", + Value: delta, + }})) + return err +} + // TailDaemonLogs returns a stream the Tailscale daemon's logs as they arrive. // Close the context to stop the stream. func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) { diff --git a/client/systray/systray.go b/client/systray/systray.go index 195a157fb1386..a87783c06ce5a 100644 --- a/client/systray/systray.go +++ b/client/systray/systray.go @@ -61,7 +61,8 @@ func (menu *Menu) Run() { case <-menu.bgCtx.Done(): } }() - go menu.lc.IncrementCounter(menu.bgCtx, "systray_start", 1) + go menu.lc.IncrementGauge(menu.bgCtx, "systray_running", 1) + defer menu.lc.IncrementGauge(menu.bgCtx, "systray_running", -1) systray.Run(menu.onReady, menu.onExit) } From 2fc247573bfaa7e1dac695e98b5a31c3a2f5217e Mon Sep 17 00:00:00 2001 From: Tom Meadows Date: Mon, 30 Jun 2025 12:08:35 +0100 Subject: [PATCH 135/263] cmd/k8s-operator: ProxyClass annotation for Services and Ingresses (#16363) * cmd/k8s-operator: ProxyClass annotation for Services and Ingresses Previously, the ProxyClass could only be configured for Services and Ingresses via a Label. This adds the ability to set it via an Annotation, but prioritizes the Label if both a Label and Annotation are set. Updates #14323 Signed-off-by: chaosinthecrd * Update cmd/k8s-operator/operator.go Co-authored-by: Tom Proctor Signed-off-by: Tom Meadows * Update cmd/k8s-operator/operator.go Signed-off-by: Tom Meadows * cmd/k8s-operator: ProxyClass annotation for Services and Ingresses Previously, the ProxyClass could only be configured for Services and Ingresses via a Label. This adds the ability to set it via an Annotation, but prioritizes the Label if both a Label and Annotation are set. Updates #14323 Signed-off-by: chaosinthecrd --------- Signed-off-by: chaosinthecrd Signed-off-by: Tom Meadows Co-authored-by: Tom Proctor --- cmd/k8s-operator/ingress.go | 1 + cmd/k8s-operator/ingress_test.go | 124 ++++++++++++++++-- cmd/k8s-operator/operator.go | 69 +++++++++- cmd/k8s-operator/operator_test.go | 202 ++++++++++++++++++++++++++++-- cmd/k8s-operator/sts.go | 18 ++- cmd/k8s-operator/svc.go | 12 +- 6 files changed, 398 insertions(+), 28 deletions(-) diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 6c50e10b2ba94..5058fd6dda8fc 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -34,6 +34,7 @@ const ( tailscaleIngressClassName = "tailscale" // ingressClass.metadata.name for tailscale IngressClass resource tailscaleIngressControllerName = "tailscale.com/ts-ingress" // ingressClass.spec.controllerName for tailscale IngressClass resource ingressClassDefaultAnnotation = "ingressclass.kubernetes.io/is-default-class" // we do not support this https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class + indexIngressProxyClass = ".metadata.annotations.ingress-proxy-class" ) type IngressReconciler struct { diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index aacf27d8e6600..e4396eb106a96 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -230,7 +230,8 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{ Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"bar.io/foo": "some-val"}, - Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}, + }}, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -285,7 +286,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { // 2. Ingress is updated to specify a ProxyClass, ProxyClass is not yet // ready, so proxy resource configuration does not change. mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { - mak.Set(&ing.ObjectMeta.Labels, LabelProxyClass, "custom-metadata") + mak.Set(&ing.ObjectMeta.Labels, LabelAnnotationProxyClass, "custom-metadata") }) expectReconciled(t, ingR, "default", "test") expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) @@ -299,7 +300,8 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: pc.Generation, - }}} + }}, + } }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = pc.Name @@ -309,7 +311,7 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { // Ingress gets reconciled and the custom ProxyClass configuration is // removed from the proxy resources. mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { - delete(ing.ObjectMeta.Labels, LabelProxyClass) + delete(ing.ObjectMeta.Labels, LabelAnnotationProxyClass) }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = "" @@ -325,14 +327,15 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: 1, - }}}, + }}, + }, } crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}} // Create fake client with ProxyClass, IngressClass, Ingress with metrics ProxyClass, and Service ing := ingress() ing.Labels = map[string]string{ - LabelProxyClass: "metrics", + LabelAnnotationProxyClass: "metrics", } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -421,6 +424,113 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { // ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here. } +func TestIngressProxyClassAnnotation(t *testing.T) { + cl := tstest.NewClock(tstest.ClockOpts{}) + zl := zap.Must(zap.NewDevelopment()) + + pcLEStaging, pcLEStagingFalse, _ := proxyClassesForLEStagingTest() + + testCases := []struct { + name string + proxyClassAnnotation string + proxyClassLabel string + proxyClassDefault string + expectedProxyClass string + expectEvents []string + }{ + { + name: "via_label", + proxyClassLabel: pcLEStaging.Name, + expectedProxyClass: pcLEStaging.Name, + }, + { + name: "via_annotation", + proxyClassAnnotation: pcLEStaging.Name, + expectedProxyClass: pcLEStaging.Name, + }, + { + name: "via_default", + proxyClassDefault: pcLEStaging.Name, + expectedProxyClass: pcLEStaging.Name, + }, + { + name: "via_label_override_annotation", + proxyClassLabel: pcLEStaging.Name, + proxyClassAnnotation: pcLEStagingFalse.Name, + expectedProxyClass: pcLEStaging.Name, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + builder := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme) + + builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse). + WithStatusSubresource(pcLEStaging, pcLEStagingFalse) + + fc := builder.Build() + + if tt.proxyClassAnnotation != "" || tt.proxyClassLabel != "" || tt.proxyClassDefault != "" { + name := tt.proxyClassDefault + if name == "" { + name = tt.proxyClassLabel + if name == "" { + name = tt.proxyClassAnnotation + } + } + setProxyClassReady(t, fc, cl, name) + } + + mustCreate(t, fc, ingressClass()) + mustCreate(t, fc, service()) + ing := ingress() + if tt.proxyClassLabel != "" { + ing.Labels = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassLabel, + } + } + if tt.proxyClassAnnotation != "" { + ing.Annotations = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassAnnotation, + } + } + mustCreate(t, fc, ing) + + ingR := &IngressReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: &fakeTSClient{}, + tsnetServer: &fakeTSNetServer{certDomains: []string{"test-host"}}, + defaultTags: []string{"tag:test"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale:test", + }, + logger: zl.Sugar(), + defaultProxyClass: tt.proxyClassDefault, + } + + expectReconciled(t, ingR, "default", "test") + + _, shortName := findGenName(t, fc, "default", "test", "ingress") + sts := &appsv1.StatefulSet{} + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + switch tt.expectedProxyClass { + case pcLEStaging.Name: + verifyEnvVar(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint) + case pcLEStagingFalse.Name: + verifyEnvVarNotPresent(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL") + default: + t.Fatalf("unexpected expected ProxyClass %q", tt.expectedProxyClass) + } + }) + } +} + func TestIngressLetsEncryptStaging(t *testing.T) { cl := tstest.NewClock(tstest.ClockOpts{}) zl := zap.Must(zap.NewDevelopment()) @@ -452,7 +562,7 @@ func TestIngressLetsEncryptStaging(t *testing.T) { ing := ingress() if tt.proxyClassPerResource != "" { ing.Labels = map[string]string{ - LabelProxyClass: tt.proxyClassPerResource, + LabelAnnotationProxyClass: tt.proxyClassPerResource, } } mustCreate(t, fc, ing) diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index cd1ae8158d01c..b33dcd114d32b 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -54,6 +54,7 @@ import ( "tailscale.com/tsnet" "tailscale.com/tstime" "tailscale.com/types/logger" + "tailscale.com/util/set" "tailscale.com/version" ) @@ -307,6 +308,7 @@ func runReconcilers(opts reconcilerOpts) { proxyPriorityClassName: opts.proxyPriorityClassName, tsFirewallMode: opts.proxyFirewallMode, } + err = builder. ControllerManagedBy(mgr). Named("service-reconciler"). @@ -327,6 +329,10 @@ func runReconcilers(opts reconcilerOpts) { if err != nil { startlog.Fatalf("could not create service reconciler: %v", err) } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexServiceProxyClass, indexProxyClass); err != nil { + startlog.Fatalf("failed setting up ProxyClass indexer for Services: %v", err) + } + ingressChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("ingress")) // If a ProxyClassChanges, enqueue all Ingresses labeled with that // ProxyClass's name. @@ -351,6 +357,10 @@ func runReconcilers(opts reconcilerOpts) { if err != nil { startlog.Fatalf("could not create ingress reconciler: %v", err) } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(networkingv1.Ingress), indexIngressProxyClass, indexProxyClass); err != nil { + startlog.Fatalf("failed setting up ProxyClass indexer for Ingresses: %v", err) + } + lc, err := opts.tsServer.LocalClient() if err != nil { startlog.Fatalf("could not get local client: %v", err) @@ -797,6 +807,16 @@ func managedResourceHandlerForType(typ string) handler.MapFunc { } } +// indexProxyClass is used to select ProxyClass-backed objects which are +// locally indexed in the cache for efficient listing without requiring labels. +func indexProxyClass(o client.Object) []string { + if !hasProxyClassAnnotation(o) { + return nil + } + + return []string{o.GetAnnotations()[LabelAnnotationProxyClass]} +} + // proxyClassHandlerForSvc returns a handler that, for a given ProxyClass, // returns a list of reconcile requests for all Services labeled with // tailscale.com/proxy-class: . @@ -804,16 +824,37 @@ func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger) handle return func(ctx context.Context, o client.Object) []reconcile.Request { svcList := new(corev1.ServiceList) labels := map[string]string{ - LabelProxyClass: o.GetName(), + LabelAnnotationProxyClass: o.GetName(), } + if err := cl.List(ctx, svcList, client.MatchingLabels(labels)); err != nil { logger.Debugf("error listing Services for ProxyClass: %v", err) return nil } + reqs := make([]reconcile.Request, 0) + seenSvcs := make(set.Set[string]) for _, svc := range svcList.Items { reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)}) + seenSvcs.Add(fmt.Sprintf("%s/%s", svc.Namespace, svc.Name)) } + + svcAnnotationList := new(corev1.ServiceList) + if err := cl.List(ctx, svcAnnotationList, client.MatchingFields{indexServiceProxyClass: o.GetName()}); err != nil { + logger.Debugf("error listing Services for ProxyClass: %v", err) + return nil + } + + for _, svc := range svcAnnotationList.Items { + nsname := fmt.Sprintf("%s/%s", svc.Namespace, svc.Name) + if seenSvcs.Contains(nsname) { + continue + } + + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)}) + seenSvcs.Add(nsname) + } + return reqs } } @@ -825,16 +866,36 @@ func proxyClassHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) ha return func(ctx context.Context, o client.Object) []reconcile.Request { ingList := new(networkingv1.IngressList) labels := map[string]string{ - LabelProxyClass: o.GetName(), + LabelAnnotationProxyClass: o.GetName(), } if err := cl.List(ctx, ingList, client.MatchingLabels(labels)); err != nil { logger.Debugf("error listing Ingresses for ProxyClass: %v", err) return nil } + reqs := make([]reconcile.Request, 0) + seenIngs := make(set.Set[string]) for _, ing := range ingList.Items { reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) + seenIngs.Add(fmt.Sprintf("%s/%s", ing.Namespace, ing.Name)) + } + + ingAnnotationList := new(networkingv1.IngressList) + if err := cl.List(ctx, ingAnnotationList, client.MatchingFields{indexIngressProxyClass: o.GetName()}); err != nil { + logger.Debugf("error listing Ingreses for ProxyClass: %v", err) + return nil + } + + for _, ing := range ingAnnotationList.Items { + nsname := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name) + if seenIngs.Contains(nsname) { + continue + } + + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) + seenIngs.Add(nsname) } + return reqs } } @@ -1500,6 +1561,10 @@ func hasProxyGroupAnnotation(obj client.Object) bool { return obj.GetAnnotations()[AnnotationProxyGroup] != "" } +func hasProxyClassAnnotation(obj client.Object) bool { + return obj.GetAnnotations()[LabelAnnotationProxyClass] != "" +} + func id(ctx context.Context, lc *local.Client) (string, error) { st, err := lc.StatusWithoutPeers(ctx) if err != nil { diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index ff6ba4f952749..a9f08c18b4793 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -7,6 +7,7 @@ package main import ( "context" + "encoding/json" "fmt" "testing" "time" @@ -20,8 +21,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/k8s-operator/apis/v1alpha1" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" "tailscale.com/net/dns/resolvconffile" @@ -1121,6 +1124,182 @@ func TestCustomPriorityClassName(t *testing.T) { expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) } +func TestServiceProxyClassAnnotation(t *testing.T) { + cl := tstest.NewClock(tstest.ClockOpts{}) + zl := zap.Must(zap.NewDevelopment()) + + pcIfNotPresent := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "if-not-present", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &v1alpha1.Container{ + ImagePullPolicy: corev1.PullIfNotPresent, + }, + }, + }, + }, + } + + pcAlways := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "always", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &v1alpha1.Container{ + ImagePullPolicy: corev1.PullAlways, + }, + }, + }, + }, + } + + builder := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme) + builder = builder.WithObjects(pcIfNotPresent, pcAlways). + WithStatusSubresource(pcIfNotPresent, pcAlways) + fc := builder.Build() + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.20.30.40", + Type: corev1.ServiceTypeLoadBalancer, + }, + } + + mustCreate(t, fc, svc) + + testCases := []struct { + name string + proxyClassAnnotation string + proxyClassLabel string + proxyClassDefault string + expectedProxyClass string + expectEvents []string + }{ + { + name: "via_label", + proxyClassLabel: pcIfNotPresent.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + { + name: "via_annotation", + proxyClassAnnotation: pcIfNotPresent.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + { + name: "via_default", + proxyClassDefault: pcIfNotPresent.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + { + name: "via_label_override_annotation", + proxyClassLabel: pcIfNotPresent.Name, + proxyClassAnnotation: pcAlways.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + ft := &fakeTSClient{} + + if tt.proxyClassAnnotation != "" || tt.proxyClassLabel != "" || tt.proxyClassDefault != "" { + name := tt.proxyClassDefault + if name == "" { + name = tt.proxyClassLabel + if name == "" { + name = tt.proxyClassAnnotation + } + } + setProxyClassReady(t, fc, cl, name) + } + + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + defaultProxyClass: tt.proxyClassDefault, + logger: zl.Sugar(), + clock: cl, + isDefaultLoadBalancer: true, + } + + if tt.proxyClassLabel != "" { + svc.Labels = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassLabel, + } + } + if tt.proxyClassAnnotation != "" { + svc.Annotations = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassAnnotation, + } + } + + mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) { + s.Labels = svc.Labels + s.Annotations = svc.Annotations + }) + + expectReconciled(t, sr, "default", "test") + + list := &corev1.ServiceList{} + fc.List(context.Background(), list, client.InNamespace("default")) + + for _, i := range list.Items { + t.Logf("found service %s", i.Name) + } + + slist := &corev1.SecretList{} + fc.List(context.Background(), slist, client.InNamespace("operator-ns")) + for _, i := range slist.Items { + l, _ := json.Marshal(i.Labels) + t.Logf("found secret %q with labels %q ", i.Name, string(l)) + } + + _, shortName := findGenName(t, fc, "default", "test", "svc") + sts := &appsv1.StatefulSet{} + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + switch tt.expectedProxyClass { + case pcIfNotPresent.Name: + for _, cont := range sts.Spec.Template.Spec.Containers { + if cont.Name == "tailscale" && cont.ImagePullPolicy != corev1.PullIfNotPresent { + t.Fatalf("ImagePullPolicy %q does not match ProxyClass %q with value %q", cont.ImagePullPolicy, pcIfNotPresent.Name, pcIfNotPresent.Spec.StatefulSet.Pod.TailscaleContainer.ImagePullPolicy) + } + } + case pcAlways.Name: + for _, cont := range sts.Spec.Template.Spec.Containers { + if cont.Name == "tailscale" && cont.ImagePullPolicy != corev1.PullAlways { + t.Fatalf("ImagePullPolicy %q does not match ProxyClass %q with value %q", cont.ImagePullPolicy, pcAlways.Name, pcAlways.Spec.StatefulSet.Pod.TailscaleContainer.ImagePullPolicy) + } + } + default: + t.Fatalf("unexpected expected ProxyClass %q", tt.expectedProxyClass) + } + }) + } +} + func TestProxyClassForService(t *testing.T) { // Setup pc := &tsapi.ProxyClass{ @@ -1132,7 +1311,9 @@ func TestProxyClassForService(t *testing.T) { StatefulSet: &tsapi.StatefulSet{ Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"bar.io/foo": "some-val"}, - Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}, + }, + }, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -1194,7 +1375,7 @@ func TestProxyClassForService(t *testing.T) { // pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not // yet ready, so no changes are actually applied to the proxy resources. mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) { - mak.Set(&svc.Labels, LabelProxyClass, "custom-metadata") + mak.Set(&svc.Labels, LabelAnnotationProxyClass, "custom-metadata") }) expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) @@ -1209,7 +1390,8 @@ func TestProxyClassForService(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: pc.Generation, - }}} + }}, + } }) opts.proxyClass = pc.Name expectReconciled(t, sr, "default", "test") @@ -1220,7 +1402,7 @@ func TestProxyClassForService(t *testing.T) { // configuration from the ProxyClass is removed from the cluster // resources. mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) { - delete(svc.Labels, LabelProxyClass) + delete(svc.Labels, LabelAnnotationProxyClass) }) opts.proxyClass = "" expectReconciled(t, sr, "default", "test") @@ -1439,7 +1621,8 @@ func Test_serviceHandlerForIngress(t *testing.T) { IngressClassName: ptr.To(tailscaleIngressClassName), Rules: []networkingv1.IngressRule{{IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{ Paths: []networkingv1.HTTPIngressPath{ - {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}}}, + {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}}, + }, }}}}, }, }) @@ -1466,7 +1649,8 @@ func Test_serviceHandlerForIngress(t *testing.T) { Spec: networkingv1.IngressSpec{ Rules: []networkingv1.IngressRule{{IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{ Paths: []networkingv1.HTTPIngressPath{ - {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "non-ts-backend"}}}}, + {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "non-ts-backend"}}}, + }, }}}}, }, }) @@ -1565,6 +1749,7 @@ func Test_clusterDomainFromResolverConf(t *testing.T) { }) } } + func Test_authKeyRemoval(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} @@ -1711,14 +1896,15 @@ func Test_metricsResourceCreation(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: 1, - }}}, + }}, + }, } svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", UID: types.UID("1234-UID"), - Labels: map[string]string{LabelProxyClass: "metrics"}, + Labels: map[string]string{LabelAnnotationProxyClass: "metrics"}, }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 3e3d2d5903a7b..a943ae97179a1 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -50,7 +50,7 @@ const ( // LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or // cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for // the Ingress or Service. - LabelProxyClass = "tailscale.com/proxy-class" + LabelAnnotationProxyClass = "tailscale.com/proxy-class" FinalizerName = "tailscale.com/finalizer" @@ -1127,6 +1127,22 @@ func nameForService(svc *corev1.Service) string { return svc.Namespace + "-" + svc.Name } +// proxyClassForObject returns the proxy class for the given object. If the +// object does not have a proxy class label, it returns the default proxy class +func proxyClassForObject(o client.Object, proxyDefaultClass string) string { + proxyClass, exists := o.GetLabels()[LabelAnnotationProxyClass] + if exists { + return proxyClass + } + + proxyClass, exists = o.GetAnnotations()[LabelAnnotationProxyClass] + if exists { + return proxyClass + } + + return proxyDefaultClass +} + func isValidFirewallMode(m string) bool { return m == "auto" || m == "nftables" || m == "iptables" } diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index c880f59f5012a..f8c9af23990e9 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -41,6 +41,8 @@ const ( reasonProxyInvalid = "ProxyInvalid" reasonProxyFailed = "ProxyFailed" reasonProxyPending = "ProxyPending" + + indexServiceProxyClass = ".metadata.annotations.service-proxy-class" ) type ServiceReconciler struct { @@ -438,16 +440,6 @@ func tailnetTargetAnnotation(svc *corev1.Service) string { return svc.Annotations[annotationTailnetTargetIPOld] } -// proxyClassForObject returns the proxy class for the given object. If the -// object does not have a proxy class label, it returns the default proxy class -func proxyClassForObject(o client.Object, proxyDefaultClass string) string { - proxyClass, exists := o.GetLabels()[LabelProxyClass] - if !exists { - proxyClass = proxyDefaultClass - } - return proxyClass -} - func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) { proxyClass := new(tsapi.ProxyClass) if err := cl.Get(ctx, types.NamespacedName{Name: name}, proxyClass); err != nil { From 47e77565c63aa1af9d0de27a38281bc0fcc02250 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 30 Jun 2025 12:12:57 -0700 Subject: [PATCH 136/263] wgengine/magicsock: avoid handshaking relay endpoints that are trusted (#16412) Changes to our src/address family can trigger blackholes. This commit also adds a missing set of trustBestAddrUntil when setting a UDP relay path as bestAddr. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 52 +++++++--- wgengine/magicsock/magicsock.go | 6 +- wgengine/magicsock/relaymanager.go | 131 ++++++++++++++++-------- wgengine/magicsock/relaymanager_test.go | 4 +- 4 files changed, 130 insertions(+), 63 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 9edc6403e6132..af4666665b0fb 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -100,25 +100,33 @@ type endpoint struct { relayCapable bool // whether the node is capable of speaking via a [tailscale.com/net/udprelay.Server] } -// relayEndpointReady determines whether the given relay addr should be -// installed as de.bestAddr. It is only called by [relayManager] once it has -// determined addr is functional via [disco.Pong] reception. -func (de *endpoint) relayEndpointReady(addr epAddr, latency time.Duration) { +// udpRelayEndpointReady determines whether the given relay [addrQuality] should +// be installed as de.bestAddr. It is only called by [relayManager] once it has +// determined maybeBest is functional via [disco.Pong] reception. +func (de *endpoint) udpRelayEndpointReady(maybeBest addrQuality) { de.c.mu.Lock() defer de.c.mu.Unlock() de.mu.Lock() defer de.mu.Unlock() - maybeBetter := addrQuality{addr, latency, pingSizeToPktLen(0, addr)} - if !betterAddr(maybeBetter, de.bestAddr) { + if maybeBest.relayServerDisco.Compare(de.bestAddr.relayServerDisco) == 0 { + // TODO(jwhited): add some observability for this case, e.g. did we + // flip transports during a de.bestAddr transition from untrusted to + // trusted? + // + // If these are equal we must set maybeBest as bestAddr, otherwise we + // could leave a stale bestAddr if it goes over a different + // address family or src. + } else if !betterAddr(maybeBest, de.bestAddr) { return } - // Promote maybeBetter to bestAddr. + // Promote maybeBest to bestAddr. // TODO(jwhited): collapse path change logging with endpoint.handlePongConnLocked() - de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v", de.publicKey.ShortString(), de.discoShort(), maybeBetter.epAddr, maybeBetter.wireMTU) - de.setBestAddrLocked(maybeBetter) - de.c.peerMap.setNodeKeyForEpAddr(addr, de.publicKey) + de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v", de.publicKey.ShortString(), de.discoShort(), maybeBest.epAddr, maybeBest.wireMTU) + de.setBestAddrLocked(maybeBest) + de.trustBestAddrUntil = mono.Now().Add(trustUDPAddrDuration) + de.c.peerMap.setNodeKeyForEpAddr(maybeBest.epAddr, de.publicKey) } func (de *endpoint) setBestAddrLocked(v addrQuality) { @@ -871,7 +879,9 @@ func (de *endpoint) discoverUDPRelayPathsLocked(now mono.Time) { // TODO(jwhited): return early if there are no relay servers set, otherwise // we spin up and down relayManager.runLoop unnecessarily. de.lastUDPRelayPathDiscovery = now - de.c.relayManager.allocateAndHandshakeAllServers(de) + lastBest := de.bestAddr + lastBestIsTrusted := mono.Now().Before(de.trustBestAddrUntil) + de.c.relayManager.startUDPRelayPathDiscoveryFor(de, lastBest, lastBestIsTrusted) } // wantUDPRelayPathDiscoveryLocked reports whether we should kick off UDP relay @@ -1714,7 +1724,16 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd // Promote this pong response to our current best address if it's lower latency. // TODO(bradfitz): decide how latency vs. preference order affects decision if !isDerp { - thisPong := addrQuality{sp.to, latency, tstun.WireMTU(pingSizeToPktLen(sp.size, sp.to))} + thisPong := addrQuality{ + epAddr: sp.to, + latency: latency, + wireMTU: pingSizeToPktLen(sp.size, sp.to), + } + // TODO(jwhited): consider checking de.trustBestAddrUntil as well. If + // de.bestAddr is untrusted we may want to clear it, otherwise we could + // get stuck with a forever untrusted bestAddr that blackholes, since + // we don't clear direct UDP paths on disco ping timeout (see + // discoPingTimeout). if betterAddr(thisPong, de.bestAddr) { if src.vni.isSet() { // This would be unexpected. Switching to a Geneve-encapsulated @@ -1765,14 +1784,17 @@ func (e epAddr) String() string { return fmt.Sprintf("%v:vni:%d", e.ap.String(), e.vni.get()) } -// addrQuality is an [epAddr] with an associated latency and path mtu. +// addrQuality is an [epAddr], an optional [key.DiscoPublic] if a relay server +// is associated, a round-trip latency measurement, and path mtu. type addrQuality struct { epAddr - latency time.Duration - wireMTU tstun.WireMTU + relayServerDisco key.DiscoPublic // only relevant if epAddr.vni.isSet(), otherwise zero value + latency time.Duration + wireMTU tstun.WireMTU } func (a addrQuality) String() string { + // TODO(jwhited): consider including relayServerDisco return fmt.Sprintf("%v@%v+%v", a.epAddr, a.latency, a.wireMTU) } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 553543b0f496d..0933c5be251a8 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2137,6 +2137,8 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake } ep.mu.Lock() relayCapable := ep.relayCapable + lastBest := ep.bestAddr + lastBestIsTrusted := mono.Now().Before(ep.trustBestAddrUntil) ep.mu.Unlock() if isVia && !relayCapable { c.logf("magicsock: disco: ignoring %s from %v; %v is not known to be relay capable", msgType, sender.ShortString(), sender.ShortString()) @@ -2156,7 +2158,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake c.discoShort, epDisco.short, via.ServerDisco.ShortString(), ep.publicKey.ShortString(), derpStr(src.String()), len(via.AddrPorts)) - c.relayManager.handleCallMeMaybeVia(ep, via) + c.relayManager.handleCallMeMaybeVia(ep, lastBest, lastBestIsTrusted, via) } else { c.dlogf("[v1] magicsock: disco: %v<-%v (%v, %v) got call-me-maybe, %d endpoints", c.discoShort, epDisco.short, @@ -2254,7 +2256,7 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src epAddr, di *discoInfo, derpN // We have no [endpoint] in the [peerMap] for this relay [epAddr] // using it as a bestAddr. [relayManager] might be in the middle of // probing it or attempting to set it as best via - // [endpoint.relayEndpointReady()]. Make [relayManager] aware. + // [endpoint.udpRelayEndpointReady()]. Make [relayManager] aware. c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(c, dm, di, src) return } diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 6418a43641200..1c173c01ac138 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -50,7 +50,7 @@ type relayManager struct { // =================================================================== // The following chan fields serve event inputs to a single goroutine, // runLoop(). - allocateHandshakeCh chan *endpoint + startDiscoveryCh chan endpointWithLastBest allocateWorkDoneCh chan relayEndpointAllocWorkDoneEvent handshakeWorkDoneCh chan relayEndpointHandshakeWorkDoneEvent cancelWorkCh chan *endpoint @@ -77,8 +77,8 @@ type serverDiscoVNI struct { // relayHandshakeWork serves to track in-progress relay handshake work for a // [udprelay.ServerEndpoint]. This structure is immutable once initialized. type relayHandshakeWork struct { - ep *endpoint - se udprelay.ServerEndpoint + wlb endpointWithLastBest + se udprelay.ServerEndpoint // handshakeServerEndpoint() always writes to doneCh (len 1) when it // returns. It may end up writing the same event afterward to @@ -97,7 +97,7 @@ type relayHandshakeWork struct { // [disco.CallMeMaybeVia] reception. This structure is immutable once // initialized. type newRelayServerEndpointEvent struct { - ep *endpoint + wlb endpointWithLastBest se udprelay.ServerEndpoint server netip.AddrPort // zero value if learned via [disco.CallMeMaybeVia] } @@ -142,9 +142,9 @@ func (r *relayManager) runLoop() { for { select { - case ep := <-r.allocateHandshakeCh: - if !r.hasActiveWorkForEndpointRunLoop(ep) { - r.allocateAllServersRunLoop(ep) + case startDiscovery := <-r.startDiscoveryCh: + if !r.hasActiveWorkForEndpointRunLoop(startDiscovery.ep) { + r.allocateAllServersRunLoop(startDiscovery) } if !r.hasActiveWorkRunLoop() { return @@ -153,7 +153,7 @@ func (r *relayManager) runLoop() { work, ok := r.allocWorkByEndpoint[done.work.ep] if ok && work == done.work { // Verify the work in the map is the same as the one that we're - // cleaning up. New events on r.allocateHandshakeCh can + // cleaning up. New events on r.startDiscoveryCh can // overwrite pre-existing keys. delete(r.allocWorkByEndpoint, done.work.ep) } @@ -237,7 +237,7 @@ func (r *relayManager) init() { r.handshakeWorkByServerDiscoVNI = make(map[serverDiscoVNI]*relayHandshakeWork) r.handshakeWorkAwaitingPong = make(map[*relayHandshakeWork]addrPortVNI) r.addrPortVNIToHandshakeWork = make(map[addrPortVNI]*relayHandshakeWork) - r.allocateHandshakeCh = make(chan *endpoint) + r.startDiscoveryCh = make(chan endpointWithLastBest) r.allocateWorkDoneCh = make(chan relayEndpointAllocWorkDoneEvent) r.handshakeWorkDoneCh = make(chan relayEndpointHandshakeWorkDoneEvent) r.cancelWorkCh = make(chan *endpoint) @@ -273,7 +273,7 @@ func (r *relayManager) ensureDiscoInfoFor(work *relayHandshakeWork) { di.di = &discoInfo{ discoKey: work.se.ServerDisco, discoShort: work.se.ServerDisco.ShortString(), - sharedKey: work.ep.c.discoPrivate.Shared(work.se.ServerDisco), + sharedKey: work.wlb.ep.c.discoPrivate.Shared(work.se.ServerDisco), } } } @@ -306,7 +306,7 @@ func (r *relayManager) discoInfo(serverDisco key.DiscoPublic) (_ *discoInfo, ok return nil, false } -func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, dm *disco.CallMeMaybeVia) { +func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, lastBest addrQuality, lastBestIsTrusted bool, dm *disco.CallMeMaybeVia) { se := udprelay.ServerEndpoint{ ServerDisco: dm.ServerDisco, LamportID: dm.LamportID, @@ -316,7 +316,11 @@ func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, dm *disco.CallMeMaybeV se.BindLifetime.Duration = dm.BindLifetime se.SteadyStateLifetime.Duration = dm.SteadyStateLifetime relayManagerInputEvent(r, nil, &r.newServerEndpointCh, newRelayServerEndpointEvent{ - ep: ep, + wlb: endpointWithLastBest{ + ep: ep, + lastBest: lastBest, + lastBestIsTrusted: lastBestIsTrusted, + }, se: se, }) } @@ -360,11 +364,19 @@ func relayManagerInputEvent[T any](r *relayManager, ctx context.Context, eventCh } } -// allocateAndHandshakeAllServers kicks off allocation and handshaking of relay -// endpoints for 'ep' on all known relay servers if there is no outstanding -// work. -func (r *relayManager) allocateAndHandshakeAllServers(ep *endpoint) { - relayManagerInputEvent(r, nil, &r.allocateHandshakeCh, ep) +// endpointWithLastBest represents an [*endpoint], its last bestAddr, and if +// the last bestAddr was trusted (see endpoint.trustBestAddrUntil) at the time +// of init. This structure is immutable once initialized. +type endpointWithLastBest struct { + ep *endpoint + lastBest addrQuality + lastBestIsTrusted bool +} + +// startUDPRelayPathDiscoveryFor starts UDP relay path discovery for ep on all +// known relay servers if ep has no in-progress work. +func (r *relayManager) startUDPRelayPathDiscoveryFor(ep *endpoint, lastBest addrQuality, lastBestIsTrusted bool) { + relayManagerInputEvent(r, nil, &r.startDiscoveryCh, endpointWithLastBest{ep, lastBest, lastBestIsTrusted}) } // stopWork stops all outstanding allocation & handshaking work for 'ep'. @@ -432,7 +444,7 @@ func (r *relayManager) handleRxHandshakeDiscoMsgRunLoop(event relayHandshakeDisc r.addrPortVNIToHandshakeWork[apv] = work case *disco.Ping: // Always TX a pong. We might not have any associated work if ping - // reception raced with our call to [endpoint.relayEndpointReady()], so + // reception raced with our call to [endpoint.udpRelayEndpointReady()], so // err on the side of enabling the remote side to use this path. // // Conn.handlePingLocked() makes efforts to suppress duplicate pongs @@ -473,7 +485,7 @@ func (r *relayManager) handleRxHandshakeDiscoMsgRunLoop(event relayHandshakeDisc } func (r *relayManager) handleHandshakeWorkDoneRunLoop(done relayEndpointHandshakeWorkDoneEvent) { - byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[done.work.ep] + byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[done.work.wlb.ep] if !ok { return } @@ -483,7 +495,7 @@ func (r *relayManager) handleHandshakeWorkDoneRunLoop(done relayEndpointHandshak } delete(byServerDisco, done.work.se.ServerDisco) if len(byServerDisco) == 0 { - delete(r.handshakeWorkByEndpointByServerDisco, done.work.ep) + delete(r.handshakeWorkByEndpointByServerDisco, done.work.wlb.ep) } delete(r.handshakeWorkByServerDiscoVNI, serverDiscoVNI{done.work.se.ServerDisco, done.work.se.VNI}) apv, ok := r.handshakeWorkAwaitingPong[work] @@ -499,10 +511,15 @@ func (r *relayManager) handleHandshakeWorkDoneRunLoop(done relayEndpointHandshak vni := virtualNetworkID{} vni.set(done.work.se.VNI) addr := epAddr{ap: done.pongReceivedFrom, vni: vni} - // ep.relayEndpointReady() must be called in a new goroutine to prevent + // ep.udpRelayEndpointReady() must be called in a new goroutine to prevent // deadlocks as it acquires [endpoint] & [Conn] mutexes. See [relayManager] // docs for details. - go done.work.ep.relayEndpointReady(addr, done.latency) + go done.work.wlb.ep.udpRelayEndpointReady(addrQuality{ + epAddr: addr, + relayServerDisco: done.work.se.ServerDisco, + latency: done.latency, + wireMTU: pingSizeToPktLen(0, addr), + }) } func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelayServerEndpointEvent) { @@ -525,7 +542,7 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay } // Check for duplicate work by [*endpoint] + server disco. - byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[newServerEndpoint.ep] + byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[newServerEndpoint.wlb.ep] if ok { existingWork, ok := byServerDisco[newServerEndpoint.se.ServerDisco] if ok { @@ -569,10 +586,40 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay } } + if newServerEndpoint.server.IsValid() { + // Send a [disco.CallMeMaybeVia] to the remote peer if we allocated this + // endpoint, regardless of if we start a handshake below. + go r.sendCallMeMaybeVia(newServerEndpoint.wlb.ep, newServerEndpoint.se) + } + + lastBestMatchingServer := newServerEndpoint.se.ServerDisco.Compare(newServerEndpoint.wlb.lastBest.relayServerDisco) == 0 + if lastBestMatchingServer && newServerEndpoint.wlb.lastBestIsTrusted { + // This relay server endpoint is the same as [endpoint]'s bestAddr at + // the time UDP relay path discovery was started, and it was also a + // trusted path (see endpoint.trustBestAddrUntil), so return early. + // + // If we were to start a new handshake, there is a chance that we + // cause [endpoint] to blackhole some packets on its bestAddr if we end + // up shifting to a new address family or src, e.g. IPv4 to IPv6, due to + // the window of time between the handshake completing, and our call to + // udpRelayEndpointReady(). The relay server can only forward packets + // from us on a single [epAddr]. + return + } + + // TODO(jwhited): if lastBest is untrusted, consider some strategies + // to reduce the chance we blackhole if it were to transition to + // trusted during/before the new handshake: + // 1. Start by attempting a handshake with only lastBest.epAddr. If + // that fails then try the remaining [epAddr]s. + // 2. Signal bestAddr trust transitions between [endpoint] and + // [relayManager] in order to prevent a handshake from starting + // and/or stop one that is running. + // We're ready to start a new handshake. ctx, cancel := context.WithCancel(context.Background()) work := &relayHandshakeWork{ - ep: newServerEndpoint.ep, + wlb: newServerEndpoint.wlb, se: newServerEndpoint.se, rxDiscoMsgCh: make(chan relayHandshakeDiscoMsgEvent), doneCh: make(chan relayEndpointHandshakeWorkDoneEvent, 1), @@ -581,16 +628,11 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay } if byServerDisco == nil { byServerDisco = make(map[key.DiscoPublic]*relayHandshakeWork) - r.handshakeWorkByEndpointByServerDisco[newServerEndpoint.ep] = byServerDisco + r.handshakeWorkByEndpointByServerDisco[newServerEndpoint.wlb.ep] = byServerDisco } byServerDisco[newServerEndpoint.se.ServerDisco] = work r.handshakeWorkByServerDiscoVNI[sdv] = work - if newServerEndpoint.server.IsValid() { - // Send CallMeMaybeVia to the remote peer if we allocated this endpoint. - go r.sendCallMeMaybeVia(work.ep, work.se) - } - r.handshakeGeneration++ if r.handshakeGeneration == 0 { // generation must be nonzero r.handshakeGeneration++ @@ -633,7 +675,8 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork, generat work.cancel() }() - epDisco := work.ep.disco.Load() + ep := work.wlb.ep + epDisco := ep.disco.Load() if epDisco == nil { return } @@ -653,7 +696,7 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork, generat for _, addrPort := range work.se.AddrPorts { if addrPort.IsValid() { sentBindAny = true - go work.ep.c.sendDiscoMessage(epAddr{ap: addrPort, vni: vni}, key.NodePublic{}, work.se.ServerDisco, bind, discoVerboseLog) + go ep.c.sendDiscoMessage(epAddr{ap: addrPort, vni: vni}, key.NodePublic{}, work.se.ServerDisco, bind, discoVerboseLog) } } if !sentBindAny { @@ -684,15 +727,15 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork, generat sentPingAt[txid] = time.Now() ping := &disco.Ping{ TxID: txid, - NodeKey: work.ep.c.publicKeyAtomic.Load(), + NodeKey: ep.c.publicKeyAtomic.Load(), } go func() { if withAnswer != nil { answer := &disco.BindUDPRelayEndpointAnswer{BindUDPRelayEndpointCommon: common} answer.Challenge = *withAnswer - work.ep.c.sendDiscoMessage(epAddr{ap: to, vni: vni}, key.NodePublic{}, work.se.ServerDisco, answer, discoVerboseLog) + ep.c.sendDiscoMessage(epAddr{ap: to, vni: vni}, key.NodePublic{}, work.se.ServerDisco, answer, discoVerboseLog) } - work.ep.c.sendDiscoMessage(epAddr{ap: to, vni: vni}, key.NodePublic{}, epDisco.key, ping, discoVerboseLog) + ep.c.sendDiscoMessage(epAddr{ap: to, vni: vni}, key.NodePublic{}, epDisco.key, ping, discoVerboseLog) }() } @@ -760,17 +803,17 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork, generat } } -func (r *relayManager) allocateAllServersRunLoop(ep *endpoint) { +func (r *relayManager) allocateAllServersRunLoop(wlb endpointWithLastBest) { if len(r.serversByAddrPort) == 0 { return } ctx, cancel := context.WithCancel(context.Background()) - started := &relayEndpointAllocWork{ep: ep, cancel: cancel, wg: &sync.WaitGroup{}} + started := &relayEndpointAllocWork{ep: wlb.ep, cancel: cancel, wg: &sync.WaitGroup{}} for k := range r.serversByAddrPort { started.wg.Add(1) - go r.allocateSingleServer(ctx, started.wg, k, ep) + go r.allocateSingleServer(ctx, started.wg, k, wlb) } - r.allocWorkByEndpoint[ep] = started + r.allocWorkByEndpoint[wlb.ep] = started go func() { started.wg.Wait() relayManagerInputEvent(r, ctx, &r.allocateWorkDoneCh, relayEndpointAllocWorkDoneEvent{work: started}) @@ -829,25 +872,25 @@ func doAllocate(ctx context.Context, server netip.AddrPort, discoKeys [2]key.Dis } } -func (r *relayManager) allocateSingleServer(ctx context.Context, wg *sync.WaitGroup, server netip.AddrPort, ep *endpoint) { +func (r *relayManager) allocateSingleServer(ctx context.Context, wg *sync.WaitGroup, server netip.AddrPort, wlb endpointWithLastBest) { // TODO(jwhited): introduce client metrics counters for notable failures defer wg.Done() - remoteDisco := ep.disco.Load() + remoteDisco := wlb.ep.disco.Load() if remoteDisco == nil { return } firstTry := true for { - se, err := doAllocate(ctx, server, [2]key.DiscoPublic{ep.c.discoPublic, remoteDisco.key}) + se, err := doAllocate(ctx, server, [2]key.DiscoPublic{wlb.ep.c.discoPublic, remoteDisco.key}) if err == nil { relayManagerInputEvent(r, ctx, &r.newServerEndpointCh, newRelayServerEndpointEvent{ - ep: ep, + wlb: wlb, se: se, server: server, // we allocated this endpoint (vs CallMeMaybeVia reception), mark it as such }) return } - ep.c.logf("[v1] magicsock: relayManager: error allocating endpoint on %v for %v: %v", server, ep.discoShort(), err) + wlb.ep.c.logf("[v1] magicsock: relayManager: error allocating endpoint on %v for %v: %v", server, wlb.ep.discoShort(), err) var notReady errNotReady if firstTry && errors.As(err, ¬Ready) { select { diff --git a/wgengine/magicsock/relaymanager_test.go b/wgengine/magicsock/relaymanager_test.go index de282b4990637..8feff2f3d5ca8 100644 --- a/wgengine/magicsock/relaymanager_test.go +++ b/wgengine/magicsock/relaymanager_test.go @@ -14,7 +14,7 @@ import ( func TestRelayManagerInitAndIdle(t *testing.T) { rm := relayManager{} - rm.allocateAndHandshakeAllServers(&endpoint{}) + rm.startUDPRelayPathDiscoveryFor(&endpoint{}, addrQuality{}, false) <-rm.runLoopStoppedCh rm = relayManager{} @@ -22,7 +22,7 @@ func TestRelayManagerInitAndIdle(t *testing.T) { <-rm.runLoopStoppedCh rm = relayManager{} - rm.handleCallMeMaybeVia(&endpoint{c: &Conn{discoPrivate: key.NewDisco()}}, &disco.CallMeMaybeVia{ServerDisco: key.NewDisco().Public()}) + rm.handleCallMeMaybeVia(&endpoint{c: &Conn{discoPrivate: key.NewDisco()}}, addrQuality{}, false, &disco.CallMeMaybeVia{ServerDisco: key.NewDisco().Public()}) <-rm.runLoopStoppedCh rm = relayManager{} From 6a9bf9172b6fa6dc645b5ea960b98014f389533d Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Mon, 30 Jun 2025 13:43:16 -0500 Subject: [PATCH 137/263] ipn/ipnlocal: add verbose Taildrive logging on client side This allows logging the following Taildrive behavior from the client's perspective when --verbose=1: - Initialization of Taildrive remotes for every peer - Peer availability checks - All HTTP requests to peers (not just GET and PUT) Updates tailscale/corp#29702 Signed-off-by: Percy Wegmann --- ipn/ipnlocal/drive.go | 6 ++++++ ipn/ipnlocal/local.go | 35 +++++++++++++++++++++-------------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/ipn/ipnlocal/drive.go b/ipn/ipnlocal/drive.go index 6a6f9bcd2b24a..8c2f339bb271a 100644 --- a/ipn/ipnlocal/drive.go +++ b/ipn/ipnlocal/drive.go @@ -306,10 +306,12 @@ func (b *LocalBackend) updateDrivePeersLocked(nm *netmap.NetworkMap) { } func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Remote { + b.logf("[v1] taildrive: setting up drive remotes from peers") driveRemotes := make([]*drive.Remote, 0, len(nm.Peers)) for _, p := range nm.Peers { peerID := p.ID() url := fmt.Sprintf("%s/%s", peerAPIBase(nm, p), taildrivePrefix[1:]) + b.logf("[v1] taildrive: appending remote for peer %d: %s", peerID, url) driveRemotes = append(driveRemotes, &drive.Remote{ Name: p.DisplayName(false), URL: url, @@ -320,6 +322,7 @@ func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Rem cn := b.currentNode() peer, ok := cn.NodeByID(peerID) if !ok { + b.logf("[v1] taildrive: Available(): peer %d not found", peerID) return false } @@ -332,14 +335,17 @@ func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Rem // The netmap.Peers slice is not updated in all cases. // It should be fixed now that we use PeerByIDOk. if !peer.Online().Get() { + b.logf("[v1] taildrive: Available(): peer %d offline", peerID) return false } // Check that the peer is allowed to share with us. if cn.PeerHasCap(peer, tailcfg.PeerCapabilityTaildriveSharer) { + b.logf("[v1] taildrive: Available(): peer %d available", peerID) return true } + b.logf("[v1] taildrive: Available(): peer %d not allowed to share", peerID) return false }, }) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 9cec088f1f28b..29d09400b0d8d 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1459,7 +1459,7 @@ func (b *LocalBackend) WhoIs(proto string, ipp netip.AddrPort) (n tailcfg.NodeVi } n, ok = cn.NodeByID(nid) if !ok { - return zero, u, false + return zero, u, false } up, ok := cn.UserByID(n.User()) if !ok { @@ -5960,6 +5960,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { // the number of bytesRead. type responseBodyWrapper struct { io.ReadCloser + logVerbose bool bytesRx int64 bytesTx int64 log logger.Logf @@ -5981,8 +5982,22 @@ func (rbw *responseBodyWrapper) logAccess(err string) { // Some operating systems create and copy lots of 0 length hidden files for // tracking various states. Omit these to keep logs from being too verbose. - if rbw.contentLength > 0 { - rbw.log("taildrive: access: %s from %s to %s: status-code=%d ext=%q content-type=%q content-length=%.f tx=%.f rx=%.f err=%q", rbw.method, rbw.selfNodeKey, rbw.shareNodeKey, rbw.statusCode, rbw.fileExtension, rbw.contentType, roundTraffic(rbw.contentLength), roundTraffic(rbw.bytesTx), roundTraffic(rbw.bytesRx), err) + if rbw.logVerbose || rbw.contentLength > 0 { + levelPrefix := "" + if rbw.logVerbose { + levelPrefix = "[v1] " + } + rbw.log( + "%staildrive: access: %s from %s to %s: status-code=%d ext=%q content-type=%q content-length=%.f tx=%.f rx=%.f err=%q", + levelPrefix, + rbw.method, + rbw.selfNodeKey, + rbw.shareNodeKey, + rbw.statusCode, + rbw.fileExtension, + rbw.contentType, + roundTraffic(rbw.contentLength), + roundTraffic(rbw.bytesTx), roundTraffic(rbw.bytesRx), err) } } @@ -6037,17 +6052,8 @@ func (dt *driveTransport) RoundTrip(req *http.Request) (resp *http.Response, err defer func() { contentType := "unknown" - switch req.Method { - case httpm.PUT: - if ct := req.Header.Get("Content-Type"); ct != "" { - contentType = ct - } - case httpm.GET: - if ct := resp.Header.Get("Content-Type"); ct != "" { - contentType = ct - } - default: - return + if ct := req.Header.Get("Content-Type"); ct != "" { + contentType = ct } dt.b.mu.Lock() @@ -6061,6 +6067,7 @@ func (dt *driveTransport) RoundTrip(req *http.Request) (resp *http.Response, err rbw := responseBodyWrapper{ log: dt.b.logf, + logVerbose: req.Method != httpm.GET && req.Method != httpm.PUT, // other requests like PROPFIND are quite chatty, so we log those at verbose level method: req.Method, bytesTx: int64(bw.bytesRead), selfNodeKey: selfNodeKey, From 454d856be853c713e5e916f13f75cf183de2c94e Mon Sep 17 00:00:00 2001 From: Percy Wegmann Date: Tue, 1 Jul 2025 09:03:54 -0500 Subject: [PATCH 138/263] drive,ipn/ipnlocal: calculate peer taildrive URLs on-demand Instead of calculating the PeerAPI URL at the time that we add the peer, we now calculate it on every access to the peer. This way, if we initially did not have a shared address family with the peer, but later do, this allows us to access the peer at that point. This follows the pattern from other places where we access the peer API, which also calculate the URL on an as-needed basis. Additionally, we now show peers as not Available when we can't get a peer API URL. Lastly, this moves some of the more frequent verbose Taildrive logging from [v1] to [v2] level. Updates #29702 Signed-off-by: Percy Wegmann --- drive/driveimpl/drive_test.go | 2 +- drive/driveimpl/local_impl.go | 2 +- drive/local.go | 2 +- ipn/ipnlocal/drive.go | 27 +++++++++++++++++++-------- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/drive/driveimpl/drive_test.go b/drive/driveimpl/drive_test.go index e7dd832918cec..cff55fbb2c858 100644 --- a/drive/driveimpl/drive_test.go +++ b/drive/driveimpl/drive_test.go @@ -524,7 +524,7 @@ func (s *system) addRemote(name string) string { for name, r := range s.remotes { remotes = append(remotes, &drive.Remote{ Name: name, - URL: fmt.Sprintf("http://%s", r.l.Addr()), + URL: func() string { return fmt.Sprintf("http://%s", r.l.Addr()) }, }) } s.local.fs.SetRemotes( diff --git a/drive/driveimpl/local_impl.go b/drive/driveimpl/local_impl.go index 8cdf60179aa0b..871d033431038 100644 --- a/drive/driveimpl/local_impl.go +++ b/drive/driveimpl/local_impl.go @@ -81,7 +81,7 @@ func (s *FileSystemForLocal) SetRemotes(domain string, remotes []*drive.Remote, Name: remote.Name, Available: remote.Available, }, - BaseURL: func() (string, error) { return remote.URL, nil }, + BaseURL: func() (string, error) { return remote.URL(), nil }, Transport: transport, }) } diff --git a/drive/local.go b/drive/local.go index aff79a57bd9b2..052efb3f97ecf 100644 --- a/drive/local.go +++ b/drive/local.go @@ -17,7 +17,7 @@ import ( // Remote represents a remote Taildrive node. type Remote struct { Name string - URL string + URL func() string Available func() bool } diff --git a/ipn/ipnlocal/drive.go b/ipn/ipnlocal/drive.go index 8c2f339bb271a..d77481903fc09 100644 --- a/ipn/ipnlocal/drive.go +++ b/ipn/ipnlocal/drive.go @@ -309,20 +309,26 @@ func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Rem b.logf("[v1] taildrive: setting up drive remotes from peers") driveRemotes := make([]*drive.Remote, 0, len(nm.Peers)) for _, p := range nm.Peers { - peerID := p.ID() - url := fmt.Sprintf("%s/%s", peerAPIBase(nm, p), taildrivePrefix[1:]) - b.logf("[v1] taildrive: appending remote for peer %d: %s", peerID, url) + peer := p + peerID := peer.ID() + peerKey := peer.Key().ShortString() + b.logf("[v1] taildrive: appending remote for peer %s", peerKey) driveRemotes = append(driveRemotes, &drive.Remote{ Name: p.DisplayName(false), - URL: url, + URL: func() string { + url := fmt.Sprintf("%s/%s", b.currentNode().PeerAPIBase(peer), taildrivePrefix[1:]) + b.logf("[v2] taildrive: url for peer %s: %s", peerKey, url) + return url + }, Available: func() bool { // Peers are available to Taildrive if: // - They are online + // - Their PeerAPI is reachable // - They are allowed to share at least one folder with us cn := b.currentNode() peer, ok := cn.NodeByID(peerID) if !ok { - b.logf("[v1] taildrive: Available(): peer %d not found", peerID) + b.logf("[v2] taildrive: Available(): peer %s not found", peerKey) return false } @@ -335,17 +341,22 @@ func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Rem // The netmap.Peers slice is not updated in all cases. // It should be fixed now that we use PeerByIDOk. if !peer.Online().Get() { - b.logf("[v1] taildrive: Available(): peer %d offline", peerID) + b.logf("[v2] taildrive: Available(): peer %s offline", peerKey) + return false + } + + if b.currentNode().PeerAPIBase(peer) == "" { + b.logf("[v2] taildrive: Available(): peer %s PeerAPI unreachable", peerKey) return false } // Check that the peer is allowed to share with us. if cn.PeerHasCap(peer, tailcfg.PeerCapabilityTaildriveSharer) { - b.logf("[v1] taildrive: Available(): peer %d available", peerID) + b.logf("[v2] taildrive: Available(): peer %s available", peerKey) return true } - b.logf("[v1] taildrive: Available(): peer %d not allowed to share", peerID) + b.logf("[v2] taildrive: Available(): peer %s not allowed to share", peerKey) return false }, }) From d15b2312c4fb7b8ea1f98c5c80f7f72aed784b5d Mon Sep 17 00:00:00 2001 From: kari-ts <135075563+kari-ts@users.noreply.github.com> Date: Tue, 1 Jul 2025 09:28:48 -0700 Subject: [PATCH 139/263] tailcfg: add CapabilityOwner (#16426) We would like to start sending whether a node is a Tailnet owner in netmap responses so that clients can determine what information to display to a user who wants to request account deletion. Updates tailscale/corp#30016 Signed-off-by: kari-ts --- ipn/ipnlocal/local_test.go | 14 ++++++++++++++ tailcfg/tailcfg.go | 1 + 2 files changed, 15 insertions(+) diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 16dbef62a4190..47e5fa37d11cc 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -826,10 +826,21 @@ func TestStatusPeerCapabilities(t *testing.T) { tailcfg.CapabilityAdmin: {`{"test": "true}`}, }), }).View(), + (&tailcfg.Node{ + ID: 3, + StableID: "baz", + Key: makeNodeKeyFromID(3), + Hostinfo: (&tailcfg.Hostinfo{}).View(), + Capabilities: []tailcfg.NodeCapability{tailcfg.CapabilityOwner}, + CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{ + tailcfg.CapabilityOwner: nil, + }), + }).View(), }, expectedPeerCapabilities: map[tailcfg.StableNodeID][]tailcfg.NodeCapability{ tailcfg.StableNodeID("foo"): {tailcfg.CapabilitySSH}, tailcfg.StableNodeID("bar"): {tailcfg.CapabilityAdmin}, + tailcfg.StableNodeID("baz"): {tailcfg.CapabilityOwner}, }, expectedPeerCapMap: map[tailcfg.StableNodeID]tailcfg.NodeCapMap{ tailcfg.StableNodeID("foo"): (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{ @@ -838,6 +849,9 @@ func TestStatusPeerCapabilities(t *testing.T) { tailcfg.StableNodeID("bar"): (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{ tailcfg.CapabilityAdmin: {`{"test": "true}`}, }), + tailcfg.StableNodeID("baz"): (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{ + tailcfg.CapabilityOwner: nil, + }), }, }, { diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index fb7d54c388619..4b1217d4e9596 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2367,6 +2367,7 @@ type NodeCapability string const ( CapabilityFileSharing NodeCapability = "https://tailscale.com/cap/file-sharing" CapabilityAdmin NodeCapability = "https://tailscale.com/cap/is-admin" + CapabilityOwner NodeCapability = "https://tailscale.com/cap/is-owner" CapabilitySSH NodeCapability = "https://tailscale.com/cap/ssh" // feature enabled/available CapabilitySSHRuleIn NodeCapability = "https://tailscale.com/cap/ssh-rule-in" // some SSH rule reach this node CapabilityDataPlaneAuditLogs NodeCapability = "https://tailscale.com/cap/data-plane-audit-logs" // feature enabled From d2edf7133a078880995deb184ae66211efb07b34 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 2 Jul 2025 09:23:54 -0700 Subject: [PATCH 140/263] wgengine/magicsock: remove references to rucPtr (#16441) It used to be a **RebindingUDPConn, now it's just a *RebindingUDPConn. Updates #cleanup Signed-off-by: Jordan Whited --- wgengine/magicsock/magicsock.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 0933c5be251a8..89111b7a0485f 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3197,9 +3197,9 @@ func (c *Conn) listenPacket(network string, port uint16) (nettype.PacketConn, er return nettype.MakePacketListenerWithNetIP(netns.Listener(c.logf, c.netMon)).ListenPacket(ctx, network, addr) } -// bindSocket initializes rucPtr if necessary and binds a UDP socket to it. +// bindSocket binds a UDP socket to ruc. // Network indicates the UDP socket type; it must be "udp4" or "udp6". -// If rucPtr had an existing UDP socket bound, it closes that socket. +// If ruc had an existing UDP socket bound, it closes that socket. // The caller is responsible for informing the portMapper of any changes. // If curPortFate is set to dropCurrentPort, no attempt is made to reuse // the current port. From 172e26b3e3cf70455161609379da1820f6065f77 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Wed, 2 Jul 2025 10:52:00 -0700 Subject: [PATCH 141/263] tailcfg: report StateEncrypted in Hostinfo (#16434) Report whether the client is configured with state encryption (which varies by platform and can be optional on some). Wire it up to `--encrypt-state` in tailscaled, which is set for Linux/Windows, and set defaults for other platforms. Macsys will also report this if full Keychain migration is done. Updates #15830 Signed-off-by: Andrew Lytvynov --- feature/tpm/tpm.go | 2 ++ ipn/ipnlocal/local.go | 27 +++++++++++++++++++++++++++ ipn/store.go | 6 ++++++ tailcfg/tailcfg.go | 9 ++++++++- tailcfg/tailcfg_clone.go | 1 + tailcfg/tailcfg_test.go | 1 + tailcfg/tailcfg_view.go | 2 ++ 7 files changed, 47 insertions(+), 1 deletion(-) diff --git a/feature/tpm/tpm.go b/feature/tpm/tpm.go index 5ec084effa27d..9499ed02a8b2f 100644 --- a/feature/tpm/tpm.go +++ b/feature/tpm/tpm.go @@ -159,6 +159,8 @@ func newStore(logf logger.Logf, path string) (ipn.StateStore, error) { // tpmStore is an ipn.StateStore that stores the state in a secretbox-encrypted // file using a TPM-sealed symmetric key. type tpmStore struct { + ipn.EncryptedStateStore + logf logger.Logf path string key [32]byte diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 29d09400b0d8d..9c16d55af45f6 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -2244,6 +2244,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error { hostinfo.Userspace.Set(b.sys.IsNetstack()) hostinfo.UserspaceRouter.Set(b.sys.IsNetstackRouter()) hostinfo.AppConnector.Set(b.appConnector != nil) + hostinfo.StateEncrypted = b.stateEncrypted() b.logf.JSON(1, "Hostinfo", hostinfo) // TODO(apenwarr): avoid the need to reinit controlclient. @@ -7801,3 +7802,29 @@ func (b *LocalBackend) vipServicesFromPrefsLocked(prefs ipn.PrefsView) []*tailcf var ( metricCurrentWatchIPNBus = clientmetric.NewGauge("localbackend_current_watch_ipn_bus") ) + +func (b *LocalBackend) stateEncrypted() opt.Bool { + switch runtime.GOOS { + case "android", "ios": + return opt.NewBool(true) + case "darwin": + switch { + case version.IsMacAppStore(): + return opt.NewBool(true) + case version.IsMacSysExt(): + // MacSys still stores its state in plaintext on disk in addition to + // the Keychain. A future release will clean up the on-disk state + // files. + // TODO(#15830): always return true here once MacSys is fully migrated. + sp, _ := syspolicy.GetBoolean(syspolicy.EncryptState, false) + return opt.NewBool(sp) + default: + // Probably self-compiled tailscaled, we don't use the Keychain + // there. + return opt.NewBool(false) + } + default: + _, ok := b.store.(ipn.EncryptedStateStore) + return opt.NewBool(ok) + } +} diff --git a/ipn/store.go b/ipn/store.go index 550aa8cba819a..9da5288c0d371 100644 --- a/ipn/store.go +++ b/ipn/store.go @@ -113,3 +113,9 @@ func ReadStoreInt(store StateStore, id StateKey) (int64, error) { func PutStoreInt(store StateStore, id StateKey, val int64) error { return WriteState(store, id, fmt.Appendf(nil, "%d", val)) } + +// EncryptedStateStore is a marker interface implemented by StateStores that +// encrypt data at rest. +type EncryptedStateStore interface { + stateStoreIsEncrypted() +} diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 4b1217d4e9596..10b157ac15642 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -162,7 +162,8 @@ type CapabilityVersion int // - 115: 2025-03-07: Client understands DERPRegion.NoMeasureNoHome. // - 116: 2025-05-05: Client serves MagicDNS "AAAA" if NodeAttrMagicDNSPeerAAAA set on self node // - 117: 2025-05-28: Client understands DisplayMessages (structured health messages), but not necessarily PrimaryAction. -const CurrentCapabilityVersion CapabilityVersion = 117 +// - 118: 2025-07-01: Client sends Hostinfo.StateEncrypted to report whether the state file is encrypted at rest (#15830) +const CurrentCapabilityVersion CapabilityVersion = 118 // ID is an integer ID for a user, node, or login allocated by the // control plane. @@ -878,6 +879,12 @@ type Hostinfo struct { Location *Location `json:",omitempty"` TPM *TPMInfo `json:",omitempty"` // TPM device metadata, if available + // StateEncrypted reports whether the node state is stored encrypted on + // disk. The actual mechanism is platform-specific: + // * Apple nodes use the Keychain + // * Linux and Windows nodes use the TPM + // * Android apps use EncryptedSharedPreferences + StateEncrypted opt.Bool `json:",omitempty"` // NOTE: any new fields containing pointers in this type // require changes to Hostinfo.Equal. diff --git a/tailcfg/tailcfg_clone.go b/tailcfg/tailcfg_clone.go index 2c7941d51d7e3..412e1f38d18bc 100644 --- a/tailcfg/tailcfg_clone.go +++ b/tailcfg/tailcfg_clone.go @@ -188,6 +188,7 @@ var _HostinfoCloneNeedsRegeneration = Hostinfo(struct { ServicesHash string Location *Location TPM *TPMInfo + StateEncrypted opt.Bool }{}) // Clone makes a deep copy of NetInfo. diff --git a/tailcfg/tailcfg_test.go b/tailcfg/tailcfg_test.go index 60e86794a195c..e8e86cdb139bd 100644 --- a/tailcfg/tailcfg_test.go +++ b/tailcfg/tailcfg_test.go @@ -69,6 +69,7 @@ func TestHostinfoEqual(t *testing.T) { "ServicesHash", "Location", "TPM", + "StateEncrypted", } if have := fieldsOf(reflect.TypeFor[Hostinfo]()); !reflect.DeepEqual(have, hiHandles) { t.Errorf("Hostinfo.Equal check might be out of sync\nfields: %q\nhandled: %q\n", diff --git a/tailcfg/tailcfg_view.go b/tailcfg/tailcfg_view.go index c76654887f8ab..7e82cd871c64a 100644 --- a/tailcfg/tailcfg_view.go +++ b/tailcfg/tailcfg_view.go @@ -303,6 +303,7 @@ func (v HostinfoView) ServicesHash() string { return v.ж.Serv func (v HostinfoView) Location() LocationView { return v.ж.Location.View() } func (v HostinfoView) TPM() views.ValuePointer[TPMInfo] { return views.ValuePointerOf(v.ж.TPM) } +func (v HostinfoView) StateEncrypted() opt.Bool { return v.ж.StateEncrypted } func (v HostinfoView) Equal(v2 HostinfoView) bool { return v.ж.Equal(v2.ж) } // A compilation failure here means this code must be regenerated, with the command at the top of this file. @@ -346,6 +347,7 @@ var _HostinfoViewNeedsRegeneration = Hostinfo(struct { ServicesHash string Location *Location TPM *TPMInfo + StateEncrypted opt.Bool }{}) // View returns a read-only view of NetInfo. From f9e7131772ffc85016921fe099791ffb467cc681 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 2 Jul 2025 13:27:30 -0700 Subject: [PATCH 142/263] wgengine/magicsock: make lazyEndpoint load bearing for UDP relay (#16435) Cryptokey Routing identification is now required to set an [epAddr] into the peerMap for Geneve-encapsulated [epAddr]s. Updates tailscale/corp#27502 Updates tailscale/corp#29422 Updates tailscale/corp#30042 Signed-off-by: Jordan Whited --- go.mod | 2 +- go.sum | 4 ++-- wgengine/magicsock/endpoint.go | 1 - wgengine/magicsock/magicsock.go | 28 ++++++++++++++++++++++++---- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 0d031d0baa6c2..5bf04fedaba2e 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/tailscale/setec v0.0.0-20250205144240-8898a29c3fbb github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 - github.com/tailscale/wireguard-go v0.0.0-20250530210235-65cd6eed7d7f + github.com/tailscale/wireguard-go v0.0.0-20250701223756-24483d7a0003 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e github.com/tc-hib/winres v0.2.1 github.com/tcnksm/go-httpstat v0.2.0 diff --git a/go.sum b/go.sum index 6f44cd86eb068..f9910bb59bb4d 100644 --- a/go.sum +++ b/go.sum @@ -975,8 +975,8 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 h1:U github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y= -github.com/tailscale/wireguard-go v0.0.0-20250530210235-65cd6eed7d7f h1:vg3PmQdq1BbB2V81iC1VBICQtfwbVGZ/4A/p7QKXTK0= -github.com/tailscale/wireguard-go v0.0.0-20250530210235-65cd6eed7d7f/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= +github.com/tailscale/wireguard-go v0.0.0-20250701223756-24483d7a0003 h1:chIzUDKxR0nXQQra0j41aqiiFNICs0FIC5ZCwDO7z3k= +github.com/tailscale/wireguard-go v0.0.0-20250701223756-24483d7a0003/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg= github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA= diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index af4666665b0fb..0569341ff4ab3 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -126,7 +126,6 @@ func (de *endpoint) udpRelayEndpointReady(maybeBest addrQuality) { de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v", de.publicKey.ShortString(), de.discoShort(), maybeBest.epAddr, maybeBest.wireMTU) de.setBestAddrLocked(maybeBest) de.trustBestAddrUntil = mono.Now().Add(trustUDPAddrDuration) - de.c.peerMap.setNodeKeyForEpAddr(maybeBest.epAddr, de.publicKey) } func (de *endpoint) setBestAddrLocked(v addrQuality) { diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 89111b7a0485f..174345a84ac87 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -1695,8 +1695,13 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach c.mu.Unlock() if !ok { if c.controlKnobs != nil && c.controlKnobs.DisableCryptorouting.Load() { + // Note: UDP relay is dependent on cryptorouting enablement. We + // only update Geneve-encapsulated [epAddr]s in the [peerMap] + // via [lazyEndpoint]. return nil, 0, false } + // TODO(jwhited): reuse [lazyEndpoint] across calls to receiveIP() + // for the same batch & [epAddr] src. return &lazyEndpoint{c: c, src: src}, size, true } cache.epAddr = src @@ -1704,6 +1709,8 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach cache.gen = de.numStopAndReset() ep = de } + // TODO(jwhited): consider the implications of not recording this receive + // activity due to an early [lazyEndpoint] return above. now := mono.Now() ep.lastRecvUDPAny.StoreAtomic(now) ep.noteRecvActivity(src, now) @@ -3793,14 +3800,27 @@ func (le *lazyEndpoint) DstIP() netip.Addr { return netip.Addr{} } func (le *lazyEndpoint) SrcToString() string { return le.src.String() } func (le *lazyEndpoint) DstToString() string { return "dst" } func (le *lazyEndpoint) DstToBytes() []byte { return nil } -func (le *lazyEndpoint) GetPeerEndpoint(peerPublicKey [32]byte) conn.Endpoint { + +// FromPeer implements [conn.PeerAwareEndpoint]. We return a [*lazyEndpoint] in +// our [conn.ReceiveFunc]s when we are unable to identify the peer at WireGuard +// packet reception time, pre-decryption. If wireguard-go successfully decrypts +// the packet it calls us here, and we update our [peerMap] in order to +// associate le.src with peerPublicKey. +func (le *lazyEndpoint) FromPeer(peerPublicKey [32]byte) { pubKey := key.NodePublicFromRaw32(mem.B(peerPublicKey[:])) le.c.mu.Lock() defer le.c.mu.Unlock() ep, ok := le.c.peerMap.endpointForNodeKey(pubKey) if !ok { - return nil + return } - le.c.logf("magicsock: lazyEndpoint.GetPeerEndpoint(%v) found: %v", pubKey.ShortString(), ep.nodeAddr) - return ep + // TODO(jwhited): Consider [lazyEndpoint] effectiveness as a means to make + // this the sole call site for setNodeKeyForEpAddr. If this is the sole + // call site, and we always update the mapping based on successful + // Cryptokey Routing identification events, then we can go ahead and make + // [epAddr]s singular per peer (like they are for Geneve-encapsulated ones + // already). + // See http://go/corp/29422 & http://go/corp/30042 + le.c.peerMap.setNodeKeyForEpAddr(le.src, pubKey) + le.c.logf("magicsock: lazyEndpoint.FromPeer(%v) setting epAddr(%v) in peerMap for node(%v)", pubKey.ShortString(), le.src, ep.nodeAddr) } From eb03d42fe60acce0e7efacc3a026b26bfb56897c Mon Sep 17 00:00:00 2001 From: David Bond Date: Wed, 2 Jul 2025 21:42:31 +0100 Subject: [PATCH 143/263] cmd/k8s-operator: Allow configuration of login server (#16432) This commit modifies the kubernetes operator to allow for customisation of the tailscale login url. This provides some data locality for people that want to configure it. This value is set in the `loginServer` helm value and is propagated down to all resources managed by the operator. The only exception to this is recorder nodes, where additional changes are required to support modifying the url. Updates https://github.com/tailscale/corp/issues/29847 Signed-off-by: David Bond --- cmd/k8s-operator/connector.go | 5 +++-- .../deploy/chart/templates/deployment.yaml | 2 ++ cmd/k8s-operator/deploy/chart/values.yaml | 3 +++ cmd/k8s-operator/deploy/manifests/operator.yaml | 2 ++ cmd/k8s-operator/ingress.go | 2 ++ cmd/k8s-operator/operator.go | 11 ++++++++--- cmd/k8s-operator/proxygroup.go | 10 ++++++++-- cmd/k8s-operator/sts.go | 9 +++++++++ cmd/k8s-operator/svc.go | 2 ++ cmd/k8s-operator/tsclient.go | 14 +++++++++++--- 10 files changed, 50 insertions(+), 10 deletions(-) diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index c243036cbabd9..8406a1156fc8f 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -7,6 +7,7 @@ package main import ( "context" + "errors" "fmt" "net/netip" "slices" @@ -14,8 +15,6 @@ import ( "sync" "time" - "errors" - "go.uber.org/zap" xslices "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" @@ -26,6 +25,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -199,6 +199,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge }, ProxyClassName: proxyClass, proxyType: proxyTypeConnector, + LoginServer: a.ssr.loginServer, } if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 { diff --git a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml index 1b9b97186b6ca..8deba7dab0139 100644 --- a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml @@ -68,6 +68,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OPERATOR_LOGIN_SERVER + value: {{ .Values.operatorConfig.loginServer }} - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE diff --git a/cmd/k8s-operator/deploy/chart/values.yaml b/cmd/k8s-operator/deploy/chart/values.yaml index 2d1effc255dc5..af941425a5006 100644 --- a/cmd/k8s-operator/deploy/chart/values.yaml +++ b/cmd/k8s-operator/deploy/chart/values.yaml @@ -72,6 +72,9 @@ operatorConfig: # - name: EXTRA_VAR2 # value: "value2" + # URL of the control plane to be used by all resources managed by the operator. + loginServer: "" + # In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here ingressClass: enabled: true diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index fa18a5debeaa9..4f1faf104cfc6 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -5124,6 +5124,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OPERATOR_LOGIN_SERVER + value: null - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 5058fd6dda8fc..d6277093824fb 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/ipn" "tailscale.com/kube/kubetypes" "tailscale.com/types/opt" @@ -219,6 +220,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ChildResourceLabels: crl, ProxyClassName: proxyClass, proxyType: proxyTypeIngressResource, + LoginServer: a.ssr.loginServer, } if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" { diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index b33dcd114d32b..e5f7d932cc876 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -43,6 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager/signals" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/client/local" "tailscale.com/client/tailscale" "tailscale.com/hostinfo" @@ -144,18 +145,20 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) { hostname = defaultEnv("OPERATOR_HOSTNAME", "tailscale-operator") kubeSecret = defaultEnv("OPERATOR_SECRET", "") operatorTags = defaultEnv("OPERATOR_INITIAL_TAGS", "tag:k8s-operator") + loginServer = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/") ) startlog := zlog.Named("startup") if clientIDPath == "" || clientSecretPath == "" { startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set") } - tsc, err := newTSClient(context.Background(), clientIDPath, clientSecretPath) + tsc, err := newTSClient(context.Background(), clientIDPath, clientSecretPath, loginServer) if err != nil { startlog.Fatalf("error creating Tailscale client: %v", err) } s := &tsnet.Server{ - Hostname: hostname, - Logf: zlog.Named("tailscaled").Debugf, + Hostname: hostname, + Logf: zlog.Named("tailscaled").Debugf, + ControlURL: loginServer, } if p := os.Getenv("TS_PORT"); p != "" { port, err := strconv.ParseUint(p, 10, 16) @@ -307,6 +310,7 @@ func runReconcilers(opts reconcilerOpts) { proxyImage: opts.proxyImage, proxyPriorityClassName: opts.proxyPriorityClassName, tsFirewallMode: opts.proxyFirewallMode, + loginServer: opts.tsServer.ControlURL, } err = builder. @@ -639,6 +643,7 @@ func runReconcilers(opts reconcilerOpts) { defaultTags: strings.Split(opts.proxyTags, ","), tsFirewallMode: opts.proxyFirewallMode, defaultProxyClass: opts.defaultProxyClass, + loginServer: opts.tsServer.ControlURL, }) if err != nil { startlog.Fatalf("could not create ProxyGroup reconciler: %v", err) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index bedf06ba0ac28..1b622c920d22d 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/client/tailscale" "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" @@ -84,6 +85,7 @@ type ProxyGroupReconciler struct { defaultTags []string tsFirewallMode string defaultProxyClass string + loginServer string mu sync.Mutex // protects following egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge @@ -709,7 +711,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p return nil, err } - configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[replicaName], existingAdvertiseServices) + configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[replicaName], existingAdvertiseServices, r.loginServer) if err != nil { return nil, fmt.Errorf("error creating tailscaled config: %w", err) } @@ -859,7 +861,7 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) } -func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string) (tailscaledConfigs, error) { +func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string, loginServer string) (tailscaledConfigs, error) { conf := &ipn.ConfigVAlpha{ Version: "alpha0", AcceptDNS: "false", @@ -870,6 +872,10 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a AuthKey: authKey, } + if loginServer != "" { + conf.ServerURL = &loginServer + } + if pg.Spec.HostnamePrefix != "" { conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx)) } diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index a943ae97179a1..193acad87ff0e 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -27,6 +27,7 @@ import ( "k8s.io/apiserver/pkg/storage/names" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" + "tailscale.com/client/tailscale" "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" @@ -138,6 +139,9 @@ type tailscaleSTSConfig struct { ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one) + + // LoginServer denotes the URL of the control plane that should be used by the proxy. + LoginServer string } type connector struct { @@ -162,6 +166,7 @@ type tailscaleSTSReconciler struct { proxyImage string proxyPriorityClassName string tsFirewallMode string + loginServer string } func (sts tailscaleSTSReconciler) validate() error { @@ -910,6 +915,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co AppConnector: &ipn.AppConnectorPrefs{Advertise: false}, } + if stsC.LoginServer != "" { + conf.ServerURL = &stsC.LoginServer + } + if stsC.Connector != nil { routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode) if err != nil { diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index f8c9af23990e9..52c8bec7ff32a 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -23,6 +23,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -270,6 +271,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga Tags: tags, ChildResourceLabels: crl, ProxyClassName: proxyClass, + LoginServer: a.ssr.loginServer, } sts.proxyType = proxyTypeEgress if a.shouldExpose(svc) { diff --git a/cmd/k8s-operator/tsclient.go b/cmd/k8s-operator/tsclient.go index f49f84af96ed4..a94d55afed604 100644 --- a/cmd/k8s-operator/tsclient.go +++ b/cmd/k8s-operator/tsclient.go @@ -12,6 +12,7 @@ import ( "golang.org/x/oauth2/clientcredentials" "tailscale.com/internal/client/tailscale" + "tailscale.com/ipn" "tailscale.com/tailcfg" ) @@ -19,10 +20,9 @@ import ( // call should be performed on the default tailnet for the provided credentials. const ( defaultTailnet = "-" - defaultBaseURL = "https://api.tailscale.com" ) -func newTSClient(ctx context.Context, clientIDPath, clientSecretPath string) (tsClient, error) { +func newTSClient(ctx context.Context, clientIDPath, clientSecretPath, loginServer string) (tsClient, error) { clientID, err := os.ReadFile(clientIDPath) if err != nil { return nil, fmt.Errorf("error reading client ID %q: %w", clientIDPath, err) @@ -31,14 +31,22 @@ func newTSClient(ctx context.Context, clientIDPath, clientSecretPath string) (ts if err != nil { return nil, fmt.Errorf("reading client secret %q: %w", clientSecretPath, err) } + const tokenURLPath = "/api/v2/oauth/token" + tokenURL := fmt.Sprintf("%s%s", ipn.DefaultControlURL, tokenURLPath) + if loginServer != "" { + tokenURL = fmt.Sprintf("%s%s", loginServer, tokenURLPath) + } credentials := clientcredentials.Config{ ClientID: string(clientID), ClientSecret: string(clientSecret), - TokenURL: "https://login.tailscale.com/api/v2/oauth/token", + TokenURL: tokenURL, } c := tailscale.NewClient(defaultTailnet, nil) c.UserAgent = "tailscale-k8s-operator" c.HTTPClient = credentials.Client(ctx) + if loginServer != "" { + c.BaseURL = loginServer + } return c, nil } From 77d19604f449ac65092e232c93d28f9e686df161 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Wed, 2 Jul 2025 14:32:21 -0700 Subject: [PATCH 144/263] derp/derphttp: fix DERP TLS client server name inclusion in URL form When dialed with just an URL and no node, the recent proxy fixes caused a regression where there was no TLS server name being included. Updates #16222 Updates #16223 Signed-off-by: James Tucker Co-Authored-by: Jordan Whited --- derp/derphttp/derphttp_client.go | 4 +++- derp/derphttp/derphttp_test.go | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/derp/derphttp/derphttp_client.go b/derp/derphttp/derphttp_client.go index 7385f0ad1b46f..704b8175d07c6 100644 --- a/derp/derphttp/derphttp_client.go +++ b/derp/derphttp/derphttp_client.go @@ -648,12 +648,14 @@ func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.C func (c *Client) tlsClient(nc net.Conn, node *tailcfg.DERPNode) *tls.Conn { tlsConf := tlsdial.Config(c.HealthTracker, c.TLSConfig) + // node is allowed to be nil here, tlsServerName falls back to using the URL + // if node is nil. + tlsConf.ServerName = c.tlsServerName(node) if node != nil { if node.InsecureForTests { tlsConf.InsecureSkipVerify = true tlsConf.VerifyConnection = nil } - tlsConf.ServerName = c.tlsServerName(node) if node.CertName != "" { if suf, ok := strings.CutPrefix(node.CertName, "sha256-raw:"); ok { tlsdial.SetConfigExpectedCertHash(tlsConf, suf) diff --git a/derp/derphttp/derphttp_test.go b/derp/derphttp/derphttp_test.go index 7f0a7e3334abf..bb33e60232357 100644 --- a/derp/derphttp/derphttp_test.go +++ b/derp/derphttp/derphttp_test.go @@ -590,3 +590,39 @@ func TestManualDial(t *testing.T) { t.Fatalf("rc.Connect: %v", err) } } + +func TestURLDial(t *testing.T) { + if !*liveNetworkTest { + t.Skip("skipping live network test without --live-net-tests") + } + dm := &tailcfg.DERPMap{} + res, err := http.Get("https://controlplane.tailscale.com/derpmap/default") + if err != nil { + t.Fatalf("fetching DERPMap: %v", err) + } + defer res.Body.Close() + if err := json.NewDecoder(res.Body).Decode(dm); err != nil { + t.Fatalf("decoding DERPMap: %v", err) + } + + // find a valid target DERP host to test against + var hostname string + for _, reg := range dm.Regions { + for _, node := range reg.Nodes { + if !node.STUNOnly && node.CanPort80 && node.CertName == "" || node.CertName == node.HostName { + hostname = node.HostName + break + } + } + if hostname != "" { + break + } + } + netMon := netmon.NewStatic() + c, err := NewClient(key.NewNode(), "https://"+hostname+"/", t.Logf, netMon) + defer c.Close() + + if err := c.Connect(context.Background()); err != nil { + t.Fatalf("rc.Connect: %v", err) + } +} From 3a4b439c62ba30f882e50a08ae4b93f087501847 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 2 Jul 2025 20:38:39 -0700 Subject: [PATCH 145/263] feature/relayserver,net/udprelay: add IPv6 support (#16442) Updates tailscale/corp#27502 Updates tailscale/corp#30043 Signed-off-by: Jordan Whited --- feature/relayserver/relayserver.go | 2 +- net/udprelay/server.go | 133 +++++++++++++++++-------- net/udprelay/server_test.go | 154 ++++++++++++++++------------- 3 files changed, 178 insertions(+), 111 deletions(-) diff --git a/feature/relayserver/relayserver.go b/feature/relayserver/relayserver.go index 4634f3ac27151..5a82a9d117bd7 100644 --- a/feature/relayserver/relayserver.go +++ b/feature/relayserver/relayserver.go @@ -137,7 +137,7 @@ func (e *extension) relayServerOrInit() (relayServer, error) { return nil, errors.New("TAILSCALE_USE_WIP_CODE envvar is not set") } var err error - e.server, _, err = udprelay.NewServer(e.logf, *e.port, nil) + e.server, err = udprelay.NewServer(e.logf, *e.port, nil) if err != nil { return nil, err } diff --git a/net/udprelay/server.go b/net/udprelay/server.go index e32f8917c520c..d2661e59feba4 100644 --- a/net/udprelay/server.go +++ b/net/udprelay/server.go @@ -57,7 +57,10 @@ type Server struct { bindLifetime time.Duration steadyStateLifetime time.Duration bus *eventbus.Bus - uc *net.UDPConn + uc4 *net.UDPConn // always non-nil + uc4Port uint16 // always nonzero + uc6 *net.UDPConn // may be nil if IPv6 bind fails during initialization + uc6Port uint16 // may be zero if IPv6 bind fails during initialization closeOnce sync.Once wg sync.WaitGroup closeCh chan struct{} @@ -278,13 +281,11 @@ func (e *serverEndpoint) isBound() bool { e.boundAddrPorts[1].IsValid() } -// NewServer constructs a [Server] listening on 0.0.0.0:'port'. IPv6 is not yet -// supported. Port may be 0, and what ultimately gets bound is returned as -// 'boundPort'. If len(overrideAddrs) > 0 these will be used in place of dynamic -// discovery, which is useful to override in tests. -// -// TODO: IPv6 support -func NewServer(logf logger.Logf, port int, overrideAddrs []netip.Addr) (s *Server, boundPort uint16, err error) { +// NewServer constructs a [Server] listening on port. If port is zero, then +// port selection is left up to the host networking stack. If +// len(overrideAddrs) > 0 these will be used in place of dynamic discovery, +// which is useful to override in tests. +func NewServer(logf logger.Logf, port int, overrideAddrs []netip.Addr) (s *Server, err error) { s = &Server{ logf: logger.WithPrefix(logf, "relayserver"), disco: key.NewDisco(), @@ -306,30 +307,36 @@ func NewServer(logf logger.Logf, port int, overrideAddrs []netip.Addr) (s *Serve s.bus = bus netMon, err := netmon.New(s.bus, logf) if err != nil { - return nil, 0, err + return nil, err } s.netChecker = &netcheck.Client{ NetMon: netMon, Logf: logger.WithPrefix(logf, "relayserver: netcheck:"), SendPacket: func(b []byte, addrPort netip.AddrPort) (int, error) { - return s.uc.WriteToUDPAddrPort(b, addrPort) + if addrPort.Addr().Is4() { + return s.uc4.WriteToUDPAddrPort(b, addrPort) + } else if s.uc6 != nil { + return s.uc6.WriteToUDPAddrPort(b, addrPort) + } else { + return 0, errors.New("IPv6 socket is not bound") + } }, } - boundPort, err = s.listenOn(port) + err = s.listenOn(port) if err != nil { - return nil, 0, err + return nil, err } - s.wg.Add(1) - go s.packetReadLoop() - s.wg.Add(1) - go s.endpointGCLoop() if len(overrideAddrs) > 0 { addrPorts := make(set.Set[netip.AddrPort], len(overrideAddrs)) for _, addr := range overrideAddrs { if addr.IsValid() { - addrPorts.Add(netip.AddrPortFrom(addr, boundPort)) + if addr.Is4() { + addrPorts.Add(netip.AddrPortFrom(addr, s.uc4Port)) + } else if s.uc6 != nil { + addrPorts.Add(netip.AddrPortFrom(addr, s.uc6Port)) + } } } s.addrPorts = addrPorts.Slice() @@ -337,7 +344,17 @@ func NewServer(logf logger.Logf, port int, overrideAddrs []netip.Addr) (s *Serve s.wg.Add(1) go s.addrDiscoveryLoop() } - return s, boundPort, nil + + s.wg.Add(1) + go s.packetReadLoop(s.uc4) + if s.uc6 != nil { + s.wg.Add(1) + go s.packetReadLoop(s.uc6) + } + s.wg.Add(1) + go s.endpointGCLoop() + + return s, nil } func (s *Server) addrDiscoveryLoop() { @@ -351,14 +368,17 @@ func (s *Server) addrDiscoveryLoop() { addrPorts.Make() // get local addresses - localPort := s.uc.LocalAddr().(*net.UDPAddr).Port ips, _, err := netmon.LocalAddresses() if err != nil { return nil, err } for _, ip := range ips { if ip.IsValid() { - addrPorts.Add(netip.AddrPortFrom(ip, uint16(localPort))) + if ip.Is4() { + addrPorts.Add(netip.AddrPortFrom(ip, s.uc4Port)) + } else { + addrPorts.Add(netip.AddrPortFrom(ip, s.uc6Port)) + } } } @@ -413,24 +433,52 @@ func (s *Server) addrDiscoveryLoop() { } } -func (s *Server) listenOn(port int) (uint16, error) { - uc, err := net.ListenUDP("udp4", &net.UDPAddr{Port: port}) - if err != nil { - return 0, err - } - // TODO: set IP_PKTINFO sockopt - _, boundPortStr, err := net.SplitHostPort(uc.LocalAddr().String()) - if err != nil { - s.uc.Close() - return 0, err - } - boundPort, err := strconv.ParseUint(boundPortStr, 10, 16) - if err != nil { - s.uc.Close() - return 0, err +// listenOn binds an IPv4 and IPv6 socket to port. We consider it successful if +// we manage to bind the IPv4 socket. +// +// The requested port may be zero, in which case port selection is left up to +// the host networking stack. We make no attempt to bind a consistent port +// across IPv4 and IPv6 if the requested port is zero. +// +// TODO: make these "re-bindable" in similar fashion to magicsock as a means to +// deal with EDR software closing them. http://go/corp/30118 +func (s *Server) listenOn(port int) error { + for _, network := range []string{"udp4", "udp6"} { + uc, err := net.ListenUDP(network, &net.UDPAddr{Port: port}) + if err != nil { + if network == "udp4" { + return err + } else { + s.logf("ignoring IPv6 bind failure: %v", err) + break + } + } + // TODO: set IP_PKTINFO sockopt + _, boundPortStr, err := net.SplitHostPort(uc.LocalAddr().String()) + if err != nil { + uc.Close() + if s.uc4 != nil { + s.uc4.Close() + } + return err + } + portUint, err := strconv.ParseUint(boundPortStr, 10, 16) + if err != nil { + uc.Close() + if s.uc4 != nil { + s.uc4.Close() + } + return err + } + if network == "udp4" { + s.uc4 = uc + s.uc4Port = uint16(portUint) + } else { + s.uc6 = uc + s.uc6Port = uint16(portUint) + } } - s.uc = uc - return uint16(boundPort), nil + return nil } // Close closes the server. @@ -438,7 +486,10 @@ func (s *Server) Close() error { s.closeOnce.Do(func() { s.mu.Lock() defer s.mu.Unlock() - s.uc.Close() + s.uc4.Close() + if s.uc6 != nil { + s.uc6.Close() + } close(s.closeCh) s.wg.Wait() clear(s.byVNI) @@ -507,7 +558,7 @@ func (s *Server) handlePacket(from netip.AddrPort, b []byte, uw udpWriter) { e.handlePacket(from, gh, b, uw, s.discoPublic) } -func (s *Server) packetReadLoop() { +func (s *Server) packetReadLoop(uc *net.UDPConn) { defer func() { s.wg.Done() s.Close() @@ -515,11 +566,11 @@ func (s *Server) packetReadLoop() { b := make([]byte, 1<<16-1) for { // TODO: extract laddr from IP_PKTINFO for use in reply - n, from, err := s.uc.ReadFromUDPAddrPort(b) + n, from, err := uc.ReadFromUDPAddrPort(b) if err != nil { return } - s.handlePacket(from, b[:n], s.uc) + s.handlePacket(from, b[:n], uc) } } diff --git a/net/udprelay/server_test.go b/net/udprelay/server_test.go index 3fcb9b8b198c2..8c0c5aff66027 100644 --- a/net/udprelay/server_test.go +++ b/net/udprelay/server_test.go @@ -29,7 +29,7 @@ type testClient struct { func newTestClient(t *testing.T, vni uint32, serverEndpoint netip.AddrPort, local key.DiscoPrivate, remote, server key.DiscoPublic) *testClient { rAddr := &net.UDPAddr{IP: serverEndpoint.Addr().AsSlice(), Port: int(serverEndpoint.Port())} - uc, err := net.DialUDP("udp4", nil, rAddr) + uc, err := net.DialUDP("udp", nil, rAddr) if err != nil { t.Fatal(err) } @@ -180,85 +180,101 @@ func TestServer(t *testing.T) { discoA := key.NewDisco() discoB := key.NewDisco() - ipv4LoopbackAddr := netip.MustParseAddr("127.0.0.1") - - server, _, err := NewServer(t.Logf, 0, []netip.Addr{ipv4LoopbackAddr}) - if err != nil { - t.Fatal(err) + cases := []struct { + name string + overrideAddrs []netip.Addr + }{ + { + name: "over ipv4", + overrideAddrs: []netip.Addr{netip.MustParseAddr("127.0.0.1")}, + }, + { + name: "over ipv6", + overrideAddrs: []netip.Addr{netip.MustParseAddr("::1")}, + }, } - defer server.Close() - endpoint, err := server.AllocateEndpoint(discoA.Public(), discoB.Public()) - if err != nil { - t.Fatal(err) - } - dupEndpoint, err := server.AllocateEndpoint(discoA.Public(), discoB.Public()) - if err != nil { - t.Fatal(err) - } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + server, err := NewServer(t.Logf, 0, tt.overrideAddrs) + if err != nil { + t.Fatal(err) + } + defer server.Close() - // We expect the same endpoint details pre-handshake. - if diff := cmp.Diff(dupEndpoint, endpoint, cmpopts.EquateComparable(netip.AddrPort{}, key.DiscoPublic{})); diff != "" { - t.Fatalf("wrong dupEndpoint (-got +want)\n%s", diff) - } + endpoint, err := server.AllocateEndpoint(discoA.Public(), discoB.Public()) + if err != nil { + t.Fatal(err) + } + dupEndpoint, err := server.AllocateEndpoint(discoA.Public(), discoB.Public()) + if err != nil { + t.Fatal(err) + } - if len(endpoint.AddrPorts) != 1 { - t.Fatalf("unexpected endpoint.AddrPorts: %v", endpoint.AddrPorts) - } - tcA := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoA, discoB.Public(), endpoint.ServerDisco) - defer tcA.close() - tcB := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoB, discoA.Public(), endpoint.ServerDisco) - defer tcB.close() + // We expect the same endpoint details pre-handshake. + if diff := cmp.Diff(dupEndpoint, endpoint, cmpopts.EquateComparable(netip.AddrPort{}, key.DiscoPublic{})); diff != "" { + t.Fatalf("wrong dupEndpoint (-got +want)\n%s", diff) + } - tcA.handshake(t) - tcB.handshake(t) + if len(endpoint.AddrPorts) != 1 { + t.Fatalf("unexpected endpoint.AddrPorts: %v", endpoint.AddrPorts) + } + tcA := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoA, discoB.Public(), endpoint.ServerDisco) + defer tcA.close() + tcB := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoB, discoA.Public(), endpoint.ServerDisco) + defer tcB.close() - dupEndpoint, err = server.AllocateEndpoint(discoA.Public(), discoB.Public()) - if err != nil { - t.Fatal(err) - } - // We expect the same endpoint details post-handshake. - if diff := cmp.Diff(dupEndpoint, endpoint, cmpopts.EquateComparable(netip.AddrPort{}, key.DiscoPublic{})); diff != "" { - t.Fatalf("wrong dupEndpoint (-got +want)\n%s", diff) - } + tcA.handshake(t) + tcB.handshake(t) - txToB := []byte{1, 2, 3} - tcA.writeDataPkt(t, txToB) - rxFromA := tcB.readDataPkt(t) - if !bytes.Equal(txToB, rxFromA) { - t.Fatal("unexpected msg A->B") - } + dupEndpoint, err = server.AllocateEndpoint(discoA.Public(), discoB.Public()) + if err != nil { + t.Fatal(err) + } + // We expect the same endpoint details post-handshake. + if diff := cmp.Diff(dupEndpoint, endpoint, cmpopts.EquateComparable(netip.AddrPort{}, key.DiscoPublic{})); diff != "" { + t.Fatalf("wrong dupEndpoint (-got +want)\n%s", diff) + } - txToA := []byte{4, 5, 6} - tcB.writeDataPkt(t, txToA) - rxFromB := tcA.readDataPkt(t) - if !bytes.Equal(txToA, rxFromB) { - t.Fatal("unexpected msg B->A") - } + txToB := []byte{1, 2, 3} + tcA.writeDataPkt(t, txToB) + rxFromA := tcB.readDataPkt(t) + if !bytes.Equal(txToB, rxFromA) { + t.Fatal("unexpected msg A->B") + } - tcAOnNewPort := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoA, discoB.Public(), endpoint.ServerDisco) - tcAOnNewPort.handshakeGeneration = tcA.handshakeGeneration + 1 - defer tcAOnNewPort.close() + txToA := []byte{4, 5, 6} + tcB.writeDataPkt(t, txToA) + rxFromB := tcA.readDataPkt(t) + if !bytes.Equal(txToA, rxFromB) { + t.Fatal("unexpected msg B->A") + } - // Handshake client A on a new source IP:port, verify we receive packets on the new binding - tcAOnNewPort.handshake(t) - txToAOnNewPort := []byte{7, 8, 9} - tcB.writeDataPkt(t, txToAOnNewPort) - rxFromB = tcAOnNewPort.readDataPkt(t) - if !bytes.Equal(txToAOnNewPort, rxFromB) { - t.Fatal("unexpected msg B->A") - } + tcAOnNewPort := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoA, discoB.Public(), endpoint.ServerDisco) + tcAOnNewPort.handshakeGeneration = tcA.handshakeGeneration + 1 + defer tcAOnNewPort.close() + + // Handshake client A on a new source IP:port, verify we receive packets on the new binding + tcAOnNewPort.handshake(t) + txToAOnNewPort := []byte{7, 8, 9} + tcB.writeDataPkt(t, txToAOnNewPort) + rxFromB = tcAOnNewPort.readDataPkt(t) + if !bytes.Equal(txToAOnNewPort, rxFromB) { + t.Fatal("unexpected msg B->A") + } - tcBOnNewPort := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoB, discoA.Public(), endpoint.ServerDisco) - tcBOnNewPort.handshakeGeneration = tcB.handshakeGeneration + 1 - defer tcBOnNewPort.close() + tcBOnNewPort := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoB, discoA.Public(), endpoint.ServerDisco) + tcBOnNewPort.handshakeGeneration = tcB.handshakeGeneration + 1 + defer tcBOnNewPort.close() - // Handshake client B on a new source IP:port, verify we receive packets on the new binding - tcBOnNewPort.handshake(t) - txToBOnNewPort := []byte{7, 8, 9} - tcAOnNewPort.writeDataPkt(t, txToBOnNewPort) - rxFromA = tcBOnNewPort.readDataPkt(t) - if !bytes.Equal(txToBOnNewPort, rxFromA) { - t.Fatal("unexpected msg A->B") + // Handshake client B on a new source IP:port, verify we receive packets on the new binding + tcBOnNewPort.handshake(t) + txToBOnNewPort := []byte{7, 8, 9} + tcAOnNewPort.writeDataPkt(t, txToBOnNewPort) + rxFromA = tcBOnNewPort.readDataPkt(t) + if !bytes.Equal(txToBOnNewPort, rxFromA) { + t.Fatal("unexpected msg A->B") + } + }) } } From 5dc11d50f787026055a0125f536e87287ce6899e Mon Sep 17 00:00:00 2001 From: David Bond Date: Thu, 3 Jul 2025 15:53:35 +0100 Subject: [PATCH 146/263] cmd/k8s-operator: Set login server on tsrecorder nodes (#16443) This commit modifies the recorder node reconciler to include the environment variable added in https://github.com/tailscale/corp/pull/30058 which allows for configuration of the coordination server. Updates https://github.com/tailscale/corp/issues/29847 Signed-off-by: David Bond --- cmd/k8s-operator/operator.go | 10 +++++++--- cmd/k8s-operator/tsrecorder.go | 3 ++- cmd/k8s-operator/tsrecorder_specs.go | 10 +++++++--- cmd/k8s-operator/tsrecorder_specs_test.go | 4 ++-- cmd/k8s-operator/tsrecorder_test.go | 8 ++++++-- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index e5f7d932cc876..276de411c45cb 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -82,6 +82,7 @@ func main() { tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") defaultProxyClass = defaultEnv("PROXY_DEFAULT_CLASS", "") isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false) + loginServer = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/") ) var opts []kzap.Opts @@ -115,7 +116,7 @@ func main() { hostinfo.SetApp(kubetypes.AppAPIServerProxy) } - s, tsc := initTSNet(zlog) + s, tsc := initTSNet(zlog, loginServer) defer s.Close() restConfig := config.GetConfigOrDie() apiproxy.MaybeLaunchAPIServerProxy(zlog, restConfig, s, mode) @@ -131,6 +132,7 @@ func main() { proxyTags: tags, proxyFirewallMode: tsFirewallMode, defaultProxyClass: defaultProxyClass, + loginServer: loginServer, } runReconcilers(rOpts) } @@ -138,14 +140,13 @@ func main() { // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the // CLIENT_ID_FILE and CLIENT_SECRET_FILE environment variables to authenticate // with Tailscale. -func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) { +func initTSNet(zlog *zap.SugaredLogger, loginServer string) (*tsnet.Server, tsClient) { var ( clientIDPath = defaultEnv("CLIENT_ID_FILE", "") clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "") hostname = defaultEnv("OPERATOR_HOSTNAME", "tailscale-operator") kubeSecret = defaultEnv("OPERATOR_SECRET", "") operatorTags = defaultEnv("OPERATOR_INITIAL_TAGS", "tag:k8s-operator") - loginServer = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/") ) startlog := zlog.Named("startup") if clientIDPath == "" || clientSecretPath == "" { @@ -610,6 +611,7 @@ func runReconcilers(opts reconcilerOpts) { l: opts.log.Named("recorder-reconciler"), clock: tstime.DefaultClock{}, tsClient: opts.tsClient, + loginServer: opts.loginServer, }) if err != nil { startlog.Fatalf("could not create Recorder reconciler: %v", err) @@ -693,6 +695,8 @@ type reconcilerOpts struct { // class for proxies that do not have a ProxyClass set. // this is defined by an operator env variable. defaultProxyClass string + // loginServer is the coordination server URL that should be used by managed resources. + loginServer string } // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each diff --git a/cmd/k8s-operator/tsrecorder.go b/cmd/k8s-operator/tsrecorder.go index cbabc1d89e475..ec95ecf40dab5 100644 --- a/cmd/k8s-operator/tsrecorder.go +++ b/cmd/k8s-operator/tsrecorder.go @@ -59,6 +59,7 @@ type RecorderReconciler struct { clock tstime.Clock tsNamespace string tsClient tsClient + loginServer string mu sync.Mutex // protects following recorders set.Slice[types.UID] // for recorders gauge @@ -202,7 +203,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco }); err != nil { return fmt.Errorf("error creating RoleBinding: %w", err) } - ss := tsrStatefulSet(tsr, r.tsNamespace) + ss := tsrStatefulSet(tsr, r.tsNamespace, r.loginServer) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { s.ObjectMeta.Labels = ss.ObjectMeta.Labels s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations diff --git a/cmd/k8s-operator/tsrecorder_specs.go b/cmd/k8s-operator/tsrecorder_specs.go index 7c6e80aed56fd..f5eedc2a1d1da 100644 --- a/cmd/k8s-operator/tsrecorder_specs.go +++ b/cmd/k8s-operator/tsrecorder_specs.go @@ -17,7 +17,7 @@ import ( "tailscale.com/version" ) -func tsrStatefulSet(tsr *tsapi.Recorder, namespace string) *appsv1.StatefulSet { +func tsrStatefulSet(tsr *tsapi.Recorder, namespace string, loginServer string) *appsv1.StatefulSet { return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: tsr.Name, @@ -59,7 +59,7 @@ func tsrStatefulSet(tsr *tsapi.Recorder, namespace string) *appsv1.StatefulSet { ImagePullPolicy: tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy, Resources: tsr.Spec.StatefulSet.Pod.Container.Resources, SecurityContext: tsr.Spec.StatefulSet.Pod.Container.SecurityContext, - Env: env(tsr), + Env: env(tsr, loginServer), EnvFrom: func() []corev1.EnvFromSource { if tsr.Spec.Storage.S3 == nil || tsr.Spec.Storage.S3.Credentials.Secret.Name == "" { return nil @@ -201,7 +201,7 @@ func tsrStateSecret(tsr *tsapi.Recorder, namespace string) *corev1.Secret { } } -func env(tsr *tsapi.Recorder) []corev1.EnvVar { +func env(tsr *tsapi.Recorder, loginServer string) []corev1.EnvVar { envs := []corev1.EnvVar{ { Name: "TS_AUTHKEY", @@ -239,6 +239,10 @@ func env(tsr *tsapi.Recorder) []corev1.EnvVar { Name: "TSRECORDER_HOSTNAME", Value: "$(POD_NAME)", }, + { + Name: "TSRECORDER_LOGIN_SERVER", + Value: loginServer, + }, } for _, env := range tsr.Spec.StatefulSet.Pod.Container.Env { diff --git a/cmd/k8s-operator/tsrecorder_specs_test.go b/cmd/k8s-operator/tsrecorder_specs_test.go index 94a8a816c69f5..49332d09b6a08 100644 --- a/cmd/k8s-operator/tsrecorder_specs_test.go +++ b/cmd/k8s-operator/tsrecorder_specs_test.go @@ -90,7 +90,7 @@ func TestRecorderSpecs(t *testing.T) { }, } - ss := tsrStatefulSet(tsr, tsNamespace) + ss := tsrStatefulSet(tsr, tsNamespace, tsLoginServer) // StatefulSet-level. if diff := cmp.Diff(ss.Annotations, tsr.Spec.StatefulSet.Annotations); diff != "" { @@ -124,7 +124,7 @@ func TestRecorderSpecs(t *testing.T) { } // Container-level. - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, env(tsr)); diff != "" { + if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, env(tsr, tsLoginServer)); diff != "" { t.Errorf("(-got +want):\n%s", diff) } if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Image, tsr.Spec.StatefulSet.Pod.Container.Image); diff != "" { diff --git a/cmd/k8s-operator/tsrecorder_test.go b/cmd/k8s-operator/tsrecorder_test.go index e6d56ef2f04c6..990bd68193e8b 100644 --- a/cmd/k8s-operator/tsrecorder_test.go +++ b/cmd/k8s-operator/tsrecorder_test.go @@ -25,7 +25,10 @@ import ( "tailscale.com/tstest" ) -const tsNamespace = "tailscale" +const ( + tsNamespace = "tailscale" + tsLoginServer = "example.tailscale.com" +) func TestRecorder(t *testing.T) { tsr := &tsapi.Recorder{ @@ -51,6 +54,7 @@ func TestRecorder(t *testing.T) { recorder: fr, l: zl.Sugar(), clock: cl, + loginServer: tsLoginServer, } t.Run("invalid_spec_gives_an_error_condition", func(t *testing.T) { @@ -234,7 +238,7 @@ func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recor role := tsrRole(tsr, tsNamespace) roleBinding := tsrRoleBinding(tsr, tsNamespace) serviceAccount := tsrServiceAccount(tsr, tsNamespace) - statefulSet := tsrStatefulSet(tsr, tsNamespace) + statefulSet := tsrStatefulSet(tsr, tsNamespace, tsLoginServer) if shouldExist { expectEqual(t, fc, auth) From 1a2185b1ee2d96ade04fb9f4e43eff5915b9b22a Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Wed, 2 Jul 2025 19:06:54 -0500 Subject: [PATCH 147/263] ipn/ipnlocal: rename setAutoExitNodeIDLockedOnEntry to pickNewAutoExitNode; drop old function Currently, (*LocalBackend).pickNewAutoExitNode() is just a wrapper around setAutoExitNodeIDLockedOnEntry that sends a prefs-change notification at the end. It doesn't need to do that, since setPrefsLockedOnEntry already sends the notification (setAutoExitNodeIDLockedOnEntry calls it via editPrefsLockedOnEntry). This PR removes the old pickNewAutoExitNode function and renames setAutoExitNodeIDLockedOnEntry to pickNewAutoExitNode for clarity. Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 9c16d55af45f6..bea5085b7761c 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -2001,20 +2001,6 @@ func mutationsAreWorthyOfTellingIPNBus(muts []netmap.NodeMutation) bool { return false } -// pickNewAutoExitNode picks a new automatic exit node if needed. -func (b *LocalBackend) pickNewAutoExitNode() { - unlock := b.lockAndGetUnlock() - defer unlock() - - newPrefs := b.setAutoExitNodeIDLockedOnEntry(unlock) - if !newPrefs.Valid() { - // Unchanged. - return - } - - b.send(ipn.Notify{Prefs: &newPrefs}) -} - // setExitNodeID updates prefs to reference an exit node by ID, rather // than by IP. It returns whether prefs was mutated. func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap) (prefsChanged bool) { @@ -5840,40 +5826,37 @@ func (b *LocalBackend) setNetInfo(ni *tailcfg.NetInfo) { } cc.SetNetInfo(ni) if refresh { - unlock := b.lockAndGetUnlock() - defer unlock() - b.setAutoExitNodeIDLockedOnEntry(unlock) + b.pickNewAutoExitNode() } } -func (b *LocalBackend) setAutoExitNodeIDLockedOnEntry(unlock unlockOnce) (newPrefs ipn.PrefsView) { - var zero ipn.PrefsView +// pickNewAutoExitNode picks a new automatic exit node if needed. +func (b *LocalBackend) pickNewAutoExitNode() { + unlock := b.lockAndGetUnlock() defer unlock() prefs := b.pm.CurrentPrefs() if !prefs.Valid() { b.logf("[unexpected]: received tailnet exit node ID pref change callback but current prefs are nil") - return zero + return } prefsClone := prefs.AsStruct() newSuggestion, err := b.suggestExitNodeLocked(nil) if err != nil { b.logf("setAutoExitNodeID: %v", err) - return zero + return } if prefsClone.ExitNodeID == newSuggestion.ID { - return zero + return } prefsClone.ExitNodeID = newSuggestion.ID - newPrefs, err = b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ + _, err = b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ Prefs: *prefsClone, ExitNodeIDSet: true, }, unlock) if err != nil { b.logf("setAutoExitNodeID: failed to apply exit node ID preference: %v", err) - return zero } - return newPrefs } // setNetMapLocked updates the LocalBackend state to reflect the newly From 56d772bd63e5caf711ec7ffe63967d05e33307df Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Wed, 2 Jul 2025 19:16:39 -0500 Subject: [PATCH 148/263] ipn/ipnlocal: simplify pickNewAutoExitNode (*profileManager).CurrentPrefs() is always valid. Additionally, there's no value in cloning and passing the full ipn.Prefs when editing preferences. Instead, ipn.MaskedPrefs should only have ExitNodeID set. Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index bea5085b7761c..adc0af5cdac36 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -5835,23 +5835,16 @@ func (b *LocalBackend) pickNewAutoExitNode() { unlock := b.lockAndGetUnlock() defer unlock() - prefs := b.pm.CurrentPrefs() - if !prefs.Valid() { - b.logf("[unexpected]: received tailnet exit node ID pref change callback but current prefs are nil") - return - } - prefsClone := prefs.AsStruct() newSuggestion, err := b.suggestExitNodeLocked(nil) if err != nil { b.logf("setAutoExitNodeID: %v", err) return } - if prefsClone.ExitNodeID == newSuggestion.ID { + if b.pm.CurrentPrefs().ExitNodeID() == newSuggestion.ID { return } - prefsClone.ExitNodeID = newSuggestion.ID _, err = b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ - Prefs: *prefsClone, + Prefs: ipn.Prefs{ExitNodeID: newSuggestion.ID}, ExitNodeIDSet: true, }, unlock) if err != nil { From 6ecc25b26a8edf191cfbebe2f16254468b1f1695 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Thu, 3 Jul 2025 11:50:27 -0500 Subject: [PATCH 149/263] ipn/ipnlocal: skip TestUpdateNetmapDeltaAutoExitNode suggestExitNode never checks whether an exit node candidate is online. It also accepts a full netmap, which doesn't include changes from delta updates. The test can't work correctly until both issues are fixed. Previously, it passed only because the test itself is flawed. It doesn't succeed because the currently selected node goes offline and a new one is chosen. Instead, it succeeds because lastSuggestedExitNode is incorrect, and suggestExitNode picks the correct node the first time it runs, based on the DERP map and the netcheck report. The node in exitNodeIDWant just happens to be the optimal choice. Fixing SuggestExitNode requires refactoring its callers first, which in turn reveals the flawed test, as suggestExitNode ends up being called slightly earlier. In this PR, we update the test to correctly fail due to existing bugs in SuggestExitNode, and temporarily skip it until those issues are addressed in a future commit. Updates #16455 Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local_test.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 47e5fa37d11cc..06acd85ce6023 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -1918,8 +1918,10 @@ func TestSetExitNodeIDPolicy(t *testing.T) { } func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { - peer1 := makePeer(1, withCap(26), withSuggest(), withExitRoutes()) - peer2 := makePeer(2, withCap(26), withSuggest(), withExitRoutes()) + t.Skip("TODO(tailscale/tailscale#16455): suggestExitNode does not check for online status of exit nodes") + + peer1 := makePeer(1, withCap(26), withSuggest(), withOnline(true), withExitRoutes()) + peer2 := makePeer(2, withCap(26), withSuggest(), withOnline(true), withExitRoutes()) derpMap := &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ 1: { @@ -1958,8 +1960,10 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { }{ { // selected auto exit node goes offline - name: "exit-node-goes-offline", - lastSuggestedExitNode: peer1.StableID(), + name: "exit-node-goes-offline", + // PreferredDERP is 2, and it's also the region with the lowest latency. + // So, peer2 should be selected as the exit node. + lastSuggestedExitNode: peer2.StableID(), netmap: &netmap.NetworkMap{ Peers: []tailcfg.NodeView{ peer1, @@ -1970,14 +1974,14 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { muts: []*tailcfg.PeerChange{ { NodeID: 1, - Online: ptr.To(false), + Online: ptr.To(true), }, { NodeID: 2, - Online: ptr.To(true), + Online: ptr.To(false), // the selected exit node goes offline }, }, - exitNodeIDWant: peer2.StableID(), + exitNodeIDWant: peer1.StableID(), report: report, }, { @@ -1994,7 +1998,7 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { muts: []*tailcfg.PeerChange{ { NodeID: 1, - Online: ptr.To(false), + Online: ptr.To(false), // a different exit node goes offline }, { NodeID: 2, From 009882298135672522e0fa9dac1b9fe32a71581a Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Thu, 3 Jul 2025 11:51:27 -0500 Subject: [PATCH 150/263] ipn/ipnlocal: update suggestExitNode to skip offline candidates and fix TestSetControlClientStatusAutoExitNode TestSetControlClientStatusAutoExitNode is broken similarly to TestUpdateNetmapDeltaAutoExitNode as suggestExitNode didn't previously check the online status of exit nodes, and similarly to the other test it succeeded because the test itself is also broken. However, it is easier to fix as it sends out a full netmap update rather than a delta peer update, so it doesn't depend on the same refactoring as TestSetControlClientStatusAutoExitNode. Updates #16455 Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 2 +- ipn/ipnlocal/local_test.go | 24 ++++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index adc0af5cdac36..8889fa90b634f 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -7433,7 +7433,7 @@ func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSug } candidates := make([]tailcfg.NodeView, 0, len(netMap.Peers)) for _, peer := range netMap.Peers { - if !peer.Valid() { + if !peer.Valid() || !peer.Online().Get() { continue } if allowList != nil && !allowList.Contains(peer.StableID()) { diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 06acd85ce6023..ca968ccd76619 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -2166,8 +2166,8 @@ func TestAutoExitNodeSetNetInfoCallback(t *testing.T) { } func TestSetControlClientStatusAutoExitNode(t *testing.T) { - peer1 := makePeer(1, withCap(26), withSuggest(), withExitRoutes(), withNodeKey()) - peer2 := makePeer(2, withCap(26), withSuggest(), withExitRoutes(), withNodeKey()) + peer1 := makePeer(1, withCap(26), withSuggest(), withExitRoutes(), withOnline(true), withNodeKey()) + peer2 := makePeer(2, withCap(26), withSuggest(), withExitRoutes(), withOnline(true), withNodeKey()) derpMap := &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ 1: { @@ -2210,22 +2210,25 @@ func TestSetControlClientStatusAutoExitNode(t *testing.T) { )) syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore) b.currentNode().SetNetMap(nm) - b.lastSuggestedExitNode = peer1.StableID() + // Peer 2 should be the initial exit node, as it's better than peer 1 + // in terms of latency and DERP region. + b.lastSuggestedExitNode = peer2.StableID() b.sys.MagicSock.Get().SetLastNetcheckReportForTest(b.ctx, report) b.SetPrefsForTest(b.pm.CurrentPrefs().AsStruct()) - firstExitNode := b.Prefs().ExitNodeID() - newPeer1 := makePeer(1, withCap(26), withSuggest(), withExitRoutes(), withOnline(false), withNodeKey()) + offlinePeer2 := makePeer(2, withCap(26), withSuggest(), withExitRoutes(), withOnline(false), withNodeKey()) updatedNetmap := &netmap.NetworkMap{ Peers: []tailcfg.NodeView{ - newPeer1, - peer2, + peer1, + offlinePeer2, }, DERPMap: derpMap, } b.SetControlClientStatus(b.cc, controlclient.Status{NetMap: updatedNetmap}) - lastExitNode := b.Prefs().ExitNodeID() - if firstExitNode == lastExitNode { - t.Errorf("did not switch exit nodes despite auto exit node going offline") + // But now that peer 2 is offline, we should switch to peer 1. + wantExitNode := peer1.StableID() + gotExitNode := b.Prefs().ExitNodeID() + if gotExitNode != wantExitNode { + t.Errorf("did not switch exit nodes despite auto exit node going offline: got %q; want %q", gotExitNode, wantExitNode) } } @@ -3289,6 +3292,7 @@ func makePeer(id tailcfg.NodeID, opts ...peerOptFunc) tailcfg.NodeView { Key: makeNodeKeyFromID(id), StableID: tailcfg.StableNodeID(fmt.Sprintf("stable%d", id)), Name: fmt.Sprintf("peer%d", id), + Online: ptr.To(true), HomeDERP: int(id), } for _, opt := range opts { From a8055b5f40c625777e6e13dd504a110c223bc8fb Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Thu, 3 Jul 2025 12:21:29 -0500 Subject: [PATCH 151/263] cmd/tailscale/cli,ipn,ipn/ipnlocal: add AutoExitNode preference for automatic exit node selection With this change, policy enforcement and exit node resolution can happen in separate steps, since enforcement no longer depends on resolving the suggested exit node. This keeps policy enforcement synchronous (e.g., when switching profiles), while allowing exit node resolution to be asynchronous on netmap updates, link changes, etc. Additionally, the new preference will be used to let GUIs and CLIs switch back to "auto" mode after a manual exit node override, which is necessary for tailscale/corp#29969. Updates tailscale/corp#29969 Updates #16459 Signed-off-by: Nick Khyl --- cmd/tailscale/cli/cli_test.go | 4 + ipn/ipn_clone.go | 1 + ipn/ipn_view.go | 2 + ipn/ipnlocal/local.go | 189 +++++++++--- ipn/ipnlocal/local_test.go | 527 ++++++++++++++++++++++++++++++++-- ipn/ipnlocal/state_test.go | 106 +++++-- ipn/prefs.go | 45 +++ ipn/prefs_test.go | 12 + 8 files changed, 793 insertions(+), 93 deletions(-) diff --git a/cmd/tailscale/cli/cli_test.go b/cmd/tailscale/cli/cli_test.go index 9aa3693fd92c5..48121c7d912d9 100644 --- a/cmd/tailscale/cli/cli_test.go +++ b/cmd/tailscale/cli/cli_test.go @@ -971,6 +971,10 @@ func TestPrefFlagMapping(t *testing.T) { // Used internally by LocalBackend as part of exit node usage toggling. // No CLI flag for this. continue + case "AutoExitNode": + // TODO(nickkhyl): should be handled by tailscale {set,up} --exit-node. + // See tailscale/tailscale#16459. + continue } t.Errorf("unexpected new ipn.Pref field %q is not handled by up.go (see addPrefFlagMapping and checkForAccidentalSettingReverts)", prefName) } diff --git a/ipn/ipn_clone.go b/ipn/ipn_clone.go index 65438444e162f..3d67efc6fd33b 100644 --- a/ipn/ipn_clone.go +++ b/ipn/ipn_clone.go @@ -74,6 +74,7 @@ var _PrefsCloneNeedsRegeneration = Prefs(struct { RouteAll bool ExitNodeID tailcfg.StableNodeID ExitNodeIP netip.Addr + AutoExitNode ExitNodeExpression InternalExitNodePrior tailcfg.StableNodeID ExitNodeAllowLANAccess bool CorpDNS bool diff --git a/ipn/ipn_view.go b/ipn/ipn_view.go index 871270b8564f1..1d31ced9d3847 100644 --- a/ipn/ipn_view.go +++ b/ipn/ipn_view.go @@ -135,6 +135,7 @@ func (v PrefsView) ControlURL() string { return v.ж.Co func (v PrefsView) RouteAll() bool { return v.ж.RouteAll } func (v PrefsView) ExitNodeID() tailcfg.StableNodeID { return v.ж.ExitNodeID } func (v PrefsView) ExitNodeIP() netip.Addr { return v.ж.ExitNodeIP } +func (v PrefsView) AutoExitNode() ExitNodeExpression { return v.ж.AutoExitNode } func (v PrefsView) InternalExitNodePrior() tailcfg.StableNodeID { return v.ж.InternalExitNodePrior } func (v PrefsView) ExitNodeAllowLANAccess() bool { return v.ж.ExitNodeAllowLANAccess } func (v PrefsView) CorpDNS() bool { return v.ж.CorpDNS } @@ -179,6 +180,7 @@ var _PrefsViewNeedsRegeneration = Prefs(struct { RouteAll bool ExitNodeID tailcfg.StableNodeID ExitNodeIP netip.Addr + AutoExitNode ExitNodeExpression InternalExitNodePrior tailcfg.StableNodeID ExitNodeAllowLANAccess bool CorpDNS bool diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 8889fa90b634f..21057c0e675db 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -912,13 +912,14 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) { hadPAC := b.prevIfState.HasPAC() b.prevIfState = ifst b.pauseOrResumeControlClientLocked() - if delta.Major && shouldAutoExitNode() { + prefs := b.pm.CurrentPrefs() + if delta.Major && prefs.AutoExitNode().IsSet() { b.refreshAutoExitNode = true } var needReconfig bool // If the network changed and we're using an exit node and allowing LAN access, we may need to reconfigure. - if delta.Major && b.pm.CurrentPrefs().ExitNodeID() != "" && b.pm.CurrentPrefs().ExitNodeAllowLANAccess() { + if delta.Major && prefs.ExitNodeID() != "" && prefs.ExitNodeAllowLANAccess() { b.logf("linkChange: in state %v; updating LAN routes", b.state) needReconfig = true } @@ -941,8 +942,8 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) { // If the local network configuration has changed, our filter may // need updating to tweak default routes. - b.updateFilterLocked(b.pm.CurrentPrefs()) - updateExitNodeUsageWarning(b.pm.CurrentPrefs(), delta.New, b.health) + b.updateFilterLocked(prefs) + updateExitNodeUsageWarning(prefs, delta.New, b.health) cn := b.currentNode() nm := cn.NetMap() @@ -1623,17 +1624,17 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control prefsChanged = true } } - if shouldAutoExitNode() { + if applySysPolicy(prefs, b.overrideAlwaysOn) { + prefsChanged = true + } + if prefs.AutoExitNode.IsSet() { // Re-evaluate exit node suggestion in case circumstances have changed. _, err := b.suggestExitNodeLocked(curNetMap) if err != nil && !errors.Is(err, ErrNoPreferredDERP) { b.logf("SetControlClientStatus failed to select auto exit node: %v", err) } } - if applySysPolicy(prefs, b.lastSuggestedExitNode, b.overrideAlwaysOn) { - prefsChanged = true - } - if setExitNodeID(prefs, curNetMap) { + if setExitNodeID(prefs, b.lastSuggestedExitNode, curNetMap) { prefsChanged = true } @@ -1800,7 +1801,7 @@ var preferencePolicies = []preferencePolicyInfo{ // applySysPolicy overwrites configured preferences with policies that may be // configured by the system administrator in an OS-specific way. -func applySysPolicy(prefs *ipn.Prefs, lastSuggestedExitNode tailcfg.StableNodeID, overrideAlwaysOn bool) (anyChange bool) { +func applySysPolicy(prefs *ipn.Prefs, overrideAlwaysOn bool) (anyChange bool) { if controlURL, err := syspolicy.GetString(syspolicy.ControlURL, prefs.ControlURL); err == nil && prefs.ControlURL != controlURL { prefs.ControlURL = controlURL anyChange = true @@ -1839,21 +1840,51 @@ func applySysPolicy(prefs *ipn.Prefs, lastSuggestedExitNode tailcfg.StableNodeID if exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, ""); exitNodeIDStr != "" { exitNodeID := tailcfg.StableNodeID(exitNodeIDStr) - if shouldAutoExitNode() && lastSuggestedExitNode != "" { - exitNodeID = lastSuggestedExitNode - } - // Note: when exitNodeIDStr == "auto" && lastSuggestedExitNode == "", - // then exitNodeID is now "auto" which will never match a peer's node ID. - // When there is no a peer matching the node ID, traffic will blackhole, - // preventing accidental non-exit-node usage when a policy is in effect that requires an exit node. - if prefs.ExitNodeID != exitNodeID || prefs.ExitNodeIP.IsValid() { + + // Try to parse the policy setting value as an "auto:"-prefixed [ipn.ExitNodeExpression], + // and update prefs if it differs from the current one. + // This includes cases where it was previously an expression but no longer is, + // or where it wasn't before but now is. + autoExitNode, useAutoExitNode := parseAutoExitNodeID(exitNodeID) + if prefs.AutoExitNode != autoExitNode { + prefs.AutoExitNode = autoExitNode + anyChange = true + } + // Additionally, if the specified exit node ID is an expression, + // meaning an exit node is required but we don't yet have a valid exit node ID, + // we should set exitNodeID to a value that is never a valid [tailcfg.StableNodeID], + // to install a blackhole route and prevent accidental non-exit-node usage + // until the expression is evaluated and an actual exit node is selected. + // We use "auto:any" for this purpose, primarily for compatibility with + // older clients (in case a user downgrades to an earlier version) + // and GUIs/CLIs that have special handling for it. + if useAutoExitNode { + exitNodeID = unresolvedExitNodeID + } + + // If the current exit node ID doesn't match the one enforced by the policy setting, + // and the policy either requires a specific exit node ID, + // or requires an auto exit node ID and the current one isn't allowed, + // then update the exit node ID. + if prefs.ExitNodeID != exitNodeID { + if !useAutoExitNode || !isAllowedAutoExitNodeID(prefs.ExitNodeID) { + prefs.ExitNodeID = exitNodeID + anyChange = true + } + } + + // If the exit node IP is set, clear it. When ExitNodeIP is set in the prefs, + // it takes precedence over the ExitNodeID. + if prefs.ExitNodeIP.IsValid() { + prefs.ExitNodeIP = netip.Addr{} anyChange = true } - prefs.ExitNodeID = exitNodeID - prefs.ExitNodeIP = netip.Addr{} } else if exitNodeIPStr, _ := syspolicy.GetString(syspolicy.ExitNodeIP, ""); exitNodeIPStr != "" { - exitNodeIP, err := netip.ParseAddr(exitNodeIPStr) - if exitNodeIP.IsValid() && err == nil { + if prefs.AutoExitNode != "" { + prefs.AutoExitNode = "" // mutually exclusive with ExitNodeIP + anyChange = true + } + if exitNodeIP, err := netip.ParseAddr(exitNodeIPStr); err == nil { if prefs.ExitNodeID != "" || prefs.ExitNodeIP != exitNodeIP { anyChange = true } @@ -1901,7 +1932,7 @@ func (b *LocalBackend) registerSysPolicyWatch() (unregister func(), err error) { func (b *LocalBackend) applySysPolicy() (_ ipn.PrefsView, anyChange bool) { unlock := b.lockAndGetUnlock() prefs := b.pm.CurrentPrefs().AsStruct() - if !applySysPolicy(prefs, b.lastSuggestedExitNode, b.overrideAlwaysOn) { + if !applySysPolicy(prefs, b.overrideAlwaysOn) { unlock.UnlockEarly() return prefs.View(), false } @@ -1957,8 +1988,8 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo // If auto exit nodes are enabled and our exit node went offline, // we need to schedule picking a new one. // TODO(nickkhyl): move the auto exit node logic to a feature package. - if shouldAutoExitNode() { - exitNodeID := b.pm.prefs.ExitNodeID() + if prefs := b.pm.CurrentPrefs(); prefs.AutoExitNode().IsSet() { + exitNodeID := prefs.ExitNodeID() for _, m := range muts { mo, ok := m.(netmap.NodeMutationOnline) if !ok || mo.Online { @@ -2001,9 +2032,27 @@ func mutationsAreWorthyOfTellingIPNBus(muts []netmap.NodeMutation) bool { return false } -// setExitNodeID updates prefs to reference an exit node by ID, rather +// setExitNodeID updates prefs to either use the suggestedExitNodeID if AutoExitNode is enabled, +// or resolve ExitNodeIP to an ID and use that. It returns whether prefs was mutated. +func setExitNodeID(prefs *ipn.Prefs, suggestedExitNodeID tailcfg.StableNodeID, nm *netmap.NetworkMap) (prefsChanged bool) { + if prefs.AutoExitNode.IsSet() { + newExitNodeID := cmp.Or(suggestedExitNodeID, unresolvedExitNodeID) + if prefs.ExitNodeID != newExitNodeID { + prefs.ExitNodeID = newExitNodeID + prefsChanged = true + } + if prefs.ExitNodeIP.IsValid() { + prefs.ExitNodeIP = netip.Addr{} + prefsChanged = true + } + return prefsChanged + } + return resolveExitNodeIP(prefs, nm) +} + +// resolveExitNodeIP updates prefs to reference an exit node by ID, rather // than by IP. It returns whether prefs was mutated. -func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap) (prefsChanged bool) { +func resolveExitNodeIP(prefs *ipn.Prefs, nm *netmap.NetworkMap) (prefsChanged bool) { if nm == nil { // No netmap, can't resolve anything. return false @@ -2265,8 +2314,8 @@ func (b *LocalBackend) Start(opts ipn.Options) error { // And also apply syspolicy settings to the current profile. // This is important in two cases: when opts.UpdatePrefs is not nil, // and when Always Mode is enabled and we need to set WantRunning to true. - if newp := b.pm.CurrentPrefs().AsStruct(); applySysPolicy(newp, b.lastSuggestedExitNode, b.overrideAlwaysOn) { - setExitNodeID(newp, cn.NetMap()) + if newp := b.pm.CurrentPrefs().AsStruct(); applySysPolicy(newp, b.overrideAlwaysOn) { + setExitNodeID(newp, b.lastSuggestedExitNode, cn.NetMap()) b.pm.setPrefsNoPermCheck(newp.View()) } prefs := b.pm.CurrentPrefs() @@ -4187,12 +4236,23 @@ func (b *LocalBackend) SetUseExitNodeEnabled(v bool) (ipn.PrefsView, error) { mp := &ipn.MaskedPrefs{} if v { mp.ExitNodeIDSet = true - mp.ExitNodeID = tailcfg.StableNodeID(p0.InternalExitNodePrior()) + mp.ExitNodeID = p0.InternalExitNodePrior() + if expr, ok := parseAutoExitNodeID(mp.ExitNodeID); ok { + mp.AutoExitNodeSet = true + mp.AutoExitNode = expr + mp.ExitNodeID = unresolvedExitNodeID + } } else { mp.ExitNodeIDSet = true mp.ExitNodeID = "" + mp.AutoExitNodeSet = true + mp.AutoExitNode = "" mp.InternalExitNodePriorSet = true - mp.InternalExitNodePrior = p0.ExitNodeID() + if p0.AutoExitNode().IsSet() { + mp.InternalExitNodePrior = tailcfg.StableNodeID(autoExitNodePrefix + p0.AutoExitNode()) + } else { + mp.InternalExitNodePrior = p0.ExitNodeID() + } } return b.editPrefsLockedOnEntry(mp, unlock) } @@ -4229,6 +4289,13 @@ func (b *LocalBackend) EditPrefsAs(mp *ipn.MaskedPrefs, actor ipnauth.Actor) (ip mp.InternalExitNodePriorSet = true } + // Disable automatic exit node selection if the user explicitly sets + // ExitNodeID or ExitNodeIP. + if mp.ExitNodeIDSet || mp.ExitNodeIPSet { + mp.AutoExitNodeSet = true + mp.AutoExitNode = "" + } + // Acquire the lock before checking the profile access to prevent // TOCTOU issues caused by the current profile changing between the // check and the actual edit. @@ -4428,9 +4495,14 @@ func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce) // applySysPolicy returns whether it updated newp, // but everything in this function treats b.prefs as completely new // anyway, so its return value can be ignored here. - applySysPolicy(newp, b.lastSuggestedExitNode, b.overrideAlwaysOn) + applySysPolicy(newp, b.overrideAlwaysOn) + if newp.AutoExitNode.IsSet() { + if _, err := b.suggestExitNodeLocked(nil); err != nil { + b.logf("failed to select auto exit node: %v", err) + } + } // setExitNodeID does likewise. No-op if no exit node resolution is needed. - setExitNodeID(newp, netMap) + setExitNodeID(newp, b.lastSuggestedExitNode, netMap) // We do this to avoid holding the lock while doing everything else. @@ -7630,10 +7702,53 @@ func longLatDistance(fromLat, fromLong, toLat, toLong float64) float64 { return earthRadiusMeters * c } -// shouldAutoExitNode checks for the auto exit node MDM policy. -func shouldAutoExitNode() bool { - exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, "") - return exitNodeIDStr == "auto:any" +const ( + // autoExitNodePrefix is the prefix used in [syspolicy.ExitNodeID] values + // to indicate that the string following the prefix is an [ipn.ExitNodeExpression]. + autoExitNodePrefix = "auto:" + + // unresolvedExitNodeID is a special [tailcfg.StableNodeID] value + // used as an exit node ID to install a blackhole route, preventing + // accidental non-exit-node usage until the [ipn.ExitNodeExpression] + // is evaluated and an actual exit node is selected. + // + // We use "auto:any" for compatibility with older, pre-[ipn.ExitNodeExpression] + // clients that have been using "auto:any" for this purpose for a long time. + unresolvedExitNodeID tailcfg.StableNodeID = "auto:any" +) + +// isAutoExitNodeID reports whether the given [tailcfg.StableNodeID] is +// actually an "auto:"-prefixed [ipn.ExitNodeExpression]. +func isAutoExitNodeID(id tailcfg.StableNodeID) bool { + _, ok := parseAutoExitNodeID(id) + return ok +} + +// parseAutoExitNodeID attempts to parse the given [tailcfg.StableNodeID] +// as an [ExitNodeExpression]. +// +// It returns the parsed expression and true on success, +// or an empty string and false if the input does not appear to be +// an [ExitNodeExpression] (i.e., it doesn't start with "auto:"). +// +// It is mainly used to parse the [syspolicy.ExitNodeID] value +// when it is set to "auto:" (e.g., auto:any). +func parseAutoExitNodeID(id tailcfg.StableNodeID) (_ ipn.ExitNodeExpression, ok bool) { + if expr, ok := strings.CutPrefix(string(id), autoExitNodePrefix); ok && expr != "" { + return ipn.ExitNodeExpression(expr), true + } + return "", false +} + +func isAllowedAutoExitNodeID(exitNodeID tailcfg.StableNodeID) bool { + if exitNodeID == "" { + return false // an exit node is required + } + if nodes, _ := syspolicy.GetStringArray(syspolicy.AllowedSuggestedExitNodes, nil); nodes != nil { + return slices.Contains(nodes, string(exitNodeID)) + + } + return true // no policy configured; allow all exit nodes } // startAutoUpdate triggers an auto-update attempt. The actual update happens diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index ca968ccd76619..5c9c9f2fab4a9 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" memro "go4.org/mem" "go4.org/netipx" "golang.org/x/net/dns/dnsmessage" @@ -590,6 +591,391 @@ func TestSetUseExitNodeEnabled(t *testing.T) { } } +func makeExitNode(id tailcfg.NodeID, opts ...peerOptFunc) tailcfg.NodeView { + return makePeer(id, append([]peerOptFunc{withCap(26), withSuggest(), withExitRoutes()}, opts...)...) +} + +func TestConfigureExitNode(t *testing.T) { + controlURL := "https://localhost:1/" + exitNode1 := makeExitNode(1, withName("node-1"), withDERP(1), withAddresses(netip.MustParsePrefix("100.64.1.1/32"))) + exitNode2 := makeExitNode(2, withName("node-2"), withDERP(2), withAddresses(netip.MustParsePrefix("100.64.1.2/32"))) + selfNode := makeExitNode(3, withName("node-3"), withDERP(1), withAddresses(netip.MustParsePrefix("100.64.1.3/32"))) + clientNetmap := buildNetmapWithPeers(selfNode, exitNode1, exitNode2) + + report := &netcheck.Report{ + RegionLatency: map[int]time.Duration{ + 1: 5 * time.Millisecond, + 2: 10 * time.Millisecond, + }, + PreferredDERP: 1, + } + + tests := []struct { + name string + prefs ipn.Prefs + netMap *netmap.NetworkMap + report *netcheck.Report + changePrefs *ipn.MaskedPrefs + useExitNodeEnabled *bool + exitNodeIDPolicy *tailcfg.StableNodeID + exitNodeIPPolicy *netip.Addr + wantPrefs ipn.Prefs + }{ + { + name: "exit-node-id-via-prefs", // set exit node ID via prefs + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ExitNodeID: exitNode1.StableID()}, + ExitNodeIDSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + }, + { + name: "exit-node-ip-via-prefs", // set exit node IP via prefs (should be resolved to an ID) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ExitNodeIP: exitNode1.Addresses().At(0).Addr()}, + ExitNodeIPSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + }, + { + name: "auto-exit-node-via-prefs/any", // set auto exit node via prefs + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "any"}, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + AutoExitNode: "any", + }, + }, + { + name: "auto-exit-node-via-prefs/set-exit-node-id-via-prefs", // setting exit node ID explicitly should disable auto exit node + prefs: ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: "any", + ExitNodeID: exitNode1.StableID(), + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ExitNodeID: exitNode2.StableID()}, + ExitNodeIDSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), + AutoExitNode: "", // should be unset + }, + }, + { + name: "auto-exit-node-via-prefs/any/no-report", // set auto exit node via prefs, but no report means we can't resolve the exit node ID + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "any"}, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: unresolvedExitNodeID, // cannot resolve; traffic will be dropped + AutoExitNode: "any", + }, + }, + { + name: "auto-exit-node-via-prefs/any/no-netmap", // similarly, but without a netmap (no exit node should be selected) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "any"}, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: unresolvedExitNodeID, // cannot resolve; traffic will be dropped + AutoExitNode: "any", + }, + }, + { + name: "auto-exit-node-via-prefs/foo", // set auto exit node via prefs with an unknown/unsupported expression + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "foo"}, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" + AutoExitNode: "foo", + }, + }, + { + name: "auto-exit-node-via-prefs/off", // toggle the exit node off after it was set to "any" + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "any"}, + AutoExitNodeSet: true, + }, + useExitNodeEnabled: ptr.To(false), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: "", + AutoExitNode: "", + InternalExitNodePrior: "auto:any", + }, + }, + { + name: "auto-exit-node-via-prefs/on", // toggle the exit node on + prefs: ipn.Prefs{ + ControlURL: controlURL, + InternalExitNodePrior: "auto:any", + }, + netMap: clientNetmap, + report: report, + useExitNodeEnabled: ptr.To(true), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + AutoExitNode: "any", + InternalExitNodePrior: "auto:any", + }, + }, + { + name: "id-via-policy", // set exit node ID via syspolicy + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + exitNodeIDPolicy: ptr.To(exitNode1.StableID()), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + }, + { + name: "id-via-policy/cannot-override-via-prefs/by-id", // syspolicy should take precedence over prefs + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + exitNodeIDPolicy: ptr.To(exitNode1.StableID()), + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + ExitNodeID: exitNode2.StableID(), // this should be ignored + }, + ExitNodeIDSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + }, + { + name: "id-via-policy/cannot-override-via-prefs/by-ip", // syspolicy should take precedence over prefs + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + exitNodeIDPolicy: ptr.To(exitNode1.StableID()), + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + ExitNodeIP: exitNode2.Addresses().At(0).Addr(), // this should be ignored + }, + ExitNodeIPSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + }, + { + name: "id-via-policy/cannot-override-via-prefs/by-auto-expr", // syspolicy should take precedence over prefs + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + exitNodeIDPolicy: ptr.To(exitNode1.StableID()), + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + AutoExitNode: "any", // this should be ignored + }, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + }, + { + name: "ip-via-policy", // set exit node IP via syspolicy (should be resolved to an ID) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + exitNodeIPPolicy: ptr.To(exitNode2.Addresses().At(0).Addr()), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), + }, + }, + { + name: "auto-any-via-policy", // set auto exit node via syspolicy (an exit node should be selected) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + AutoExitNode: "any", + }, + }, + { + name: "auto-any-via-policy/no-report", // set auto exit node via syspolicy without a netcheck report (no exit node should be selected) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: nil, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: unresolvedExitNodeID, + AutoExitNode: "any", + }, + }, + { + name: "auto-any-via-policy/no-netmap", // similarly, but without a netmap (no exit node should be selected) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: nil, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: unresolvedExitNodeID, + AutoExitNode: "any", + }, + }, + { + name: "auto-foo-via-policy", // set auto exit node via syspolicy with an unknown/unsupported expression + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:foo")), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" + AutoExitNode: "foo", + }, + }, + { + name: "auto-any-via-policy/toggle-off", // cannot toggle off the exit node if it was set via syspolicy + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + useExitNodeEnabled: ptr.To(false), // should be ignored + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // still enforced by the policy setting + AutoExitNode: "any", + InternalExitNodePrior: "auto:any", + }, + }, + } + syspolicy.RegisterWellKnownSettingsForTest(t) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Configure policy settings, if any. + var settings []source.TestSetting[string] + if tt.exitNodeIDPolicy != nil { + settings = append(settings, source.TestSettingOf(syspolicy.ExitNodeID, string(*tt.exitNodeIDPolicy))) + } + if tt.exitNodeIPPolicy != nil { + settings = append(settings, source.TestSettingOf(syspolicy.ExitNodeIP, tt.exitNodeIPPolicy.String())) + } + if settings != nil { + syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, source.NewTestStoreOf(t, settings...)) + } else { + // No syspolicy settings, so don't register a store. + // This allows the test to run in parallel with other tests. + t.Parallel() + } + + // Create a new LocalBackend with the given prefs. + // Any syspolicy settings will be applied to the initial prefs. + lb := newTestLocalBackend(t) + lb.SetPrefsForTest(tt.prefs.Clone()) + // Then set the netcheck report and netmap, if any. + if tt.report != nil { + lb.MagicConn().SetLastNetcheckReportForTest(t.Context(), tt.report) + } + if tt.netMap != nil { + lb.SetControlClientStatus(lb.cc, controlclient.Status{NetMap: tt.netMap}) + } + + // If we have a changePrefs, apply it. + if tt.changePrefs != nil { + lb.EditPrefs(tt.changePrefs) + } + + // If we need to flip exit node toggle on or off, do it. + if tt.useExitNodeEnabled != nil { + lb.SetUseExitNodeEnabled(*tt.useExitNodeEnabled) + } + + // Now check the prefs. + opts := []cmp.Option{ + cmpopts.EquateComparable(netip.Addr{}, netip.Prefix{}), + } + if diff := cmp.Diff(&tt.wantPrefs, lb.Prefs().AsStruct(), opts...); diff != "" { + t.Errorf("Prefs(+got -want): %v", diff) + } + }) + } +} + func TestInternalAndExternalInterfaces(t *testing.T) { type interfacePrefix struct { i netmon.Interface @@ -1646,6 +2032,7 @@ func TestSetExitNodeIDPolicy(t *testing.T) { prefs *ipn.Prefs exitNodeIPWant string exitNodeIDWant string + autoExitNodeWant ipn.ExitNodeExpression prefsChanged bool nm *netmap.NetworkMap lastSuggestedExitNode tailcfg.StableNodeID @@ -1850,19 +2237,38 @@ func TestSetExitNodeIDPolicy(t *testing.T) { }, }, { - name: "ExitNodeID key is set to auto and last suggested exit node is populated", + name: "ExitNodeID key is set to auto:any and last suggested exit node is populated", exitNodeIDKey: true, exitNodeID: "auto:any", lastSuggestedExitNode: "123", exitNodeIDWant: "123", + autoExitNodeWant: "any", prefsChanged: true, }, { - name: "ExitNodeID key is set to auto and last suggested exit node is not populated", - exitNodeIDKey: true, - exitNodeID: "auto:any", - prefsChanged: true, - exitNodeIDWant: "auto:any", + name: "ExitNodeID key is set to auto:any and last suggested exit node is not populated", + exitNodeIDKey: true, + exitNodeID: "auto:any", + exitNodeIDWant: "auto:any", + autoExitNodeWant: "any", + prefsChanged: true, + }, + { + name: "ExitNodeID key is set to auto:foo and last suggested exit node is populated", + exitNodeIDKey: true, + exitNodeID: "auto:foo", + lastSuggestedExitNode: "123", + exitNodeIDWant: "123", + autoExitNodeWant: "foo", + prefsChanged: true, + }, + { + name: "ExitNodeID key is set to auto:foo and last suggested exit node is not populated", + exitNodeIDKey: true, + exitNodeID: "auto:foo", + exitNodeIDWant: "auto:any", // should be "auto:any" for compatibility with existing clients + autoExitNodeWant: "foo", + prefsChanged: true, }, } @@ -1893,7 +2299,7 @@ func TestSetExitNodeIDPolicy(t *testing.T) { b.pm = pm b.lastSuggestedExitNode = test.lastSuggestedExitNode prefs := b.pm.prefs.AsStruct() - if changed := applySysPolicy(prefs, test.lastSuggestedExitNode, false) || setExitNodeID(prefs, test.nm); changed != test.prefsChanged { + if changed := applySysPolicy(prefs, false) || setExitNodeID(prefs, test.lastSuggestedExitNode, test.nm); changed != test.prefsChanged { t.Errorf("wanted prefs changed %v, got prefs changed %v", test.prefsChanged, changed) } @@ -1903,15 +2309,18 @@ func TestSetExitNodeIDPolicy(t *testing.T) { // preferences to change. b.SetPrefsForTest(pm.CurrentPrefs().AsStruct()) - if got := b.pm.prefs.ExitNodeID(); got != tailcfg.StableNodeID(test.exitNodeIDWant) { - t.Errorf("got %v want %v", got, test.exitNodeIDWant) + if got := b.Prefs().ExitNodeID(); got != tailcfg.StableNodeID(test.exitNodeIDWant) { + t.Errorf("ExitNodeID: got %q; want %q", got, test.exitNodeIDWant) } - if got := b.pm.prefs.ExitNodeIP(); test.exitNodeIPWant == "" { + if got := b.Prefs().ExitNodeIP(); test.exitNodeIPWant == "" { if got.String() != "invalid IP" { - t.Errorf("got %v want invalid IP", got) + t.Errorf("ExitNodeIP: got %v want invalid IP", got) } } else if got.String() != test.exitNodeIPWant { - t.Errorf("got %v want %v", got, test.exitNodeIPWant) + t.Errorf("ExitNodeIP: got %q; want %q", got, test.exitNodeIPWant) + } + if got := b.Prefs().AutoExitNode(); got != test.autoExitNodeWant { + t.Errorf("AutoExitNode: got %q; want %q", got, test.autoExitNodeWant) } }) } @@ -2459,7 +2868,7 @@ func TestApplySysPolicy(t *testing.T) { t.Run("unit", func(t *testing.T) { prefs := tt.prefs.Clone() - gotAnyChange := applySysPolicy(prefs, "", false) + gotAnyChange := applySysPolicy(prefs, false) if gotAnyChange && prefs.Equals(&tt.prefs) { t.Errorf("anyChange but prefs is unchanged: %v", prefs.Pretty()) @@ -2607,7 +3016,7 @@ func TestPreferencePolicyInfo(t *testing.T) { prefs := defaultPrefs.AsStruct() pp.set(prefs, tt.initialValue) - gotAnyChange := applySysPolicy(prefs, "", false) + gotAnyChange := applySysPolicy(prefs, false) if gotAnyChange != tt.wantChange { t.Errorf("anyChange=%v, want %v", gotAnyChange, tt.wantChange) @@ -3288,12 +3697,14 @@ type peerOptFunc func(*tailcfg.Node) func makePeer(id tailcfg.NodeID, opts ...peerOptFunc) tailcfg.NodeView { node := &tailcfg.Node{ - ID: id, - Key: makeNodeKeyFromID(id), - StableID: tailcfg.StableNodeID(fmt.Sprintf("stable%d", id)), - Name: fmt.Sprintf("peer%d", id), - Online: ptr.To(true), - HomeDERP: int(id), + ID: id, + Key: makeNodeKeyFromID(id), + DiscoKey: makeDiscoKeyFromID(id), + StableID: tailcfg.StableNodeID(fmt.Sprintf("stable%d", id)), + Name: fmt.Sprintf("peer%d", id), + Online: ptr.To(true), + MachineAuthorized: true, + HomeDERP: int(id), } for _, opt := range opts { opt(node) @@ -3363,6 +3774,12 @@ func withNodeKey() peerOptFunc { } } +func withAddresses(addresses ...netip.Prefix) peerOptFunc { + return func(n *tailcfg.Node) { + n.Addresses = append(n.Addresses, addresses...) + } +} + func deterministicRegionForTest(t testing.TB, want views.Slice[int], use int) selectRegionFunc { t.Helper() @@ -4065,9 +4482,9 @@ func TestShouldAutoExitNode(t *testing.T) { expectedBool: false, }, { - name: "auto prefix invalid suffix", + name: "auto prefix unknown suffix", exitNodeIDPolicyValue: "auto:foo", - expectedBool: false, + expectedBool: true, // "auto:{unknown}" is treated as "auto:any" }, } @@ -4075,12 +4492,7 @@ func TestShouldAutoExitNode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - policyStore := source.NewTestStoreOf(t, source.TestSettingOf( - syspolicy.ExitNodeID, tt.exitNodeIDPolicyValue, - )) - syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore) - - got := shouldAutoExitNode() + got := isAutoExitNodeID(tailcfg.StableNodeID(tt.exitNodeIDPolicyValue)) if got != tt.expectedBool { t.Fatalf("expected %v got %v for %v policy value", tt.expectedBool, got, tt.exitNodeIDPolicyValue) } @@ -4088,6 +4500,65 @@ func TestShouldAutoExitNode(t *testing.T) { } } +func TestParseAutoExitNodeID(t *testing.T) { + tests := []struct { + name string + exitNodeID string + wantOk bool + wantExpr ipn.ExitNodeExpression + }{ + { + name: "empty expr", + exitNodeID: "", + wantOk: false, + wantExpr: "", + }, + { + name: "no auto prefix", + exitNodeID: "foo", + wantOk: false, + wantExpr: "", + }, + { + name: "auto:any", + exitNodeID: "auto:any", + wantOk: true, + wantExpr: ipn.AnyExitNode, + }, + { + name: "auto:foo", + exitNodeID: "auto:foo", + wantOk: true, + wantExpr: "foo", + }, + { + name: "auto prefix but empty suffix", + exitNodeID: "auto:", + wantOk: false, + wantExpr: "", + }, + { + name: "auto prefix no colon", + exitNodeID: "auto", + wantOk: false, + wantExpr: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotExpr, gotOk := parseAutoExitNodeID(tailcfg.StableNodeID(tt.exitNodeID)) + if gotOk != tt.wantOk || gotExpr != tt.wantExpr { + if tt.wantOk { + t.Fatalf("got %v (%q); want %v (%q)", gotOk, gotExpr, tt.wantOk, tt.wantExpr) + } else { + t.Fatalf("got %v (%q); want false", gotOk, gotExpr) + } + } + }) + } +} + func TestEnableAutoUpdates(t *testing.T) { lb := newTestLocalBackend(t) diff --git a/ipn/ipnlocal/state_test.go b/ipn/ipnlocal/state_test.go index eb36643856f82..f0ac5f9442704 100644 --- a/ipn/ipnlocal/state_test.go +++ b/ipn/ipnlocal/state_test.go @@ -6,6 +6,7 @@ package ipnlocal import ( "context" "errors" + "fmt" "net/netip" "strings" "sync" @@ -1108,10 +1109,17 @@ func TestEngineReconfigOnStateChange(t *testing.T) { enableLogging := false connect := &ipn.MaskedPrefs{Prefs: ipn.Prefs{WantRunning: true}, WantRunningSet: true} disconnect := &ipn.MaskedPrefs{Prefs: ipn.Prefs{WantRunning: false}, WantRunningSet: true} - node1 := testNetmapForNode(1, "node-1", []netip.Prefix{netip.MustParsePrefix("100.64.1.1/32")}) - node2 := testNetmapForNode(2, "node-2", []netip.Prefix{netip.MustParsePrefix("100.64.1.2/32")}) - node3 := testNetmapForNode(3, "node-3", []netip.Prefix{netip.MustParsePrefix("100.64.1.3/32")}) - node3.Peers = []tailcfg.NodeView{node1.SelfNode, node2.SelfNode} + node1 := buildNetmapWithPeers( + makePeer(1, withName("node-1"), withAddresses(netip.MustParsePrefix("100.64.1.1/32"))), + ) + node2 := buildNetmapWithPeers( + makePeer(2, withName("node-2"), withAddresses(netip.MustParsePrefix("100.64.1.2/32"))), + ) + node3 := buildNetmapWithPeers( + makePeer(3, withName("node-3"), withAddresses(netip.MustParsePrefix("100.64.1.3/32"))), + node1.SelfNode, + node2.SelfNode, + ) routesWithQuad100 := func(extra ...netip.Prefix) []netip.Prefix { return append(extra, netip.MustParsePrefix("100.100.100.100/32")) } @@ -1380,33 +1388,75 @@ func TestEngineReconfigOnStateChange(t *testing.T) { } } -func testNetmapForNode(userID tailcfg.UserID, name string, addresses []netip.Prefix) *netmap.NetworkMap { +func buildNetmapWithPeers(self tailcfg.NodeView, peers ...tailcfg.NodeView) *netmap.NetworkMap { const ( - domain = "example.com" - magicDNSSuffix = ".test.ts.net" + firstAutoUserID = tailcfg.UserID(10000) + domain = "example.com" + magicDNSSuffix = ".test.ts.net" ) - user := &tailcfg.UserProfile{ - ID: userID, - DisplayName: name, - LoginName: strings.Join([]string{name, domain}, "@"), - } - self := &tailcfg.Node{ - ID: tailcfg.NodeID(1000 + userID), - StableID: tailcfg.StableNodeID("stable-" + name), - User: user.ID, - Name: name + magicDNSSuffix, - Addresses: addresses, - MachineAuthorized: true, - } - self.Key = makeNodeKeyFromID(self.ID) - self.DiscoKey = makeDiscoKeyFromID(self.ID) + + users := make(map[tailcfg.UserID]tailcfg.UserProfileView) + makeUserForNode := func(n *tailcfg.Node) { + var user *tailcfg.UserProfile + if n.User == 0 { + n.User = firstAutoUserID + tailcfg.UserID(n.ID) + user = &tailcfg.UserProfile{ + DisplayName: n.Name, + LoginName: n.Name, + } + } else if _, ok := users[n.User]; !ok { + user = &tailcfg.UserProfile{ + DisplayName: fmt.Sprintf("User %d", n.User), + LoginName: fmt.Sprintf("user-%d", n.User), + } + } + if user != nil { + user.ID = n.User + user.LoginName = strings.Join([]string{user.LoginName, domain}, "@") + users[n.User] = user.View() + } + } + + derpmap := &tailcfg.DERPMap{ + Regions: make(map[int]*tailcfg.DERPRegion), + } + makeDERPRegionForNode := func(n *tailcfg.Node) { + if n.HomeDERP == 0 { + return // no DERP region + } + if _, ok := derpmap.Regions[n.HomeDERP]; !ok { + r := &tailcfg.DERPRegion{ + RegionID: n.HomeDERP, + RegionName: fmt.Sprintf("Region %d", n.HomeDERP), + } + r.Nodes = append(r.Nodes, &tailcfg.DERPNode{ + Name: fmt.Sprintf("%da", n.HomeDERP), + RegionID: n.HomeDERP, + }) + derpmap.Regions[n.HomeDERP] = r + } + } + + updateNode := func(n tailcfg.NodeView) tailcfg.NodeView { + mut := n.AsStruct() + makeUserForNode(mut) + makeDERPRegionForNode(mut) + mut.Name = mut.Name + magicDNSSuffix + return mut.View() + } + + self = updateNode(self) + for i := range peers { + peers[i] = updateNode(peers[i]) + } + return &netmap.NetworkMap{ - SelfNode: self.View(), - Name: self.Name, - Domain: domain, - UserProfiles: map[tailcfg.UserID]tailcfg.UserProfileView{ - user.ID: user.View(), - }, + SelfNode: self, + Name: self.Name(), + Domain: domain, + Peers: peers, + UserProfiles: users, + DERPMap: derpmap, } } diff --git a/ipn/prefs.go b/ipn/prefs.go index 01275a7e25bdc..77cea0493af16 100644 --- a/ipn/prefs.go +++ b/ipn/prefs.go @@ -94,6 +94,25 @@ type Prefs struct { ExitNodeID tailcfg.StableNodeID ExitNodeIP netip.Addr + // AutoExitNode is an optional expression that specifies whether and how + // tailscaled should pick an exit node automatically. + // + // If specified, tailscaled will use an exit node based on the expression, + // and will re-evaluate the selection periodically as network conditions, + // available exit nodes, or policy settings change. A blackhole route will + // be installed to prevent traffic from escaping to the local network until + // an exit node is selected. It takes precedence over ExitNodeID and ExitNodeIP. + // + // If empty, tailscaled will not automatically select an exit node. + // + // If the specified expression is invalid or unsupported by the client, + // it falls back to the behavior of [AnyExitNode]. + // + // As of 2025-07-02, the only supported value is [AnyExitNode]. + // It's a string rather than a boolean to allow future extensibility + // (e.g., AutoExitNode = "mullvad" or AutoExitNode = "geo:us"). + AutoExitNode ExitNodeExpression `json:",omitempty"` + // InternalExitNodePrior is the most recently used ExitNodeID in string form. It is set by // the backend on transition from exit node on to off and used by the // backend. @@ -325,6 +344,7 @@ type MaskedPrefs struct { RouteAllSet bool `json:",omitempty"` ExitNodeIDSet bool `json:",omitempty"` ExitNodeIPSet bool `json:",omitempty"` + AutoExitNodeSet bool `json:",omitempty"` InternalExitNodePriorSet bool `json:",omitempty"` // Internal; can't be set by LocalAPI clients ExitNodeAllowLANAccessSet bool `json:",omitempty"` CorpDNSSet bool `json:",omitempty"` @@ -533,6 +553,9 @@ func (p *Prefs) pretty(goos string) string { } else if !p.ExitNodeID.IsZero() { fmt.Fprintf(&sb, "exit=%v lan=%t ", p.ExitNodeID, p.ExitNodeAllowLANAccess) } + if p.AutoExitNode.IsSet() { + fmt.Fprintf(&sb, "auto=%v ", p.AutoExitNode) + } if len(p.AdvertiseRoutes) > 0 || goos == "linux" { fmt.Fprintf(&sb, "routes=%v ", p.AdvertiseRoutes) } @@ -609,6 +632,7 @@ func (p *Prefs) Equals(p2 *Prefs) bool { p.RouteAll == p2.RouteAll && p.ExitNodeID == p2.ExitNodeID && p.ExitNodeIP == p2.ExitNodeIP && + p.AutoExitNode == p2.AutoExitNode && p.InternalExitNodePrior == p2.InternalExitNodePrior && p.ExitNodeAllowLANAccess == p2.ExitNodeAllowLANAccess && p.CorpDNS == p2.CorpDNS && @@ -804,6 +828,7 @@ func isRemoteIP(st *ipnstate.Status, ip netip.Addr) bool { func (p *Prefs) ClearExitNode() { p.ExitNodeID = "" p.ExitNodeIP = netip.Addr{} + p.AutoExitNode = "" } // ExitNodeLocalIPError is returned when the requested IP address for an exit @@ -1043,3 +1068,23 @@ func (p *LoginProfile) Equals(p2 *LoginProfile) bool { p.LocalUserID == p2.LocalUserID && p.ControlURL == p2.ControlURL } + +// ExitNodeExpression is a string that specifies how an exit node +// should be selected. An empty string means that no exit node +// should be selected. +// +// As of 2025-07-02, the only supported value is [AnyExitNode]. +type ExitNodeExpression string + +// AnyExitNode indicates that the exit node should be automatically +// selected from the pool of available exit nodes, excluding any +// disallowed by policy (e.g., [syspolicy.AllowedSuggestedExitNodes]). +// The exact implementation is subject to change, but exit nodes +// offering the best performance will be preferred. +const AnyExitNode ExitNodeExpression = "any" + +// IsSet reports whether the expression is non-empty and can be used +// to select an exit node. +func (e ExitNodeExpression) IsSet() bool { + return e != "" +} diff --git a/ipn/prefs_test.go b/ipn/prefs_test.go index d28d161db422e..268ea206c137f 100644 --- a/ipn/prefs_test.go +++ b/ipn/prefs_test.go @@ -40,6 +40,7 @@ func TestPrefsEqual(t *testing.T) { "RouteAll", "ExitNodeID", "ExitNodeIP", + "AutoExitNode", "InternalExitNodePrior", "ExitNodeAllowLANAccess", "CorpDNS", @@ -150,6 +151,17 @@ func TestPrefsEqual(t *testing.T) { true, }, + { + &Prefs{AutoExitNode: ""}, + &Prefs{AutoExitNode: "auto:any"}, + false, + }, + { + &Prefs{AutoExitNode: "auto:any"}, + &Prefs{AutoExitNode: "auto:any"}, + true, + }, + { &Prefs{}, &Prefs{ExitNodeAllowLANAccess: true}, From c46145b99e4157d89df807dc64133e31d855cf09 Mon Sep 17 00:00:00 2001 From: David Bond Date: Fri, 4 Jul 2025 12:19:23 +0100 Subject: [PATCH 152/263] cmd/k8s-operator: Move login server value to top-level (#16470) This commit modifies the operator helm chart values to bring the newly added `loginServer` field to the top level. We felt as though it was a bit confusing to be at the `operatorConfig` level as this value modifies the behaviour or the operator, api server & all resources that the operator manages. Updates https://github.com/tailscale/corp/issues/29847 Signed-off-by: David Bond --- cmd/k8s-operator/deploy/chart/templates/deployment.yaml | 2 +- cmd/k8s-operator/deploy/chart/values.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml index 8deba7dab0139..01a290c076368 100644 --- a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml @@ -69,7 +69,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: OPERATOR_LOGIN_SERVER - value: {{ .Values.operatorConfig.loginServer }} + value: {{ .Values.loginServer }} - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE diff --git a/cmd/k8s-operator/deploy/chart/values.yaml b/cmd/k8s-operator/deploy/chart/values.yaml index af941425a5006..0ba8d045a858d 100644 --- a/cmd/k8s-operator/deploy/chart/values.yaml +++ b/cmd/k8s-operator/deploy/chart/values.yaml @@ -9,6 +9,9 @@ oauth: {} # clientId: "" # clientSecret: "" +# URL of the control plane to be used by all resources managed by the operator. +loginServer: "" + # Secret volume. # If set it defines the volume the oauth secrets will be mounted from. # The volume needs to contain two files named `client_id` and `client_secret`. @@ -72,9 +75,6 @@ operatorConfig: # - name: EXTRA_VAR2 # value: "value2" - # URL of the control plane to be used by all resources managed by the operator. - loginServer: "" - # In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here ingressClass: enabled: true From 639fed6856722bad94762b48546cd84331f12b97 Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Fri, 4 Jul 2025 16:06:22 +0100 Subject: [PATCH 153/263] Dockerfile,build_docker.sh: add a note on how to build local images (#16471) Updates#cleanup Signed-off-by: Irbe Krumina --- Dockerfile | 9 +++++++++ build_docker.sh | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/Dockerfile b/Dockerfile index 015022e49fc28..fbc0d1194ffc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,15 @@ # Tailscale images are currently built using https://github.com/tailscale/mkctr, # and the build script can be found in ./build_docker.sh. # +# If you want to build local images for testing, you can use make. +# +# To build a Tailscale image and push to the local docker registry: +# +# $ REPO=local/tailscale TAGS=v0.0.1 PLATFORM=local make publishdevimage +# +# To build a Tailscale image and push to a remote docker registry: +# +# $ REPO=//tailscale TAGS=v0.0.1 make publishdevimage # # This Dockerfile includes all the tailscale binaries. # diff --git a/build_docker.sh b/build_docker.sh index bdc9dc08609fa..7840dc89775d3 100755 --- a/build_docker.sh +++ b/build_docker.sh @@ -6,6 +6,16 @@ # hash of this repository as produced by ./cmd/mkversion. # This is the image build mechanim used to build the official Tailscale # container images. +# +# If you want to build local images for testing, you can use make, which provides few convenience wrappers around this script. +# +# To build a Tailscale image and push to the local docker registry: + +# $ REPO=local/tailscale TAGS=v0.0.1 PLATFORM=local make publishdevimage +# +# To build a Tailscale image and push to a remote docker registry: +# +# $ REPO=//tailscale TAGS=v0.0.1 make publishdevimage set -eu From 92a114c66d296704d48045ee12c0fe28bb7f5b6c Mon Sep 17 00:00:00 2001 From: Dylan Bargatze Date: Fri, 4 Jul 2025 12:48:38 -0400 Subject: [PATCH 154/263] tailcfg, feature/relayserver, wgengine/magicsock: invert UDP relay server nodeAttrs (#16444) Inverts the nodeAttrs related to UDP relay client/server enablement to disablement, and fixes up the corresponding logic that uses them. Also updates the doc comments on both nodeAttrs. Fixes tailscale/corp#30024 Signed-off-by: Dylan Bargatze --- feature/relayserver/relayserver.go | 18 +++++++++--------- tailcfg/tailcfg.go | 21 ++++++++++++++------- wgengine/magicsock/magicsock.go | 2 +- wgengine/magicsock/magicsock_test.go | 3 --- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/feature/relayserver/relayserver.go b/feature/relayserver/relayserver.go index 5a82a9d117bd7..f4a533193999e 100644 --- a/feature/relayserver/relayserver.go +++ b/feature/relayserver/relayserver.go @@ -50,11 +50,11 @@ func newExtension(logf logger.Logf, _ ipnext.SafeBackend) (ipnext.Extension, err type extension struct { logf logger.Logf - mu sync.Mutex // guards the following fields - shutdown bool - port *int // ipn.Prefs.RelayServerPort, nil if disabled - hasNodeAttrRelayServer bool // tailcfg.NodeAttrRelayServer - server relayServer // lazily initialized + mu sync.Mutex // guards the following fields + shutdown bool + port *int // ipn.Prefs.RelayServerPort, nil if disabled + hasNodeAttrDisableRelayServer bool // tailcfg.NodeAttrDisableRelayServer + server relayServer // lazily initialized } // relayServer is the interface of [udprelay.Server]. @@ -81,8 +81,8 @@ func (e *extension) Init(host ipnext.Host) error { func (e *extension) selfNodeViewChanged(nodeView tailcfg.NodeView) { e.mu.Lock() defer e.mu.Unlock() - e.hasNodeAttrRelayServer = nodeView.HasCap(tailcfg.NodeAttrRelayServer) - if !e.hasNodeAttrRelayServer && e.server != nil { + e.hasNodeAttrDisableRelayServer = nodeView.HasCap(tailcfg.NodeAttrDisableRelayServer) + if e.hasNodeAttrDisableRelayServer && e.server != nil { e.server.Close() e.server = nil } @@ -130,8 +130,8 @@ func (e *extension) relayServerOrInit() (relayServer, error) { if e.port == nil { return nil, errors.New("relay server is not configured") } - if !e.hasNodeAttrRelayServer { - return nil, errors.New("no relay:server node attribute") + if e.hasNodeAttrDisableRelayServer { + return nil, errors.New("disable-relay-server node attribute is present") } if !envknob.UseWIPCode() { return nil, errors.New("TAILSCALE_USE_WIP_CODE envvar is not set") diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 10b157ac15642..d97f60a8acb84 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2602,13 +2602,20 @@ const ( // peer node list. NodeAttrNativeIPV4 NodeCapability = "native-ipv4" - // NodeAttrRelayServer permits the node to act as an underlay UDP relay - // server. There are no expected values for this key in NodeCapMap. - NodeAttrRelayServer NodeCapability = "relay:server" - - // NodeAttrRelayClient permits the node to act as an underlay UDP relay - // client. There are no expected values for this key in NodeCapMap. - NodeAttrRelayClient NodeCapability = "relay:client" + // NodeAttrDisableRelayServer prevents the node from acting as an underlay + // UDP relay server. There are no expected values for this key; the key + // only needs to be present in [NodeCapMap] to take effect. + NodeAttrDisableRelayServer NodeCapability = "disable-relay-server" + + // NodeAttrDisableRelayClient prevents the node from allocating UDP relay + // server endpoints itself; the node may still bind into and relay traffic + // using endpoints allocated by its peers. This attribute can be added to + // the node dynamically; if added while the node is already running, the + // node will be unable to allocate UDP relay server endpoints after it next + // updates its network map. There are no expected values for this key in + // [NodeCapMap]; the key only needs to be present in [NodeCapMap] to take + // effect. + NodeAttrDisableRelayClient NodeCapability = "disable-relay-client" // NodeAttrMagicDNSPeerAAAA is a capability that tells the node's MagicDNS // server to answer AAAA queries about its peers. See tailscale/tailscale#1152. diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 174345a84ac87..5719b20f9ab6f 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2703,7 +2703,7 @@ func (c *Conn) onNodeViewsUpdate(update NodeViewsUpdate) { peersChanged := c.updateNodes(update) relayClientEnabled := update.SelfNode.Valid() && - update.SelfNode.HasCap(tailcfg.NodeAttrRelayClient) && + !update.SelfNode.HasCap(tailcfg.NodeAttrDisableRelayClient) && envknob.UseWIPCode() c.mu.Lock() diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 8aa9a09d2c15a..c388e9ed15d00 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -3408,9 +3408,6 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { } peerOnlyIPv4 := &tailcfg.Node{ Cap: math.MinInt32, - CapMap: map[tailcfg.NodeCapability][]tailcfg.RawMessage{ - tailcfg.NodeAttrRelayServer: nil, - }, Addresses: []netip.Prefix{ netip.MustParsePrefix("2.2.2.2/32"), }, From 079134d3c0f51ad27e502e70a172e10326c70d3d Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Mon, 7 Jul 2025 00:40:56 +0100 Subject: [PATCH 155/263] cmd/k8s-operator: always set ProxyGroup status conditions (#16429) Refactors setting status into its own top-level function to make it easier to ensure we _always_ set the status if it's changed on every reconcile. Previously, it was possible to have stale status if some earlier part of the provision logic failed. Updates #16327 Change-Id: Idab0cfc15ae426cf6914a82f0d37a5cc7845236b Signed-off-by: Tom Proctor --- .../crds/tailscale.com_proxygroups.yaml | 5 +- .../deploy/manifests/operator.yaml | 5 +- cmd/k8s-operator/proxygroup.go | 300 +++++++++--------- cmd/k8s-operator/proxygroup_test.go | 66 ++-- k8s-operator/api.md | 2 +- .../apis/v1alpha1/types_proxygroup.go | 6 +- 6 files changed, 212 insertions(+), 172 deletions(-) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index f695e989d7b85..c426c8427a507 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -124,7 +124,10 @@ spec: conditions: description: |- List of status conditions to indicate the status of the ProxyGroup - resources. Known condition types are `ProxyGroupReady`. + resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`. + `ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled + and ready. `ProxyGroupAvailable` indicates that at least one proxy is + ready to serve traffic. type: array items: description: Condition contains details for one aspect of the current state of this API Resource. diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 4f1faf104cfc6..2888575692594 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -2953,7 +2953,10 @@ spec: conditions: description: |- List of status conditions to indicate the status of the ProxyGroup - resources. Known condition types are `ProxyGroupReady`. + resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`. + `ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled + and ready. `ProxyGroupAvailable` indicates that at least one proxy is + ready to serve traffic. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 1b622c920d22d..c44de09a7fc45 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -13,6 +13,7 @@ import ( "net/http" "net/netip" "slices" + "sort" "strings" "sync" @@ -48,7 +49,6 @@ const ( reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed" reasonProxyGroupReady = "ProxyGroupReady" reasonProxyGroupCreating = "ProxyGroupCreating" - reasonProxyGroupInvalid = "ProxyGroupInvalid" // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" @@ -132,17 +132,15 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } oldPGStatus := pg.Status.DeepCopy() - setStatusReady := func(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger) - if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) { - // An error encountered here should get returned by the Reconcile function. - if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { - err = errors.Join(err, updateErr) - } - } - return reconcile.Result{}, err - } + staticEndpoints, nrr, err := r.reconcilePG(ctx, pg, logger) + return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints)) +} +// reconcilePG handles all reconciliation of a ProxyGroup that is not marked +// for deletion. It is separated out from Reconcile to make a clear separation +// between reconciling the ProxyGroup, and posting the status of its created +// resources onto the ProxyGroup status field. +func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (map[string][]netip.AddrPort, *notReadyReason, error) { if !slices.Contains(pg.Finalizers, FinalizerName) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, @@ -150,18 +148,11 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ // operation is underway. logger.Infof("ensuring ProxyGroup is set up") pg.Finalizers = append(pg.Finalizers, FinalizerName) - if err = r.Update(ctx, pg); err != nil { - err = fmt.Errorf("error adding finalizer: %w", err) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, reasonProxyGroupCreationFailed) + if err := r.Update(ctx, pg); err != nil { + return r.notReadyErrf(pg, "error adding finalizer: %w", err) } } - if err = r.validate(pg); err != nil { - message := fmt.Sprintf("ProxyGroup is invalid: %s", err) - r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupInvalid, message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupInvalid, message) - } - proxyClassName := r.defaultProxyClass if pg.Spec.ProxyClass != "" { proxyClassName = pg.Spec.ProxyClass @@ -172,78 +163,33 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ proxyClass = new(tsapi.ProxyClass) err := r.Get(ctx, types.NamespacedName{Name: proxyClassName}, proxyClass) if apierrors.IsNotFound(err) { - err = nil - message := fmt.Sprintf("the ProxyGroup's ProxyClass %s does not (yet) exist", proxyClassName) - logger.Info(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) + msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q does not (yet) exist", proxyClassName) + logger.Info(msg) + return r.notReady(reasonProxyGroupCreating, msg) } if err != nil { - err = fmt.Errorf("error getting ProxyGroup's ProxyClass %s: %s", proxyClassName, err) - r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error()) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error()) + return r.notReadyErrf(pg, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err) } validateProxyClassForPG(logger, pg, proxyClass) if !tsoperator.ProxyClassIsReady(proxyClass) { - message := fmt.Sprintf("the ProxyGroup's ProxyClass %s is not yet in a ready state, waiting...", proxyClassName) - logger.Info(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) + msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName) + logger.Info(msg) + return r.notReady(reasonProxyGroupCreating, msg) } } - isProvisioned, err := r.maybeProvision(ctx, pg, proxyClass) + staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass) if err != nil { - reason := reasonProxyGroupCreationFailed - msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", err) if strings.Contains(err.Error(), optimisticLockErrorMsg) { - reason = reasonProxyGroupCreating - msg = fmt.Sprintf("optimistic lock error, retrying: %s", err) - err = nil + msg := fmt.Sprintf("optimistic lock error, retrying: %s", nrr.message) logger.Info(msg) + return r.notReady(reasonProxyGroupCreating, msg) } else { - r.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg) - } - - return setStatusReady(pg, metav1.ConditionFalse, reason, msg) - } - - if !isProvisioned { - if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) { - // An error encountered here should get returned by the Reconcile function. - if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { - return reconcile.Result{}, errors.Join(err, updateErr) - } + return nil, nrr, err } - return } - desiredReplicas := int(pgReplicas(pg)) - - // Set ProxyGroupAvailable condition. - status := metav1.ConditionFalse - reason := reasonProxyGroupCreating - message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas) - if len(pg.Status.Devices) > 0 { - status = metav1.ConditionTrue - if len(pg.Status.Devices) == desiredReplicas { - reason = reasonProxyGroupReady - } - } - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, pg.Generation, r.clock, logger) - - // Set ProxyGroupReady condition. - if len(pg.Status.Devices) < desiredReplicas { - logger.Debug(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) - } - - if len(pg.Status.Devices) > desiredReplicas { - message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(pg.Status.Devices)-desiredReplicas) - logger.Debug(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) - } - - logger.Info("ProxyGroup resources synced") - return setStatusReady(pg, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady) + return staticEndpoints, nrr, nil } // validateProxyClassForPG applies custom validation logic for ProxyClass applied to ProxyGroup. @@ -271,7 +217,7 @@ func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc } } -func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (isProvisioned bool, err error) { +func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) { logger := r.logger(pg.Name) r.mu.Lock() r.ensureAddedToGaugeForProxyGroup(pg) @@ -280,31 +226,30 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro svcToNodePorts := make(map[string]uint16) var tailscaledPort *uint16 if proxyClass != nil && proxyClass.Spec.StaticEndpoints != nil { + var err error svcToNodePorts, tailscaledPort, err = r.ensureNodePortServiceCreated(ctx, pg, proxyClass) if err != nil { - wrappedErr := fmt.Errorf("error provisioning NodePort Services for static endpoints: %w", err) var allocatePortErr *allocatePortsErr if errors.As(err, &allocatePortErr) { reason := reasonProxyGroupCreationFailed - msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", wrappedErr) - r.setStatusReady(pg, metav1.ConditionFalse, reason, msg, logger) - return false, nil + msg := fmt.Sprintf("error provisioning NodePort Services for static endpoints: %v", err) + r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg) + return r.notReady(reason, msg) } - return false, wrappedErr + return r.notReadyErrf(pg, "error provisioning NodePort Services for static endpoints: %w", err) } } staticEndpoints, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass, svcToNodePorts) if err != nil { - wrappedErr := fmt.Errorf("error provisioning config Secrets: %w", err) var selectorErr *FindStaticEndpointErr if errors.As(err, &selectorErr) { reason := reasonProxyGroupCreationFailed - msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", wrappedErr) - r.setStatusReady(pg, metav1.ConditionFalse, reason, msg, logger) - return false, nil + msg := fmt.Sprintf("error provisioning config Secrets: %v", err) + r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg) + return r.notReady(reason, msg) } - return false, wrappedErr + return r.notReadyErrf(pg, "error provisioning config Secrets: %w", err) } // State secrets are precreated so we can use the ProxyGroup CR as their owner ref. @@ -315,7 +260,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences }); err != nil { - return false, fmt.Errorf("error provisioning state Secrets: %w", err) + return r.notReadyErrf(pg, "error provisioning state Secrets: %w", err) } } sa := pgServiceAccount(pg, r.tsNamespace) @@ -324,7 +269,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences }); err != nil { - return false, fmt.Errorf("error provisioning ServiceAccount: %w", err) + return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err) } role := pgRole(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { @@ -333,7 +278,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences r.Rules = role.Rules }); err != nil { - return false, fmt.Errorf("error provisioning Role: %w", err) + return r.notReadyErrf(pg, "error provisioning Role: %w", err) } roleBinding := pgRoleBinding(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { @@ -343,7 +288,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.RoleRef = roleBinding.RoleRef r.Subjects = roleBinding.Subjects }); err != nil { - return false, fmt.Errorf("error provisioning RoleBinding: %w", err) + return r.notReadyErrf(pg, "error provisioning RoleBinding: %w", err) } if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { cm, hp := pgEgressCM(pg, r.tsNamespace) @@ -352,7 +297,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp) }); err != nil { - return false, fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err) + return r.notReadyErrf(pg, "error provisioning egress ConfigMap %q: %w", cm.Name, err) } } if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { @@ -361,28 +306,27 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro existing.ObjectMeta.Labels = cm.ObjectMeta.Labels existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences }); err != nil { - return false, fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err) + return r.notReadyErrf(pg, "error provisioning ingress ConfigMap %q: %w", cm.Name, err) } } ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, tailscaledPort, proxyClass) if err != nil { - return false, fmt.Errorf("error generating StatefulSet spec: %w", err) + return r.notReadyErrf(pg, "error generating StatefulSet spec: %w", err) } cfg := &tailscaleSTSConfig{ proxyType: string(pg.Spec.Type), } ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger) - updateSS := func(s *appsv1.StatefulSet) { + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { s.Spec = ss.Spec - s.ObjectMeta.Labels = ss.ObjectMeta.Labels s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences + }); err != nil { + return r.notReadyErrf(pg, "error provisioning StatefulSet: %w", err) } - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, updateSS); err != nil { - return false, fmt.Errorf("error provisioning StatefulSet: %w", err) - } + mo := &metricsOpts{ tsNamespace: r.tsNamespace, proxyStsName: pg.Name, @@ -390,21 +334,67 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro proxyType: "proxygroup", } if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil { - return false, fmt.Errorf("error reconciling metrics resources: %w", err) + return r.notReadyErrf(pg, "error reconciling metrics resources: %w", err) } if err := r.cleanupDanglingResources(ctx, pg, proxyClass); err != nil { - return false, fmt.Errorf("error cleaning up dangling resources: %w", err) + return r.notReadyErrf(pg, "error cleaning up dangling resources: %w", err) } - devices, err := r.getDeviceInfo(ctx, staticEndpoints, pg) + logger.Info("ProxyGroup resources synced") + + return staticEndpoints, nil, nil +} + +func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, oldPGStatus *tsapi.ProxyGroupStatus, nrr *notReadyReason, endpoints map[string][]netip.AddrPort) (err error) { + defer func() { + if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) { + if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { + err = errors.Join(err, updateErr) + } + } + }() + + devices, err := r.getRunningProxies(ctx, pg, endpoints) if err != nil { - return false, fmt.Errorf("failed to get device info: %w", err) + return fmt.Errorf("failed to list running proxies: %w", err) } pg.Status.Devices = devices - return true, nil + desiredReplicas := int(pgReplicas(pg)) + + // Set ProxyGroupAvailable condition. + status := metav1.ConditionFalse + reason := reasonProxyGroupCreating + message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(devices), desiredReplicas) + if len(devices) > 0 { + status = metav1.ConditionTrue + if len(devices) == desiredReplicas { + reason = reasonProxyGroupReady + } + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger) + + // Set ProxyGroupReady condition. + status = metav1.ConditionFalse + reason = reasonProxyGroupCreating + switch { + case nrr != nil: + // If we failed earlier, that reason takes precedence. + reason = nrr.reason + message = nrr.message + case len(devices) < desiredReplicas: + case len(devices) > desiredReplicas: + message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(devices)-desiredReplicas) + default: + status = metav1.ConditionTrue + reason = reasonProxyGroupReady + message = reasonProxyGroupReady + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger) + + return nil } // getServicePortsForProxyGroups returns a map of ProxyGroup Service names to their NodePorts, @@ -484,15 +474,15 @@ func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, tailscaledPort := getRandomPort() svcs := []*corev1.Service{} for i := range pgReplicas(pg) { - replicaName := pgNodePortServiceName(pg.Name, i) + nodePortSvcName := pgNodePortServiceName(pg.Name, i) svc := &corev1.Service{} - err := r.Get(ctx, types.NamespacedName{Name: replicaName, Namespace: r.tsNamespace}, svc) + err := r.Get(ctx, types.NamespacedName{Name: nodePortSvcName, Namespace: r.tsNamespace}, svc) if err != nil && !apierrors.IsNotFound(err) { - return nil, nil, fmt.Errorf("error getting Kubernetes Service %q: %w", replicaName, err) + return nil, nil, fmt.Errorf("error getting Kubernetes Service %q: %w", nodePortSvcName, err) } if apierrors.IsNotFound(err) { - svcs = append(svcs, pgNodePortService(pg, replicaName, r.tsNamespace)) + svcs = append(svcs, pgNodePortService(pg, nodePortSvcName, r.tsNamespace)) } else { // NOTE: if we can we want to recover the random port used for tailscaled, // as well as the NodePort previously used for that Service @@ -638,7 +628,7 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass, svcToNodePorts map[string]uint16) (endpoints map[string][]netip.AddrPort, err error) { logger := r.logger(pg.Name) - endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg)) + endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg)) // keyed by Service name. for i := range pgReplicas(pg) { cfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -691,14 +681,15 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p } } - replicaName := pgNodePortServiceName(pg.Name, i) + nodePortSvcName := pgNodePortServiceName(pg.Name, i) if len(svcToNodePorts) > 0 { - port, ok := svcToNodePorts[replicaName] + replicaName := fmt.Sprintf("%s-%d", pg.Name, i) + port, ok := svcToNodePorts[nodePortSvcName] if !ok { return nil, fmt.Errorf("could not find configured NodePort for ProxyGroup replica %q", replicaName) } - endpoints[replicaName], err = r.findStaticEndpoints(ctx, existingCfgSecret, proxyClass, port, logger) + endpoints[nodePortSvcName], err = r.findStaticEndpoints(ctx, existingCfgSecret, proxyClass, port, logger) if err != nil { return nil, fmt.Errorf("could not find static endpoints for replica %q: %w", replicaName, err) } @@ -711,7 +702,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p return nil, err } - configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[replicaName], existingAdvertiseServices, r.loginServer) + configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer) if err != nil { return nil, fmt.Errorf("error creating tailscaled config: %w", err) } @@ -910,16 +901,14 @@ func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error) return conf.AdvertiseServices, nil } -func (r *ProxyGroupReconciler) validate(_ *tsapi.ProxyGroup) error { - return nil -} - // getNodeMetadata gets metadata for all the pods owned by this ProxyGroup by // querying their state Secrets. It may not return the same number of items as // specified in the ProxyGroup spec if e.g. it is getting scaled up or down, or // some pods have failed to write state. +// +// The returned metadata will contain an entry for each state Secret that exists. func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup) (metadata []nodeMetadata, _ error) { - // List all state secrets owned by this ProxyGroup. + // List all state Secrets owned by this ProxyGroup. secrets := &corev1.SecretList{} if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, "state"))); err != nil { return nil, fmt.Errorf("failed to list state Secrets: %w", err) @@ -930,20 +919,20 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr return nil, fmt.Errorf("unexpected secret %s was labelled as owned by the ProxyGroup %s: %w", secret.Name, pg.Name, err) } + nm := nodeMetadata{ + ordinal: ordinal, + stateSecret: &secret, + } + prefs, ok, err := getDevicePrefs(&secret) if err != nil { return nil, err } - if !ok { - continue + if ok { + nm.tsID = prefs.Config.NodeID + nm.dnsName = prefs.Config.UserProfile.LoginName } - nm := nodeMetadata{ - ordinal: ordinal, - stateSecret: &secret, - tsID: prefs.Config.NodeID, - dnsName: prefs.Config.UserProfile.LoginName, - } pod := &corev1.Pod{} if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: fmt.Sprintf("%s-%d", pg.Name, ordinal)}, pod); err != nil && !apierrors.IsNotFound(err) { return nil, err @@ -953,23 +942,36 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr metadata = append(metadata, nm) } + // Sort for predictable ordering and status. + sort.Slice(metadata, func(i, j int) bool { + return metadata[i].ordinal < metadata[j].ordinal + }) + return metadata, nil } -func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, staticEndpoints map[string][]netip.AddrPort, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) { +// getRunningProxies will return status for all proxy Pods whose state Secret +// has an up to date Pod UID and at least a hostname. +func (r *ProxyGroupReconciler) getRunningProxies(ctx context.Context, pg *tsapi.ProxyGroup, staticEndpoints map[string][]netip.AddrPort) (devices []tsapi.TailnetDevice, _ error) { metadata, err := r.getNodeMetadata(ctx, pg) if err != nil { return nil, err } - for _, m := range metadata { - if !strings.EqualFold(string(m.stateSecret.Data[kubetypes.KeyPodUID]), m.podUID) { + for i, m := range metadata { + if m.podUID == "" || !strings.EqualFold(string(m.stateSecret.Data[kubetypes.KeyPodUID]), m.podUID) { // Current Pod has not yet written its UID to the state Secret, data may // be stale. continue } device := tsapi.TailnetDevice{} + if hostname, _, ok := strings.Cut(string(m.stateSecret.Data[kubetypes.KeyDeviceFQDN]), "."); ok { + device.Hostname = hostname + } else { + continue + } + if ipsB := m.stateSecret.Data[kubetypes.KeyDeviceIPs]; len(ipsB) > 0 { ips := []string{} if err := json.Unmarshal(ipsB, &ips); err != nil { @@ -978,11 +980,10 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, staticEndpoint device.TailnetIPs = ips } - if hostname, _, ok := strings.Cut(string(m.stateSecret.Data[kubetypes.KeyDeviceFQDN]), "."); ok { - device.Hostname = hostname - } - - if ep, ok := staticEndpoints[device.Hostname]; ok && len(ep) > 0 { + // TODO(tomhjp): This is our input to the proxy, but we should instead + // read this back from the proxy's state in some way to more accurately + // reflect its status. + if ep, ok := staticEndpoints[pgNodePortServiceName(pg.Name, int32(i))]; ok && len(ep) > 0 { eps := make([]string, 0, len(ep)) for _, e := range ep { eps = append(eps, e.String()) @@ -999,13 +1000,28 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, staticEndpoint type nodeMetadata struct { ordinal int stateSecret *corev1.Secret - // podUID is the UID of the current Pod or empty if the Pod does not exist. - podUID string - tsID tailcfg.StableNodeID - dnsName string + podUID string // or empty if the Pod no longer exists. + tsID tailcfg.StableNodeID + dnsName string +} + +func (r *ProxyGroupReconciler) notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason, error) { + return nil, ¬ReadyReason{ + reason: reason, + message: msg, + }, nil +} + +func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) { + err := fmt.Errorf(format, a...) + r.recorder.Event(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error()) + return nil, ¬ReadyReason{ + reason: reasonProxyGroupCreationFailed, + message: err.Error(), + }, err } -func (pr *ProxyGroupReconciler) setStatusReady(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason string, msg string, logger *zap.SugaredLogger) { - pr.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, msg, pg.Generation, pr.clock, logger) +type notReadyReason struct { + reason string + message string } diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 87b04a434c102..bd69b49a8978d 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -6,7 +6,6 @@ package main import ( - "context" "encoding/json" "fmt" "net/netip" @@ -22,6 +21,7 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" @@ -207,7 +207,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { }, }, expectedIPs: []netip.Addr{}, - expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning NodePort Services for static endpoints: failed to allocate NodePorts to ProxyGroup Services: not enough available ports to allocate all replicas (needed 4, got 3). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass \"default-pc\" must have bigger range allocated"}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning NodePort Services for static endpoints: failed to allocate NodePorts to ProxyGroup Services: not enough available ports to allocate all replicas (needed 4, got 3). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass \"default-pc\" must have bigger range allocated"}, expectedErr: "", expectStatefulSet: false, }, @@ -265,7 +265,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { {name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}}, labels: map[string]string{"zone": "eu-central"}}, }, expectedIPs: []netip.Addr{}, - expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning config Secrets: could not find static endpoints for replica \"test-0\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""}, expectedErr: "", expectStatefulSet: false, }, @@ -309,7 +309,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { }, }, expectedIPs: []netip.Addr{}, - expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to find any `status.addresses` of type \"ExternalIP\" on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass \"default-pc\""}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning config Secrets: could not find static endpoints for replica \"test-0\": failed to find any `status.addresses` of type \"ExternalIP\" on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass \"default-pc\""}, expectedErr: "", expectStatefulSet: false, }, @@ -576,7 +576,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { }, }, expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, - expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning ProxyGroup resources: error provisioning config Secrets: could not find static endpoints for replica \"test-0-nodeport\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning config Secrets: could not find static endpoints for replica \"test-0\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""}, expectStatefulSet: true, }, }, @@ -659,7 +659,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { Address: addr.ip, }) } - if err := fc.Create(context.Background(), no); err != nil { + if err := fc.Create(t.Context(), no); err != nil { t.Fatalf("failed to create node %q: %v", n.name, err) } createdNodes = append(createdNodes, *no) @@ -670,11 +670,11 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { pg.Spec.Replicas = r.replicas pc.Spec.StaticEndpoints = r.staticEndpointConfig - createOrUpdate(context.Background(), fc, "", pg, func(o *tsapi.ProxyGroup) { + createOrUpdate(t.Context(), fc, "", pg, func(o *tsapi.ProxyGroup) { o.Spec.Replicas = pg.Spec.Replicas }) - createOrUpdate(context.Background(), fc, "", pc, func(o *tsapi.ProxyClass) { + createOrUpdate(t.Context(), fc, "", pc, func(o *tsapi.ProxyClass) { o.Spec.StaticEndpoints = pc.Spec.StaticEndpoints }) @@ -686,7 +686,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { expectEvents(t, fr, r.expectedEvents) sts := &appsv1.StatefulSet{} - err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts) + err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts) if r.expectStatefulSet { if err != nil { t.Fatalf("failed to get StatefulSet: %v", err) @@ -694,7 +694,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { for j := range 2 { sec := &corev1.Secret{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: fmt.Sprintf("%s-%d-config", pg.Name, j)}, sec); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: fmt.Sprintf("%s-%d-config", pg.Name, j)}, sec); err != nil { t.Fatalf("failed to get state Secret for replica %d: %v", j, err) } @@ -740,7 +740,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { } pgroup := &tsapi.ProxyGroup{} - err = fc.Get(context.Background(), client.ObjectKey{Name: pg.Name}, pgroup) + err = fc.Get(t.Context(), client.ObjectKey{Name: pg.Name}, pgroup) if err != nil { t.Fatalf("failed to get ProxyGroup %q: %v", pg.Name, err) } @@ -762,7 +762,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { // node cleanup between reconciles // we created a new set of nodes for each for _, n := range createdNodes { - err := fc.Delete(context.Background(), &n) + err := fc.Delete(t.Context(), &n) if err != nil && !apierrors.IsNotFound(err) { t.Fatalf("failed to delete node: %v", err) } @@ -784,14 +784,14 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { clock: cl, } - if err := fc.Delete(context.Background(), pg); err != nil { + if err := fc.Delete(t.Context(), pg); err != nil { t.Fatalf("error deleting ProxyGroup: %v", err) } expectReconciled(t, reconciler, "", pg.Name) expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name) - if err := fc.Delete(context.Background(), pc); err != nil { + if err := fc.Delete(t.Context(), pc); err != nil { t.Fatalf("error deleting ProxyClass: %v", err) } expectMissing[tsapi.ProxyClass](t, fc, "", pc.Name) @@ -855,7 +855,8 @@ func TestProxyGroup(t *testing.T) { t.Run("proxyclass_not_ready", func(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, false, pc) }) @@ -870,7 +871,7 @@ func TestProxyGroup(t *testing.T) { LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, }}, } - if err := fc.Status().Update(context.Background(), pc); err != nil { + if err := fc.Status().Update(t.Context(), pc); err != nil { t.Fatal(err) } @@ -978,7 +979,7 @@ func TestProxyGroup(t *testing.T) { }) t.Run("delete_and_cleanup", func(t *testing.T) { - if err := fc.Delete(context.Background(), pg); err != nil { + if err := fc.Delete(t.Context(), pg); err != nil { t.Fatal(err) } @@ -1049,7 +1050,7 @@ func TestProxyGroupTypes(t *testing.T) { verifyProxyGroupCounts(t, reconciler, 0, 1) sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupEgress) @@ -1059,7 +1060,7 @@ func TestProxyGroupTypes(t *testing.T) { // Verify that egress configuration has been set up. cm := &corev1.ConfigMap{} cmName := fmt.Sprintf("%s-egress-config", pg.Name) - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: cmName}, cm); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: cmName}, cm); err != nil { t.Fatalf("failed to get ConfigMap: %v", err) } @@ -1135,7 +1136,7 @@ func TestProxyGroupTypes(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } @@ -1155,7 +1156,7 @@ func TestProxyGroupTypes(t *testing.T) { Replicas: ptr.To[int32](0), }, } - if err := fc.Create(context.Background(), pg); err != nil { + if err := fc.Create(t.Context(), pg); err != nil { t.Fatal(err) } @@ -1163,7 +1164,7 @@ func TestProxyGroupTypes(t *testing.T) { verifyProxyGroupCounts(t, reconciler, 1, 2) sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress) @@ -1306,7 +1307,7 @@ func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsap func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name string) *tsapi.ProxyClass { t.Helper() pc := &tsapi.ProxyClass{} - if err := fc.Get(context.Background(), client.ObjectKey{Name: name}, pc); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Name: name}, pc); err != nil { t.Fatal(err) } pc.Status = tsapi.ProxyClassStatus{ @@ -1319,7 +1320,7 @@ func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name s ObservedGeneration: pc.Generation, }}, } - if err := fc.Status().Update(context.Background(), pc); err != nil { + if err := fc.Status().Update(t.Context(), pc); err != nil { t.Fatal(err) } return pc @@ -1398,7 +1399,7 @@ func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) { t.Helper() secrets := &corev1.SecretList{} - if err := fc.List(context.Background(), secrets); err != nil { + if err := fc.List(t.Context(), secrets); err != nil { t.Fatal(err) } @@ -1413,6 +1414,7 @@ func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) { } func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup) { + t.Helper() const key = "profile-abc" for i := range pgReplicas(pg) { bytes, err := json.Marshal(map[string]any{ @@ -1424,6 +1426,17 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG t.Fatal(err) } + podUID := fmt.Sprintf("pod-uid-%d", i) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", pg.Name, i), + Namespace: "tailscale", + UID: types.UID(podUID), + }, + } + if _, err := createOrUpdate(t.Context(), fc, "tailscale", pod, nil); err != nil { + t.Fatalf("failed to create or update Pod %s: %v", pod.Name, err) + } mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) { s.Data = map[string][]byte{ currentProfileKey: []byte(key), @@ -1433,6 +1446,7 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG // TODO(tomhjp): We have two different mechanisms to retrieve device IDs. // Consolidate on this one. kubetypes.KeyDeviceID: []byte(fmt.Sprintf("nodeid-%d", i)), + kubetypes.KeyPodUID: []byte(podUID), } }) } @@ -1512,7 +1526,7 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { // Verify that the StatefulSet created for ProxyGrup has // the expected setting for the staging endpoint. sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index aba5f9e2df4b2..18bf1cb50400f 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -658,7 +658,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the ProxyGroup
          resources. Known condition types are `ProxyGroupReady`. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the ProxyGroup
          resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`.
          `ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled
          and ready. `ProxyGroupAvailable` indicates that at least one proxy is
          ready to serve traffic. | | | | `devices` _[TailnetDevice](#tailnetdevice) array_ | List of tailnet devices associated with the ProxyGroup StatefulSet. | | | diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index 17b13064bb4fc..5edb47f0da6c3 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -88,7 +88,11 @@ type ProxyGroupSpec struct { type ProxyGroupStatus struct { // List of status conditions to indicate the status of the ProxyGroup - // resources. Known condition types are `ProxyGroupReady`. + // resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`. + // `ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled + // and ready. `ProxyGroupAvailable` indicates that at least one proxy is + // ready to serve traffic. + // // +listType=map // +listMapKey=type // +optional From 4f3355e4997500cef05a7189e6a325c8a687730e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 6 Jul 2025 22:25:18 -0600 Subject: [PATCH 156/263] .github: Bump github/codeql-action from 3.29.0 to 3.29.1 (#16423) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.29.0 to 3.29.1. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/ce28f5bb42b7a9f2c824e633a3f6ee835bab6858...39edc492dbe16b1465b0cafca41432d857bdb31a) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.29.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2b471e943318f..610b93b610ea3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -55,7 +55,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0 + uses: github/codeql-action/init@39edc492dbe16b1465b0cafca41432d857bdb31a # v3.29.1 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0 + uses: github/codeql-action/autobuild@39edc492dbe16b1465b0cafca41432d857bdb31a # v3.29.1 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -80,4 +80,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0 + uses: github/codeql-action/analyze@39edc492dbe16b1465b0cafca41432d857bdb31a # v3.29.1 From 84eac7b8de99e0d6bad73f2b7998ede7228f2a2a Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 7 Jul 2025 12:12:59 +0100 Subject: [PATCH 157/263] cmd/k8s-operator: Allow custom ingress class names (#16472) This commit modifies the k8s operator to allow for customisation of the ingress class name via a new `OPERATOR_INGRESS_CLASS_NAME` environment variable. For backwards compatibility, this defaults to `tailscale`. When using helm, a new `ingress.name` value is provided that will set this environment variable and modify the name of the deployed `IngressClass` resource. Fixes https://github.com/tailscale/tailscale/issues/16248 Signed-off-by: David Bond --- .../deploy/chart/templates/deployment.yaml | 2 + .../deploy/chart/templates/ingressclass.yaml | 2 +- cmd/k8s-operator/deploy/chart/values.yaml | 4 ++ .../deploy/manifests/operator.yaml | 2 + cmd/k8s-operator/ingress-for-pg.go | 21 ++++++----- cmd/k8s-operator/ingress-for-pg_test.go | 22 +++++++---- cmd/k8s-operator/ingress.go | 10 ++--- cmd/k8s-operator/ingress_test.go | 23 ++++++++---- cmd/k8s-operator/operator.go | 37 +++++++++++-------- cmd/k8s-operator/operator_test.go | 12 +++--- 10 files changed, 83 insertions(+), 52 deletions(-) diff --git a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml index 01a290c076368..51d0a88c36671 100644 --- a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml @@ -70,6 +70,8 @@ spec: fieldPath: metadata.namespace - name: OPERATOR_LOGIN_SERVER value: {{ .Values.loginServer }} + - name: OPERATOR_INGRESS_CLASS_NAME + value: {{ .Values.ingressClass.name }} - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE diff --git a/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml b/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml index 208d58ee10f08..54851955db67a 100644 --- a/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: - name: tailscale # class name currently can not be changed + name: {{ .Values.ingressClass.name }} annotations: {} # we do not support default IngressClass annotation https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class spec: controller: tailscale.com/ts-ingress # controller name currently can not be changed diff --git a/cmd/k8s-operator/deploy/chart/values.yaml b/cmd/k8s-operator/deploy/chart/values.yaml index 0ba8d045a858d..2926f6d0759f2 100644 --- a/cmd/k8s-operator/deploy/chart/values.yaml +++ b/cmd/k8s-operator/deploy/chart/values.yaml @@ -77,6 +77,10 @@ operatorConfig: # In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here ingressClass: + # Allows for customization of the ingress class name used by the operator to identify ingresses to reconcile. This does + # not allow multiple operator instances to manage different ingresses, but provides an onboarding route for users that + # may have previously set up ingress classes named "tailscale" prior to using the operator. + name: "tailscale" enabled: true # proxyConfig contains configuraton that will be applied to any ingress/egress diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 2888575692594..cdf301318f923 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -5129,6 +5129,8 @@ spec: fieldPath: metadata.namespace - name: OPERATOR_LOGIN_SERVER value: null + - name: OPERATOR_INGRESS_CLASS_NAME + value: tailscale - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 09417fd0c8878..79bad92be080e 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -68,14 +68,15 @@ var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGRes type HAIngressReconciler struct { client.Client - recorder record.EventRecorder - logger *zap.SugaredLogger - tsClient tsClient - tsnetServer tsnetServer - tsNamespace string - lc localClient - defaultTags []string - operatorID string // stableID of the operator's Tailscale device + recorder record.EventRecorder + logger *zap.SugaredLogger + tsClient tsClient + tsnetServer tsnetServer + tsNamespace string + lc localClient + defaultTags []string + operatorID string // stableID of the operator's Tailscale device + ingressClassName string mu sync.Mutex // protects following // managedIngresses is a set of all ingress resources that we're currently @@ -162,7 +163,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin return false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err) } - if err := validateIngressClass(ctx, r.Client); err != nil { + if err := validateIngressClass(ctx, r.Client, r.ingressClassName); err != nil { logger.Infof("error validating tailscale IngressClass: %v.", err) return false, nil } @@ -645,7 +646,7 @@ func (r *HAIngressReconciler) tailnetCertDomain(ctx context.Context) (string, er func (r *HAIngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool { isTSIngress := ing != nil && ing.Spec.IngressClassName != nil && - *ing.Spec.IngressClassName == tailscaleIngressClassName + *ing.Spec.IngressClassName == r.ingressClassName pgAnnot := ing.Annotations[AnnotationProxyGroup] return isTSIngress && pgAnnot != "" } diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 2308514f3af9c..d29368caef59d 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -438,7 +438,12 @@ func TestValidateIngress(t *testing.T) { WithObjects(tt.ing). WithLists(&networkingv1.IngressList{Items: tt.existingIngs}). Build() + r := &HAIngressReconciler{Client: fc} + if tt.ing.Spec.IngressClassName != nil { + r.ingressClassName = *tt.ing.Spec.IngressClassName + } + err := r.validateIngress(context.Background(), tt.ing, tt.pg) if (err == nil && tt.wantErr != "") || (err != nil && err.Error() != tt.wantErr) { t.Errorf("validateIngress() error = %v, wantErr %v", err, tt.wantErr) @@ -841,14 +846,15 @@ func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeT } ingPGR := &HAIngressReconciler{ - Client: fc, - tsClient: ft, - defaultTags: []string{"tag:k8s"}, - tsNamespace: "operator-ns", - tsnetServer: fakeTsnetServer, - logger: zl.Sugar(), - recorder: record.NewFakeRecorder(10), - lc: lc, + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + tsNamespace: "operator-ns", + tsnetServer: fakeTsnetServer, + logger: zl.Sugar(), + recorder: record.NewFakeRecorder(10), + lc: lc, + ingressClassName: tsIngressClass.Name, } return ingPGR, fc, ft diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index d6277093824fb..d66cf9116f14a 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -32,7 +32,6 @@ import ( ) const ( - tailscaleIngressClassName = "tailscale" // ingressClass.metadata.name for tailscale IngressClass resource tailscaleIngressControllerName = "tailscale.com/ts-ingress" // ingressClass.spec.controllerName for tailscale IngressClass resource ingressClassDefaultAnnotation = "ingressclass.kubernetes.io/is-default-class" // we do not support this https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class indexIngressProxyClass = ".metadata.annotations.ingress-proxy-class" @@ -52,6 +51,7 @@ type IngressReconciler struct { managedIngresses set.Slice[types.UID] defaultProxyClass string + ingressClassName string } var ( @@ -132,7 +132,7 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare // This function adds a finalizer to ing, ensuring that we can handle orderly // deprovisioning later. func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error { - if err := validateIngressClass(ctx, a.Client); err != nil { + if err := validateIngressClass(ctx, a.Client, a.ingressClassName); err != nil { logger.Warnf("error validating tailscale IngressClass: %v. In future this might be a terminal error.", err) } if !slices.Contains(ing.Finalizers, FinalizerName) { @@ -266,17 +266,17 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga func (a *IngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool { return ing != nil && ing.Spec.IngressClassName != nil && - *ing.Spec.IngressClassName == tailscaleIngressClassName && + *ing.Spec.IngressClassName == a.ingressClassName && ing.Annotations[AnnotationProxyGroup] == "" } // validateIngressClass attempts to validate that 'tailscale' IngressClass // included in Tailscale installation manifests exists and has not been modified // to attempt to enable features that we do not support. -func validateIngressClass(ctx context.Context, cl client.Client) error { +func validateIngressClass(ctx context.Context, cl client.Client, ingressClassName string) error { ic := &networkingv1.IngressClass{ ObjectMeta: metav1.ObjectMeta{ - Name: tailscaleIngressClassName, + Name: ingressClassName, }, } if err := cl.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) { diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index e4396eb106a96..fe4d90c785c47 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -36,7 +36,8 @@ func TestTailscaleIngress(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: ft, @@ -120,7 +121,8 @@ func TestTailscaleIngressHostname(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: ft, @@ -245,7 +247,8 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: ft, @@ -350,7 +353,8 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: ft, @@ -498,7 +502,8 @@ func TestIngressProxyClassAnnotation(t *testing.T) { mustCreate(t, fc, ing) ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: &fakeTSClient{}, @@ -568,7 +573,8 @@ func TestIngressLetsEncryptStaging(t *testing.T) { mustCreate(t, fc, ing) ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: &fakeTSClient{}, @@ -675,8 +681,9 @@ func TestEmptyPath(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - recorder: fr, - Client: fc, + recorder: fr, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: ft, diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 276de411c45cb..96b3b37ad0340 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -83,6 +83,7 @@ func main() { defaultProxyClass = defaultEnv("PROXY_DEFAULT_CLASS", "") isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false) loginServer = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/") + ingressClassName = defaultEnv("OPERATOR_INGRESS_CLASS_NAME", "tailscale") ) var opts []kzap.Opts @@ -133,6 +134,7 @@ func main() { proxyFirewallMode: tsFirewallMode, defaultProxyClass: defaultProxyClass, loginServer: loginServer, + ingressClassName: ingressClassName, } runReconcilers(rOpts) } @@ -343,7 +345,7 @@ func runReconcilers(opts reconcilerOpts) { // ProxyClass's name. proxyClassFilterForIngress := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForIngress(mgr.GetClient(), startlog)) // Enque Ingress if a managed Service or backend Service associated with a tailscale Ingress changes. - svcHandlerForIngress := handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngress(mgr.GetClient(), startlog)) + svcHandlerForIngress := handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngress(mgr.GetClient(), startlog, opts.ingressClassName)) err = builder. ControllerManagedBy(mgr). For(&networkingv1.Ingress{}). @@ -358,6 +360,7 @@ func runReconcilers(opts reconcilerOpts) { Client: mgr.GetClient(), logger: opts.log.Named("ingress-reconciler"), defaultProxyClass: opts.defaultProxyClass, + ingressClassName: opts.ingressClassName, }) if err != nil { startlog.Fatalf("could not create ingress reconciler: %v", err) @@ -379,19 +382,20 @@ func runReconcilers(opts reconcilerOpts) { ControllerManagedBy(mgr). For(&networkingv1.Ingress{}). Named("ingress-pg-reconciler"). - Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))). + Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog, opts.ingressClassName))). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(HAIngressesFromSecret(mgr.GetClient(), startlog))). Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter). Complete(&HAIngressReconciler{ - recorder: eventRecorder, - tsClient: opts.tsClient, - tsnetServer: opts.tsServer, - defaultTags: strings.Split(opts.proxyTags, ","), - Client: mgr.GetClient(), - logger: opts.log.Named("ingress-pg-reconciler"), - lc: lc, - operatorID: id, - tsNamespace: opts.tailscaleNamespace, + recorder: eventRecorder, + tsClient: opts.tsClient, + tsnetServer: opts.tsServer, + defaultTags: strings.Split(opts.proxyTags, ","), + Client: mgr.GetClient(), + logger: opts.log.Named("ingress-pg-reconciler"), + lc: lc, + operatorID: id, + tsNamespace: opts.tailscaleNamespace, + ingressClassName: opts.ingressClassName, }) if err != nil { startlog.Fatalf("could not create ingress-pg-reconciler: %v", err) @@ -697,6 +701,9 @@ type reconcilerOpts struct { defaultProxyClass string // loginServer is the coordination server URL that should be used by managed resources. loginServer string + // ingressClassName is the name of the ingress class used by reconcilers of Ingress resources. This defaults + // to "tailscale" but can be customised. + ingressClassName string } // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each @@ -1015,7 +1022,7 @@ func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) // The Services of interest are backend Services for tailscale Ingress and // managed Services for an StatefulSet for a proxy configured for tailscale // Ingress -func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { +func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger, ingressClassName string) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { if isManagedByType(o, "ingress") { ingName := parentFromObjectLabels(o) @@ -1028,7 +1035,7 @@ func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handl } reqs := make([]reconcile.Request, 0) for _, ing := range ingList.Items { - if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName { + if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != ingressClassName { return nil } if hasProxyGroupAnnotation(&ing) { @@ -1533,7 +1540,7 @@ func indexPGIngresses(o client.Object) []string { // serviceHandlerForIngressPG returns a handler for Service events that ensures that if the Service // associated with an event is a backend Service for a tailscale Ingress with ProxyGroup annotation, // the associated Ingress gets reconciled. -func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { +func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger, ingressClassName string) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { ingList := networkingv1.IngressList{} if err := cl.List(ctx, &ingList, client.InNamespace(o.GetNamespace())); err != nil { @@ -1542,7 +1549,7 @@ func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) han } reqs := make([]reconcile.Request, 0) for _, ing := range ingList.Items { - if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName { + if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != ingressClassName { continue } if !hasProxyGroupAnnotation(&ing) { diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index a9f08c18b4793..1f700f13a4fc0 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -1549,6 +1549,8 @@ func Test_isMagicDNSName(t *testing.T) { } func Test_serviceHandlerForIngress(t *testing.T) { + const tailscaleIngressClassName = "tailscale" + fc := fake.NewFakeClient() zl, err := zap.NewDevelopment() if err != nil { @@ -1578,7 +1580,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { } mustCreate(t, fc, svc1) wantReqs := []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-1", Name: "ing-1"}}} - gotReqs := serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), svc1) + gotReqs := serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), svc1) if diff := cmp.Diff(gotReqs, wantReqs); diff != "" { t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff) } @@ -1605,7 +1607,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { } mustCreate(t, fc, backendSvc) wantReqs = []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-2", Name: "ing-2"}}} - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), backendSvc) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), backendSvc) if diff := cmp.Diff(gotReqs, wantReqs); diff != "" { t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff) } @@ -1634,7 +1636,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { } mustCreate(t, fc, backendSvc2) wantReqs = []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-3", Name: "ing-3"}}} - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), backendSvc2) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), backendSvc2) if diff := cmp.Diff(gotReqs, wantReqs); diff != "" { t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff) } @@ -1661,7 +1663,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { }, } mustCreate(t, fc, nonTSBackend) - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), nonTSBackend) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), nonTSBackend) if len(gotReqs) > 0 { t.Errorf("unexpected reconcile request for a Service that does not belong to a Tailscale Ingress: %#+v\n", gotReqs) } @@ -1675,7 +1677,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { }, } mustCreate(t, fc, someSvc) - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), someSvc) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), someSvc) if len(gotReqs) > 0 { t.Errorf("unexpected reconcile request for a Service that does not belong to any Ingress: %#+v\n", gotReqs) } From 540eb0563803e86fd08369d242e0aff4db5fee32 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 7 Jul 2025 08:45:13 -0700 Subject: [PATCH 158/263] wgengine/magicsock: make Conn.Send() lazyEndpoint aware (#16465) A lazyEndpoint may end up on this TX codepath when wireguard-go is deemed "under load" and ends up transmitting a cookie reply using the received conn.Endpoint. Updates tailscale/corp#20732 Updates tailscale/corp#30042 Signed-off-by: Jordan Whited --- wgengine/magicsock/magicsock.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 5719b20f9ab6f..8d3b2d082c633 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -1363,12 +1363,18 @@ func (c *Conn) Send(buffs [][]byte, ep conn.Endpoint, offset int) (err error) { metricSendDataNetworkDown.Add(n) return errNetworkDown } - if ep, ok := ep.(*endpoint); ok { + switch ep := ep.(type) { + case *endpoint: return ep.send(buffs, offset) + case *lazyEndpoint: + // A [*lazyEndpoint] may end up on this TX codepath when wireguard-go is + // deemed "under handshake load" and ends up transmitting a cookie reply + // using the received [conn.Endpoint] in [device.SendHandshakeCookie]. + if ep.src.ap.Addr().Is6() { + return c.pconn6.WriteBatchTo(buffs, ep.src, offset) + } + return c.pconn4.WriteBatchTo(buffs, ep.src, offset) } - // If it's not of type *endpoint, it's probably *lazyEndpoint, which means - // we don't actually know who the peer is and we're waiting for wireguard-go - // to switch the endpoint. See go/corp/20732. return nil } @@ -1702,6 +1708,11 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach } // TODO(jwhited): reuse [lazyEndpoint] across calls to receiveIP() // for the same batch & [epAddr] src. + // + // TODO(jwhited): implement [lazyEndpoint] integration to call + // [endpoint.noteRecvActivity], which triggers just-in-time + // wireguard-go configuration of the peer, prior to peer lookup + // within wireguard-go. return &lazyEndpoint{c: c, src: src}, size, true } cache.epAddr = src @@ -1709,8 +1720,6 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach cache.gen = de.numStopAndReset() ep = de } - // TODO(jwhited): consider the implications of not recording this receive - // activity due to an early [lazyEndpoint] return above. now := mono.Now() ep.lastRecvUDPAny.StoreAtomic(now) ep.noteRecvActivity(src, now) From 3b32cc758647bde17c9e3fef36086439ba1bb7e8 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 7 Jul 2025 09:38:10 -0700 Subject: [PATCH 159/263] wgengine/magicsock: simplify Geneve-encapsulated disco.Ping handling (#16448) Just make [relayManager] always handle it, there's no benefit to checking bestAddr's. Also, remove passing of disco.Pong to [relayManager] in endpoint.handlePongConnLocked(), which is redundant with the callsite in Conn.handleDiscoMessage(). Conn.handleDiscoMessage() already passes to [relayManager] if the txID us not known to any [*endpoint]. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 7 -- wgengine/magicsock/magicsock.go | 129 +++++++++++------------- wgengine/magicsock/relaymanager.go | 7 +- wgengine/magicsock/relaymanager_test.go | 2 +- 4 files changed, 61 insertions(+), 84 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 0569341ff4ab3..4780c7f37a781 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1656,13 +1656,6 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAdd de.mu.Lock() defer de.mu.Unlock() - if src.vni.isSet() && src != de.bestAddr.epAddr { - // "src" is not our bestAddr, but [relayManager] might be in the - // middle of probing it, awaiting pong reception. Make it aware. - de.c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(de.c, m, di, src) - return false - } - isDerp := src.ap.Addr() == tailcfg.DerpMagicIPAddr sp, ok := de.sentPing[m.TxID] diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 8d3b2d082c633..37de4668ab394 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2103,7 +2103,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake c.logf("[unexpected] %T packets should not come from a relay server with Geneve control bit set", dm) return } - c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(c, challenge, di, src) + c.relayManager.handleGeneveEncapDiscoMsg(c, challenge, di, src) return } @@ -2125,7 +2125,10 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake return true }) if !knownTxID && src.vni.isSet() { - c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(c, dm, di, src) + // If it's an unknown TxID, and it's Geneve-encapsulated, then + // make [relayManager] aware. It might be in the middle of probing + // src. + c.relayManager.handleGeneveEncapDiscoMsg(c, dm, di, src) } case *disco.CallMeMaybe, *disco.CallMeMaybeVia: var via *disco.CallMeMaybeVia @@ -2233,6 +2236,35 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src epAddr, di *discoInfo, derpN di.lastPingTime = time.Now() isDerp := src.ap.Addr() == tailcfg.DerpMagicIPAddr + if src.vni.isSet() { + if isDerp { + c.logf("[unexpected] got Geneve-encapsulated disco ping from %v/%v over DERP", src, derpNodeSrc) + return + } + + // [relayManager] is always responsible for handling (replying) to + // Geneve-encapsulated [disco.Ping] messages in the interest of + // simplicity. It might be in the middle of probing src, so it must be + // made aware. + c.relayManager.handleGeneveEncapDiscoMsg(c, dm, di, src) + return + } + + // This is a naked [disco.Ping] without a VNI. + + // If we can figure out with certainty which node key this disco + // message is for, eagerly update our [epAddr]<>node and disco<>node + // mappings to make p2p path discovery faster in simple + // cases. Without this, disco would still work, but would be + // reliant on DERP call-me-maybe to establish the disco<>node + // mapping, and on subsequent disco handlePongConnLocked to establish + // the IP:port<>disco mapping. + if nk, ok := c.unambiguousNodeKeyOfPingLocked(dm, di.discoKey, derpNodeSrc); ok { + if !isDerp { + c.peerMap.setNodeKeyForEpAddr(src, nk) + } + } + // numNodes tracks how many nodes (node keys) are associated with the disco // key tied to this inbound ping. Multiple nodes may share the same disco // key in the case of node sharing and users switching accounts. @@ -2244,81 +2276,34 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src epAddr, di *discoInfo, derpN // a dstKey if the dst ip:port is DERP. dstKey := derpNodeSrc - switch { - case src.vni.isSet(): - if isDerp { - c.logf("[unexpected] got Geneve-encapsulated disco ping from %v/%v over DERP", src, derpNodeSrc) - return - } - - var bestEpAddr epAddr - var discoKey key.DiscoPublic - ep, ok := c.peerMap.endpointForEpAddr(src) - if ok { - ep.mu.Lock() - bestEpAddr = ep.bestAddr.epAddr - ep.mu.Unlock() - disco := ep.disco.Load() - if disco != nil { - discoKey = disco.key + // Remember this route if not present. + var dup bool + if isDerp { + if ep, ok := c.peerMap.endpointForNodeKey(derpNodeSrc); ok { + if ep.addCandidateEndpoint(src.ap, dm.TxID) { + return } - } - - if src == bestEpAddr && discoKey == di.discoKey { - // We have an associated endpoint with src as its bestAddr. Set - // numNodes so we TX a pong further down. numNodes = 1 - } else { - // We have no [endpoint] in the [peerMap] for this relay [epAddr] - // using it as a bestAddr. [relayManager] might be in the middle of - // probing it or attempting to set it as best via - // [endpoint.udpRelayEndpointReady()]. Make [relayManager] aware. - c.relayManager.handleGeneveEncapDiscoMsgNotBestAddr(c, dm, di, src) - return - } - default: // no VNI - // If we can figure out with certainty which node key this disco - // message is for, eagerly update our [epAddr]<>node and disco<>node - // mappings to make p2p path discovery faster in simple - // cases. Without this, disco would still work, but would be - // reliant on DERP call-me-maybe to establish the disco<>node - // mapping, and on subsequent disco handlePongConnLocked to establish - // the IP:port<>disco mapping. - if nk, ok := c.unambiguousNodeKeyOfPingLocked(dm, di.discoKey, derpNodeSrc); ok { - if !isDerp { - c.peerMap.setNodeKeyForEpAddr(src, nk) - } } - - // Remember this route if not present. - var dup bool - if isDerp { - if ep, ok := c.peerMap.endpointForNodeKey(derpNodeSrc); ok { - if ep.addCandidateEndpoint(src.ap, dm.TxID) { - return - } - numNodes = 1 - } - } else { - c.peerMap.forEachEndpointWithDiscoKey(di.discoKey, func(ep *endpoint) (keepGoing bool) { - if ep.addCandidateEndpoint(src.ap, dm.TxID) { - dup = true - return false - } - numNodes++ - if numNodes == 1 && dstKey.IsZero() { - dstKey = ep.publicKey - } - return true - }) - if dup { - return + } else { + c.peerMap.forEachEndpointWithDiscoKey(di.discoKey, func(ep *endpoint) (keepGoing bool) { + if ep.addCandidateEndpoint(src.ap, dm.TxID) { + dup = true + return false } - if numNodes > 1 { - // Zero it out if it's ambiguous, so sendDiscoMessage logging - // isn't confusing. - dstKey = key.NodePublic{} + numNodes++ + if numNodes == 1 && dstKey.IsZero() { + dstKey = ep.publicKey } + return true + }) + if dup { + return + } + if numNodes > 1 { + // Zero it out if it's ambiguous, so sendDiscoMessage logging + // isn't confusing. + dstKey = key.NodePublic{} } } diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 1c173c01ac138..c8c9ed41b7b82 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -325,10 +325,9 @@ func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, lastBest addrQuality, }) } -// handleGeneveEncapDiscoMsgNotBestAddr handles reception of Geneve-encapsulated -// disco messages if they are not associated with any known -// [*endpoint.bestAddr]. -func (r *relayManager) handleGeneveEncapDiscoMsgNotBestAddr(conn *Conn, dm disco.Message, di *discoInfo, src epAddr) { +// handleGeneveEncapDiscoMsg handles reception of Geneve-encapsulated disco +// messages. +func (r *relayManager) handleGeneveEncapDiscoMsg(conn *Conn, dm disco.Message, di *discoInfo, src epAddr) { relayManagerInputEvent(r, nil, &r.rxHandshakeDiscoMsgCh, relayHandshakeDiscoMsgEvent{conn: conn, msg: dm, disco: di.discoKey, from: src.ap, vni: src.vni.get(), at: time.Now()}) } diff --git a/wgengine/magicsock/relaymanager_test.go b/wgengine/magicsock/relaymanager_test.go index 8feff2f3d5ca8..8f92360122d0e 100644 --- a/wgengine/magicsock/relaymanager_test.go +++ b/wgengine/magicsock/relaymanager_test.go @@ -26,7 +26,7 @@ func TestRelayManagerInitAndIdle(t *testing.T) { <-rm.runLoopStoppedCh rm = relayManager{} - rm.handleGeneveEncapDiscoMsgNotBestAddr(&Conn{discoPrivate: key.NewDisco()}, &disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, epAddr{}) + rm.handleGeneveEncapDiscoMsg(&Conn{discoPrivate: key.NewDisco()}, &disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, epAddr{}) <-rm.runLoopStoppedCh rm = relayManager{} From a84d58015ce875863a266dacdfb1ffd65de1615d Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 7 Jul 2025 10:06:38 -0700 Subject: [PATCH 160/263] wgengine/magicsock: fix lazyEndpoint DstIP() vs SrcIP() (#16453) These were flipped. DstIP() and DstIPBytes() are used internally by wireguard-go as part of a handshake DoS mitigation strategy. Updates tailscale/corp#20732 Updates tailscale/corp#30042 Signed-off-by: Jordan Whited --- wgengine/magicsock/magicsock.go | 34 +++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 37de4668ab394..a7eab36786f12 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3774,12 +3774,12 @@ func (c *Conn) SetLastNetcheckReportForTest(ctx context.Context, report *netchec c.lastNetCheckReport.Store(report) } -// lazyEndpoint is a wireguard conn.Endpoint for when magicsock received a +// lazyEndpoint is a wireguard [conn.Endpoint] for when magicsock received a // non-disco (presumably WireGuard) packet from a UDP address from which we -// can't map to a Tailscale peer. But Wireguard most likely can, once it -// decrypts it. So we implement the conn.PeerAwareEndpoint interface +// can't map to a Tailscale peer. But WireGuard most likely can, once it +// decrypts it. So we implement the [conn.PeerAwareEndpoint] interface // from https://github.com/tailscale/wireguard-go/pull/27 to allow WireGuard -// to tell us who it is later and get the correct conn.Endpoint. +// to tell us who it is later and get the correct [conn.Endpoint]. type lazyEndpoint struct { c *Conn src epAddr @@ -3788,12 +3788,26 @@ type lazyEndpoint struct { var _ conn.PeerAwareEndpoint = (*lazyEndpoint)(nil) var _ conn.Endpoint = (*lazyEndpoint)(nil) -func (le *lazyEndpoint) ClearSrc() {} -func (le *lazyEndpoint) SrcIP() netip.Addr { return le.src.ap.Addr() } -func (le *lazyEndpoint) DstIP() netip.Addr { return netip.Addr{} } -func (le *lazyEndpoint) SrcToString() string { return le.src.String() } -func (le *lazyEndpoint) DstToString() string { return "dst" } -func (le *lazyEndpoint) DstToBytes() []byte { return nil } +func (le *lazyEndpoint) ClearSrc() {} +func (le *lazyEndpoint) SrcIP() netip.Addr { return netip.Addr{} } + +// DstIP returns the remote address of the peer. +// +// Note: DstIP is used internally by wireguard-go as part of handshake DoS +// mitigation. +func (le *lazyEndpoint) DstIP() netip.Addr { return le.src.ap.Addr() } + +func (le *lazyEndpoint) SrcToString() string { return "" } +func (le *lazyEndpoint) DstToString() string { return le.src.String() } + +// DstToBytes returns a binary representation of the remote address of the peer. +// +// Note: DstToBytes is used internally by wireguard-go as part of handshake DoS +// mitigation. +func (le *lazyEndpoint) DstToBytes() []byte { + b, _ := le.src.ap.MarshalBinary() + return b +} // FromPeer implements [conn.PeerAwareEndpoint]. We return a [*lazyEndpoint] in // our [conn.ReceiveFunc]s when we are unable to identify the peer at WireGuard From 04d24cdbd4b551d95f85ca3b9b36ef147503d2b7 Mon Sep 17 00:00:00 2001 From: Naman Sood Date: Mon, 7 Jul 2025 15:36:16 -0400 Subject: [PATCH 161/263] wgengine/netstack: correctly proxy half-closed TCP connections TCP connections are two unidirectional data streams, and if one of these streams closes, we should not assume the other half is closed as well. For example, if an HTTP client closes its write half of the connection early, it may still be expecting to receive data on its read half, so we should keep the server -> client half of the connection open, while terminating the client -> server half. Fixes tailscale/corp#29837. Signed-off-by: Naman Sood --- wgengine/netstack/netstack.go | 43 ++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/wgengine/netstack/netstack.go b/wgengine/netstack/netstack.go index dab692ead4aa7..d97c669463d78 100644 --- a/wgengine/netstack/netstack.go +++ b/wgengine/netstack/netstack.go @@ -1435,6 +1435,13 @@ func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) { } } +// tcpCloser is an interface to abstract around various TCPConn types that +// allow closing of the read and write streams independently of each other. +type tcpCloser interface { + CloseRead() error + CloseWrite() error +} + func (ns *Impl) forwardTCP(getClient func(...tcpip.SettableSocketOption) *gonet.TCPConn, clientRemoteIP netip.Addr, wq *waiter.Queue, dialAddr netip.AddrPort) (handled bool) { dialAddrStr := dialAddr.String() if debugNetstack() { @@ -1501,18 +1508,48 @@ func (ns *Impl) forwardTCP(getClient func(...tcpip.SettableSocketOption) *gonet. } defer client.Close() + // As of 2025-07-03, backend is always either a net.TCPConn + // from stdDialer.DialContext (which has the requisite functions), + // or nil from hangDialer in tests (in which case we would have + // errored out by now), so this conversion should always succeed. + backendTCPCloser, backendIsTCPCloser := backend.(tcpCloser) connClosed := make(chan error, 2) go func() { _, err := io.Copy(backend, client) + if err != nil { + err = fmt.Errorf("client -> backend: %w", err) + } connClosed <- err + err = nil + if backendIsTCPCloser { + err = backendTCPCloser.CloseWrite() + } + err = errors.Join(err, client.CloseRead()) + if err != nil { + ns.logf("client -> backend close connection: %v", err) + } }() go func() { _, err := io.Copy(client, backend) + if err != nil { + err = fmt.Errorf("backend -> client: %w", err) + } connClosed <- err + err = nil + if backendIsTCPCloser { + err = backendTCPCloser.CloseRead() + } + err = errors.Join(err, client.CloseWrite()) + if err != nil { + ns.logf("backend -> client close connection: %v", err) + } }() - err = <-connClosed - if err != nil { - ns.logf("proxy connection closed with error: %v", err) + // Wait for both ends of the connection to close. + for range 2 { + err = <-connClosed + if err != nil { + ns.logf("proxy connection closed with error: %v", err) + } } ns.logf("[v2] netstack: forwarder connection to %s closed", dialAddrStr) return From 3e01652e4dba619d475cc98e691c0e1d155969ae Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Thu, 3 Jul 2025 14:25:33 -0500 Subject: [PATCH 162/263] ipn/ipnlocal: add (*LocalBackend).RefreshExitNode In this PR, we add (*LocalBackend).RefreshExitNode which determines which exit node to use based on the current prefs and netmap and switches to it if needed. It supports both scenarios when an exit node is specified by IP (rather than ID) and needs to be resolved once the netmap is ready as well as auto exit nodes. We then use it in (*LocalBackend).SetControlClientStatus when the netmap changes, and wherever (*LocalBackend).pickNewAutoExitNode was previously used. Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 77 +++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 21057c0e675db..a69b7dd5a1289 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1627,16 +1627,6 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control if applySysPolicy(prefs, b.overrideAlwaysOn) { prefsChanged = true } - if prefs.AutoExitNode.IsSet() { - // Re-evaluate exit node suggestion in case circumstances have changed. - _, err := b.suggestExitNodeLocked(curNetMap) - if err != nil && !errors.Is(err, ErrNoPreferredDERP) { - b.logf("SetControlClientStatus failed to select auto exit node: %v", err) - } - } - if setExitNodeID(prefs, b.lastSuggestedExitNode, curNetMap) { - prefsChanged = true - } // Until recently, we did not store the account's tailnet name. So check if this is the case, // and backfill it on incoming status update. @@ -1653,6 +1643,8 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control }); err != nil { b.logf("Failed to save new controlclient state: %v", err) } + + b.sendToLocked(ipn.Notify{Prefs: ptr.To(prefs.View())}, allClients) } // initTKALocked is dependent on CurrentProfile.ID, which is initialized @@ -1695,16 +1687,17 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control b.mu.Unlock() // Now complete the lock-free parts of what we started while locked. - if prefsChanged { - b.send(ipn.Notify{Prefs: ptr.To(prefs.View())}) - } - if st.NetMap != nil { + // Check and update the exit node if needed, now that we have a new netmap. + b.RefreshExitNode() + if envknob.NoLogsNoSupport() && st.NetMap.HasCap(tailcfg.CapabilityDataPlaneAuditLogs) { msg := "tailnet requires logging to be enabled. Remove --no-logs-no-support from tailscaled command line." b.health.SetLocalLogConfigHealth(errors.New(msg)) // Connecting to this tailnet without logging is forbidden; boot us outta here. b.mu.Lock() + // Get the current prefs again, since we unlocked above. + prefs := b.pm.CurrentPrefs().AsStruct() prefs.WantRunning = false p := prefs.View() if err := b.pm.SetPrefs(p, ipn.NetworkProfile{ @@ -1999,7 +1992,7 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo if !ok || n.StableID() != exitNodeID { continue } - b.goTracker.Go(b.pickNewAutoExitNode) + b.goTracker.Go(b.RefreshExitNode) break } } @@ -5898,30 +5891,50 @@ func (b *LocalBackend) setNetInfo(ni *tailcfg.NetInfo) { } cc.SetNetInfo(ni) if refresh { - b.pickNewAutoExitNode() + b.RefreshExitNode() } } -// pickNewAutoExitNode picks a new automatic exit node if needed. -func (b *LocalBackend) pickNewAutoExitNode() { - unlock := b.lockAndGetUnlock() - defer unlock() +// RefreshExitNode determines which exit node to use based on the current +// prefs and netmap and switches to it if needed. +func (b *LocalBackend) RefreshExitNode() { + if b.resolveExitNode() { + b.authReconfig() + } +} - newSuggestion, err := b.suggestExitNodeLocked(nil) - if err != nil { - b.logf("setAutoExitNodeID: %v", err) - return +// resolveExitNode determines which exit node to use based on the current +// prefs and netmap. It updates the exit node ID in the prefs if needed, +// sends a notification to clients, and returns true if the exit node has changed. +// +// It is the caller's responsibility to reconfigure routes and actually +// start using the selected exit node, if needed. +// +// b.mu must not be held. +func (b *LocalBackend) resolveExitNode() (changed bool) { + b.mu.Lock() + defer b.mu.Unlock() + + nm := b.currentNode().NetMap() + prefs := b.pm.CurrentPrefs().AsStruct() + if prefs.AutoExitNode.IsSet() { + _, err := b.suggestExitNodeLocked(nil) + if err != nil && !errors.Is(err, ErrNoPreferredDERP) { + b.logf("failed to select auto exit node: %v", err) + } } - if b.pm.CurrentPrefs().ExitNodeID() == newSuggestion.ID { - return + if !setExitNodeID(prefs, b.lastSuggestedExitNode, nm) { + return false // no changes } - _, err = b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ - Prefs: ipn.Prefs{ExitNodeID: newSuggestion.ID}, - ExitNodeIDSet: true, - }, unlock) - if err != nil { - b.logf("setAutoExitNodeID: failed to apply exit node ID preference: %v", err) + + if err := b.pm.SetPrefs(prefs.View(), ipn.NetworkProfile{ + MagicDNSName: nm.MagicDNSSuffix(), + DomainName: nm.DomainName(), + }); err != nil { + b.logf("failed to save exit node changes: %v", err) } + b.sendToLocked(ipn.Notify{Prefs: ptr.To(prefs.View())}, allClients) + return true } // setNetMapLocked updates the LocalBackend state to reflect the newly From 4c1c0bac8dcaa717c9909d7b5c9c9991223e9f5f Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Thu, 3 Jul 2025 14:32:28 -0500 Subject: [PATCH 163/263] ipn/ipnlocal: plumb nodeBackend into suggestExitNode to support delta updates, such as online status changes Now that (*LocalBackend).suggestExitNodeLocked is never called with a non-current netmap (the netMap parameter is always nil, indicating that the current netmap should be used), we can remove the unused parameter. Additionally, instead of suggestExitNodeLocked passing the most recent full netmap to suggestExitNode, we now pass the current nodeBackend so it can access peers with delta updates applied. Finally, with that fixed, we no longer need to skip TestUpdateNetmapDeltaAutoExitNode. Updates tailscale/corp#29969 Fixes #16455 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 41 +++++++++++++++----------------------- ipn/ipnlocal/local_test.go | 9 ++++++--- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index a69b7dd5a1289..5fbb0bd9849d9 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1947,10 +1947,7 @@ func (b *LocalBackend) sysPolicyChanged(policy *rsop.PolicyChange) { if policy.HasChanged(syspolicy.AllowedSuggestedExitNodes) { b.refreshAllowedSuggestions() // Re-evaluate exit node suggestion now that the policy setting has changed. - b.mu.Lock() - _, err := b.suggestExitNodeLocked(nil) - b.mu.Unlock() - if err != nil && !errors.Is(err, ErrNoPreferredDERP) { + if _, err := b.SuggestExitNode(); err != nil && !errors.Is(err, ErrNoPreferredDERP) { b.logf("failed to select auto exit node: %v", err) } // If [syspolicy.ExitNodeID] is set to `auto:any`, the suggested exit node ID @@ -4490,7 +4487,7 @@ func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce) // anyway, so its return value can be ignored here. applySysPolicy(newp, b.overrideAlwaysOn) if newp.AutoExitNode.IsSet() { - if _, err := b.suggestExitNodeLocked(nil); err != nil { + if _, err := b.suggestExitNodeLocked(); err != nil { b.logf("failed to select auto exit node: %v", err) } } @@ -5918,7 +5915,7 @@ func (b *LocalBackend) resolveExitNode() (changed bool) { nm := b.currentNode().NetMap() prefs := b.pm.CurrentPrefs().AsStruct() if prefs.AutoExitNode.IsSet() { - _, err := b.suggestExitNodeLocked(nil) + _, err := b.suggestExitNodeLocked() if err != nil && !errors.Is(err, ErrNoPreferredDERP) { b.logf("failed to select auto exit node: %v", err) } @@ -7445,19 +7442,12 @@ var ErrNoPreferredDERP = errors.New("no preferred DERP, try again later") // Peers are selected based on having a DERP home that is the lowest latency to this device. For peers // without a DERP home, we look for geographic proximity to this device's DERP home. // -// netMap is an optional netmap to use that overrides b.netMap (needed for SetControlClientStatus before b.netMap is updated). -// If netMap is nil, then b.netMap is used. -// // b.mu.lock() must be held. -func (b *LocalBackend) suggestExitNodeLocked(netMap *netmap.NetworkMap) (response apitype.ExitNodeSuggestionResponse, err error) { - // netMap is an optional netmap to use that overrides b.netMap (needed for SetControlClientStatus before b.netMap is updated). If netMap is nil, then b.netMap is used. - if netMap == nil { - netMap = b.NetMap() - } +func (b *LocalBackend) suggestExitNodeLocked() (response apitype.ExitNodeSuggestionResponse, err error) { lastReport := b.MagicConn().GetLastNetcheckReport(b.ctx) prevSuggestion := b.lastSuggestedExitNode - res, err := suggestExitNode(lastReport, netMap, prevSuggestion, randomRegion, randomNode, b.getAllowedSuggestions()) + res, err := suggestExitNode(lastReport, b.currentNode(), prevSuggestion, randomRegion, randomNode, b.getAllowedSuggestions()) if err != nil { return res, err } @@ -7468,7 +7458,7 @@ func (b *LocalBackend) suggestExitNodeLocked(netMap *netmap.NetworkMap) (respons func (b *LocalBackend) SuggestExitNode() (response apitype.ExitNodeSuggestionResponse, err error) { b.mu.Lock() defer b.mu.Unlock() - return b.suggestExitNodeLocked(nil) + return b.suggestExitNodeLocked() } // getAllowedSuggestions returns a set of exit nodes permitted by the most recent @@ -7512,22 +7502,23 @@ func fillAllowedSuggestions() set.Set[tailcfg.StableNodeID] { return s } -func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSuggestion tailcfg.StableNodeID, selectRegion selectRegionFunc, selectNode selectNodeFunc, allowList set.Set[tailcfg.StableNodeID]) (res apitype.ExitNodeSuggestionResponse, err error) { +func suggestExitNode(report *netcheck.Report, nb *nodeBackend, prevSuggestion tailcfg.StableNodeID, selectRegion selectRegionFunc, selectNode selectNodeFunc, allowList set.Set[tailcfg.StableNodeID]) (res apitype.ExitNodeSuggestionResponse, err error) { + netMap := nb.NetMap() if report == nil || report.PreferredDERP == 0 || netMap == nil || netMap.DERPMap == nil { return res, ErrNoPreferredDERP } - candidates := make([]tailcfg.NodeView, 0, len(netMap.Peers)) - for _, peer := range netMap.Peers { + // Use [nodeBackend.AppendMatchingPeers] instead of the netmap directly, + // since the netmap doesn't include delta updates (e.g., home DERP or Online + // status changes) from the control plane since the last full update. + candidates := nb.AppendMatchingPeers(nil, func(peer tailcfg.NodeView) bool { if !peer.Valid() || !peer.Online().Get() { - continue + return false } if allowList != nil && !allowList.Contains(peer.StableID()) { - continue - } - if peer.CapMap().Contains(tailcfg.NodeAttrSuggestExitNode) && tsaddr.ContainsExitRoutes(peer.AllowedIPs()) { - candidates = append(candidates, peer) + return false } - } + return peer.CapMap().Contains(tailcfg.NodeAttrSuggestExitNode) && tsaddr.ContainsExitRoutes(peer.AllowedIPs()) + }) if len(candidates) == 0 { return res, nil } diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 5c9c9f2fab4a9..5c9adfb5fc386 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -57,6 +57,7 @@ import ( "tailscale.com/types/ptr" "tailscale.com/types/views" "tailscale.com/util/dnsname" + "tailscale.com/util/eventbus" "tailscale.com/util/mak" "tailscale.com/util/must" "tailscale.com/util/set" @@ -2327,8 +2328,6 @@ func TestSetExitNodeIDPolicy(t *testing.T) { } func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { - t.Skip("TODO(tailscale/tailscale#16455): suggestExitNode does not check for online status of exit nodes") - peer1 := makePeer(1, withCap(26), withSuggest(), withOnline(true), withExitRoutes()) peer2 := makePeer(2, withCap(26), withSuggest(), withOnline(true), withExitRoutes()) derpMap := &tailcfg.DERPMap{ @@ -4278,7 +4277,11 @@ func TestSuggestExitNode(t *testing.T) { allowList = set.SetOf(tt.allowPolicy) } - got, err := suggestExitNode(tt.lastReport, tt.netMap, tt.lastSuggestion, selectRegion, selectNode, allowList) + nb := newNodeBackend(t.Context(), eventbus.New()) + defer nb.shutdown(errShutdown) + nb.SetNetMap(tt.netMap) + + got, err := suggestExitNode(tt.lastReport, nb, tt.lastSuggestion, selectRegion, selectNode, allowList) if got.Name != tt.wantName { t.Errorf("name=%v, want %v", got.Name, tt.wantName) } From 381fdcc3f17f406bb8c5a711b562a23aaef6c98f Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Thu, 3 Jul 2025 20:32:30 -0500 Subject: [PATCH 164/263] ipn/ipnlocal,util/syspolicy/source: retain existing exit node when using auto exit node, if it's allowed by policy In this PR, we update setExitNodeID to retain the existing exit node if auto exit node is enabled, the current exit node is allowed by policy, and no suggested exit node is available yet. Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 15 +++- ipn/ipnlocal/local_test.go | 110 ++++++++++++++++++++++++++-- util/syspolicy/source/test_store.go | 7 ++ 3 files changed, 125 insertions(+), 7 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 5fbb0bd9849d9..6120c52c68a06 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -2026,7 +2026,20 @@ func mutationsAreWorthyOfTellingIPNBus(muts []netmap.NodeMutation) bool { // or resolve ExitNodeIP to an ID and use that. It returns whether prefs was mutated. func setExitNodeID(prefs *ipn.Prefs, suggestedExitNodeID tailcfg.StableNodeID, nm *netmap.NetworkMap) (prefsChanged bool) { if prefs.AutoExitNode.IsSet() { - newExitNodeID := cmp.Or(suggestedExitNodeID, unresolvedExitNodeID) + var newExitNodeID tailcfg.StableNodeID + if !suggestedExitNodeID.IsZero() { + // If we have a suggested exit node, use it. + newExitNodeID = suggestedExitNodeID + } else if isAllowedAutoExitNodeID(prefs.ExitNodeID) { + // If we don't have a suggested exit node, but the prefs already + // specify an allowed auto exit node ID, retain it. + newExitNodeID = prefs.ExitNodeID + } else { + // Otherwise, use [unresolvedExitNodeID] to install a blackhole route, + // preventing traffic from leaking to the local network until an actual + // exit node is selected. + newExitNodeID = unresolvedExitNodeID + } if prefs.ExitNodeID != newExitNodeID { prefs.ExitNodeID = newExitNodeID prefsChanged = true diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 5c9adfb5fc386..c9bad838e9cdb 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -620,6 +620,7 @@ func TestConfigureExitNode(t *testing.T) { useExitNodeEnabled *bool exitNodeIDPolicy *tailcfg.StableNodeID exitNodeIPPolicy *netip.Addr + exitNodeAllowedIDs []tailcfg.StableNodeID // nil if all IDs are allowed for auto exit nodes wantPrefs ipn.Prefs }{ { @@ -894,6 +895,91 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "any", }, }, + { + name: "auto-any-via-policy/no-netmap/with-existing", // set auto exit node via syspolicy without a netmap, but with a previously set exit node ID + prefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), // should be retained + }, + netMap: nil, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowedIDs: nil, // not configured, so all exit node IDs are implicitly allowed + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), + AutoExitNode: "any", + }, + }, + { + name: "auto-any-via-policy/no-netmap/with-allowed-existing", // same, but now with a syspolicy setting that explicitly allows the existing exit node ID + prefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), // should be retained + }, + netMap: nil, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowedIDs: []tailcfg.StableNodeID{ + exitNode2.StableID(), // the current exit node ID is allowed + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), + AutoExitNode: "any", + }, + }, + { + name: "auto-any-via-policy/no-netmap/with-disallowed-existing", // same, but now with a syspolicy setting that does not allow the existing exit node ID + prefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), // not allowed by [syspolicy.AllowedSuggestedExitNodes] + }, + netMap: nil, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowedIDs: []tailcfg.StableNodeID{ + exitNode1.StableID(), // a different exit node ID; the current one is not allowed + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: unresolvedExitNodeID, // we don't have a netmap yet, and the current exit node ID is not allowed; block traffic + AutoExitNode: "any", + }, + }, + { + name: "auto-any-via-policy/with-netmap/with-allowed-existing", // same, but now with a syspolicy setting that does not allow the existing exit node ID + prefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // not allowed by [syspolicy.AllowedSuggestedExitNodes] + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowedIDs: []tailcfg.StableNodeID{ + exitNode2.StableID(), // a different exit node ID; the current one is not allowed + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), // we have a netmap; switch to the best allowed exit node + AutoExitNode: "any", + }, + }, + { + name: "auto-any-via-policy/with-netmap/switch-to-better", // if all exit nodes are allowed, switch to the best one once we have a netmap + prefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // switch to the best exit node + AutoExitNode: "any", + }, + }, { name: "auto-foo-via-policy", // set auto exit node via syspolicy with an unknown/unsupported expression prefs: ipn.Prefs{ @@ -929,19 +1015,23 @@ func TestConfigureExitNode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Configure policy settings, if any. - var settings []source.TestSetting[string] + store := source.NewTestStore(t) if tt.exitNodeIDPolicy != nil { - settings = append(settings, source.TestSettingOf(syspolicy.ExitNodeID, string(*tt.exitNodeIDPolicy))) + store.SetStrings(source.TestSettingOf(syspolicy.ExitNodeID, string(*tt.exitNodeIDPolicy))) } if tt.exitNodeIPPolicy != nil { - settings = append(settings, source.TestSettingOf(syspolicy.ExitNodeIP, tt.exitNodeIPPolicy.String())) + store.SetStrings(source.TestSettingOf(syspolicy.ExitNodeIP, tt.exitNodeIPPolicy.String())) } - if settings != nil { - syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, source.NewTestStoreOf(t, settings...)) - } else { + if tt.exitNodeAllowedIDs != nil { + store.SetStringLists(source.TestSettingOf(syspolicy.AllowedSuggestedExitNodes, toStrings(tt.exitNodeAllowedIDs))) + } + if store.IsEmpty() { // No syspolicy settings, so don't register a store. // This allows the test to run in parallel with other tests. t.Parallel() + } else { + // Register the store for syspolicy settings to make them available to the LocalBackend. + syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, store) } // Create a new LocalBackend with the given prefs. @@ -6127,3 +6217,11 @@ func TestDisplayMessageIPNBus(t *testing.T) { }) } } + +func toStrings[T ~string](in []T) []string { + out := make([]string, len(in)) + for i, v := range in { + out[i] = string(v) + } + return out +} diff --git a/util/syspolicy/source/test_store.go b/util/syspolicy/source/test_store.go index 4b175611fef0d..efaf4cd5a7c0f 100644 --- a/util/syspolicy/source/test_store.go +++ b/util/syspolicy/source/test_store.go @@ -154,6 +154,13 @@ func (s *TestStore) RegisterChangeCallback(callback func()) (unregister func(), }, nil } +// IsEmpty reports whether the store does not contain any settings. +func (s *TestStore) IsEmpty() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.mr) == 0 +} + // ReadString implements [Store]. func (s *TestStore) ReadString(key setting.Key) (string, error) { defer s.recordRead(key, setting.StringValue) From cb7b49941eae3a933c4c5b7dc56398bce24d7e08 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Thu, 3 Jul 2025 19:37:56 -0500 Subject: [PATCH 165/263] ipn/ipnlocal: add (*LocalBackend).reconcilePrefsLocked We have several places where we call applySysPolicy, suggestExitNodeLocked, and setExitNodeID. While there are cases where we want to resolve the exit node specifically, such as when network conditions change or a new netmap is received, we typically need to perform all three steps. For example, enforcing policy settings may enable auto exit nodes or set an ExitNodeIP, which in turn requires picking a suggested exit node or resolving the IP to an ID, respectively. In this PR, we introduce (*LocalBackend).resolveExitNodeInPrefsLocked and (*LocalBackend).reconcilePrefsLocked, with the latter calling both applySysPolicy and resolveExitNodeInPrefsLocked. Consolidating these steps into a single extensibility point would also make it easier to support future hooks registered by ipnext extensions. Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 113 ++++++++++++++++++++++++------------- ipn/ipnlocal/local_test.go | 2 +- 2 files changed, 76 insertions(+), 39 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 6120c52c68a06..0ee249dfb732d 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1624,7 +1624,11 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control prefsChanged = true } } - if applySysPolicy(prefs, b.overrideAlwaysOn) { + // We primarily need this to apply syspolicy to the prefs if an implicit profile + // switch is about to happen. + // TODO(nickkhyl): remove this once we improve handling of implicit profile switching + // in tailscale/corp#28014 and we apply syspolicy when the switch actually happens. + if b.reconcilePrefsLocked(prefs) { prefsChanged = true } @@ -1911,21 +1915,21 @@ func (b *LocalBackend) registerSysPolicyWatch() (unregister func(), err error) { if unregister, err = syspolicy.RegisterChangeCallback(b.sysPolicyChanged); err != nil { return nil, fmt.Errorf("syspolicy: LocalBacked failed to register policy change callback: %v", err) } - if prefs, anyChange := b.applySysPolicy(); anyChange { + if prefs, anyChange := b.reconcilePrefs(); anyChange { b.logf("syspolicy: changed initial profile prefs: %v", prefs.Pretty()) } b.refreshAllowedSuggestions() return unregister, nil } -// applySysPolicy overwrites the current profile's preferences with policies +// reconcilePrefs overwrites the current profile's preferences with policies // that may be configured by the system administrator in an OS-specific way. // // b.mu must not be held. -func (b *LocalBackend) applySysPolicy() (_ ipn.PrefsView, anyChange bool) { +func (b *LocalBackend) reconcilePrefs() (_ ipn.PrefsView, anyChange bool) { unlock := b.lockAndGetUnlock() prefs := b.pm.CurrentPrefs().AsStruct() - if !applySysPolicy(prefs, b.overrideAlwaysOn) { + if !b.reconcilePrefsLocked(prefs) { unlock.UnlockEarly() return prefs.View(), false } @@ -1954,7 +1958,7 @@ func (b *LocalBackend) sysPolicyChanged(policy *rsop.PolicyChange) { // will be used when [applySysPolicy] updates the current profile's prefs. } - if prefs, anyChange := b.applySysPolicy(); anyChange { + if prefs, anyChange := b.reconcilePrefs(); anyChange { b.logf("syspolicy: changed profile prefs: %v", prefs.Pretty()) } } @@ -2302,26 +2306,32 @@ func (b *LocalBackend) Start(opts ipn.Options) error { b.setStateLocked(ipn.NoState) cn := b.currentNode() + + prefsChanged := false + newPrefs := b.pm.CurrentPrefs().AsStruct() if opts.UpdatePrefs != nil { - oldPrefs := b.pm.CurrentPrefs() - newPrefs := opts.UpdatePrefs.Clone() - newPrefs.Persist = oldPrefs.Persist().AsStruct() - pv := newPrefs.View() - if err := b.pm.SetPrefs(pv, cn.NetworkProfile()); err != nil { - b.logf("failed to save UpdatePrefs state: %v", err) + newPrefs = opts.UpdatePrefs.Clone() + prefsChanged = true + } + // Apply any syspolicy overrides, resolve exit node ID, etc. + // As of 2025-07-03, this is primarily needed in two cases: + // - when opts.UpdatePrefs is not nil + // - when Always Mode is enabled and we need to set WantRunning to true + if b.reconcilePrefsLocked(newPrefs) { + prefsChanged = true + } + if prefsChanged { + // Neither opts.UpdatePrefs nor prefs reconciliation + // is allowed to modify Persist; retain the old value. + newPrefs.Persist = b.pm.CurrentPrefs().Persist().AsStruct() + if err := b.pm.SetPrefs(newPrefs.View(), cn.NetworkProfile()); err != nil { + b.logf("failed to save updated and reconciled prefs: %v", err) } } + prefs := newPrefs.View() // Reset the always-on override whenever Start is called. b.resetAlwaysOnOverrideLocked() - // And also apply syspolicy settings to the current profile. - // This is important in two cases: when opts.UpdatePrefs is not nil, - // and when Always Mode is enabled and we need to set WantRunning to true. - if newp := b.pm.CurrentPrefs().AsStruct(); applySysPolicy(newp, b.overrideAlwaysOn) { - setExitNodeID(newp, b.lastSuggestedExitNode, cn.NetMap()) - b.pm.setPrefsNoPermCheck(newp.View()) - } - prefs := b.pm.CurrentPrefs() b.setAtomicValuesFromPrefsLocked(prefs) wantRunning := prefs.WantRunning() @@ -4495,17 +4505,11 @@ func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce) if oldp.Valid() { newp.Persist = oldp.Persist().AsStruct() // caller isn't allowed to override this } - // applySysPolicy returns whether it updated newp, - // but everything in this function treats b.prefs as completely new + // Apply reconciliation to the prefs, such as policy overrides, + // exit node resolution, and so on. The call returns whether it updated + // newp, but everything in this function treats newp as completely new // anyway, so its return value can be ignored here. - applySysPolicy(newp, b.overrideAlwaysOn) - if newp.AutoExitNode.IsSet() { - if _, err := b.suggestExitNodeLocked(); err != nil { - b.logf("failed to select auto exit node: %v", err) - } - } - // setExitNodeID does likewise. No-op if no exit node resolution is needed. - setExitNodeID(newp, b.lastSuggestedExitNode, netMap) + b.reconcilePrefsLocked(newp) // We do this to avoid holding the lock while doing everything else. @@ -5927,14 +5931,8 @@ func (b *LocalBackend) resolveExitNode() (changed bool) { nm := b.currentNode().NetMap() prefs := b.pm.CurrentPrefs().AsStruct() - if prefs.AutoExitNode.IsSet() { - _, err := b.suggestExitNodeLocked() - if err != nil && !errors.Is(err, ErrNoPreferredDERP) { - b.logf("failed to select auto exit node: %v", err) - } - } - if !setExitNodeID(prefs, b.lastSuggestedExitNode, nm) { - return false // no changes + if !b.resolveExitNodeInPrefsLocked(prefs) { + return } if err := b.pm.SetPrefs(prefs.View(), ipn.NetworkProfile{ @@ -5947,6 +5945,45 @@ func (b *LocalBackend) resolveExitNode() (changed bool) { return true } +// reconcilePrefsLocked applies policy overrides, exit node resolution, +// and other post-processing to the prefs, and reports whether the prefs +// were modified as a result. +// +// It must not perform any reconfiguration, as the prefs are not yet effective. +// +// b.mu must be held. +func (b *LocalBackend) reconcilePrefsLocked(prefs *ipn.Prefs) (changed bool) { + if applySysPolicy(prefs, b.overrideAlwaysOn) { + changed = true + } + if b.resolveExitNodeInPrefsLocked(prefs) { + changed = true + } + if changed { + b.logf("prefs reconciled: %v", prefs.Pretty()) + } + return changed +} + +// resolveExitNodeInPrefsLocked determines which exit node to use +// based on the specified prefs and netmap. It updates the exit node ID +// in the prefs if needed, and returns true if the exit node has changed. +// +// b.mu must be held. +func (b *LocalBackend) resolveExitNodeInPrefsLocked(prefs *ipn.Prefs) (changed bool) { + if prefs.AutoExitNode.IsSet() { + _, err := b.suggestExitNodeLocked() + if err != nil && !errors.Is(err, ErrNoPreferredDERP) { + b.logf("failed to select auto exit node: %v", err) + } + } + if setExitNodeID(prefs, b.lastSuggestedExitNode, b.currentNode().NetMap()) { + b.logf("exit node resolved: %v", prefs.ExitNodeID) + return true + } + return false +} + // setNetMapLocked updates the LocalBackend state to reflect the newly // received nm. If nm is nil, it resets all configuration as though // Tailscale is turned off. diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index c9bad838e9cdb..3a2258cc6051f 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -2390,7 +2390,7 @@ func TestSetExitNodeIDPolicy(t *testing.T) { b.pm = pm b.lastSuggestedExitNode = test.lastSuggestedExitNode prefs := b.pm.prefs.AsStruct() - if changed := applySysPolicy(prefs, false) || setExitNodeID(prefs, test.lastSuggestedExitNode, test.nm); changed != test.prefsChanged { + if changed := b.reconcilePrefsLocked(prefs); changed != test.prefsChanged { t.Errorf("wanted prefs changed %v, got prefs changed %v", test.prefsChanged, changed) } From a6f647812901a11572b9143607ec24445574fed7 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Mon, 7 Jul 2025 11:50:59 -0500 Subject: [PATCH 166/263] util/syspolicy: add HasAnyOf to check if any specified policy settings are configured Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- util/syspolicy/syspolicy.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/util/syspolicy/syspolicy.go b/util/syspolicy/syspolicy.go index afcc28ff1fd86..a84afa5dbb6db 100644 --- a/util/syspolicy/syspolicy.go +++ b/util/syspolicy/syspolicy.go @@ -56,6 +56,27 @@ func MustRegisterStoreForTest(tb testenv.TB, name string, scope setting.PolicySc return reg } +// HasAnyOf returns whether at least one of the specified policy settings is configured, +// or an error if no keys are provided or the check fails. +func HasAnyOf(keys ...Key) (bool, error) { + if len(keys) == 0 { + return false, errors.New("at least one key must be specified") + } + policy, err := rsop.PolicyFor(setting.DefaultScope()) + if err != nil { + return false, err + } + effective := policy.Get() + for _, k := range keys { + _, err := effective.GetErr(k) + if errors.Is(err, setting.ErrNotConfigured) || errors.Is(err, setting.ErrNoSuchKey) { + continue + } + return err == nil, err // err may be nil or non-nil + } + return false, nil +} + // GetString returns a string policy setting with the specified key, // or defaultValue if it does not exist. func GetString(key Key, defaultValue string) (string, error) { From f1c7b463cd1cbc6de634a8b75a14cfeca498756f Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Mon, 7 Jul 2025 17:04:07 -0500 Subject: [PATCH 167/263] ipn/{ipnauth,ipnlocal,localapi}: make EditPrefs return an error if changing exit node is restricted by policy We extract checkEditPrefsAccessLocked, adjustEditPrefsLocked, and onEditPrefsLocked from the EditPrefs execution path, defining when each step is performed and what behavior is allowed at each stage. Currently, this is primarily used to support Always On mode, to handle the Exit Node enablement toggle, and to report prefs edit metrics. We then use it to enforce Exit Node policy settings by preventing users from setting an exit node and making EditPrefs return an error when an exit node is restricted by policy. This enforcement is also extended to the Exit Node toggle. These changes prepare for supporting Exit Node overrides when permitted by policy and preventing logout while Always On mode is enabled. In the future, implementation of these methods can be delegated to ipnext extensions via the feature hooks. Updates tailscale/corp#29969 Updates tailscale/corp#26249 Signed-off-by: Nick Khyl --- ipn/ipnauth/self.go | 12 +++ ipn/ipnlocal/local.go | 168 ++++++++++++++++++++++++++---------- ipn/ipnlocal/local_test.go | 83 +++++++++++------- ipn/localapi/localapi.go | 2 +- util/syspolicy/syspolicy.go | 5 +- 5 files changed, 191 insertions(+), 79 deletions(-) diff --git a/ipn/ipnauth/self.go b/ipn/ipnauth/self.go index 9b430dc6d915e..adee0696458d6 100644 --- a/ipn/ipnauth/self.go +++ b/ipn/ipnauth/self.go @@ -13,6 +13,11 @@ import ( // has unlimited access. var Self Actor = unrestricted{} +// TODO is a caller identity used when the operation is performed on behalf of a user, +// rather than by tailscaled itself, but the surrounding function is not yet extended +// to accept an [Actor] parameter. It grants the same unrestricted access as [Self]. +var TODO Actor = unrestricted{} + // unrestricted is an [Actor] that has unlimited access to the currently running // tailscaled instance. It's typically used for operations performed by tailscaled // on its own, or upon a request from the control plane, rather on behalf of a user. @@ -49,3 +54,10 @@ func (unrestricted) IsLocalSystem() bool { return false } // Deprecated: this method exists for compatibility with the current (as of 2025-01-28) // permission model and will be removed as we progress on tailscale/corp#18342. func (unrestricted) IsLocalAdmin(operatorUID string) bool { return false } + +// IsTailscaled reports whether the given Actor represents Tailscaled itself, +// such as [Self] or a [TODO] placeholder actor. +func IsTailscaled(a Actor) bool { + _, ok := a.(unrestricted) + return ok +} diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 0ee249dfb732d..03a0709e2d95d 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -178,6 +178,10 @@ var ( // It is used as a context cancellation cause for the old context // and can be returned when an operation is performed on it. errNodeContextChanged = errors.New("profile changed") + + // errManagedByPolicy indicates the operation is blocked + // because the target state is managed by a GP/MDM policy. + errManagedByPolicy = errors.New("managed by policy") ) // LocalBackend is the glue between the major pieces of the Tailscale @@ -3477,12 +3481,14 @@ func (b *LocalBackend) onTailnetDefaultAutoUpdate(au bool) { b.logf("using tailnet default auto-update setting: %v", au) prefsClone := prefs.AsStruct() prefsClone.AutoUpdate.Apply = opt.NewBool(au) - _, err := b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ - Prefs: *prefsClone, - AutoUpdateSet: ipn.AutoUpdatePrefsMask{ - ApplySet: true, - }, - }, unlock) + _, err := b.editPrefsLockedOnEntry( + ipnauth.Self, + &ipn.MaskedPrefs{ + Prefs: *prefsClone, + AutoUpdateSet: ipn.AutoUpdatePrefsMask{ + ApplySet: true, + }, + }, unlock) if err != nil { b.logf("failed to apply tailnet-wide default for auto-updates (%v): %v", au, err) return @@ -4224,7 +4230,7 @@ func (b *LocalBackend) checkAutoUpdatePrefsLocked(p *ipn.Prefs) error { // On success, it returns the resulting prefs (or current prefs, in the case of no change). // Setting the value to false when use of an exit node is already false is not an error, // nor is true when the exit node is already in use. -func (b *LocalBackend) SetUseExitNodeEnabled(v bool) (ipn.PrefsView, error) { +func (b *LocalBackend) SetUseExitNodeEnabled(actor ipnauth.Actor, v bool) (ipn.PrefsView, error) { unlock := b.lockAndGetUnlock() defer unlock() @@ -4267,7 +4273,7 @@ func (b *LocalBackend) SetUseExitNodeEnabled(v bool) (ipn.PrefsView, error) { mp.InternalExitNodePrior = p0.ExitNodeID() } } - return b.editPrefsLockedOnEntry(mp, unlock) + return b.editPrefsLockedOnEntry(actor, mp, unlock) } // MaybeClearAppConnector clears the routes from any AppConnector if @@ -4296,30 +4302,83 @@ func (b *LocalBackend) EditPrefsAs(mp *ipn.MaskedPrefs, actor ipnauth.Actor) (ip return ipn.PrefsView{}, errors.New("can't set Internal fields") } - // Zeroing the ExitNodeId via localAPI must also zero the prior exit node. - if mp.ExitNodeIDSet && mp.ExitNodeID == "" { + return b.editPrefsLockedOnEntry(actor, mp, b.lockAndGetUnlock()) +} + +// checkEditPrefsAccessLocked checks whether the current user has access +// to apply the prefs changes in mp. +// +// It returns an error if the user is not allowed, or nil otherwise. +// +// b.mu must be held. +func (b *LocalBackend) checkEditPrefsAccessLocked(actor ipnauth.Actor, mp *ipn.MaskedPrefs) error { + var errs []error + + if mp.RunSSHSet && mp.RunSSH && !envknob.CanSSHD() { + errs = append(errs, errors.New("Tailscale SSH server administratively disabled")) + } + + // Check if the user is allowed to disconnect Tailscale. + if mp.WantRunningSet && !mp.WantRunning && b.pm.CurrentPrefs().WantRunning() { + if err := actor.CheckProfileAccess(b.pm.CurrentProfile(), ipnauth.Disconnect, b.extHost.AuditLogger()); err != nil { + errs = append(errs, err) + } + } + + // Prevent users from changing exit node preferences + // when exit node usage is managed by policy. + if mp.ExitNodeIDSet || mp.ExitNodeIPSet || mp.AutoExitNodeSet { + // TODO(nickkhyl): Allow users to override ExitNode policy settings + // if the ExitNode.AllowUserOverride policy permits it. + // (Policy setting name and details are TBD. See tailscale/corp#29969) + isManaged, err := syspolicy.HasAnyOf(syspolicy.ExitNodeID, syspolicy.ExitNodeIP) + if err != nil { + err = fmt.Errorf("policy check failed: %w", err) + } else if isManaged { + err = errManagedByPolicy + } + if err != nil { + errs = append(errs, fmt.Errorf("exit node cannot be changed: %w", err)) + } + } + + return multierr.New(errs...) +} + +// adjustEditPrefsLocked applies additional changes to mp if necessary, +// such as zeroing out mutually exclusive fields. +// +// It must not assume that the changes in mp will actually be applied. +// +// b.mu must be held. +func (b *LocalBackend) adjustEditPrefsLocked(_ ipnauth.Actor, mp *ipn.MaskedPrefs) { + // Zeroing the ExitNodeID via localAPI must also zero the prior exit node. + if mp.ExitNodeIDSet && mp.ExitNodeID == "" && !mp.InternalExitNodePriorSet { mp.InternalExitNodePrior = "" mp.InternalExitNodePriorSet = true } // Disable automatic exit node selection if the user explicitly sets // ExitNodeID or ExitNodeIP. - if mp.ExitNodeIDSet || mp.ExitNodeIPSet { + if (mp.ExitNodeIDSet || mp.ExitNodeIPSet) && !mp.AutoExitNodeSet { mp.AutoExitNodeSet = true mp.AutoExitNode = "" } +} - // Acquire the lock before checking the profile access to prevent - // TOCTOU issues caused by the current profile changing between the - // check and the actual edit. - unlock := b.lockAndGetUnlock() - defer unlock() - if mp.WantRunningSet && !mp.WantRunning && b.pm.CurrentPrefs().WantRunning() { - if err := actor.CheckProfileAccess(b.pm.CurrentProfile(), ipnauth.Disconnect, b.extHost.AuditLogger()); err != nil { - b.logf("check profile access failed: %v", err) - return ipn.PrefsView{}, err - } - +// onEditPrefsLocked is called when prefs are edited (typically, via LocalAPI), +// just before the changes in newPrefs are set for the current profile. +// +// The changes in mp have been allowed, but the resulting [ipn.Prefs] +// have not yet been applied and may be subject to reconciliation +// by [LocalBackend.reconcilePrefsLocked], either before or after being set. +// +// This method handles preference edits, typically initiated by the user, +// as opposed to reconfiguring the backend when the final prefs are set. +// +// b.mu must be held; mp must not be mutated by this method. +func (b *LocalBackend) onEditPrefsLocked(_ ipnauth.Actor, mp *ipn.MaskedPrefs, oldPrefs, newPrefs ipn.PrefsView) { + if mp.WantRunningSet && !mp.WantRunning && oldPrefs.WantRunning() { // If a user has enough rights to disconnect, such as when [syspolicy.AlwaysOn] // is disabled, or [syspolicy.AlwaysOnOverrideWithReason] is also set and the user // provides a reason for disconnecting, then we should not force the "always on" @@ -4331,7 +4390,18 @@ func (b *LocalBackend) EditPrefsAs(mp *ipn.MaskedPrefs, actor ipnauth.Actor) (ip } } - return b.editPrefsLockedOnEntry(mp, unlock) + // This is recorded here in the EditPrefs path, not the setPrefs path on purpose. + // recordForEdit records metrics related to edits and changes, not the final state. + // If, in the future, we want to record gauge-metrics related to the state of prefs, + // that should be done in the setPrefs path. + e := prefsMetricsEditEvent{ + change: mp, + pNew: newPrefs, + pOld: oldPrefs, + node: b.currentNode(), + lastSuggestedExitNode: b.lastSuggestedExitNode, + } + e.record() } // startReconnectTimerLocked sets a timer to automatically set WantRunning to true @@ -4368,7 +4438,7 @@ func (b *LocalBackend) startReconnectTimerLocked(d time.Duration) { } mp := &ipn.MaskedPrefs{WantRunningSet: true, Prefs: ipn.Prefs{WantRunning: true}} - if _, err := b.editPrefsLockedOnEntry(mp, unlock); err != nil { + if _, err := b.editPrefsLockedOnEntry(ipnauth.Self, mp, unlock); err != nil { b.logf("failed to automatically reconnect as %q after %v: %v", cp.Name(), d, err) } else { b.logf("automatically reconnected as %q after %v", cp.Name(), d) @@ -4399,9 +4469,19 @@ func (b *LocalBackend) stopReconnectTimerLocked() { // Warning: b.mu must be held on entry, but it unlocks it on the way out. // TODO(bradfitz): redo the locking on all these weird methods like this. -func (b *LocalBackend) editPrefsLockedOnEntry(mp *ipn.MaskedPrefs, unlock unlockOnce) (ipn.PrefsView, error) { +func (b *LocalBackend) editPrefsLockedOnEntry(actor ipnauth.Actor, mp *ipn.MaskedPrefs, unlock unlockOnce) (ipn.PrefsView, error) { defer unlock() // for error paths + // Check if the changes in mp are allowed. + if err := b.checkEditPrefsAccessLocked(actor, mp); err != nil { + b.logf("EditPrefs(%v): %v", mp.Pretty(), err) + return ipn.PrefsView{}, err + } + + // Apply additional changes to mp if necessary, + // such as clearing mutually exclusive fields. + b.adjustEditPrefsLocked(actor, mp) + if mp.EggSet { mp.EggSet = false b.egg = true @@ -4416,29 +4496,18 @@ func (b *LocalBackend) editPrefsLockedOnEntry(mp *ipn.MaskedPrefs, unlock unlock b.logf("EditPrefs check error: %v", err) return ipn.PrefsView{}, err } - if p1.RunSSH && !envknob.CanSSHD() { - b.logf("EditPrefs requests SSH, but disabled by envknob; returning error") - return ipn.PrefsView{}, errors.New("Tailscale SSH server administratively disabled.") - } + if p1.View().Equals(p0) { return stripKeysFromPrefs(p0), nil } b.logf("EditPrefs: %v", mp.Pretty()) - newPrefs := b.setPrefsLockedOnEntry(p1, unlock) - // This is recorded here in the EditPrefs path, not the setPrefs path on purpose. - // recordForEdit records metrics related to edits and changes, not the final state. - // If, in the future, we want to record gauge-metrics related to the state of prefs, - // that should be done in the setPrefs path. - e := prefsMetricsEditEvent{ - change: mp, - pNew: p1.View(), - pOld: p0, - node: b.currentNode(), - lastSuggestedExitNode: b.lastSuggestedExitNode, - } - e.record() + // Perform any actions required when prefs are edited (typically by a user), + // before the modified prefs are actually set for the current profile. + b.onEditPrefsLocked(actor, mp, p0, p1.View()) + + newPrefs := b.setPrefsLockedOnEntry(p1, unlock) // Note: don't perform any actions for the new prefs here. Not // every prefs change goes through EditPrefs. Put your actions @@ -5829,11 +5898,16 @@ func (b *LocalBackend) Logout(ctx context.Context) error { // delete it later. profile := b.pm.CurrentProfile() - _, err := b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ - WantRunningSet: true, - LoggedOutSet: true, - Prefs: ipn.Prefs{WantRunning: false, LoggedOut: true}, - }, unlock) + // TODO(nickkhyl): change [LocalBackend.Logout] to accept an [ipnauth.Actor]. + // This will allow enforcing Always On mode when a user tries to log out + // while logged in and connected. See tailscale/corp#26249. + _, err := b.editPrefsLockedOnEntry( + ipnauth.TODO, + &ipn.MaskedPrefs{ + WantRunningSet: true, + LoggedOutSet: true, + Prefs: ipn.Prefs{WantRunning: false, LoggedOut: true}, + }, unlock) if err != nil { return err } diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 3a2258cc6051f..1e1b7663ab687 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -501,29 +501,30 @@ func TestLazyMachineKeyGeneration(t *testing.T) { func TestZeroExitNodeViaLocalAPI(t *testing.T) { lb := newTestLocalBackend(t) + user := &ipnauth.TestActor{} // Give it an initial exit node in use. - if _, err := lb.EditPrefs(&ipn.MaskedPrefs{ + if _, err := lb.EditPrefsAs(&ipn.MaskedPrefs{ ExitNodeIDSet: true, Prefs: ipn.Prefs{ ExitNodeID: "foo", }, - }); err != nil { + }, user); err != nil { t.Fatalf("enabling first exit node: %v", err) } // SetUseExitNodeEnabled(false) "remembers" the prior exit node. - if _, err := lb.SetUseExitNodeEnabled(false); err != nil { + if _, err := lb.SetUseExitNodeEnabled(user, false); err != nil { t.Fatal("expected failure") } // Zero the exit node - pv, err := lb.EditPrefs(&ipn.MaskedPrefs{ + pv, err := lb.EditPrefsAs(&ipn.MaskedPrefs{ ExitNodeIDSet: true, Prefs: ipn.Prefs{ ExitNodeID: "", }, - }) + }, user) if err != nil { t.Fatalf("enabling first exit node: %v", err) @@ -539,29 +540,30 @@ func TestZeroExitNodeViaLocalAPI(t *testing.T) { func TestSetUseExitNodeEnabled(t *testing.T) { lb := newTestLocalBackend(t) + user := &ipnauth.TestActor{} // Can't turn it on if it never had an old value. - if _, err := lb.SetUseExitNodeEnabled(true); err == nil { + if _, err := lb.SetUseExitNodeEnabled(user, true); err == nil { t.Fatal("expected success") } // But we can turn it off when it's already off. - if _, err := lb.SetUseExitNodeEnabled(false); err != nil { + if _, err := lb.SetUseExitNodeEnabled(user, false); err != nil { t.Fatal("expected failure") } // Give it an initial exit node in use. - if _, err := lb.EditPrefs(&ipn.MaskedPrefs{ + if _, err := lb.EditPrefsAs(&ipn.MaskedPrefs{ ExitNodeIDSet: true, Prefs: ipn.Prefs{ ExitNodeID: "foo", }, - }); err != nil { + }, user); err != nil { t.Fatalf("enabling first exit node: %v", err) } // Now turn off that exit node. - if prefs, err := lb.SetUseExitNodeEnabled(false); err != nil { + if prefs, err := lb.SetUseExitNodeEnabled(user, false); err != nil { t.Fatal("expected failure") } else { if g, w := prefs.ExitNodeID(), tailcfg.StableNodeID(""); g != w { @@ -573,7 +575,7 @@ func TestSetUseExitNodeEnabled(t *testing.T) { } // And turn it back on. - if prefs, err := lb.SetUseExitNodeEnabled(true); err != nil { + if prefs, err := lb.SetUseExitNodeEnabled(user, true); err != nil { t.Fatal("expected failure") } else { if g, w := prefs.ExitNodeID(), tailcfg.StableNodeID("foo"); g != w { @@ -585,9 +587,9 @@ func TestSetUseExitNodeEnabled(t *testing.T) { } // Verify we block setting an Internal field. - if _, err := lb.EditPrefs(&ipn.MaskedPrefs{ + if _, err := lb.EditPrefsAs(&ipn.MaskedPrefs{ InternalExitNodePriorSet: true, - }); err == nil { + }, user); err == nil { t.Fatalf("unexpected success; want an error trying to set an internal field") } } @@ -612,16 +614,18 @@ func TestConfigureExitNode(t *testing.T) { } tests := []struct { - name string - prefs ipn.Prefs - netMap *netmap.NetworkMap - report *netcheck.Report - changePrefs *ipn.MaskedPrefs - useExitNodeEnabled *bool - exitNodeIDPolicy *tailcfg.StableNodeID - exitNodeIPPolicy *netip.Addr - exitNodeAllowedIDs []tailcfg.StableNodeID // nil if all IDs are allowed for auto exit nodes - wantPrefs ipn.Prefs + name string + prefs ipn.Prefs + netMap *netmap.NetworkMap + report *netcheck.Report + changePrefs *ipn.MaskedPrefs + useExitNodeEnabled *bool + exitNodeIDPolicy *tailcfg.StableNodeID + exitNodeIPPolicy *netip.Addr + exitNodeAllowedIDs []tailcfg.StableNodeID // nil if all IDs are allowed for auto exit nodes + wantChangePrefsErr error // if non-nil, the error we expect from [LocalBackend.EditPrefsAs] + wantPrefs ipn.Prefs + wantExitNodeToggleErr error // if non-nil, the error we expect from [LocalBackend.SetUseExitNodeEnabled] }{ { name: "exit-node-id-via-prefs", // set exit node ID via prefs @@ -804,6 +808,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, + wantChangePrefsErr: errManagedByPolicy, }, { name: "id-via-policy/cannot-override-via-prefs/by-ip", // syspolicy should take precedence over prefs @@ -822,6 +827,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, + wantChangePrefsErr: errManagedByPolicy, }, { name: "id-via-policy/cannot-override-via-prefs/by-auto-expr", // syspolicy should take precedence over prefs @@ -840,6 +846,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, + wantChangePrefsErr: errManagedByPolicy, }, { name: "ip-via-policy", // set exit node IP via syspolicy (should be resolved to an ID) @@ -999,15 +1006,16 @@ func TestConfigureExitNode(t *testing.T) { prefs: ipn.Prefs{ ControlURL: controlURL, }, - netMap: clientNetmap, - report: report, - exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), - useExitNodeEnabled: ptr.To(false), // should be ignored + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + useExitNodeEnabled: ptr.To(false), // should fail with an error + wantExitNodeToggleErr: errManagedByPolicy, wantPrefs: ipn.Prefs{ ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), // still enforced by the policy setting AutoExitNode: "any", - InternalExitNodePrior: "auto:any", + InternalExitNodePrior: "", }, }, } @@ -1046,14 +1054,17 @@ func TestConfigureExitNode(t *testing.T) { lb.SetControlClientStatus(lb.cc, controlclient.Status{NetMap: tt.netMap}) } + user := &ipnauth.TestActor{} // If we have a changePrefs, apply it. if tt.changePrefs != nil { - lb.EditPrefs(tt.changePrefs) + _, err := lb.EditPrefsAs(tt.changePrefs, user) + checkError(t, err, tt.wantChangePrefsErr, true) } // If we need to flip exit node toggle on or off, do it. if tt.useExitNodeEnabled != nil { - lb.SetUseExitNodeEnabled(*tt.useExitNodeEnabled) + _, err := lb.SetUseExitNodeEnabled(user, *tt.useExitNodeEnabled) + checkError(t, err, tt.wantExitNodeToggleErr, true) } // Now check the prefs. @@ -6218,6 +6229,18 @@ func TestDisplayMessageIPNBus(t *testing.T) { } } +func checkError(tb testing.TB, got, want error, fatal bool) { + tb.Helper() + f := tb.Errorf + if fatal { + f = tb.Fatalf + } + if (want == nil) != (got == nil) || + (want != nil && got != nil && want.Error() != got.Error() && !errors.Is(got, want)) { + f("gotErr: %v; wantErr: %v", got, want) + } +} + func toStrings[T ~string](in []T) []string { out := make([]string, len(in)) for i, v := range in { diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index a90ae5d844b90..d4b4b443ef852 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -1910,7 +1910,7 @@ func (h *Handler) serveSetUseExitNodeEnabled(w http.ResponseWriter, r *http.Requ http.Error(w, "invalid 'enabled' parameter", http.StatusBadRequest) return } - prefs, err := h.b.SetUseExitNodeEnabled(v) + prefs, err := h.b.SetUseExitNodeEnabled(h.Actor, v) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/util/syspolicy/syspolicy.go b/util/syspolicy/syspolicy.go index a84afa5dbb6db..6555a58ac4564 100644 --- a/util/syspolicy/syspolicy.go +++ b/util/syspolicy/syspolicy.go @@ -72,7 +72,10 @@ func HasAnyOf(keys ...Key) (bool, error) { if errors.Is(err, setting.ErrNotConfigured) || errors.Is(err, setting.ErrNoSuchKey) { continue } - return err == nil, err // err may be nil or non-nil + if err != nil { + return false, err + } + return true, nil } return false, nil } From ea4018b757fa6f925be59f9d95011c3a7de3ee10 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Mon, 7 Jul 2025 17:21:21 -0500 Subject: [PATCH 168/263] ipn/ipnlocal: fix missing defer in testExtension.Shutdown Updates #cleanup Signed-off-by: Nick Khyl --- ipn/ipnlocal/extension_host_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipn/ipnlocal/extension_host_test.go b/ipn/ipnlocal/extension_host_test.go index f655c477fcb36..509833ff6de46 100644 --- a/ipn/ipnlocal/extension_host_test.go +++ b/ipn/ipnlocal/extension_host_test.go @@ -1230,7 +1230,7 @@ func (e *testExtension) InitCalled() bool { func (e *testExtension) Shutdown() (err error) { e.t.Helper() e.mu.Lock() - e.mu.Unlock() + defer e.mu.Unlock() if e.ShutdownHook != nil { err = e.ShutdownHook(e) } From 47f431b656d0c35aac6f97530a4daa2404bc12d6 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 7 Jul 2025 19:46:20 -0700 Subject: [PATCH 169/263] net/udprelay: fix relaying between mixed address family sockets (#16485) We can't relay a packet received over the IPv4 socket back out the same socket if destined to an IPv6 address, and vice versa. Updates tailscale/corp#30206 Signed-off-by: Jordan Whited --- net/udprelay/server.go | 42 ++++++++++--------- net/udprelay/server_test.go | 81 +++++++++++++++++++++++++++---------- 2 files changed, 83 insertions(+), 40 deletions(-) diff --git a/net/udprelay/server.go b/net/udprelay/server.go index d2661e59feba4..979ccf71765ed 100644 --- a/net/udprelay/server.go +++ b/net/udprelay/server.go @@ -112,7 +112,7 @@ type serverEndpoint struct { allocatedAt time.Time } -func (e *serverEndpoint) handleDiscoControlMsg(from netip.AddrPort, senderIndex int, discoMsg disco.Message, uw udpWriter, serverDisco key.DiscoPublic) { +func (e *serverEndpoint) handleDiscoControlMsg(from netip.AddrPort, senderIndex int, discoMsg disco.Message, conn *net.UDPConn, serverDisco key.DiscoPublic) { if senderIndex != 0 && senderIndex != 1 { return } @@ -165,7 +165,7 @@ func (e *serverEndpoint) handleDiscoControlMsg(from netip.AddrPort, senderIndex reply = serverDisco.AppendTo(reply) box := e.discoSharedSecrets[senderIndex].Seal(m.AppendMarshal(nil)) reply = append(reply, box...) - uw.WriteMsgUDPAddrPort(reply, nil, from) + conn.WriteMsgUDPAddrPort(reply, nil, from) return case *disco.BindUDPRelayEndpointAnswer: err := validateVNIAndRemoteKey(discoMsg.BindUDPRelayEndpointCommon) @@ -191,7 +191,7 @@ func (e *serverEndpoint) handleDiscoControlMsg(from netip.AddrPort, senderIndex } } -func (e *serverEndpoint) handleSealedDiscoControlMsg(from netip.AddrPort, b []byte, uw udpWriter, serverDisco key.DiscoPublic) { +func (e *serverEndpoint) handleSealedDiscoControlMsg(from netip.AddrPort, b []byte, conn *net.UDPConn, serverDisco key.DiscoPublic) { senderRaw, isDiscoMsg := disco.Source(b) if !isDiscoMsg { // Not a Disco message @@ -222,14 +222,10 @@ func (e *serverEndpoint) handleSealedDiscoControlMsg(from netip.AddrPort, b []by return } - e.handleDiscoControlMsg(from, senderIndex, discoMsg, uw, serverDisco) + e.handleDiscoControlMsg(from, senderIndex, discoMsg, conn, serverDisco) } -type udpWriter interface { - WriteMsgUDPAddrPort(b []byte, oob []byte, addr netip.AddrPort) (n, oobn int, err error) -} - -func (e *serverEndpoint) handlePacket(from netip.AddrPort, gh packet.GeneveHeader, b []byte, uw udpWriter, serverDisco key.DiscoPublic) { +func (e *serverEndpoint) handlePacket(from netip.AddrPort, gh packet.GeneveHeader, b []byte, rxSocket, otherAFSocket *net.UDPConn, serverDisco key.DiscoPublic) { if !gh.Control { if !e.isBound() { // not a control packet, but serverEndpoint isn't bound @@ -247,8 +243,16 @@ func (e *serverEndpoint) handlePacket(from netip.AddrPort, gh packet.GeneveHeade // unrecognized source return } - // relay packet - uw.WriteMsgUDPAddrPort(b, nil, to) + // Relay the packet towards the other party via the socket associated + // with the destination's address family. If source and destination + // address families are matching we tx on the same socket the packet + // was received (rxSocket), otherwise we use the "other" socket + // (otherAFSocket). [Server] makes no use of dual-stack sockets. + if from.Addr().Is4() == to.Addr().Is4() { + rxSocket.WriteMsgUDPAddrPort(b, nil, to) + } else if otherAFSocket != nil { + otherAFSocket.WriteMsgUDPAddrPort(b, nil, to) + } return } @@ -258,7 +262,7 @@ func (e *serverEndpoint) handlePacket(from netip.AddrPort, gh packet.GeneveHeade } msg := b[packet.GeneveFixedHeaderLength:] - e.handleSealedDiscoControlMsg(from, msg, uw, serverDisco) + e.handleSealedDiscoControlMsg(from, msg, rxSocket, serverDisco) } func (e *serverEndpoint) isExpired(now time.Time, bindLifetime, steadyStateLifetime time.Duration) bool { @@ -346,10 +350,10 @@ func NewServer(logf logger.Logf, port int, overrideAddrs []netip.Addr) (s *Serve } s.wg.Add(1) - go s.packetReadLoop(s.uc4) + go s.packetReadLoop(s.uc4, s.uc6) if s.uc6 != nil { s.wg.Add(1) - go s.packetReadLoop(s.uc6) + go s.packetReadLoop(s.uc6, s.uc4) } s.wg.Add(1) go s.endpointGCLoop() @@ -531,7 +535,7 @@ func (s *Server) endpointGCLoop() { } } -func (s *Server) handlePacket(from netip.AddrPort, b []byte, uw udpWriter) { +func (s *Server) handlePacket(from netip.AddrPort, b []byte, rxSocket, otherAFSocket *net.UDPConn) { if stun.Is(b) && b[1] == 0x01 { // A b[1] value of 0x01 (STUN method binding) is sufficiently // non-overlapping with the Geneve header where the LSB is always 0 @@ -555,10 +559,10 @@ func (s *Server) handlePacket(from netip.AddrPort, b []byte, uw udpWriter) { return } - e.handlePacket(from, gh, b, uw, s.discoPublic) + e.handlePacket(from, gh, b, rxSocket, otherAFSocket, s.discoPublic) } -func (s *Server) packetReadLoop(uc *net.UDPConn) { +func (s *Server) packetReadLoop(readFromSocket, otherSocket *net.UDPConn) { defer func() { s.wg.Done() s.Close() @@ -566,11 +570,11 @@ func (s *Server) packetReadLoop(uc *net.UDPConn) { b := make([]byte, 1<<16-1) for { // TODO: extract laddr from IP_PKTINFO for use in reply - n, from, err := uc.ReadFromUDPAddrPort(b) + n, from, err := readFromSocket.ReadFromUDPAddrPort(b) if err != nil { return } - s.handlePacket(from, b[:n], uc) + s.handlePacket(from, b[:n], readFromSocket, otherSocket) } } diff --git a/net/udprelay/server_test.go b/net/udprelay/server_test.go index 8c0c5aff66027..de1c293644992 100644 --- a/net/udprelay/server_test.go +++ b/net/udprelay/server_test.go @@ -181,8 +181,9 @@ func TestServer(t *testing.T) { discoB := key.NewDisco() cases := []struct { - name string - overrideAddrs []netip.Addr + name string + overrideAddrs []netip.Addr + forceClientsMixedAF bool }{ { name: "over ipv4", @@ -192,6 +193,11 @@ func TestServer(t *testing.T) { name: "over ipv6", overrideAddrs: []netip.Addr{netip.MustParseAddr("::1")}, }, + { + name: "mixed address families", + overrideAddrs: []netip.Addr{netip.MustParseAddr("127.0.0.1"), netip.MustParseAddr("::1")}, + forceClientsMixedAF: true, + }, } for _, tt := range cases { @@ -216,16 +222,47 @@ func TestServer(t *testing.T) { t.Fatalf("wrong dupEndpoint (-got +want)\n%s", diff) } - if len(endpoint.AddrPorts) != 1 { + if len(endpoint.AddrPorts) < 1 { t.Fatalf("unexpected endpoint.AddrPorts: %v", endpoint.AddrPorts) } - tcA := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoA, discoB.Public(), endpoint.ServerDisco) + tcAServerEndpointAddr := endpoint.AddrPorts[0] + tcA := newTestClient(t, endpoint.VNI, tcAServerEndpointAddr, discoA, discoB.Public(), endpoint.ServerDisco) defer tcA.close() - tcB := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoB, discoA.Public(), endpoint.ServerDisco) + tcBServerEndpointAddr := tcAServerEndpointAddr + if tt.forceClientsMixedAF { + foundMixedAF := false + for _, addr := range endpoint.AddrPorts { + if addr.Addr().Is4() != tcBServerEndpointAddr.Addr().Is4() { + tcBServerEndpointAddr = addr + foundMixedAF = true + } + } + if !foundMixedAF { + t.Fatal("force clients to mixed address families is set, but relay server lacks address family diversity") + } + } + tcB := newTestClient(t, endpoint.VNI, tcBServerEndpointAddr, discoB, discoA.Public(), endpoint.ServerDisco) defer tcB.close() - tcA.handshake(t) - tcB.handshake(t) + for i := 0; i < 2; i++ { + // We handshake both clients twice to guarantee server-side + // packet reading goroutines, which are independent across + // address families, have seen an answer from both clients + // before proceeding. This is needed because the test assumes + // that B's handshake is complete (the first send is A->B below), + // but the server may not have handled B's handshake answer + // before it handles A's data pkt towards B. + // + // Data transmissions following "re-handshakes" orient so that + // the sender is the same as the party that performed the + // handshake, for the same reasons. + // + // [magicsock.relayManager] is not prone to this issue as both + // parties transmit data packets immediately following their + // handshake answer. + tcA.handshake(t) + tcB.handshake(t) + } dupEndpoint, err = server.AllocateEndpoint(discoA.Public(), discoB.Public()) if err != nil { @@ -250,30 +287,32 @@ func TestServer(t *testing.T) { t.Fatal("unexpected msg B->A") } - tcAOnNewPort := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoA, discoB.Public(), endpoint.ServerDisco) + tcAOnNewPort := newTestClient(t, endpoint.VNI, tcAServerEndpointAddr, discoA, discoB.Public(), endpoint.ServerDisco) tcAOnNewPort.handshakeGeneration = tcA.handshakeGeneration + 1 defer tcAOnNewPort.close() - // Handshake client A on a new source IP:port, verify we receive packets on the new binding + // Handshake client A on a new source IP:port, verify we can send packets on the new binding tcAOnNewPort.handshake(t) - txToAOnNewPort := []byte{7, 8, 9} - tcB.writeDataPkt(t, txToAOnNewPort) - rxFromB = tcAOnNewPort.readDataPkt(t) - if !bytes.Equal(txToAOnNewPort, rxFromB) { - t.Fatal("unexpected msg B->A") + + fromAOnNewPort := []byte{7, 8, 9} + tcAOnNewPort.writeDataPkt(t, fromAOnNewPort) + rxFromA = tcB.readDataPkt(t) + if !bytes.Equal(fromAOnNewPort, rxFromA) { + t.Fatal("unexpected msg A->B") } - tcBOnNewPort := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoB, discoA.Public(), endpoint.ServerDisco) + tcBOnNewPort := newTestClient(t, endpoint.VNI, tcBServerEndpointAddr, discoB, discoA.Public(), endpoint.ServerDisco) tcBOnNewPort.handshakeGeneration = tcB.handshakeGeneration + 1 defer tcBOnNewPort.close() - // Handshake client B on a new source IP:port, verify we receive packets on the new binding + // Handshake client B on a new source IP:port, verify we can send packets on the new binding tcBOnNewPort.handshake(t) - txToBOnNewPort := []byte{7, 8, 9} - tcAOnNewPort.writeDataPkt(t, txToBOnNewPort) - rxFromA = tcBOnNewPort.readDataPkt(t) - if !bytes.Equal(txToBOnNewPort, rxFromA) { - t.Fatal("unexpected msg A->B") + + fromBOnNewPort := []byte{7, 8, 9} + tcBOnNewPort.writeDataPkt(t, fromBOnNewPort) + rxFromB = tcAOnNewPort.readDataPkt(t) + if !bytes.Equal(fromBOnNewPort, rxFromB) { + t.Fatal("unexpected msg B->A") } }) } From 5b0074729d38f8cc301803da06086033f53b1b93 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 8 Jul 2025 09:45:18 -0700 Subject: [PATCH 170/263] go.mod,wgengine/magicsock: implement conn.InitiationAwareEndpoint (#16486) Since a [*lazyEndpoint] makes wireguard-go responsible for peer ID, but wireguard-go may not yet be configured for said peer, we need a JIT hook around initiation message reception to call what is usually called from an [*endpoint]. Updates tailscale/corp#30042 Signed-off-by: Jordan Whited --- go.mod | 2 +- go.sum | 4 ++-- wgengine/magicsock/magicsock.go | 34 ++++++++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5bf04fedaba2e..e89a383a62726 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/tailscale/setec v0.0.0-20250205144240-8898a29c3fbb github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 - github.com/tailscale/wireguard-go v0.0.0-20250701223756-24483d7a0003 + github.com/tailscale/wireguard-go v0.0.0-20250707220504-1f398ae148a8 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e github.com/tc-hib/winres v0.2.1 github.com/tcnksm/go-httpstat v0.2.0 diff --git a/go.sum b/go.sum index f9910bb59bb4d..062af66622b85 100644 --- a/go.sum +++ b/go.sum @@ -975,8 +975,8 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 h1:U github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y= -github.com/tailscale/wireguard-go v0.0.0-20250701223756-24483d7a0003 h1:chIzUDKxR0nXQQra0j41aqiiFNICs0FIC5ZCwDO7z3k= -github.com/tailscale/wireguard-go v0.0.0-20250701223756-24483d7a0003/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= +github.com/tailscale/wireguard-go v0.0.0-20250707220504-1f398ae148a8 h1:Yjg/+1VVRcdY3DL9fs8g+QnZ1aizotU0pp0VSOSCuTQ= +github.com/tailscale/wireguard-go v0.0.0-20250707220504-1f398ae148a8/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg= github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA= diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index a7eab36786f12..fbfcf0b41565a 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3777,17 +3777,45 @@ func (c *Conn) SetLastNetcheckReportForTest(ctx context.Context, report *netchec // lazyEndpoint is a wireguard [conn.Endpoint] for when magicsock received a // non-disco (presumably WireGuard) packet from a UDP address from which we // can't map to a Tailscale peer. But WireGuard most likely can, once it -// decrypts it. So we implement the [conn.PeerAwareEndpoint] interface -// from https://github.com/tailscale/wireguard-go/pull/27 to allow WireGuard -// to tell us who it is later and get the correct [conn.Endpoint]. +// decrypts it. So we implement the [conn.InitiationAwareEndpoint] and +// [conn.PeerAwareEndpoint] interfaces, to allow WireGuard to tell us who it is +// later, just-in-time to configure the peer, and set the associated [epAddr] +// in the [peerMap]. Future receives on the associated [epAddr] will then be +// resolvable directly to an [*endpoint]. type lazyEndpoint struct { c *Conn src epAddr } +var _ conn.InitiationAwareEndpoint = (*lazyEndpoint)(nil) var _ conn.PeerAwareEndpoint = (*lazyEndpoint)(nil) var _ conn.Endpoint = (*lazyEndpoint)(nil) +// InitiationMessagePublicKey implements [conn.InitiationAwareEndpoint]. +// wireguard-go calls us here if we passed it a [*lazyEndpoint] for an +// initiation message, for which it might not have the relevant peer configured, +// enabling us to just-in-time configure it and note its activity via +// [*endpoint.noteRecvActivity], before it performs peer lookup and attempts +// decryption. +// +// Reception of all other WireGuard message types implies pre-existing knowledge +// of the peer by wireguard-go for it to do useful work. See +// [userspaceEngine.maybeReconfigWireguardLocked] & +// [userspaceEngine.noteRecvActivity] for more details around just-in-time +// wireguard-go peer (de)configuration. +func (le *lazyEndpoint) InitiationMessagePublicKey(peerPublicKey [32]byte) { + pubKey := key.NodePublicFromRaw32(mem.B(peerPublicKey[:])) + le.c.mu.Lock() + defer le.c.mu.Unlock() + ep, ok := le.c.peerMap.endpointForNodeKey(pubKey) + if !ok { + return + } + now := mono.Now() + ep.lastRecvUDPAny.StoreAtomic(now) + ep.noteRecvActivity(le.src, now) +} + func (le *lazyEndpoint) ClearSrc() {} func (le *lazyEndpoint) SrcIP() netip.Addr { return netip.Addr{} } From 1fe82d6ef5f48a85ce7ba6ce388a6c29f112b2cb Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Tue, 8 Jul 2025 14:37:13 -0500 Subject: [PATCH 171/263] cmd/tailscale/cli,ipn/ipnlocal: restrict logout when AlwaysOn mode is enabled In this PR, we start passing a LocalAPI actor to (*LocalBackend).Logout to make it subject to the same access check as disconnects made via tailscale down or the GUI. We then update the CLI to allow `tailscale logout` to accept a reason, similar to `tailscale down`. Updates tailscale/corp#26249 Signed-off-by: Nick Khyl --- cmd/tailscale/cli/logout.go | 12 ++++++++++++ cmd/tsconnect/wasm/wasm_js.go | 3 ++- ipn/ipnlocal/local.go | 9 +++------ ipn/ipnlocal/state_test.go | 7 ++++--- ipn/localapi/localapi.go | 2 +- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/cmd/tailscale/cli/logout.go b/cmd/tailscale/cli/logout.go index 0c2007a66ab1b..fbc39473026a1 100644 --- a/cmd/tailscale/cli/logout.go +++ b/cmd/tailscale/cli/logout.go @@ -5,12 +5,18 @@ package cli import ( "context" + "flag" "fmt" "strings" "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/client/tailscale/apitype" ) +var logoutArgs struct { + reason string +} + var logoutCmd = &ffcli.Command{ Name: "logout", ShortUsage: "tailscale logout", @@ -22,11 +28,17 @@ the current node key, forcing a future use of it to cause a reauthentication. `), Exec: runLogout, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("logout") + fs.StringVar(&logoutArgs.reason, "reason", "", "reason for the logout, if required by a policy") + return fs + })(), } func runLogout(ctx context.Context, args []string) error { if len(args) > 0 { return fmt.Errorf("too many non-flag arguments: %q", args) } + ctx = apitype.RequestReasonKey.WithValue(ctx, logoutArgs.reason) return localClient.Logout(ctx) } diff --git a/cmd/tsconnect/wasm/wasm_js.go b/cmd/tsconnect/wasm/wasm_js.go index 779a87e49dec9..ebf7284aa0d43 100644 --- a/cmd/tsconnect/wasm/wasm_js.go +++ b/cmd/tsconnect/wasm/wasm_js.go @@ -27,6 +27,7 @@ import ( "golang.org/x/crypto/ssh" "tailscale.com/control/controlclient" "tailscale.com/ipn" + "tailscale.com/ipn/ipnauth" "tailscale.com/ipn/ipnlocal" "tailscale.com/ipn/ipnserver" "tailscale.com/ipn/store/mem" @@ -336,7 +337,7 @@ func (i *jsIPN) logout() { go func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - i.lb.Logout(ctx) + i.lb.Logout(ctx, ipnauth.Self) }() } diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 03a0709e2d95d..8fbce4631368a 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1077,7 +1077,7 @@ func (b *LocalBackend) Shutdown() { ctx, cancel := context.WithTimeout(b.ctx, 5*time.Second) defer cancel() t0 := time.Now() - err := b.Logout(ctx) // best effort + err := b.Logout(ctx, ipnauth.Self) // best effort td := time.Since(t0).Round(time.Millisecond) if err != nil { b.logf("failed to log out ephemeral node on shutdown after %v: %v", td, err) @@ -5884,7 +5884,7 @@ func (b *LocalBackend) ShouldHandleViaIP(ip netip.Addr) bool { // Logout logs out the current profile, if any, and waits for the logout to // complete. -func (b *LocalBackend) Logout(ctx context.Context) error { +func (b *LocalBackend) Logout(ctx context.Context, actor ipnauth.Actor) error { unlock := b.lockAndGetUnlock() defer unlock() @@ -5898,11 +5898,8 @@ func (b *LocalBackend) Logout(ctx context.Context) error { // delete it later. profile := b.pm.CurrentProfile() - // TODO(nickkhyl): change [LocalBackend.Logout] to accept an [ipnauth.Actor]. - // This will allow enforcing Always On mode when a user tries to log out - // while logged in and connected. See tailscale/corp#26249. _, err := b.editPrefsLockedOnEntry( - ipnauth.TODO, + actor, &ipn.MaskedPrefs{ WantRunningSet: true, LoggedOutSet: true, diff --git a/ipn/ipnlocal/state_test.go b/ipn/ipnlocal/state_test.go index f0ac5f9442704..c29589acc698c 100644 --- a/ipn/ipnlocal/state_test.go +++ b/ipn/ipnlocal/state_test.go @@ -21,6 +21,7 @@ import ( "tailscale.com/control/controlclient" "tailscale.com/envknob" "tailscale.com/ipn" + "tailscale.com/ipn/ipnauth" "tailscale.com/ipn/ipnstate" "tailscale.com/ipn/store/mem" "tailscale.com/net/dns" @@ -607,7 +608,7 @@ func TestStateMachine(t *testing.T) { store.awaitWrite() t.Logf("\n\nLogout") notifies.expect(5) - b.Logout(context.Background()) + b.Logout(context.Background(), ipnauth.Self) { nn := notifies.drain(5) previousCC.assertCalls("pause", "Logout", "unpause", "Shutdown") @@ -637,7 +638,7 @@ func TestStateMachine(t *testing.T) { // A second logout should be a no-op as we are in the NeedsLogin state. t.Logf("\n\nLogout2") notifies.expect(0) - b.Logout(context.Background()) + b.Logout(context.Background(), ipnauth.Self) { notifies.drain(0) cc.assertCalls() @@ -650,7 +651,7 @@ func TestStateMachine(t *testing.T) { // AuthCantContinue state. t.Logf("\n\nLogout3") notifies.expect(3) - b.Logout(context.Background()) + b.Logout(context.Background(), ipnauth.Self) { notifies.drain(0) cc.assertCalls() diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index d4b4b443ef852..60ed89b3b2ad3 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -1460,7 +1460,7 @@ func (h *Handler) serveLogout(w http.ResponseWriter, r *http.Request) { http.Error(w, "want POST", http.StatusBadRequest) return } - err := h.b.Logout(r.Context()) + err := h.b.Logout(r.Context(), h.Actor) if err == nil { w.WriteHeader(http.StatusNoContent) return From 9bf99741ddb42cf3a2dec644cdff0f8cf9b99265 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Mon, 7 Jul 2025 19:02:10 -0500 Subject: [PATCH 172/263] ipn/ipnlocal: refactor resolveExitNodeInPrefsLocked, setExitNodeID and resolveExitNodeIP Now that resolveExitNodeInPrefsLocked is the only caller of setExitNodeID, and setExitNodeID is the only caller of resolveExitNodeIP, we can restructure the code with resolveExitNodeInPrefsLocked now calling both resolveAutoExitNodeLocked and resolveExitNodeIPLocked directly. This prepares for factoring out resolveAutoExitNodeLocked and related auto-exit-node logic into an ipnext extension in a future commit. While there, we also update exit node by IP lookup to use (*nodeBackend).NodeByAddr and (*nodeBackend).NodeByID instead of iterating over all peers in the most recent netmap. Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 106 ++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 8fbce4631368a..221edad92147c 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -2030,47 +2030,48 @@ func mutationsAreWorthyOfTellingIPNBus(muts []netmap.NodeMutation) bool { return false } -// setExitNodeID updates prefs to either use the suggestedExitNodeID if AutoExitNode is enabled, -// or resolve ExitNodeIP to an ID and use that. It returns whether prefs was mutated. -func setExitNodeID(prefs *ipn.Prefs, suggestedExitNodeID tailcfg.StableNodeID, nm *netmap.NetworkMap) (prefsChanged bool) { - if prefs.AutoExitNode.IsSet() { - var newExitNodeID tailcfg.StableNodeID - if !suggestedExitNodeID.IsZero() { - // If we have a suggested exit node, use it. - newExitNodeID = suggestedExitNodeID - } else if isAllowedAutoExitNodeID(prefs.ExitNodeID) { - // If we don't have a suggested exit node, but the prefs already - // specify an allowed auto exit node ID, retain it. - newExitNodeID = prefs.ExitNodeID - } else { - // Otherwise, use [unresolvedExitNodeID] to install a blackhole route, - // preventing traffic from leaking to the local network until an actual - // exit node is selected. - newExitNodeID = unresolvedExitNodeID - } - if prefs.ExitNodeID != newExitNodeID { - prefs.ExitNodeID = newExitNodeID - prefsChanged = true - } - if prefs.ExitNodeIP.IsValid() { - prefs.ExitNodeIP = netip.Addr{} - prefsChanged = true - } - return prefsChanged +// resolveAutoExitNodeLocked computes a suggested exit node and updates prefs +// to use it if AutoExitNode is enabled, and reports whether prefs was mutated. +// +// b.mu must be held. +func (b *LocalBackend) resolveAutoExitNodeLocked(prefs *ipn.Prefs) (prefsChanged bool) { + if !prefs.AutoExitNode.IsSet() { + return false + } + if _, err := b.suggestExitNodeLocked(); err != nil && !errors.Is(err, ErrNoPreferredDERP) { + b.logf("failed to select auto exit node: %v", err) // non-fatal, see below + } + var newExitNodeID tailcfg.StableNodeID + if !b.lastSuggestedExitNode.IsZero() { + // If we have a suggested exit node, use it. + newExitNodeID = b.lastSuggestedExitNode + } else if isAllowedAutoExitNodeID(prefs.ExitNodeID) { + // If we don't have a suggested exit node, but the prefs already + // specify an allowed auto exit node ID, retain it. + newExitNodeID = prefs.ExitNodeID + } else { + // Otherwise, use [unresolvedExitNodeID] to install a blackhole route, + // preventing traffic from leaking to the local network until an actual + // exit node is selected. + newExitNodeID = unresolvedExitNodeID + } + if prefs.ExitNodeID != newExitNodeID { + prefs.ExitNodeID = newExitNodeID + prefsChanged = true } - return resolveExitNodeIP(prefs, nm) + if prefs.ExitNodeIP.IsValid() { + prefs.ExitNodeIP = netip.Addr{} + prefsChanged = true + } + return prefsChanged } -// resolveExitNodeIP updates prefs to reference an exit node by ID, rather +// resolveExitNodeIPLocked updates prefs to reference an exit node by ID, rather // than by IP. It returns whether prefs was mutated. -func resolveExitNodeIP(prefs *ipn.Prefs, nm *netmap.NetworkMap) (prefsChanged bool) { - if nm == nil { - // No netmap, can't resolve anything. - return false - } - - // If we have a desired IP on file, try to find the corresponding - // node. +// +// b.mu must be held. +func (b *LocalBackend) resolveExitNodeIPLocked(prefs *ipn.Prefs) (prefsChanged bool) { + // If we have a desired IP on file, try to find the corresponding node. if !prefs.ExitNodeIP.IsValid() { return false } @@ -2081,20 +2082,19 @@ func resolveExitNodeIP(prefs *ipn.Prefs, nm *netmap.NetworkMap) (prefsChanged bo prefsChanged = true } - oldExitNodeID := prefs.ExitNodeID - for _, peer := range nm.Peers { - for _, addr := range peer.Addresses().All() { - if !addr.IsSingleIP() || addr.Addr() != prefs.ExitNodeIP { - continue - } + cn := b.currentNode() + if nid, ok := cn.NodeByAddr(prefs.ExitNodeIP); ok { + if node, ok := cn.NodeByID(nid); ok { // Found the node being referenced, upgrade prefs to // reference it directly for next time. - prefs.ExitNodeID = peer.StableID() + prefs.ExitNodeID = node.StableID() prefs.ExitNodeIP = netip.Addr{} - return prefsChanged || oldExitNodeID != prefs.ExitNodeID + // Cleared ExitNodeIP, so prefs changed + // even if the ID stayed the same. + prefsChanged = true + } } - return prefsChanged } @@ -6042,17 +6042,13 @@ func (b *LocalBackend) reconcilePrefsLocked(prefs *ipn.Prefs) (changed bool) { // // b.mu must be held. func (b *LocalBackend) resolveExitNodeInPrefsLocked(prefs *ipn.Prefs) (changed bool) { - if prefs.AutoExitNode.IsSet() { - _, err := b.suggestExitNodeLocked() - if err != nil && !errors.Is(err, ErrNoPreferredDERP) { - b.logf("failed to select auto exit node: %v", err) - } + if b.resolveAutoExitNodeLocked(prefs) { + changed = true } - if setExitNodeID(prefs, b.lastSuggestedExitNode, b.currentNode().NetMap()) { - b.logf("exit node resolved: %v", prefs.ExitNodeID) - return true + if b.resolveExitNodeIPLocked(prefs) { + changed = true } - return false + return changed } // setNetMapLocked updates the LocalBackend state to reflect the newly From 2c630e126b84b537053947b579f0b44623deb496 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Mon, 7 Jul 2025 19:05:41 -0500 Subject: [PATCH 173/263] ipn/ipnlocal: make applySysPolicy a method on LocalBackend Now that applySysPolicy is only called by (*LocalBackend).reconcilePrefsLocked, we can make it a method to avoid passing state via parameters and to support future extensibility. Also factor out exit node-specific logic into applyExitNodeSysPolicyLocked. Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 52 ++++++++++++++++++++++++-------------- ipn/ipnlocal/local_test.go | 6 +++-- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 221edad92147c..9ed9522ab259c 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1800,9 +1800,11 @@ var preferencePolicies = []preferencePolicyInfo{ }, } -// applySysPolicy overwrites configured preferences with policies that may be +// applySysPolicyLocked overwrites configured preferences with policies that may be // configured by the system administrator in an OS-specific way. -func applySysPolicy(prefs *ipn.Prefs, overrideAlwaysOn bool) (anyChange bool) { +// +// b.mu must be held. +func (b *LocalBackend) applySysPolicyLocked(prefs *ipn.Prefs) (anyChange bool) { if controlURL, err := syspolicy.GetString(syspolicy.ControlURL, prefs.ControlURL); err == nil && prefs.ControlURL != controlURL { prefs.ControlURL = controlURL anyChange = true @@ -1839,6 +1841,34 @@ func applySysPolicy(prefs *ipn.Prefs, overrideAlwaysOn bool) (anyChange bool) { } } + if b.applyExitNodeSysPolicyLocked(prefs) { + anyChange = true + } + + if alwaysOn, _ := syspolicy.GetBoolean(syspolicy.AlwaysOn, false); alwaysOn && !b.overrideAlwaysOn && !prefs.WantRunning { + prefs.WantRunning = true + anyChange = true + } + + for _, opt := range preferencePolicies { + if po, err := syspolicy.GetPreferenceOption(opt.key); err == nil { + curVal := opt.get(prefs.View()) + newVal := po.ShouldEnable(curVal) + if curVal != newVal { + opt.set(prefs, newVal) + anyChange = true + } + } + } + + return anyChange +} + +// applyExitNodeSysPolicyLocked applies the exit node policy settings to prefs +// and reports whether any change was made. +// +// b.mu must be held. +func (b *LocalBackend) applyExitNodeSysPolicyLocked(prefs *ipn.Prefs) (anyChange bool) { if exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, ""); exitNodeIDStr != "" { exitNodeID := tailcfg.StableNodeID(exitNodeIDStr) @@ -1894,22 +1924,6 @@ func applySysPolicy(prefs *ipn.Prefs, overrideAlwaysOn bool) (anyChange bool) { } } - if alwaysOn, _ := syspolicy.GetBoolean(syspolicy.AlwaysOn, false); alwaysOn && !overrideAlwaysOn && !prefs.WantRunning { - prefs.WantRunning = true - anyChange = true - } - - for _, opt := range preferencePolicies { - if po, err := syspolicy.GetPreferenceOption(opt.key); err == nil { - curVal := opt.get(prefs.View()) - newVal := po.ShouldEnable(curVal) - if curVal != newVal { - opt.set(prefs, newVal) - anyChange = true - } - } - } - return anyChange } @@ -6024,7 +6038,7 @@ func (b *LocalBackend) resolveExitNode() (changed bool) { // // b.mu must be held. func (b *LocalBackend) reconcilePrefsLocked(prefs *ipn.Prefs) (changed bool) { - if applySysPolicy(prefs, b.overrideAlwaysOn) { + if b.applySysPolicyLocked(prefs) { changed = true } if b.resolveExitNodeInPrefsLocked(prefs) { diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 1e1b7663ab687..b8526a4fcc70e 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -2968,7 +2968,8 @@ func TestApplySysPolicy(t *testing.T) { t.Run("unit", func(t *testing.T) { prefs := tt.prefs.Clone() - gotAnyChange := applySysPolicy(prefs, false) + lb := newTestLocalBackend(t) + gotAnyChange := lb.applySysPolicyLocked(prefs) if gotAnyChange && prefs.Equals(&tt.prefs) { t.Errorf("anyChange but prefs is unchanged: %v", prefs.Pretty()) @@ -3116,7 +3117,8 @@ func TestPreferencePolicyInfo(t *testing.T) { prefs := defaultPrefs.AsStruct() pp.set(prefs, tt.initialValue) - gotAnyChange := applySysPolicy(prefs, false) + lb := newTestLocalBackend(t) + gotAnyChange := lb.applySysPolicyLocked(prefs) if gotAnyChange != tt.wantChange { t.Errorf("anyChange=%v, want %v", gotAnyChange, tt.wantChange) From 740b77df594f649830d151f64700caea5c341e60 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Tue, 8 Jul 2025 16:08:28 -0500 Subject: [PATCH 174/263] ipn/ipnlocal,util/syspolicy: add support for ExitNode.AllowOverride policy setting When the policy setting is enabled, it allows users to override the exit node enforced by the ExitNodeID or ExitNodeIP policy. It's primarily intended for use when ExitNodeID is set to auto:any, but it can also be used with specific exit nodes. It does not allow disabling exit node usage entirely. Once the exit node policy is overridden, it will not be enforced again until the policy changes, the user connects or disconnects Tailscale, switches profiles, or disables the override. Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 119 ++++++++- ipn/ipnlocal/local_test.go | 312 ++++++++++++++++++++++++ util/syspolicy/policy_keys.go | 10 + util/syspolicy/rsop/change_callbacks.go | 5 + 4 files changed, 434 insertions(+), 12 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 9ed9522ab259c..c54cb32d3125c 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -414,6 +414,19 @@ type LocalBackend struct { // reconnectTimer is used to schedule a reconnect by setting [ipn.Prefs.WantRunning] // to true after a delay, or nil if no reconnect is scheduled. reconnectTimer tstime.TimerController + + // overrideExitNodePolicy is whether the user has overridden the exit node policy + // by manually selecting an exit node, as allowed by [syspolicy.AllowExitNodeOverride]. + // + // If true, the [syspolicy.ExitNodeID] and [syspolicy.ExitNodeIP] policy settings are ignored, + // and the suggested exit node is not applied automatically. + // + // It is cleared when the user switches back to the state required by policy (typically, auto:any), + // or when switching profiles, connecting/disconnecting Tailscale, restarting the client, + // or on similar events. + // + // See tailscale/corp#29969. + overrideExitNodePolicy bool } // HealthTracker returns the health tracker for the backend. @@ -1841,7 +1854,8 @@ func (b *LocalBackend) applySysPolicyLocked(prefs *ipn.Prefs) (anyChange bool) { } } - if b.applyExitNodeSysPolicyLocked(prefs) { + // Only apply the exit node policy if the user hasn't overridden it. + if !b.overrideExitNodePolicy && b.applyExitNodeSysPolicyLocked(prefs) { anyChange = true } @@ -1957,7 +1971,7 @@ func (b *LocalBackend) reconcilePrefs() (_ ipn.PrefsView, anyChange bool) { // sysPolicyChanged is a callback triggered by syspolicy when it detects // a change in one or more syspolicy settings. func (b *LocalBackend) sysPolicyChanged(policy *rsop.PolicyChange) { - if policy.HasChanged(syspolicy.AlwaysOn) || policy.HasChanged(syspolicy.AlwaysOnOverrideWithReason) { + if policy.HasChangedAnyOf(syspolicy.AlwaysOn, syspolicy.AlwaysOnOverrideWithReason) { // If the AlwaysOn or the AlwaysOnOverrideWithReason policy has changed, // we should reset the overrideAlwaysOn flag, as the override might // no longer be valid. @@ -1966,6 +1980,14 @@ func (b *LocalBackend) sysPolicyChanged(policy *rsop.PolicyChange) { b.mu.Unlock() } + if policy.HasChangedAnyOf(syspolicy.ExitNodeID, syspolicy.ExitNodeIP, syspolicy.AllowExitNodeOverride) { + // Reset the exit node override if a policy that enforces exit node usage + // or allows the user to override automatic exit node selection has changed. + b.mu.Lock() + b.overrideExitNodePolicy = false + b.mu.Unlock() + } + if policy.HasChanged(syspolicy.AllowedSuggestedExitNodes) { b.refreshAllowedSuggestions() // Re-evaluate exit node suggestion now that the policy setting has changed. @@ -4320,12 +4342,12 @@ func (b *LocalBackend) EditPrefsAs(mp *ipn.MaskedPrefs, actor ipnauth.Actor) (ip } // checkEditPrefsAccessLocked checks whether the current user has access -// to apply the prefs changes in mp. +// to apply the changes in mp to the given prefs. // // It returns an error if the user is not allowed, or nil otherwise. // // b.mu must be held. -func (b *LocalBackend) checkEditPrefsAccessLocked(actor ipnauth.Actor, mp *ipn.MaskedPrefs) error { +func (b *LocalBackend) checkEditPrefsAccessLocked(actor ipnauth.Actor, prefs ipn.PrefsView, mp *ipn.MaskedPrefs) error { var errs []error if mp.RunSSHSet && mp.RunSSH && !envknob.CanSSHD() { @@ -4342,14 +4364,18 @@ func (b *LocalBackend) checkEditPrefsAccessLocked(actor ipnauth.Actor, mp *ipn.M // Prevent users from changing exit node preferences // when exit node usage is managed by policy. if mp.ExitNodeIDSet || mp.ExitNodeIPSet || mp.AutoExitNodeSet { - // TODO(nickkhyl): Allow users to override ExitNode policy settings - // if the ExitNode.AllowUserOverride policy permits it. - // (Policy setting name and details are TBD. See tailscale/corp#29969) isManaged, err := syspolicy.HasAnyOf(syspolicy.ExitNodeID, syspolicy.ExitNodeIP) if err != nil { err = fmt.Errorf("policy check failed: %w", err) } else if isManaged { - err = errManagedByPolicy + // Allow users to override ExitNode policy settings and select an exit node manually + // if permitted by [syspolicy.AllowExitNodeOverride]. + // + // Disabling exit node usage entirely is not allowed. + allowExitNodeOverride, _ := syspolicy.GetBoolean(syspolicy.AllowExitNodeOverride, false) + if !allowExitNodeOverride || b.changeDisablesExitNodeLocked(prefs, mp) { + err = errManagedByPolicy + } } if err != nil { errs = append(errs, fmt.Errorf("exit node cannot be changed: %w", err)) @@ -4359,19 +4385,70 @@ func (b *LocalBackend) checkEditPrefsAccessLocked(actor ipnauth.Actor, mp *ipn.M return multierr.New(errs...) } +// changeDisablesExitNodeLocked reports whether applying the change +// to the given prefs would disable exit node usage. +// +// In other words, it returns true if prefs.ExitNodeID is non-empty +// initially, but would become empty after applying the given change. +// +// It applies the same adjustments and resolves the exit node in the prefs +// as done during actual edits. While not optimal performance-wise, +// changing the exit node via LocalAPI isn't a hot path, and reusing +// the same logic ensures consistency and simplifies maintenance. +// +// b.mu must be held. +func (b *LocalBackend) changeDisablesExitNodeLocked(prefs ipn.PrefsView, change *ipn.MaskedPrefs) bool { + if !change.AutoExitNodeSet && !change.ExitNodeIDSet && !change.ExitNodeIPSet { + // The change does not affect exit node usage. + return false + } + + if prefs.ExitNodeID() == "" { + // Exit node usage is already disabled. + // Note that we do not check for ExitNodeIP here. + // If ExitNodeIP hasn't been resolved to a node, + // it's not enabled yet. + return false + } + + // First, apply the adjustments to a copy of the changes, + // e.g., clear AutoExitNode if ExitNodeID is set. + tmpChange := ptr.To(*change) + tmpChange.Prefs = *change.Prefs.Clone() + b.adjustEditPrefsLocked(prefs, tmpChange) + + // Then apply the adjusted changes to a copy of the current prefs, + // and resolve the exit node in the prefs. + tmpPrefs := prefs.AsStruct() + tmpPrefs.ApplyEdits(tmpChange) + b.resolveExitNodeInPrefsLocked(tmpPrefs) + + // If ExitNodeID is empty after applying the changes, + // but wasn't empty before, then the change disables + // exit node usage. + return tmpPrefs.ExitNodeID == "" + +} + // adjustEditPrefsLocked applies additional changes to mp if necessary, // such as zeroing out mutually exclusive fields. // // It must not assume that the changes in mp will actually be applied. // // b.mu must be held. -func (b *LocalBackend) adjustEditPrefsLocked(_ ipnauth.Actor, mp *ipn.MaskedPrefs) { +func (b *LocalBackend) adjustEditPrefsLocked(prefs ipn.PrefsView, mp *ipn.MaskedPrefs) { // Zeroing the ExitNodeID via localAPI must also zero the prior exit node. if mp.ExitNodeIDSet && mp.ExitNodeID == "" && !mp.InternalExitNodePriorSet { mp.InternalExitNodePrior = "" mp.InternalExitNodePriorSet = true } + // Clear ExitNodeID if AutoExitNode is disabled and ExitNodeID is still unresolved. + if mp.AutoExitNodeSet && mp.AutoExitNode == "" && prefs.ExitNodeID() == unresolvedExitNodeID { + mp.ExitNodeIDSet = true + mp.ExitNodeID = "" + } + // Disable automatic exit node selection if the user explicitly sets // ExitNodeID or ExitNodeIP. if (mp.ExitNodeIDSet || mp.ExitNodeIPSet) && !mp.AutoExitNodeSet { @@ -4404,6 +4481,22 @@ func (b *LocalBackend) onEditPrefsLocked(_ ipnauth.Actor, mp *ipn.MaskedPrefs, o } } + if oldPrefs.WantRunning() != newPrefs.WantRunning() { + // Connecting to or disconnecting from Tailscale clears the override, + // unless the user is also explicitly changing the exit node (see below). + b.overrideExitNodePolicy = false + } + if mp.AutoExitNodeSet || mp.ExitNodeIDSet || mp.ExitNodeIPSet { + if allowExitNodeOverride, _ := syspolicy.GetBoolean(syspolicy.AllowExitNodeOverride, false); allowExitNodeOverride { + // If applying exit node policy settings to the new prefs results in no change, + // the user is not overriding the policy. Otherwise, it is an override. + b.overrideExitNodePolicy = b.applyExitNodeSysPolicyLocked(newPrefs.AsStruct()) + } else { + // Overrides are not allowed; clear the override flag. + b.overrideExitNodePolicy = false + } + } + // This is recorded here in the EditPrefs path, not the setPrefs path on purpose. // recordForEdit records metrics related to edits and changes, not the final state. // If, in the future, we want to record gauge-metrics related to the state of prefs, @@ -4486,15 +4579,17 @@ func (b *LocalBackend) stopReconnectTimerLocked() { func (b *LocalBackend) editPrefsLockedOnEntry(actor ipnauth.Actor, mp *ipn.MaskedPrefs, unlock unlockOnce) (ipn.PrefsView, error) { defer unlock() // for error paths + p0 := b.pm.CurrentPrefs() + // Check if the changes in mp are allowed. - if err := b.checkEditPrefsAccessLocked(actor, mp); err != nil { + if err := b.checkEditPrefsAccessLocked(actor, p0, mp); err != nil { b.logf("EditPrefs(%v): %v", mp.Pretty(), err) return ipn.PrefsView{}, err } // Apply additional changes to mp if necessary, // such as clearing mutually exclusive fields. - b.adjustEditPrefsLocked(actor, mp) + b.adjustEditPrefsLocked(p0, mp) if mp.EggSet { mp.EggSet = false @@ -4502,7 +4597,6 @@ func (b *LocalBackend) editPrefsLockedOnEntry(actor ipnauth.Actor, mp *ipn.Maske b.goTracker.Go(b.doSetHostinfoFilterServices) } - p0 := b.pm.CurrentPrefs() p1 := b.pm.CurrentPrefs().AsStruct() p1.ApplyEdits(mp) @@ -7231,6 +7325,7 @@ func (b *LocalBackend) resetForProfileChangeLockedOnEntry(unlock unlockOnce) err b.serveConfig = ipn.ServeConfigView{} b.lastSuggestedExitNode = "" b.keyExpired = false + b.overrideExitNodePolicy = false b.resetAlwaysOnOverrideLocked() b.extHost.NotifyProfileChange(b.pm.CurrentProfile(), b.pm.CurrentPrefs(), false) b.setAtomicValuesFromPrefsLocked(b.pm.CurrentPrefs()) diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index b8526a4fcc70e..8bc84b081c016 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -623,6 +623,7 @@ func TestConfigureExitNode(t *testing.T) { exitNodeIDPolicy *tailcfg.StableNodeID exitNodeIPPolicy *netip.Addr exitNodeAllowedIDs []tailcfg.StableNodeID // nil if all IDs are allowed for auto exit nodes + exitNodeAllowOverride bool // whether [syspolicy.AllowExitNodeOverride] should be set to true wantChangePrefsErr error // if non-nil, the error we expect from [LocalBackend.EditPrefsAs] wantPrefs ipn.Prefs wantExitNodeToggleErr error // if non-nil, the error we expect from [LocalBackend.SetUseExitNodeEnabled] @@ -1018,6 +1019,108 @@ func TestConfigureExitNode(t *testing.T) { InternalExitNodePrior: "", }, }, + { + name: "auto-any-via-policy/allow-override/change", // changing the exit node is allowed by [syspolicy.AllowExitNodeOverride] + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowOverride: true, // allow changing the exit node + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + ExitNodeID: exitNode2.StableID(), // change the exit node ID + }, + ExitNodeIDSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), // overridden by user + AutoExitNode: "", // cleared, as we are setting the exit node ID explicitly + }, + }, + { + name: "auto-any-via-policy/allow-override/clear", // clearing the exit node ID is not allowed by [syspolicy.AllowExitNodeOverride] + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowOverride: true, // allow changing, but not disabling, the exit node + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + ExitNodeID: "", // clearing the exit node ID disables the exit node and should not be allowed + }, + ExitNodeIDSet: true, + }, + wantChangePrefsErr: errManagedByPolicy, // edit prefs should fail with an error + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // still enforced by the policy setting + AutoExitNode: "any", + InternalExitNodePrior: "", + }, + }, + { + name: "auto-any-via-policy/allow-override/toggle-off", // similarly, toggling off the exit node is not allowed even with [syspolicy.AllowExitNodeOverride] + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowOverride: true, // allow changing, but not disabling, the exit node + useExitNodeEnabled: ptr.To(false), // should fail with an error + wantExitNodeToggleErr: errManagedByPolicy, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // still enforced by the policy setting + AutoExitNode: "any", + InternalExitNodePrior: "", + }, + }, + { + name: "auto-any-via-initial-prefs/no-netmap/clear-auto-exit-node", + prefs: ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: ipn.AnyExitNode, + }, + netMap: nil, // no netmap; exit node cannot be resolved + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + AutoExitNode: "", // clear the auto exit node + }, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: "", // cleared + ExitNodeID: "", // has never been resolved, so it should be cleared as well + }, + }, + { + name: "auto-any-via-initial-prefs/with-netmap/clear-auto-exit-node", + prefs: ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: ipn.AnyExitNode, + }, + netMap: clientNetmap, // has a netmap; exit node will be resolved + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + AutoExitNode: "", // clear the auto exit node + }, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: "", // cleared + ExitNodeID: exitNode1.StableID(), // a resolved exit node ID should be retained + }, + }, } syspolicy.RegisterWellKnownSettingsForTest(t) for _, tt := range tests { @@ -1033,6 +1136,9 @@ func TestConfigureExitNode(t *testing.T) { if tt.exitNodeAllowedIDs != nil { store.SetStringLists(source.TestSettingOf(syspolicy.AllowedSuggestedExitNodes, toStrings(tt.exitNodeAllowedIDs))) } + if tt.exitNodeAllowOverride { + store.SetBooleans(source.TestSettingOf(syspolicy.AllowExitNodeOverride, true)) + } if store.IsEmpty() { // No syspolicy settings, so don't register a store. // This allows the test to run in parallel with other tests. @@ -1078,6 +1184,212 @@ func TestConfigureExitNode(t *testing.T) { } } +func TestPrefsChangeDisablesExitNode(t *testing.T) { + tests := []struct { + name string + netMap *netmap.NetworkMap + prefs ipn.Prefs + change ipn.MaskedPrefs + wantDisablesExitNode bool + }{ + { + name: "has-exit-node-id/no-change", + prefs: ipn.Prefs{ + ExitNodeID: "test-exit-node", + }, + change: ipn.MaskedPrefs{}, + wantDisablesExitNode: false, + }, + { + name: "has-exit-node-ip/no-change", + prefs: ipn.Prefs{ + ExitNodeIP: netip.MustParseAddr("100.100.1.1"), + }, + change: ipn.MaskedPrefs{}, + wantDisablesExitNode: false, + }, + { + name: "has-auto-exit-node/no-change", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{}, + wantDisablesExitNode: false, + }, + { + name: "has-exit-node-id/non-exit-node-change", + prefs: ipn.Prefs{ + ExitNodeID: "test-exit-node", + }, + change: ipn.MaskedPrefs{ + WantRunningSet: true, + HostnameSet: true, + ExitNodeAllowLANAccessSet: true, + Prefs: ipn.Prefs{ + WantRunning: true, + Hostname: "test-hostname", + ExitNodeAllowLANAccess: true, + }, + }, + wantDisablesExitNode: false, + }, + { + name: "has-exit-node-ip/non-exit-node-change", + prefs: ipn.Prefs{ + ExitNodeIP: netip.MustParseAddr("100.100.1.1"), + }, + change: ipn.MaskedPrefs{ + WantRunningSet: true, + RouteAllSet: true, + ShieldsUpSet: true, + Prefs: ipn.Prefs{ + WantRunning: false, + RouteAll: false, + ShieldsUp: true, + }, + }, + wantDisablesExitNode: false, + }, + { + name: "has-auto-exit-node/non-exit-node-change", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{ + CorpDNSSet: true, + RouteAllSet: true, + ExitNodeAllowLANAccessSet: true, + Prefs: ipn.Prefs{ + CorpDNS: true, + RouteAll: false, + ExitNodeAllowLANAccess: true, + }, + }, + wantDisablesExitNode: false, + }, + { + name: "has-exit-node-id/change-exit-node-id", + prefs: ipn.Prefs{ + ExitNodeID: "exit-node-1", + }, + change: ipn.MaskedPrefs{ + ExitNodeIDSet: true, + Prefs: ipn.Prefs{ + ExitNodeID: "exit-node-2", + }, + }, + wantDisablesExitNode: false, // changing the exit node ID does not disable it + }, + { + name: "has-exit-node-id/enable-auto-exit-node", + prefs: ipn.Prefs{ + ExitNodeID: "exit-node-1", + }, + change: ipn.MaskedPrefs{ + AutoExitNodeSet: true, + Prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + }, + wantDisablesExitNode: false, // changing the exit node ID does not disable it + }, + { + name: "has-exit-node-id/clear-exit-node-id", + prefs: ipn.Prefs{ + ExitNodeID: "exit-node-1", + }, + change: ipn.MaskedPrefs{ + ExitNodeIDSet: true, + Prefs: ipn.Prefs{ + ExitNodeID: "", + }, + }, + wantDisablesExitNode: true, // clearing the exit node ID disables it + }, + { + name: "has-auto-exit-node/clear-exit-node-id", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{ + ExitNodeIDSet: true, + Prefs: ipn.Prefs{ + ExitNodeID: "", + }, + }, + wantDisablesExitNode: true, // clearing the exit node ID disables auto exit node as well... + }, + { + name: "has-auto-exit-node/clear-exit-node-id/but-keep-auto-exit-node", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{ + ExitNodeIDSet: true, + AutoExitNodeSet: true, + Prefs: ipn.Prefs{ + ExitNodeID: "", + AutoExitNode: ipn.AnyExitNode, + }, + }, + wantDisablesExitNode: false, // ... unless we explicitly keep the auto exit node enabled + }, + { + name: "has-auto-exit-node/clear-exit-node-ip", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{ + ExitNodeIPSet: true, + Prefs: ipn.Prefs{ + ExitNodeIP: netip.Addr{}, + }, + }, + wantDisablesExitNode: false, // auto exit node is still enabled + }, + { + name: "has-auto-exit-node/clear-auto-exit-node", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{ + AutoExitNodeSet: true, + Prefs: ipn.Prefs{ + AutoExitNode: "", + }, + }, + wantDisablesExitNode: true, // clearing the auto exit while the exit node ID is unresolved disables exit node usage + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lb := newTestLocalBackend(t) + if tt.netMap != nil { + lb.SetControlClientStatus(lb.cc, controlclient.Status{NetMap: tt.netMap}) + } + // Set the initial prefs via SetPrefsForTest + // to apply necessary adjustments. + lb.SetPrefsForTest(tt.prefs.Clone()) + initialPrefs := lb.Prefs() + + // Check whether changeDisablesExitNodeLocked correctly identifies the change. + if got := lb.changeDisablesExitNodeLocked(initialPrefs, &tt.change); got != tt.wantDisablesExitNode { + t.Errorf("disablesExitNode: got %v; want %v", got, tt.wantDisablesExitNode) + } + + // Apply the change and check if it the actual behavior matches the expectation. + gotPrefs, err := lb.EditPrefsAs(&tt.change, &ipnauth.TestActor{}) + if err != nil { + t.Fatalf("EditPrefsAs failed: %v", err) + } + gotDisabledExitNode := initialPrefs.ExitNodeID() != "" && gotPrefs.ExitNodeID() == "" + if gotDisabledExitNode != tt.wantDisablesExitNode { + t.Errorf("disabledExitNode: got %v; want %v", gotDisabledExitNode, tt.wantDisablesExitNode) + } + }) + } +} + func TestInternalAndExternalInterfaces(t *testing.T) { type interfacePrefix struct { i netmon.Interface diff --git a/util/syspolicy/policy_keys.go b/util/syspolicy/policy_keys.go index b19a3e7fec61f..cd5f8172c159a 100644 --- a/util/syspolicy/policy_keys.go +++ b/util/syspolicy/policy_keys.go @@ -54,6 +54,15 @@ const ( ExitNodeID Key = "ExitNodeID" ExitNodeIP Key = "ExitNodeIP" // default ""; if blank, no exit node is forced. Value is exit node IP. + // AllowExitNodeOverride is a boolean key that allows the user to override exit node policy settings + // and manually select an exit node. It does not allow disabling exit node usage entirely. + // It is typically used in conjunction with [ExitNodeID] set to "auto:any". + // + // Warning: This policy setting is experimental and may change, be renamed or removed in the future. + // It may also not be fully supported by all Tailscale clients until it is out of experimental status. + // See tailscale/corp#29969. + AllowExitNodeOverride Key = "ExitNode.AllowOverride" + // Keys with a string value that specifies an option: "always", "never", "user-decides". // The default is "user-decides" unless otherwise stated. Enforcement of // these policies is typically performed in ipnlocal.applySysPolicy(). GUIs @@ -173,6 +182,7 @@ const ( var implicitDefinitions = []*setting.Definition{ // Device policy settings (can only be configured on a per-device basis): setting.NewDefinition(AllowedSuggestedExitNodes, setting.DeviceSetting, setting.StringListValue), + setting.NewDefinition(AllowExitNodeOverride, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(AlwaysOn, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(AlwaysOnOverrideWithReason, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(ApplyUpdates, setting.DeviceSetting, setting.PreferenceOptionValue), diff --git a/util/syspolicy/rsop/change_callbacks.go b/util/syspolicy/rsop/change_callbacks.go index b962f30c008c1..87b45b654709d 100644 --- a/util/syspolicy/rsop/change_callbacks.go +++ b/util/syspolicy/rsop/change_callbacks.go @@ -59,6 +59,11 @@ func (c PolicyChange) HasChanged(key setting.Key) bool { } } +// HasChangedAnyOf reports whether any of the specified policy settings has changed. +func (c PolicyChange) HasChangedAnyOf(keys ...setting.Key) bool { + return slices.ContainsFunc(keys, c.HasChanged) +} + // policyChangeCallbacks are the callbacks to invoke when the effective policy changes. // It is safe for concurrent use. type policyChangeCallbacks struct { From a60e0caf6a3bc4c2801f4ca6e1630fc9409d1125 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 8 Jul 2025 19:37:09 -0700 Subject: [PATCH 175/263] wgengine/magicsock: remove conn.InitiationAwareEndpoint TODO (#16498) It was implemented in 5b0074729d38f8cc301803da06086033f53b1b93. Updates #cleanup Signed-off-by: Jordan Whited --- wgengine/magicsock/magicsock.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index fbfcf0b41565a..ab7c2102fe1ec 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -1708,11 +1708,6 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach } // TODO(jwhited): reuse [lazyEndpoint] across calls to receiveIP() // for the same batch & [epAddr] src. - // - // TODO(jwhited): implement [lazyEndpoint] integration to call - // [endpoint.noteRecvActivity], which triggers just-in-time - // wireguard-go configuration of the peer, prior to peer lookup - // within wireguard-go. return &lazyEndpoint{c: c, src: src}, size, true } cache.epAddr = src From bad17a1bfaa0ac3e62e2ebc95fca7c5c5959055b Mon Sep 17 00:00:00 2001 From: Simon Law Date: Tue, 8 Jul 2025 22:14:18 -0700 Subject: [PATCH 176/263] cmd/tailscale: format empty cities and countries as hyphens (#16495) When running `tailscale exit-node list`, an empty city or country name should be displayed as a hyphen "-". However, this only happened when there was no location at all. If a node provides a Hostinfo.Location, then the list would display exactly what was provided. This patch changes the listing so that empty cities and countries will either render the provided name or "-". Fixes #16500 Signed-off-by: Simon Law --- cmd/tailscale/cli/exitnode.go | 22 +++++++++------------- cmd/tailscale/cli/exitnode_test.go | 28 ++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/cmd/tailscale/cli/exitnode.go b/cmd/tailscale/cli/exitnode.go index ad7a8ccee5b42..b153f096d6869 100644 --- a/cmd/tailscale/cli/exitnode.go +++ b/cmd/tailscale/cli/exitnode.go @@ -131,7 +131,7 @@ func runExitNodeList(ctx context.Context, args []string) error { for _, country := range filteredPeers.Countries { for _, city := range country.Cities { for _, peer := range city.Peers { - fmt.Fprintf(w, "\n %s\t%s\t%s\t%s\t%s\t", peer.TailscaleIPs[0], strings.Trim(peer.DNSName, "."), country.Name, city.Name, peerStatus(peer)) + fmt.Fprintf(w, "\n %s\t%s\t%s\t%s\t%s\t", peer.TailscaleIPs[0], strings.Trim(peer.DNSName, "."), cmp.Or(country.Name, "-"), cmp.Or(city.Name, "-"), peerStatus(peer)) } } } @@ -202,23 +202,16 @@ type filteredCity struct { Peers []*ipnstate.PeerStatus } -const noLocationData = "-" - -var noLocation = &tailcfg.Location{ - Country: noLocationData, - CountryCode: noLocationData, - City: noLocationData, - CityCode: noLocationData, -} - // filterFormatAndSortExitNodes filters and sorts exit nodes into // alphabetical order, by country, city and then by priority if // present. +// // If an exit node has location data, and the country has more than // one city, an `Any` city is added to the country that contains the // highest priority exit node within that country. +// // For exit nodes without location data, their country fields are -// defined as '-' to indicate that the data is not available. +// defined as the empty string to indicate that the data is not available. func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string) filteredExitNodes { // first get peers into some fixed order, as code below doesn't break ties // and our input comes from a random range-over-map. @@ -229,7 +222,10 @@ func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string) countries := make(map[string]*filteredCountry) cities := make(map[string]*filteredCity) for _, ps := range peers { - loc := cmp.Or(ps.Location, noLocation) + loc := ps.Location + if loc == nil { + loc = &tailcfg.Location{} + } if filterBy != "" && !strings.EqualFold(loc.Country, filterBy) { continue @@ -259,7 +255,7 @@ func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string) } for _, country := range filteredExitNodes.Countries { - if country.Name == noLocationData { + if country.Name == "" { // Countries without location data should not // be filtered further. continue diff --git a/cmd/tailscale/cli/exitnode_test.go b/cmd/tailscale/cli/exitnode_test.go index 9d569a45a4615..cc38fd3a4d39e 100644 --- a/cmd/tailscale/cli/exitnode_test.go +++ b/cmd/tailscale/cli/exitnode_test.go @@ -74,10 +74,10 @@ func TestFilterFormatAndSortExitNodes(t *testing.T) { want := filteredExitNodes{ Countries: []*filteredCountry{ { - Name: noLocationData, + Name: "", Cities: []*filteredCity{ { - Name: noLocationData, + Name: "", Peers: []*ipnstate.PeerStatus{ ps[5], }, @@ -273,14 +273,20 @@ func TestSortByCountryName(t *testing.T) { Name: "Zimbabwe", }, { - Name: noLocationData, + Name: "", }, } sortByCountryName(fc) - if fc[0].Name != noLocationData { - t.Fatalf("sortByCountryName did not order countries by alphabetical order, got %v, want %v", fc[0].Name, noLocationData) + want := []string{"", "Albania", "Sweden", "Zimbabwe"} + var got []string + for _, c := range fc { + got = append(got, c.Name) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("sortByCountryName did not order countries by alphabetical order (-want +got):\n%s", diff) } } @@ -296,13 +302,19 @@ func TestSortByCityName(t *testing.T) { Name: "Squamish", }, { - Name: noLocationData, + Name: "", }, } sortByCityName(fc) - if fc[0].Name != noLocationData { - t.Fatalf("sortByCityName did not order cities by alphabetical order, got %v, want %v", fc[0].Name, noLocationData) + want := []string{"", "Goteborg", "Kingston", "Squamish"} + var got []string + for _, c := range fc { + got = append(got, c.Name) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("sortByCityName did not order countries by alphabetical order (-want +got):\n%s", diff) } } From 90bf0a97b3b1c042b3a6be48ec186732733f995b Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 9 Jul 2025 09:13:11 +0100 Subject: [PATCH 177/263] cmd/k8s-operator/deploy: clarify helm install notes (#16449) Based on feedback that it wasn't clear what the user is meant to do with the output of the last command, clarify that it's an optional command to explore what got created. Updates #13427 Change-Id: Iff64ec6d02dc04bf4bbebf415d7ed1a44e7dd658 Signed-off-by: Tom Proctor --- cmd/k8s-operator/deploy/chart/templates/NOTES.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/k8s-operator/deploy/chart/templates/NOTES.txt b/cmd/k8s-operator/deploy/chart/templates/NOTES.txt index 5678e597a6824..1bee6704616e6 100644 --- a/cmd/k8s-operator/deploy/chart/templates/NOTES.txt +++ b/cmd/k8s-operator/deploy/chart/templates/NOTES.txt @@ -22,4 +22,6 @@ $ kubectl explain proxyclass $ kubectl explain recorder $ kubectl explain dnsconfig -$ kubectl --namespace={{ .Release.Namespace }} get pods +If you're interested to explore what resources were created: + +$ kubectl --namespace={{ .Release.Namespace }} get all -l app.kubernetes.io/managed-by=Helm From 4dfed6b14697d1a9ab217e01fff774a3b72391df Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 9 Jul 2025 09:21:56 +0100 Subject: [PATCH 178/263] cmd/{k8s-operator,k8s-proxy}: add kube-apiserver ProxyGroup type (#16266) Adds a new k8s-proxy command to convert operator's in-process proxy to a separately deployable type of ProxyGroup: kube-apiserver. k8s-proxy reads in a new config file written by the operator, modelled on tailscaled's conffile but with some modifications to ensure multiple versions of the config can co-exist within a file. This should make it much easier to support reading that config file from a Kube Secret with a stable file name. To avoid needing to give the operator ClusterRole{,Binding} permissions, the helm chart now optionally deploys a new static ServiceAccount for the API Server proxy to use if in auth mode. Proxies deployed by kube-apiserver ProxyGroups currently work the same as the operator's in-process proxy. They do not yet leverage Tailscale Services for presenting a single HA DNS name. Updates #13358 Change-Id: Ib6ead69b2173c5e1929f3c13fb48a9a5362195d8 Signed-off-by: Tom Proctor --- Makefile | 48 ++-- build_docker.sh | 18 ++ cmd/k8s-operator/depaware.txt | 3 +- .../chart/templates/apiserverproxy-rbac.yaml | 16 +- cmd/k8s-operator/deploy/chart/values.yaml | 16 ++ .../crds/tailscale.com_proxyclasses.yaml | 46 +++- .../crds/tailscale.com_proxygroups.yaml | 19 +- .../deploy/manifests/authproxy-rbac.yaml | 9 + .../deploy/manifests/operator.yaml | 65 ++++- cmd/k8s-operator/ingress-for-pg.go | 6 +- cmd/k8s-operator/ingress-for-pg_test.go | 49 ++++ cmd/k8s-operator/operator.go | 64 ++++- cmd/k8s-operator/proxy.go | 61 +++++ cmd/k8s-operator/proxygroup.go | 194 +++++++++++--- cmd/k8s-operator/proxygroup_specs.go | 162 +++++++++++- cmd/k8s-operator/proxygroup_test.go | 249 ++++++++++++++++-- cmd/k8s-operator/sts.go | 12 +- cmd/k8s-operator/svc-for-pg.go | 60 +---- cmd/k8s-operator/svc-for-pg_test.go | 2 - cmd/k8s-proxy/k8s-proxy.go | 197 ++++++++++++++ k8s-operator/api-proxy/env.go | 29 -- k8s-operator/api-proxy/proxy.go | 187 +++++-------- k8s-operator/api.md | 40 ++- .../apis/v1alpha1/types_proxyclass.go | 22 +- .../apis/v1alpha1/types_proxygroup.go | 33 ++- .../apis/v1alpha1/zz_generated.deepcopy.go | 25 ++ kube/k8s-proxy/conf/conf.go | 101 +++++++ kube/k8s-proxy/conf/conf_test.go | 86 ++++++ kube/kubetypes/types.go | 18 +- kube/state/state.go | 97 +++++++ kube/state/state_test.go | 203 ++++++++++++++ 31 files changed, 1787 insertions(+), 350 deletions(-) create mode 100644 cmd/k8s-operator/proxy.go create mode 100644 cmd/k8s-proxy/k8s-proxy.go delete mode 100644 k8s-operator/api-proxy/env.go create mode 100644 kube/k8s-proxy/conf/conf.go create mode 100644 kube/k8s-proxy/conf/conf_test.go create mode 100644 kube/state/state.go create mode 100644 kube/state/state_test.go diff --git a/Makefile b/Makefile index 41c67c711791d..f5fc205891191 100644 --- a/Makefile +++ b/Makefile @@ -92,38 +92,38 @@ pushspk: spk ## Push and install synology package on ${SYNO_HOST} host scp tailscale.spk root@${SYNO_HOST}: ssh root@${SYNO_HOST} /usr/syno/bin/synopkg install tailscale.spk -publishdevimage: ## Build and publish tailscale image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1) +.PHONY: check-image-repo +check-image-repo: + @if [ -z "$(REPO)" ]; then \ + echo "REPO=... required; e.g. REPO=ghcr.io/$$USER/tailscale" >&2; \ + exit 1; \ + fi + @for repo in tailscale/tailscale ghcr.io/tailscale/tailscale \ + tailscale/k8s-operator ghcr.io/tailscale/k8s-operator \ + tailscale/k8s-nameserver ghcr.io/tailscale/k8s-nameserver \ + tailscale/tsidp ghcr.io/tailscale/tsidp \ + tailscale/k8s-proxy ghcr.io/tailscale/k8s-proxy; do \ + if [ "$(REPO)" = "$$repo" ]; then \ + echo "REPO=... must not be $$repo" >&2; \ + exit 1; \ + fi; \ + done + +publishdevimage: check-image-repo ## Build and publish tailscale image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=client ./build_docker.sh -publishdevoperator: ## Build and publish k8s-operator image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1) +publishdevoperator: check-image-repo ## Build and publish k8s-operator image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-operator ./build_docker.sh -publishdevnameserver: ## Build and publish k8s-nameserver image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-nameserver" || (echo "REPO=... must not be tailscale/k8s-nameserver" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-nameserver" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-nameserver" && exit 1) +publishdevnameserver: check-image-repo ## Build and publish k8s-nameserver image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-nameserver ./build_docker.sh -publishdevtsidp: ## Build and publish tsidp image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tsidp" || (echo "REPO=... must not be tailscale/tsidp" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tsidp" || (echo "REPO=... must not be ghcr.io/tailscale/tsidp" && exit 1) +publishdevtsidp: check-image-repo ## Build and publish tsidp image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=tsidp ./build_docker.sh +publishdevproxy: check-image-repo ## Build and publish k8s-proxy image to location specified by ${REPO} + TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-proxy ./build_docker.sh + .PHONY: sshintegrationtest sshintegrationtest: ## Run the SSH integration tests in various Docker containers @GOOS=linux GOARCH=amd64 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ diff --git a/build_docker.sh b/build_docker.sh index 7840dc89775d3..bdeaa8659b805 100755 --- a/build_docker.sh +++ b/build_docker.sh @@ -118,6 +118,24 @@ case "$TARGET" in --annotations="${ANNOTATIONS}" \ /usr/local/bin/tsidp ;; + k8s-proxy) + DEFAULT_REPOS="tailscale/k8s-proxy" + REPOS="${REPOS:-${DEFAULT_REPOS}}" + go run github.com/tailscale/mkctr \ + --gopaths="tailscale.com/cmd/k8s-proxy:/usr/local/bin/k8s-proxy" \ + --ldflags=" \ + -X tailscale.com/version.longStamp=${VERSION_LONG} \ + -X tailscale.com/version.shortStamp=${VERSION_SHORT} \ + -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \ + --base="${BASE}" \ + --tags="${TAGS}" \ + --gotags="ts_kube,ts_package_container" \ + --repos="${REPOS}" \ + --push="${PUSH}" \ + --target="${PLATFORM}" \ + --annotations="${ANNOTATIONS}" \ + /usr/local/bin/k8s-proxy + ;; *) echo "unknown target: $TARGET" exit 1 diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 36c5184c3b44a..f810d1b4fd62a 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -200,7 +200,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+ github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp - github.com/tailscale/hujson from tailscale.com/ipn/conffile + github.com/tailscale/hujson from tailscale.com/ipn/conffile+ L 💣 github.com/tailscale/netlink from tailscale.com/net/routetable+ L 💣 github.com/tailscale/netlink/nl from github.com/tailscale/netlink github.com/tailscale/peercred from tailscale.com/ipn/ipnauth @@ -822,6 +822,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/k8s-operator/sessionrecording/ws from tailscale.com/k8s-operator/sessionrecording tailscale.com/kube/egressservices from tailscale.com/cmd/k8s-operator tailscale.com/kube/ingressservices from tailscale.com/cmd/k8s-operator + tailscale.com/kube/k8s-proxy/conf from tailscale.com/cmd/k8s-operator tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+ tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore tailscale.com/kube/kubetypes from tailscale.com/cmd/k8s-operator+ diff --git a/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml index 072ecf6d22e2f..ad0a6fb66f51e 100644 --- a/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml @@ -1,7 +1,16 @@ # Copyright (c) Tailscale Inc & AUTHORS # SPDX-License-Identifier: BSD-3-Clause -{{ if eq .Values.apiServerProxyConfig.mode "true" }} +# If old setting used, enable both old (operator) and new (ProxyGroup) workflows. +# If new setting used, enable only new workflow. +{{ if or (eq .Values.apiServerProxyConfig.mode "true") + (eq .Values.apiServerProxyConfig.allowImpersonation "true") }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-apiserver-auth-proxy + namespace: {{ .Release.Namespace }} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -16,9 +25,14 @@ kind: ClusterRoleBinding metadata: name: tailscale-auth-proxy subjects: +{{- if eq .Values.apiServerProxyConfig.mode "true" }} - kind: ServiceAccount name: operator namespace: {{ .Release.Namespace }} +{{- end }} +- kind: ServiceAccount + name: kube-apiserver-auth-proxy + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: tailscale-auth-proxy diff --git a/cmd/k8s-operator/deploy/chart/values.yaml b/cmd/k8s-operator/deploy/chart/values.yaml index 2926f6d0759f2..cdedb92e819e4 100644 --- a/cmd/k8s-operator/deploy/chart/values.yaml +++ b/cmd/k8s-operator/deploy/chart/values.yaml @@ -92,6 +92,13 @@ ingressClass: # If you need more configuration options, take a look at ProxyClass: # https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource proxyConfig: + # Configure the proxy image to use instead of the default tailscale/tailscale:latest. + # Applying a ProxyClass with `spec.statefulSet.pod.tailscaleContainer.image` + # set will override any defaults here. + # + # Note that ProxyGroups of type "kube-apiserver" use a different default image, + # tailscale/k8s-proxy:latest, and it is currently only possible to override + # that image via the same ProxyClass field. image: # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale. repository: tailscale/tailscale @@ -115,6 +122,15 @@ proxyConfig: # Kubernetes API server. # https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy apiServerProxyConfig: + # Set to "true" to create the ClusterRole permissions required for the API + # server proxy's auth mode. In auth mode, the API server proxy impersonates + # groups and users based on tailnet ACL grants. Required for ProxyGroups of + # type "kube-apiserver" running in auth mode. + allowImpersonation: "false" # "true", "false" + + # If true or noauth, the operator will run an in-process API server proxy. + # You can deploy a ProxyGroup of type "kube-apiserver" to run a high + # availability set of API server proxies instead. mode: "false" # "true", "false", "noauth" imagePullSecrets: [] diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index fcf1b27aaf318..c5dc9c3e96a83 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -1379,12 +1379,21 @@ spec: type: string image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -1655,7 +1664,9 @@ spec: PodSecurityContext, the value specified in SecurityContext takes precedence. type: string tailscaleInitContainer: - description: Configuration for the proxy init container that enables forwarding. + description: |- + Configuration for the proxy init container that enables forwarding. + Not valid to apply to ProxyGroups of type "kube-apiserver". type: object properties: debug: @@ -1709,12 +1720,21 @@ spec: type: string image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index c426c8427a507..06c8479252873 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -77,6 +77,22 @@ spec: must not start with a dash and must be between 1 and 62 characters long. type: string pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + kubeAPIServer: + description: |- + KubeAPIServer contains configuration specific to the kube-apiserver + ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + type: object + properties: + mode: + description: |- + Mode to run the API server proxy in. Supported modes are auth and noauth. + In auth mode, requests from the tailnet proxied over to the Kubernetes + API server are additionally impersonated using the sender's tailnet identity. + If not specified, defaults to auth mode. + type: string + enum: + - auth + - noauth proxyClass: description: |- ProxyClass is the name of the ProxyClass custom resource that contains @@ -106,12 +122,13 @@ spec: pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: description: |- - Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. Type is immutable once a ProxyGroup is created. type: string enum: - egress - ingress + - kube-apiserver x-kubernetes-validations: - rule: self == oldSelf message: ProxyGroup type is immutable diff --git a/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml b/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml index ddbdda32e476e..5818fa69fff7d 100644 --- a/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml +++ b/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml @@ -1,6 +1,12 @@ # Copyright (c) Tailscale Inc & AUTHORS # SPDX-License-Identifier: BSD-3-Clause +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-apiserver-auth-proxy + namespace: tailscale +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -18,6 +24,9 @@ subjects: - kind: ServiceAccount name: operator namespace: tailscale +- kind: ServiceAccount + name: kube-apiserver-auth-proxy + namespace: tailscale roleRef: kind: ClusterRole name: tailscale-auth-proxy diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index cdf301318f923..ff3705cb343ff 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -1852,12 +1852,21 @@ spec: type: array image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -2129,7 +2138,9 @@ spec: type: object type: object tailscaleInitContainer: - description: Configuration for the proxy init container that enables forwarding. + description: |- + Configuration for the proxy init container that enables forwarding. + Not valid to apply to ProxyGroups of type "kube-apiserver". properties: debug: description: |- @@ -2182,12 +2193,21 @@ spec: type: array image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -2904,6 +2924,22 @@ spec: must not start with a dash and must be between 1 and 62 characters long. pattern: ^[a-z0-9][a-z0-9-]{0,61}$ type: string + kubeAPIServer: + description: |- + KubeAPIServer contains configuration specific to the kube-apiserver + ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + properties: + mode: + description: |- + Mode to run the API server proxy in. Supported modes are auth and noauth. + In auth mode, requests from the tailnet proxied over to the Kubernetes + API server are additionally impersonated using the sender's tailnet identity. + If not specified, defaults to auth mode. + enum: + - auth + - noauth + type: string + type: object proxyClass: description: |- ProxyClass is the name of the ProxyClass custom resource that contains @@ -2933,11 +2969,12 @@ spec: type: array type: description: |- - Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. Type is immutable once a ProxyGroup is created. enum: - egress - ingress + - kube-apiserver type: string x-kubernetes-validations: - message: ProxyGroup type is immutable diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 79bad92be080e..aaf22d471353f 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -239,7 +239,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // This checks and ensures that Tailscale Service's owner references are updated // for this Ingress and errors if that is not possible (i.e. because it // appears that the Tailscale Service has been created by a non-operator actor). - updatedAnnotations, err := r.ownerAnnotations(existingTSSvc) + updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc) if err != nil { const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different MagicDNS name at `.spec.tls.hosts[0] in the Ingress definition" msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) @@ -867,9 +867,9 @@ type OwnerRef struct { // nil, but does not contain an owner reference we return an error as this likely means // that the Service was created by somthing other than a Tailscale // Kubernetes operator. -func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) { +func ownerAnnotations(operatorID string, svc *tailscale.VIPService) (map[string]string, error) { ref := OwnerRef{ - OperatorID: r.operatorID, + OperatorID: operatorID, } if svc == nil { c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index d29368caef59d..5de86cdad573a 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -12,8 +12,10 @@ import ( "maps" "reflect" "slices" + "strings" "testing" + "github.com/google/go-cmp/cmp" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -650,6 +652,53 @@ func TestIngressPGReconciler_MultiCluster(t *testing.T) { } } +func TestOwnerAnnotations(t *testing.T) { + singleSelfOwner := map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, + } + + for name, tc := range map[string]struct { + svc *tailscale.VIPService + wantAnnotations map[string]string + wantErr string + }{ + "no_svc": { + svc: nil, + wantAnnotations: singleSelfOwner, + }, + "empty_svc": { + svc: &tailscale.VIPService{}, + wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator", + }, + "already_owner": { + svc: &tailscale.VIPService{ + Annotations: singleSelfOwner, + }, + wantAnnotations: singleSelfOwner, + }, + "add_owner": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`, + }, + }, + wantAnnotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"},{"operatorID":"self-id"}]}`, + }, + }, + } { + t.Run(name, func(t *testing.T) { + got, err := ownerAnnotations("self-id", tc.svc) + if tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("ownerAnnotations() error = %v, wantErr %v", err, tc.wantErr) + } + if diff := cmp.Diff(tc.wantAnnotations, got); diff != "" { + t.Errorf("ownerAnnotations() mismatch (-want +got):\n%s", diff) + } + }) + } +} + func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 96b3b37ad0340..870a6f8b7f37e 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -26,6 +26,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" klabels "k8s.io/apimachinery/pkg/labels" @@ -77,6 +78,7 @@ func main() { tsNamespace = defaultEnv("OPERATOR_NAMESPACE", "") tslogging = defaultEnv("OPERATOR_LOGGING", "info") image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest") + k8sProxyImage = defaultEnv("K8S_PROXY_IMAGE", "tailscale/k8s-proxy:latest") priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "") tags = defaultEnv("PROXY_TAGS", "tag:k8s") tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") @@ -110,17 +112,27 @@ func main() { // The operator can run either as a plain operator or it can // additionally act as api-server proxy // https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy. - mode := apiproxy.ParseAPIProxyMode() - if mode == apiproxy.APIServerProxyModeDisabled { + mode := parseAPIProxyMode() + if mode == apiServerProxyModeDisabled { hostinfo.SetApp(kubetypes.AppOperator) } else { - hostinfo.SetApp(kubetypes.AppAPIServerProxy) + hostinfo.SetApp(kubetypes.AppInProcessAPIServerProxy) } s, tsc := initTSNet(zlog, loginServer) defer s.Close() restConfig := config.GetConfigOrDie() - apiproxy.MaybeLaunchAPIServerProxy(zlog, restConfig, s, mode) + if mode != apiServerProxyModeDisabled { + ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, mode == apiServerProxyModeEnabled) + if err != nil { + zlog.Fatalf("error creating API server proxy: %v", err) + } + go func() { + if err := ap.Run(context.Background()); err != nil { + zlog.Fatalf("error running API server proxy: %v", err) + } + }() + } rOpts := reconcilerOpts{ log: zlog, tsServer: s, @@ -128,6 +140,7 @@ func main() { tailscaleNamespace: tsNamespace, restConfig: restConfig, proxyImage: image, + k8sProxyImage: k8sProxyImage, proxyPriorityClassName: priorityClassName, proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer, proxyTags: tags, @@ -415,7 +428,6 @@ func runReconcilers(opts reconcilerOpts) { Complete(&HAServiceReconciler{ recorder: eventRecorder, tsClient: opts.tsClient, - tsnetServer: opts.tsServer, defaultTags: strings.Split(opts.proxyTags, ","), Client: mgr.GetClient(), logger: opts.log.Named("service-pg-reconciler"), @@ -625,13 +637,14 @@ func runReconcilers(opts reconcilerOpts) { ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{}) proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog)) nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog)) + saFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(serviceAccountHandlerForProxyGroup(mgr.GetClient(), startlog)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyGroup{}). Named("proxygroup-reconciler"). Watches(&corev1.Service{}, ownedByProxyGroupFilter). Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter). Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter). - Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter). + Watches(&corev1.ServiceAccount{}, saFilterForProxyGroup). Watches(&corev1.Secret{}, ownedByProxyGroupFilter). Watches(&rbacv1.Role{}, ownedByProxyGroupFilter). Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter). @@ -645,7 +658,8 @@ func runReconcilers(opts reconcilerOpts) { tsClient: opts.tsClient, tsNamespace: opts.tailscaleNamespace, - proxyImage: opts.proxyImage, + tsProxyImage: opts.proxyImage, + k8sProxyImage: opts.k8sProxyImage, defaultTags: strings.Split(opts.proxyTags, ","), tsFirewallMode: opts.proxyFirewallMode, defaultProxyClass: opts.defaultProxyClass, @@ -668,6 +682,7 @@ type reconcilerOpts struct { tailscaleNamespace string // namespace in which operator resources will be deployed restConfig *rest.Config // config for connecting to the kube API server proxyImage string // : + k8sProxyImage string // : // proxyPriorityClassName isPriorityClass to be set for proxy Pods. This // is a legacy mechanism for cluster resource configuration options - // going forward use ProxyClass. @@ -996,8 +1011,8 @@ func nodeHandlerForProxyGroup(cl client.Client, defaultProxyClass string, logger } // proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass, -// returns a list of reconcile requests for all Connectors that have -// .spec.proxyClass set. +// returns a list of reconcile requests for all ProxyGroups that have +// .spec.proxyClass set to that ProxyClass. func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { pgList := new(tsapi.ProxyGroupList) @@ -1016,6 +1031,37 @@ func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) } } +// serviceAccountHandlerForProxyGroup returns a handler that, for a given ServiceAccount, +// returns a list of reconcile requests for all ProxyGroups that use that ServiceAccount. +// For most ProxyGroups, this will be a dedicated ServiceAccount owned by a specific +// ProxyGroup. But for kube-apiserver ProxyGroups running in auth mode, they use a shared +// static ServiceAccount named "kube-apiserver-auth-proxy". +func serviceAccountHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + pgList := new(tsapi.ProxyGroupList) + if err := cl.List(ctx, pgList); err != nil { + logger.Debugf("error listing ProxyGroups for ServiceAccount: %v", err) + return nil + } + reqs := make([]reconcile.Request, 0) + saName := o.GetName() + for _, pg := range pgList.Items { + if saName == authAPIServerProxySAName && isAuthAPIServerProxy(&pg) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + } + expectedOwner := pgOwnerReference(&pg)[0] + saOwnerRefs := o.GetOwnerReferences() + for _, ref := range saOwnerRefs { + if apiequality.Semantic.DeepEqual(ref, expectedOwner) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + break + } + } + } + return reqs + } +} + // serviceHandlerForIngress returns a handler for Service events for ingress // reconciler that ensures that if the Service associated with an event is of // interest to the reconciler, the associated Ingress(es) gets be reconciled. diff --git a/cmd/k8s-operator/proxy.go b/cmd/k8s-operator/proxy.go new file mode 100644 index 0000000000000..09a7b8c6232d2 --- /dev/null +++ b/cmd/k8s-operator/proxy.go @@ -0,0 +1,61 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "fmt" + "log" + "os" +) + +type apiServerProxyMode int + +func (a apiServerProxyMode) String() string { + switch a { + case apiServerProxyModeDisabled: + return "disabled" + case apiServerProxyModeEnabled: + return "auth" + case apiServerProxyModeNoAuth: + return "noauth" + default: + return "unknown" + } +} + +const ( + apiServerProxyModeDisabled apiServerProxyMode = iota + apiServerProxyModeEnabled + apiServerProxyModeNoAuth +) + +func parseAPIProxyMode() apiServerProxyMode { + haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != "" + haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != "" + switch { + case haveAPIProxyEnv && haveAuthProxyEnv: + log.Fatal("AUTH_PROXY (deprecated) and APISERVER_PROXY are mutually exclusive, please unset AUTH_PROXY") + case haveAuthProxyEnv: + var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated + if authProxyEnv { + return apiServerProxyModeEnabled + } + return apiServerProxyModeDisabled + case haveAPIProxyEnv: + var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth" + switch apiProxyEnv { + case "true": + return apiServerProxyModeEnabled + case "false", "": + return apiServerProxyModeDisabled + case "noauth": + return apiServerProxyModeNoAuth + default: + panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv)) + } + } + return apiServerProxyModeDisabled +} diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index c44de09a7fc45..3dfb004f1dd36 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -17,6 +17,7 @@ import ( "strings" "sync" + dockerref "github.com/distribution/reference" "go.uber.org/zap" xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" @@ -36,9 +37,11 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" + "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/tstime" + "tailscale.com/types/opt" "tailscale.com/types/ptr" "tailscale.com/util/clientmetric" "tailscale.com/util/mak" @@ -48,7 +51,9 @@ import ( const ( reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed" reasonProxyGroupReady = "ProxyGroupReady" + reasonProxyGroupAvailable = "ProxyGroupAvailable" reasonProxyGroupCreating = "ProxyGroupCreating" + reasonProxyGroupInvalid = "ProxyGroupInvalid" // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" @@ -63,12 +68,14 @@ const ( // // tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was // first introduced. - pgMinCapabilityVersion = 106 + pgMinCapabilityVersion = 106 + kubeAPIServerConfigFile = "config.hujson" ) var ( - gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount) - gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount) + gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount) + gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount) + gaugeAPIServerProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupAPIServerCount) ) // ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition. @@ -81,15 +88,17 @@ type ProxyGroupReconciler struct { // User-specified defaults from the helm installation. tsNamespace string - proxyImage string + tsProxyImage string + k8sProxyImage string defaultTags []string tsFirewallMode string defaultProxyClass string loginServer string - mu sync.Mutex // protects following - egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge - ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge + mu sync.Mutex // protects following + egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge + ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge + apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge } func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger { @@ -170,7 +179,6 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG if err != nil { return r.notReadyErrf(pg, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err) } - validateProxyClassForPG(logger, pg, proxyClass) if !tsoperator.ProxyClassIsReady(proxyClass) { msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName) logger.Info(msg) @@ -178,6 +186,10 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG } } + if err := r.validate(ctx, pg, proxyClass, logger); err != nil { + return r.notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err)) + } + staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass) if err != nil { if strings.Contains(err.Error(), optimisticLockErrorMsg) { @@ -192,11 +204,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG return staticEndpoints, nrr, nil } -// validateProxyClassForPG applies custom validation logic for ProxyClass applied to ProxyGroup. -func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) { - if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { - return - } +func (r *ProxyGroupReconciler) validate(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) error { // Our custom logic for ensuring minimum downtime ProxyGroup update rollouts relies on the local health check // beig accessible on the replica Pod IP:9002. This address can also be modified by users, via // TS_LOCAL_ADDR_PORT env var. @@ -208,13 +216,70 @@ func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc // shouldn't need to set their own). // // TODO(irbekrm): maybe disallow configuring this env var in future (in Tailscale 1.84 or later). - if hasLocalAddrPortSet(pc) { + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress && hasLocalAddrPortSet(pc) { msg := fmt.Sprintf("ProxyClass %s applied to an egress ProxyGroup has TS_LOCAL_ADDR_PORT env var set to a custom value."+ "This will disable the ProxyGroup graceful failover mechanism, so you might experience downtime when ProxyGroup pods are restarted."+ "In future we will remove the ability to set custom TS_LOCAL_ADDR_PORT for egress ProxyGroups."+ "Please raise an issue if you expect that this will cause issues for your workflow.", pc.Name) logger.Warn(msg) } + + // image is the value of pc.Spec.StatefulSet.Pod.TailscaleContainer.Image or "" + // imagePath is a slash-delimited path ending with the image name, e.g. + // "tailscale/tailscale" or maybe "k8s-proxy" if hosted at example.com/k8s-proxy. + var image, imagePath string + if pc != nil && + pc.Spec.StatefulSet != nil && + pc.Spec.StatefulSet.Pod != nil && + pc.Spec.StatefulSet.Pod.TailscaleContainer != nil && + pc.Spec.StatefulSet.Pod.TailscaleContainer.Image != "" { + image, err := dockerref.ParseNormalizedNamed(pc.Spec.StatefulSet.Pod.TailscaleContainer.Image) + if err != nil { + // Shouldn't be possible as the ProxyClass won't be marked ready + // without successfully parsing the image. + return fmt.Errorf("error parsing %q as a container image reference: %w", pc.Spec.StatefulSet.Pod.TailscaleContainer.Image, err) + } + imagePath = dockerref.Path(image) + } + + var errs []error + if isAuthAPIServerProxy(pg) { + // Validate that the static ServiceAccount already exists. + sa := &corev1.ServiceAccount{} + if err := r.Get(ctx, types.NamespacedName{Namespace: r.tsNamespace, Name: authAPIServerProxySAName}, sa); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("error validating that ServiceAccount %q exists: %w", authAPIServerProxySAName, err) + } + + errs = append(errs, fmt.Errorf("the ServiceAccount %q used for the API server proxy in auth mode does not exist but "+ + "should have been created during operator installation; use apiServerProxyConfig.allowImpersonation=true "+ + "in the helm chart, or authproxy-rbac.yaml from the static manifests", authAPIServerProxySAName)) + } + } else { + // Validate that the ServiceAccount we create won't overwrite the static one. + // TODO(tomhjp): This doesn't cover other controllers that could create a + // ServiceAccount. Perhaps should have some guards to ensure that an update + // would never change the ownership of a resource we expect to already be owned. + if pgServiceAccountName(pg) == authAPIServerProxySAName { + errs = append(errs, fmt.Errorf("the name of the ProxyGroup %q conflicts with the static ServiceAccount used for the API server proxy in auth mode", pg.Name)) + } + } + + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + if strings.HasSuffix(imagePath, "tailscale") { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "k8s-proxy", pg.Spec.Type)) + } + + if pc != nil && pc.Spec.StatefulSet != nil && pc.Spec.StatefulSet.Pod != nil && pc.Spec.StatefulSet.Pod.TailscaleInitContainer != nil { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies Tailscale init container config, but ProxyGroups of type %q do not use init containers", pc.Name, pg.Spec.Type)) + } + } else { + if strings.HasSuffix(imagePath, "k8s-proxy") { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "tailscale", pg.Spec.Type)) + } + } + + return errors.Join(errs...) } func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) { @@ -263,14 +328,21 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return r.notReadyErrf(pg, "error provisioning state Secrets: %w", err) } } - sa := pgServiceAccount(pg, r.tsNamespace) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) { - s.ObjectMeta.Labels = sa.ObjectMeta.Labels - s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations - s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences - }); err != nil { - return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err) + + // auth mode kube-apiserver ProxyGroups use a statically created + // ServiceAccount to keep ClusterRole creation permissions limited to the + // helm chart installer. + if !isAuthAPIServerProxy(pg) { + sa := pgServiceAccount(pg, r.tsNamespace) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) { + s.ObjectMeta.Labels = sa.ObjectMeta.Labels + s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations + s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences + }); err != nil { + return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err) + } } + role := pgRole(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { r.ObjectMeta.Labels = role.ObjectMeta.Labels @@ -280,6 +352,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro }); err != nil { return r.notReadyErrf(pg, "error provisioning Role: %w", err) } + roleBinding := pgRoleBinding(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels @@ -290,6 +363,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro }); err != nil { return r.notReadyErrf(pg, "error provisioning RoleBinding: %w", err) } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { cm, hp := pgEgressCM(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { @@ -300,6 +374,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return r.notReadyErrf(pg, "error provisioning egress ConfigMap %q: %w", cm.Name, err) } } + if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { cm := pgIngressCM(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { @@ -309,7 +384,12 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro return r.notReadyErrf(pg, "error provisioning ingress ConfigMap %q: %w", cm.Name, err) } } - ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, tailscaledPort, proxyClass) + + defaultImage := r.tsProxyImage + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + defaultImage = r.k8sProxyImage + } + ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass) if err != nil { return r.notReadyErrf(pg, "error generating StatefulSet spec: %w", err) } @@ -371,7 +451,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za if len(devices) > 0 { status = metav1.ConditionTrue if len(devices) == desiredReplicas { - reason = reasonProxyGroupReady + reason = reasonProxyGroupAvailable } } tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger) @@ -702,17 +782,57 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p return nil, err } - configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer) - if err != nil { - return nil, fmt.Errorf("error creating tailscaled config: %w", err) - } + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + hostname := pgHostname(pg, i) - for cap, cfg := range configs { - cfgJSON, err := json.Marshal(cfg) + if authKey == nil && existingCfgSecret != nil { + deviceAuthed := false + for _, d := range pg.Status.Devices { + if d.Hostname == hostname { + deviceAuthed = true + break + } + } + if !deviceAuthed { + existingCfg := conf.ConfigV1Alpha1{} + if err := json.Unmarshal(existingCfgSecret.Data[kubeAPIServerConfigFile], &existingCfg); err != nil { + return nil, fmt.Errorf("error unmarshalling existing config: %w", err) + } + if existingCfg.AuthKey != nil { + authKey = existingCfg.AuthKey + } + } + } + cfg := conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &conf.ConfigV1Alpha1{ + Hostname: &hostname, + State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))), + App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer), + AuthKey: authKey, + KubeAPIServer: &conf.KubeAPIServer{ + AuthMode: opt.NewBool(isAuthAPIServerProxy(pg)), + }, + }, + } + cfgB, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err) + } + mak.Set(&cfgSecret.Data, kubeAPIServerConfigFile, cfgB) + } else { + configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer) if err != nil { - return nil, fmt.Errorf("error marshalling tailscaled config: %w", err) + return nil, fmt.Errorf("error creating tailscaled config: %w", err) + } + + for cap, cfg := range configs { + cfgJSON, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("error marshalling tailscaled config: %w", err) + } + mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON) } - mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON) } if existingCfgSecret != nil { @@ -834,9 +954,12 @@ func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGr r.egressProxyGroups.Add(pg.UID) case tsapi.ProxyGroupTypeIngress: r.ingressProxyGroups.Add(pg.UID) + case tsapi.ProxyGroupTypeKubernetesAPIServer: + r.apiServerProxyGroups.Add(pg.UID) } gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) + gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) } // ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the @@ -847,9 +970,12 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro r.egressProxyGroups.Remove(pg.UID) case tsapi.ProxyGroupTypeIngress: r.ingressProxyGroups.Remove(pg.UID) + case tsapi.ProxyGroupTypeKubernetesAPIServer: + r.apiServerProxyGroups.Remove(pg.UID) } gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) + gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) } func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string, loginServer string) (tailscaledConfigs, error) { @@ -858,7 +984,7 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a AcceptDNS: "false", AcceptRoutes: "false", // AcceptRoutes defaults to true Locked: "false", - Hostname: ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)), + Hostname: ptr.To(pgHostname(pg, idx)), AdvertiseServices: oldAdvertiseServices, AuthKey: authKey, } @@ -867,10 +993,6 @@ func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, a conf.ServerURL = &loginServer } - if pg.Spec.HostnamePrefix != "" { - conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx)) - } - if shouldAcceptRoutes(pc) { conf.AcceptRoutes = "true" } diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 50d9c2d5fd8f9..5d6d0b8ef9626 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -7,6 +7,7 @@ package main import ( "fmt" + "path/filepath" "slices" "strconv" "strings" @@ -28,6 +29,9 @@ const ( // deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully. deletionGracePeriodSeconds int64 = 360 staticEndpointPortName = "static-endpoint-port" + // authAPIServerProxySAName is the ServiceAccount deployed by the helm chart + // if apiServerProxy.authEnabled is true. + authAPIServerProxySAName = "kube-apiserver-auth-proxy" ) func pgNodePortServiceName(proxyGroupName string, replica int32) string { @@ -61,6 +65,9 @@ func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *cor // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be // applied over the top after. func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + return kubeAPIServerStatefulSet(pg, namespace, image) + } ss := new(appsv1.StatefulSet) if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) @@ -167,6 +174,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Value: "$(POD_NAME)", }, { + // TODO(tomhjp): This is tsrecorder-specific and does nothing. Delete. Name: "TS_STATE", Value: "kube:$(POD_NAME)", }, @@ -264,9 +272,124 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string // gracefully. ss.Spec.Template.DeletionGracePeriodSeconds = ptr.To(deletionGracePeriodSeconds) } + return ss, nil } +func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string) (*appsv1.StatefulSet, error) { + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: pg.Name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(pgReplicas(pg)), + Selector: &metav1.LabelSelector{ + MatchLabels: pgLabels(pg.Name, nil), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: pg.Name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + DeletionGracePeriodSeconds: ptr.To[int64](10), + }, + Spec: corev1.PodSpec{ + ServiceAccountName: pgServiceAccountName(pg), + Containers: []corev1.Container{ + { + Name: mainContainerName, + Image: image, + Env: []corev1.EnvVar{ + { + // Used as default hostname and in Secret names. + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + { + // Used by kubeclient to post Events about the Pod's lifecycle. + Name: "POD_UID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.uid", + }, + }, + }, + { + // Used in an interpolated env var if metrics enabled. + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + // Included for completeness with POD_IP and easier backwards compatibility in future. + Name: "POD_IPS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, + }, + }, + { + Name: "TS_K8S_PROXY_CONFIG", + Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile), + }, + }, + VolumeMounts: func() []corev1.VolumeMount { + var mounts []corev1.VolumeMount + + // TODO(tomhjp): Read config directly from the Secret instead. + for i := range pgReplicas(pg) { + mounts = append(mounts, corev1.VolumeMount{ + Name: fmt.Sprintf("k8s-proxy-config-%d", i), + ReadOnly: true, + MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i), + }) + } + + return mounts + }(), + Ports: []corev1.ContainerPort{ + { + Name: "k8s-proxy", + ContainerPort: 443, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + Volumes: func() []corev1.Volume { + var volumes []corev1.Volume + for i := range pgReplicas(pg) { + volumes = append(volumes, corev1.Volume{ + Name: fmt.Sprintf("k8s-proxy-config-%d", i), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: pgConfigSecretName(pg.Name, i), + }, + }, + }) + } + + return volumes + }(), + }, + }, + }, + } + + return sts, nil +} + func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAccount { return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ @@ -305,8 +428,8 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role { ResourceNames: func() (secrets []string) { for i := range pgReplicas(pg) { secrets = append(secrets, - pgConfigSecretName(pg.Name, i), // Config with auth key. - fmt.Sprintf("%s-%d", pg.Name, i), // State. + pgConfigSecretName(pg.Name, i), // Config with auth key. + pgPodName(pg.Name, i), // State. ) } return secrets @@ -336,7 +459,7 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding { Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: pg.Name, + Name: pgServiceAccountName(pg), Namespace: namespace, }, }, @@ -347,6 +470,27 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding { } } +// kube-apiserver proxies in auth mode use a static ServiceAccount. Everything +// else uses a per-ProxyGroup ServiceAccount. +func pgServiceAccountName(pg *tsapi.ProxyGroup) string { + if isAuthAPIServerProxy(pg) { + return authAPIServerProxySAName + } + + return pg.Name +} + +func isAuthAPIServerProxy(pg *tsapi.ProxyGroup) bool { + if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer { + return false + } + + // The default is auth mode. + return pg.Spec.KubeAPIServer == nil || + pg.Spec.KubeAPIServer.Mode == nil || + *pg.Spec.KubeAPIServer.Mode == tsapi.APIServerProxyModeAuth +} + func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.Secret) { for i := range pgReplicas(pg) { secrets = append(secrets, &corev1.Secret{ @@ -418,6 +562,18 @@ func pgReplicas(pg *tsapi.ProxyGroup) int32 { return 2 } +func pgPodName(pgName string, i int32) string { + return fmt.Sprintf("%s-%d", pgName, i) +} + +func pgHostname(pg *tsapi.ProxyGroup, i int32) string { + if pg.Spec.HostnamePrefix != "" { + return fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, i) + } + + return fmt.Sprintf("%s-%d", pg.Name, i) +} + func pgConfigSecretName(pgName string, i int32) string { return fmt.Sprintf("%s-%d-config", pgName, i) } diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index bd69b49a8978d..c58e427aa06b6 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -629,7 +629,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, tsFirewallMode: "auto", defaultProxyClass: "default-pc", @@ -772,7 +772,7 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) { t.Run("delete_and_cleanup", func(t *testing.T) { reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, tsFirewallMode: "auto", defaultProxyClass: "default-pc", @@ -832,7 +832,7 @@ func TestProxyGroup(t *testing.T) { cl := tstest.NewClock(tstest.ClockOpts{}) reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, tsFirewallMode: "auto", defaultProxyClass: "default-pc", @@ -915,7 +915,7 @@ func TestProxyGroup(t *testing.T) { }, } tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) }) @@ -934,7 +934,7 @@ func TestProxyGroup(t *testing.T) { addNodeIDToStateSecrets(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar()) pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{ Hostname: "hostname-nodeid-2", TailnetIPs: []string{"1.2.3.4", "::1"}, @@ -952,7 +952,7 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupReady, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) }) @@ -1025,12 +1025,12 @@ func TestProxyGroupTypes(t *testing.T) { zl, _ := zap.NewDevelopment() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - proxyImage: testProxyImage, - Client: fc, - l: zl.Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + l: zl.Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), } t.Run("egress_type", func(t *testing.T) { @@ -1047,7 +1047,7 @@ func TestProxyGroupTypes(t *testing.T) { mustCreate(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 0, 1) + verifyProxyGroupCounts(t, reconciler, 0, 1, 0) sts := &appsv1.StatefulSet{} if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -1161,7 +1161,7 @@ func TestProxyGroupTypes(t *testing.T) { } expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 1, 2) + verifyProxyGroupCounts(t, reconciler, 1, 2, 0) sts := &appsv1.StatefulSet{} if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { @@ -1198,6 +1198,44 @@ func TestProxyGroupTypes(t *testing.T) { t.Errorf("unexpected volume mounts (-want +got):\n%s", diff) } }) + + t.Run("kubernetes_api_server_type", func(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-k8s-apiserver", + UID: "test-k8s-apiserver-uid", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + Replicas: ptr.To[int32](2), + KubeAPIServer: &tsapi.KubeAPIServerConfig{ + Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), + }, + }, + } + if err := fc.Create(t.Context(), pg); err != nil { + t.Fatal(err) + } + + expectReconciled(t, reconciler, "", pg.Name) + verifyProxyGroupCounts(t, reconciler, 1, 2, 1) + + sts := &appsv1.StatefulSet{} + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + // Verify the StatefulSet configuration for KubernetesAPIServer type. + if sts.Spec.Template.Spec.Containers[0].Name != mainContainerName { + t.Errorf("unexpected container name %s, want %s", sts.Spec.Template.Spec.Containers[0].Name, mainContainerName) + } + if sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort != 443 { + t.Errorf("unexpected container port %d, want 443", sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort) + } + if sts.Spec.Template.Spec.Containers[0].Ports[0].Name != "k8s-proxy" { + t.Errorf("unexpected port name %s, want k8s-proxy", sts.Spec.Template.Spec.Containers[0].Ports[0].Name) + } + }) } func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { @@ -1206,12 +1244,12 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { WithStatusSubresource(&tsapi.ProxyGroup{}). Build() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - proxyImage: testProxyImage, - Client: fc, - l: zap.Must(zap.NewDevelopment()).Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + l: zap.Must(zap.NewDevelopment()).Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), } existingServices := []string{"svc1", "svc2"} @@ -1272,6 +1310,170 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { }) } +func TestValidateProxyGroup(t *testing.T) { + type testCase struct { + typ tsapi.ProxyGroupType + pgName string + image string + noauth bool + initContainer bool + staticSAExists bool + expectedErrs int + } + + for name, tc := range map[string]testCase{ + "default_ingress": { + typ: tsapi.ProxyGroupTypeIngress, + }, + "default_kube": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + }, + "default_kube_noauth": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + noauth: true, + // Does not require the static ServiceAccount to exist. + }, + "kube_static_sa_missing": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: false, + expectedErrs: 1, + }, + "kube_noauth_would_overwrite_static_sa": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + noauth: true, + pgName: authAPIServerProxySAName, + expectedErrs: 1, + }, + "ingress_would_overwrite_static_sa": { + typ: tsapi.ProxyGroupTypeIngress, + staticSAExists: true, + pgName: authAPIServerProxySAName, + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_1": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale/tailscale", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_2": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_3": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale/tailscale:latest", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_4": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "tailscale/tailscale", + expectedErrs: 1, + }, + "k8s_proxy_image_for_ingress_pg": { + typ: tsapi.ProxyGroupTypeIngress, + image: "example.com/k8s-proxy", + expectedErrs: 1, + }, + "init_container_for_kube_pg": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + initContainer: true, + expectedErrs: 1, + }, + "init_container_for_ingress_pg": { + typ: tsapi.ProxyGroupTypeIngress, + initContainer: true, + }, + "init_container_for_egress_pg": { + typ: tsapi.ProxyGroupTypeEgress, + initContainer: true, + }, + } { + t.Run(name, func(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{}, + }, + }, + } + if tc.image != "" { + pc.Spec.StatefulSet.Pod.TailscaleContainer = &tsapi.Container{ + Image: tc.image, + } + } + if tc.initContainer { + pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{} + } + pgName := "some-pg" + if tc.pgName != "" { + pgName = tc.pgName + } + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgName, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tc.typ, + }, + } + if tc.noauth { + pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{ + Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), + } + } + + var objs []client.Object + if tc.staticSAExists { + objs = append(objs, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: authAPIServerProxySAName, + Namespace: tsNamespace, + }, + }) + } + r := ProxyGroupReconciler{ + tsNamespace: tsNamespace, + Client: fake.NewClientBuilder(). + WithObjects(objs...). + Build(), + } + + logger, _ := zap.NewDevelopment() + err := r.validate(t.Context(), pg, pc, logger.Sugar()) + if tc.expectedErrs == 0 { + if err != nil { + t.Fatalf("expected no errors, got: %v", err) + } + // Test finished. + return + } + + if err == nil { + t.Fatalf("expected %d errors, got none", tc.expectedErrs) + } + + type unwrapper interface { + Unwrap() []error + } + errs := err.(unwrapper) + if len(errs.Unwrap()) != tc.expectedErrs { + t.Fatalf("expected %d errors, got %d: %v", tc.expectedErrs, len(errs.Unwrap()), err) + } + }) + } +} + func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsapi.ProxyClass) { pcLEStaging := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{ @@ -1326,7 +1528,7 @@ func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name s return pc } -func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) { +func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress, wantAPIServer int) { t.Helper() if r.ingressProxyGroups.Len() != wantIngress { t.Errorf("expected %d ingress proxy groups, got %d", wantIngress, r.ingressProxyGroups.Len()) @@ -1334,6 +1536,9 @@ func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, if r.egressProxyGroups.Len() != wantEgress { t.Errorf("expected %d egress proxy groups, got %d", wantEgress, r.egressProxyGroups.Len()) } + if r.apiServerProxyGroups.Len() != wantAPIServer { + t.Errorf("expected %d kube-apiserver proxy groups, got %d", wantAPIServer, r.apiServerProxyGroups.Len()) + } } func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue string) { @@ -1512,7 +1717,7 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test"}, defaultProxyClass: tt.defaultProxyClass, Client: fc, diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 193acad87ff0e..fbb271800390a 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -102,6 +102,8 @@ const ( defaultLocalAddrPort = 9002 // metrics and health check port letsEncryptStagingEndpoint = "https://acme-staging-v02.api.letsencrypt.org/directory" + + mainContainerName = "tailscale" ) var ( @@ -761,7 +763,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, } if pc.Spec.UseLetsEncryptStagingEnvironment && (stsCfg.proxyType == proxyTypeIngressResource || stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress)) { for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { + if isMainContainer(&c) { ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ Name: "TS_DEBUG_ACME_DIRECTORY_URL", Value: letsEncryptStagingEndpoint, @@ -829,7 +831,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, return base } for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { + if isMainContainer(&c) { ss.Spec.Template.Spec.Containers[i] = updateContainer(wantsPod.TailscaleContainer, ss.Spec.Template.Spec.Containers[i]) break } @@ -847,7 +849,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { + if isMainContainer(&c) { if debug { ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, // Serve tailscaled's debug metrics on on @@ -902,6 +904,10 @@ func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { } } +func isMainContainer(c *corev1.Container) bool { + return c.Name == mainContainerName +} + // tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy // state and auth key and returns tailscaled config files for currently supported proxy versions. func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index 9846513c78d74..4247eaaa0040f 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -60,7 +60,6 @@ type HAServiceReconciler struct { recorder record.EventRecorder logger *zap.SugaredLogger tsClient tsClient - tsnetServer tsnetServer tsNamespace string lc localClient defaultTags []string @@ -221,7 +220,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin // This checks and ensures that Tailscale Service's owner references are updated // for this Service and errors if that is not possible (i.e. because it // appears that the Tailscale Service has been created by a non-operator actor). - updatedAnnotations, err := r.ownerAnnotations(existingTSSvc) + updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc) if err != nil { instr := fmt.Sprintf("To proceed, you can either manually delete the existing Tailscale Service or choose a different hostname with the '%s' annotaion", AnnotationHostname) msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) @@ -395,7 +394,7 @@ func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, serviceName := tailcfg.ServiceName("svc:" + hostname) // 1. Clean up the Tailscale Service. - svcChanged, err = r.cleanupTailscaleService(ctx, serviceName, logger) + svcChanged, err = cleanupTailscaleService(ctx, r.tsClient, serviceName, r.operatorID, logger) if err != nil { return false, fmt.Errorf("error deleting Tailscale Service: %w", err) } @@ -456,7 +455,7 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } - svcsChanged, err = r.cleanupTailscaleService(ctx, tailcfg.ServiceName(tsSvcName), logger) + svcsChanged, err = cleanupTailscaleService(ctx, r.tsClient, tailcfg.ServiceName(tsSvcName), r.operatorID, logger) if err != nil { return false, fmt.Errorf("deleting Tailscale Service %q: %w", tsSvcName, err) } @@ -529,8 +528,8 @@ func (r *HAServiceReconciler) tailnetCertDomain(ctx context.Context) (string, er // If a Tailscale Service is found, but contains other owner references, only removes this operator's owner reference. // If a Tailscale Service by the given name is not found or does not contain this operator's owner reference, do nothing. // It returns true if an existing Tailscale Service was updated to remove owner reference, as well as any error that occurred. -func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name tailcfg.ServiceName, logger *zap.SugaredLogger) (updated bool, err error) { - svc, err := r.tsClient.GetVIPService(ctx, name) +func cleanupTailscaleService(ctx context.Context, tsClient tsClient, name tailcfg.ServiceName, operatorID string, logger *zap.SugaredLogger) (updated bool, err error) { + svc, err := tsClient.GetVIPService(ctx, name) if isErrorFeatureFlagNotEnabled(err) { msg := fmt.Sprintf("Unable to proceed with cleanup: %s.", msgFeatureFlagNotEnabled) logger.Warn(msg) @@ -563,14 +562,14 @@ func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name // cluster before deleting the Ingress. Perhaps the comparison could be // 'if or.OperatorID == r.operatorID || or.ingressUID == r.ingressUID'. ix := slices.IndexFunc(o.OwnerRefs, func(or OwnerRef) bool { - return or.OperatorID == r.operatorID + return or.OperatorID == operatorID }) if ix == -1 { return false, nil } if len(o.OwnerRefs) == 1 { logger.Infof("Deleting Tailscale Service %q", name) - return false, r.tsClient.DeleteVIPService(ctx, name) + return false, tsClient.DeleteVIPService(ctx, name) } o.OwnerRefs = slices.Delete(o.OwnerRefs, ix, ix+1) logger.Infof("Updating Tailscale Service %q", name) @@ -579,7 +578,7 @@ func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name return false, fmt.Errorf("error marshalling updated Tailscale Service owner reference: %w", err) } svc.Annotations[ownerAnnotation] = string(json) - return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc) + return true, tsClient.CreateOrUpdateVIPService(ctx, svc) } func (a *HAServiceReconciler) backendRoutesSetup(ctx context.Context, serviceName, replicaName, pgName string, wantsCfg *ingressservices.Config, logger *zap.SugaredLogger) (bool, error) { @@ -742,49 +741,6 @@ func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName return count, nil } -// ownerAnnotations returns the updated annotations required to ensure this -// instance of the operator is included as an owner. If the Tailscale Service is not -// nil, but does not contain an owner we return an error as this likely means -// that the Tailscale Service was created by something other than a Tailscale -// Kubernetes operator. -func (r *HAServiceReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) { - ref := OwnerRef{ - OperatorID: r.operatorID, - } - if svc == nil { - c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} - json, err := json.Marshal(c) - if err != nil { - return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service owner annotation contents: %w, please report this", err) - } - return map[string]string{ - ownerAnnotation: string(json), - }, nil - } - o, err := parseOwnerAnnotation(svc) - if err != nil { - return nil, err - } - if o == nil || len(o.OwnerRefs) == 0 { - return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name) - } - if slices.Contains(o.OwnerRefs, ref) { // up to date - return svc.Annotations, nil - } - o.OwnerRefs = append(o.OwnerRefs, ref) - json, err := json.Marshal(o) - if err != nil { - return nil, fmt.Errorf("error marshalling updated owner references: %w", err) - } - - newAnnots := make(map[string]string, len(svc.Annotations)+1) - for k, v := range svc.Annotations { - newAnnots[k] = v - } - newAnnots[ownerAnnotation] = string(json) - return newAnnots, nil -} - // dnsNameForService returns the DNS name for the given Tailscale Service name. func (r *HAServiceReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) { s := svc.WithoutPrefix() diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go index e08bfd80d318b..054c3ed49f5cb 100644 --- a/cmd/k8s-operator/svc-for-pg_test.go +++ b/cmd/k8s-operator/svc-for-pg_test.go @@ -187,7 +187,6 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien if err := fc.Status().Update(context.Background(), pg); err != nil { t.Fatal(err) } - fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} ft := &fakeTSClient{} zl, err := zap.NewDevelopment() @@ -210,7 +209,6 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien clock: cl, defaultTags: []string{"tag:k8s"}, tsNamespace: "operator-ns", - tsnetServer: fakeTsnetServer, logger: zl.Sugar(), recorder: record.NewFakeRecorder(10), lc: lc, diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go new file mode 100644 index 0000000000000..6e7eadb7303b5 --- /dev/null +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -0,0 +1,197 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// k8s-proxy proxies between tailnet and Kubernetes cluster traffic. +// Currently, it only supports proxying tailnet clients to the Kubernetes API +// server. +package main + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "golang.org/x/sync/errgroup" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "tailscale.com/hostinfo" + "tailscale.com/ipn" + "tailscale.com/ipn/store" + apiproxy "tailscale.com/k8s-operator/api-proxy" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/state" + "tailscale.com/tsnet" +) + +func main() { + logger := zap.Must(zap.NewProduction()).Sugar() + defer logger.Sync() + if err := run(logger); err != nil { + logger.Fatal(err.Error()) + } +} + +func run(logger *zap.SugaredLogger) error { + var ( + configFile = os.Getenv("TS_K8S_PROXY_CONFIG") + podUID = os.Getenv("POD_UID") + ) + if configFile == "" { + return errors.New("TS_K8S_PROXY_CONFIG unset") + } + + // TODO(tomhjp): Support reloading config. + // TODO(tomhjp): Support reading config from a Secret. + cfg, err := conf.Load(configFile) + if err != nil { + return fmt.Errorf("error loading config file %q: %w", configFile, err) + } + + if cfg.Parsed.LogLevel != nil { + level, err := zapcore.ParseLevel(*cfg.Parsed.LogLevel) + if err != nil { + return fmt.Errorf("error parsing log level %q: %w", *cfg.Parsed.LogLevel, err) + } + logger = logger.WithOptions(zap.IncreaseLevel(level)) + } + + if cfg.Parsed.App != nil { + hostinfo.SetApp(*cfg.Parsed.App) + } + + st, err := getStateStore(cfg.Parsed.State, logger) + if err != nil { + return err + } + + // If Pod UID unset, assume we're running outside of a cluster/not managed + // by the operator, so no need to set additional state keys. + if podUID != "" { + if err := state.SetInitialKeys(st, podUID); err != nil { + return fmt.Errorf("error setting initial state: %w", err) + } + } + + var authKey string + if cfg.Parsed.AuthKey != nil { + authKey = *cfg.Parsed.AuthKey + } + + ts := &tsnet.Server{ + Logf: logger.Named("tsnet").Debugf, + UserLogf: logger.Named("tsnet").Infof, + Store: st, + AuthKey: authKey, + } + if cfg.Parsed.Hostname != nil { + ts.Hostname = *cfg.Parsed.Hostname + } + + // ctx to live for the lifetime of the process. + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + // Make sure we crash loop if Up doesn't complete in reasonable time. + upCtx, upCancel := context.WithTimeout(ctx, time.Minute) + defer upCancel() + if _, err := ts.Up(upCtx); err != nil { + return fmt.Errorf("error starting tailscale server: %w", err) + } + defer ts.Close() + + group, groupCtx := errgroup.WithContext(ctx) + + // Setup for updating state keys. + if podUID != "" { + lc, err := ts.LocalClient() + if err != nil { + return fmt.Errorf("error getting local client: %w", err) + } + w, err := lc.WatchIPNBus(groupCtx, ipn.NotifyInitialNetMap) + if err != nil { + return fmt.Errorf("error watching IPN bus: %w", err) + } + defer w.Close() + + group.Go(func() error { + if err := state.KeepKeysUpdated(st, w.Next); err != nil && err != groupCtx.Err() { + return fmt.Errorf("error keeping state keys updated: %w", err) + } + + return nil + }) + } + + // Setup for the API server proxy. + restConfig, err := getRestConfig(logger) + if err != nil { + return fmt.Errorf("error getting rest config: %w", err) + } + authMode := true + if cfg.Parsed.KubeAPIServer != nil { + v, ok := cfg.Parsed.KubeAPIServer.AuthMode.Get() + if ok { + authMode = v + } + } + ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, authMode) + if err != nil { + return fmt.Errorf("error creating api server proxy: %w", err) + } + + // TODO(tomhjp): Work out whether we should use TS_CERT_SHARE_MODE or not, + // and possibly issue certs upfront here before serving. + group.Go(func() error { + if err := ap.Run(groupCtx); err != nil { + return fmt.Errorf("error running API server proxy: %w", err) + } + + return nil + }) + + return group.Wait() +} + +func getStateStore(path *string, logger *zap.SugaredLogger) (ipn.StateStore, error) { + p := "mem:" + if path != nil { + p = *path + } else { + logger.Warn("No state Secret provided; using in-memory store, which will lose state on restart") + } + st, err := store.New(logger.Errorf, p) + if err != nil { + return nil, fmt.Errorf("error creating state store: %w", err) + } + + return st, nil +} + +func getRestConfig(logger *zap.SugaredLogger) (*rest.Config, error) { + restConfig, err := rest.InClusterConfig() + switch err { + case nil: + return restConfig, nil + case rest.ErrNotInCluster: + logger.Info("Not running in-cluster, falling back to kubeconfig") + default: + return nil, fmt.Errorf("error getting in-cluster config: %w", err) + } + + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, nil) + restConfig, err = clientConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("error loading kubeconfig: %w", err) + } + + return restConfig, nil +} diff --git a/k8s-operator/api-proxy/env.go b/k8s-operator/api-proxy/env.go deleted file mode 100644 index c0640ab1e16bf..0000000000000 --- a/k8s-operator/api-proxy/env.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !plan9 - -package apiproxy - -import ( - "os" - - "tailscale.com/types/opt" -) - -func defaultBool(envName string, defVal bool) bool { - vs := os.Getenv(envName) - if vs == "" { - return defVal - } - v, _ := opt.Bool(vs).Get() - return v -} - -func defaultEnv(envName, defVal string) string { - v := os.Getenv(envName) - if v == "" { - return defVal - } - return v -} diff --git a/k8s-operator/api-proxy/proxy.go b/k8s-operator/api-proxy/proxy.go index 7c7260b94af39..c3c13e7846915 100644 --- a/k8s-operator/api-proxy/proxy.go +++ b/k8s-operator/api-proxy/proxy.go @@ -6,17 +6,17 @@ package apiproxy import ( + "context" "crypto/tls" + "errors" "fmt" - "log" "net/http" "net/http/httputil" "net/netip" "net/url" - "os" "strings" + "time" - "github.com/pkg/errors" "go.uber.org/zap" "k8s.io/client-go/rest" "k8s.io/client-go/transport" @@ -37,123 +37,52 @@ var ( whoIsKey = ctxkey.New("", (*apitype.WhoIsResponse)(nil)) ) -type APIServerProxyMode int - -func (a APIServerProxyMode) String() string { - switch a { - case APIServerProxyModeDisabled: - return "disabled" - case APIServerProxyModeEnabled: - return "auth" - case APIServerProxyModeNoAuth: - return "noauth" - default: - return "unknown" - } -} - -const ( - APIServerProxyModeDisabled APIServerProxyMode = iota - APIServerProxyModeEnabled - APIServerProxyModeNoAuth -) - -func ParseAPIProxyMode() APIServerProxyMode { - haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != "" - haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != "" - switch { - case haveAPIProxyEnv && haveAuthProxyEnv: - log.Fatal("AUTH_PROXY and APISERVER_PROXY are mutually exclusive") - case haveAuthProxyEnv: - var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated - if authProxyEnv { - return APIServerProxyModeEnabled - } - return APIServerProxyModeDisabled - case haveAPIProxyEnv: - var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth" - switch apiProxyEnv { - case "true": - return APIServerProxyModeEnabled - case "false", "": - return APIServerProxyModeDisabled - case "noauth": - return APIServerProxyModeNoAuth - default: - panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv)) - } - } - return APIServerProxyModeDisabled -} - -// maybeLaunchAPIServerProxy launches the auth proxy, which is a small HTTP server -// that authenticates requests using the Tailscale LocalAPI and then proxies -// them to the kube-apiserver. -func MaybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, s *tsnet.Server, mode APIServerProxyMode) { - if mode == APIServerProxyModeDisabled { - return - } - startlog := zlog.Named("launchAPIProxy") - if mode == APIServerProxyModeNoAuth { +// NewAPIServerProxy creates a new APIServerProxy that's ready to start once Run +// is called. No network traffic will flow until Run is called. +// +// authMode controls how the proxy behaves: +// - true: the proxy is started and requests are impersonated using the +// caller's Tailscale identity and the rules defined in the tailnet ACLs. +// - false: the proxy is started and requests are passed through to the +// Kubernetes API without any auth modifications. +func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsnet.Server, authMode bool) (*APIServerProxy, error) { + if !authMode { restConfig = rest.AnonymousClientConfig(restConfig) } cfg, err := restConfig.TransportConfig() if err != nil { - startlog.Fatalf("could not get rest.TransportConfig(): %v", err) + return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err) } - // Kubernetes uses SPDY for exec and port-forward, however SPDY is - // incompatible with HTTP/2; so disable HTTP/2 in the proxy. tr := http.DefaultTransport.(*http.Transport).Clone() tr.TLSClientConfig, err = transport.TLSConfigFor(cfg) if err != nil { - startlog.Fatalf("could not get transport.TLSConfigFor(): %v", err) + return nil, fmt.Errorf("could not get transport.TLSConfigFor(): %w", err) } tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper) rt, err := transport.HTTPWrappersForConfig(cfg, tr) if err != nil { - startlog.Fatalf("could not get rest.TransportConfig(): %v", err) + return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err) } - go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode, restConfig.Host) -} -// runAPIServerProxy runs an HTTP server that authenticates requests using the -// Tailscale LocalAPI and then proxies them to the Kubernetes API. -// It listens on :443 and uses the Tailscale HTTPS certificate. -// s will be started if it is not already running. -// rt is used to proxy requests to the Kubernetes API. -// -// mode controls how the proxy behaves: -// - apiserverProxyModeDisabled: the proxy is not started. -// - apiserverProxyModeEnabled: the proxy is started and requests are impersonated using the -// caller's identity from the Tailscale LocalAPI. -// - apiserverProxyModeNoAuth: the proxy is started and requests are not impersonated and -// are passed through to the Kubernetes API. -// -// It never returns. -func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode APIServerProxyMode, host string) { - if mode == APIServerProxyModeDisabled { - return - } - ln, err := ts.Listen("tcp", ":443") + u, err := url.Parse(restConfig.Host) if err != nil { - log.Fatalf("could not listen on :443: %v", err) + return nil, fmt.Errorf("failed to parse URL %w", err) } - u, err := url.Parse(host) - if err != nil { - log.Fatalf("runAPIServerProxy: failed to parse URL %v", err) + if u.Scheme == "" || u.Host == "" { + return nil, fmt.Errorf("the API server proxy requires host and scheme but got: %q", restConfig.Host) } lc, err := ts.LocalClient() if err != nil { - log.Fatalf("could not get local client: %v", err) + return nil, fmt.Errorf("could not get local client: %w", err) } - ap := &apiserverProxy{ - log: log, + ap := &APIServerProxy{ + log: zlog, lc: lc, - mode: mode, + authMode: authMode, upstreamURL: u, ts: ts, } @@ -164,41 +93,69 @@ func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredL Transport: rt, } + return ap, nil +} + +// Run starts the HTTP server that authenticates requests using the +// Tailscale LocalAPI and then proxies them to the Kubernetes API. +// It listens on :443 and uses the Tailscale HTTPS certificate. +// +// It return when ctx is cancelled or ServeTLS fails. +func (ap *APIServerProxy) Run(ctx context.Context) error { + ln, err := ap.ts.Listen("tcp", ":443") + if err != nil { + return fmt.Errorf("could not listen on :443: %v", err) + } + mux := http.NewServeMux() mux.HandleFunc("/", ap.serveDefault) mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY) mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS) - hs := &http.Server{ + ap.hs = &http.Server{ // Kubernetes uses SPDY for exec and port-forward, however SPDY is // incompatible with HTTP/2; so disable HTTP/2 in the proxy. TLSConfig: &tls.Config{ - GetCertificate: lc.GetCertificate, + GetCertificate: ap.lc.GetCertificate, NextProtos: []string{"http/1.1"}, }, TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), Handler: mux, } - log.Infof("API server proxy in %q mode is listening on %s", mode, ln.Addr()) - if err := hs.ServeTLS(ln, "", ""); err != nil { - log.Fatalf("runAPIServerProxy: failed to serve %v", err) + + errs := make(chan error) + go func() { + ap.log.Infof("API server proxy is listening on %s with auth mode: %v", ln.Addr(), ap.authMode) + if err := ap.hs.ServeTLS(ln, "", ""); err != nil && err != http.ErrServerClosed { + errs <- fmt.Errorf("failed to serve: %w", err) + } + }() + + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return ap.hs.Shutdown(shutdownCtx) + case err := <-errs: + return err } } -// apiserverProxy is an [net/http.Handler] that authenticates requests using the Tailscale +// APIServerProxy is an [net/http.Handler] that authenticates requests using the Tailscale // LocalAPI and then proxies them to the Kubernetes API. -type apiserverProxy struct { +type APIServerProxy struct { log *zap.SugaredLogger lc *local.Client rp *httputil.ReverseProxy - mode APIServerProxyMode + authMode bool ts *tsnet.Server + hs *http.Server upstreamURL *url.URL } // serveDefault is the default handler for Kubernetes API server requests. -func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) { +func (ap *APIServerProxy) serveDefault(w http.ResponseWriter, r *http.Request) { who, err := ap.whoIs(r) if err != nil { ap.authError(w, err) @@ -210,17 +167,17 @@ func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) { // serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY, // optionally configuring the kubectl exec sessions to be recorded. -func (ap *apiserverProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) { +func (ap *APIServerProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) { ap.execForProto(w, r, ksr.SPDYProtocol) } // serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket, // optionally configuring the kubectl exec sessions to be recorded. -func (ap *apiserverProxy) serveExecWS(w http.ResponseWriter, r *http.Request) { +func (ap *APIServerProxy) serveExecWS(w http.ResponseWriter, r *http.Request) { ap.execForProto(w, r, ksr.WSProtocol) } -func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) { +func (ap *APIServerProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) { const ( podNameKey = "pod" namespaceNameKey = "namespace" @@ -282,10 +239,10 @@ func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, p ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who))) } -func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) { - r.URL.Scheme = h.upstreamURL.Scheme - r.URL.Host = h.upstreamURL.Host - if h.mode == APIServerProxyModeNoAuth { +func (ap *APIServerProxy) addImpersonationHeadersAsRequired(r *http.Request) { + r.URL.Scheme = ap.upstreamURL.Scheme + r.URL.Host = ap.upstreamURL.Host + if !ap.authMode { // If we are not providing authentication, then we are just // proxying to the Kubernetes API, so we don't need to do // anything else. @@ -310,16 +267,16 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) { } // Now add the impersonation headers that we want. - if err := addImpersonationHeaders(r, h.log); err != nil { - log.Print("failed to add impersonation headers: ", err.Error()) + if err := addImpersonationHeaders(r, ap.log); err != nil { + ap.log.Errorf("failed to add impersonation headers: %v", err) } } -func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) { +func (ap *APIServerProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) { return ap.lc.WhoIs(r.Context(), r.RemoteAddr) } -func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) { +func (ap *APIServerProxy) authError(w http.ResponseWriter, err error) { ap.log.Errorf("failed to authenticate caller: %v", err) http.Error(w, "failed to authenticate caller", http.StatusInternalServerError) } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 18bf1cb50400f..c09152da6f6c1 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -21,6 +21,21 @@ +#### APIServerProxyMode + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [auth noauth] +- Type: string + +_Appears in:_ +- [KubeAPIServerConfig](#kubeapiserverconfig) + + + #### AppConnector @@ -142,7 +157,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `env` _[Env](#env) array_ | List of environment variables to set in the container.
          https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables
          Note that environment variables provided here will take precedence
          over Tailscale-specific environment variables set by the operator,
          however running proxies with custom values for Tailscale environment
          variables (i.e TS_USERSPACE) is not recommended and might break in
          the future. | | | -| `image` _string_ | Container image name. By default images are pulled from
          docker.io/tailscale/tailscale, but the official images are also
          available at ghcr.io/tailscale/tailscale. Specifying image name here
          will override any proxy image values specified via the Kubernetes
          operator's Helm chart values or PROXY_IMAGE env var in the operator
          Deployment.
          https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | | +| `image` _string_ | Container image name. By default images are pulled from docker.io/tailscale,
          but the official images are also available at ghcr.io/tailscale.
          For all uses except on ProxyGroups of type "kube-apiserver", this image must
          be either tailscale/tailscale, or an equivalent mirror of that image.
          To apply to ProxyGroups of type "kube-apiserver", this image must be
          tailscale/k8s-proxy or a mirror of that image.
          For "tailscale/tailscale"-based proxies, specifying image name here will
          override any proxy image values specified via the Kubernetes operator's
          Helm chart values or PROXY_IMAGE env var in the operator Deployment.
          For "tailscale/k8s-proxy"-based proxies, there is currently no way to
          configure your own default, and this field is the only way to use a
          custom image.
          https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | | | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#pullpolicy-v1-core)_ | Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always.
          https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | Enum: [Always Never IfNotPresent]
          | | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#resourcerequirements-v1-core)_ | Container resource requirements.
          By default Tailscale Kubernetes operator does not apply any resource
          requirements. The amount of resources required wil depend on the
          amount of resources the operator needs to parse, usage patterns and
          cluster size.
          https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources | | | | `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#securitycontext-v1-core)_ | Container security context.
          Security context specified here will override the security context set by the operator.
          By default the operator sets the Tailscale container and the Tailscale init container to privileged
          for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
          You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
          installing device plugin in your cluster and configuring the proxies tun device to be created
          by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
          https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context | | | @@ -313,6 +328,22 @@ _Appears in:_ +#### KubeAPIServerConfig + + + +KubeAPIServerConfig contains configuration specific to the kube-apiserver ProxyGroup type. + + + +_Appears in:_ +- [ProxyGroupSpec](#proxygroupspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `mode` _[APIServerProxyMode](#apiserverproxymode)_ | Mode to run the API server proxy in. Supported modes are auth and noauth.
          In auth mode, requests from the tailnet proxied over to the Kubernetes
          API server are additionally impersonated using the sender's tailnet identity.
          If not specified, defaults to auth mode. | | Enum: [auth noauth]
          Type: string
          | + + #### LabelValue _Underlying type:_ _string_ @@ -459,7 +490,7 @@ _Appears in:_ | `annotations` _object (keys:string, values:string)_ | Annotations that will be added to the proxy Pod.
          Any annotations specified here will be merged with the default
          annotations applied to the Pod by the Tailscale Kubernetes operator.
          Annotations must be valid Kubernetes annotations.
          https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set | | | | `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#affinity-v1-core)_ | Proxy Pod's affinity rules.
          By default, the Tailscale Kubernetes operator does not apply any affinity rules.
          https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#affinity | | | | `tailscaleContainer` _[Container](#container)_ | Configuration for the proxy container running tailscale. | | | -| `tailscaleInitContainer` _[Container](#container)_ | Configuration for the proxy init container that enables forwarding. | | | +| `tailscaleInitContainer` _[Container](#container)_ | Configuration for the proxy init container that enables forwarding.
          Not valid to apply to ProxyGroups of type "kube-apiserver". | | | | `securityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#podsecuritycontext-v1-core)_ | Proxy Pod's security context.
          By default Tailscale Kubernetes operator does not apply any Pod
          security context.
          https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2 | | | | `imagePullSecrets` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#localobjectreference-v1-core) array_ | Proxy Pod's image pull Secrets.
          https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec | | | | `nodeName` _string_ | Proxy Pod's node name.
          https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling | | | @@ -638,11 +669,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress and ingress.
          Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress]
          Type: string
          | +| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.
          Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress kube-apiserver]
          Type: string
          | | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
          If you specify custom tags here, make sure you also make the operator
          an owner of these tags.
          See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
          Tags cannot be changed once a ProxyGroup device has been created.
          Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$`
          Type: string
          | | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.
          Defaults to 2. | | Minimum: 0
          | | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created
          by the ProxyGroup. Each device will have the integer number from its
          StatefulSet pod appended to this prefix to form the full hostname.
          HostnamePrefix can contain lower case letters, numbers and dashes, it
          must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$`
          Type: string
          | | `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that contains
          configuration options that should be applied to the resources created
          for this ProxyGroup. If unset, and there is no default ProxyClass
          configured, the operator will create resources with the default
          configuration. | | | +| `kubeAPIServer` _[KubeAPIServerConfig](#kubeapiserverconfig)_ | KubeAPIServer contains configuration specific to the kube-apiserver
          ProxyGroup type. This field is only used when Type is set to "kube-apiserver". | | | #### ProxyGroupStatus @@ -669,7 +701,7 @@ _Underlying type:_ _string_ _Validation:_ -- Enum: [egress ingress] +- Enum: [egress ingress kube-apiserver] - Type: string _Appears in:_ diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 9221c60f3c870..6a4114bfa83da 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -264,6 +264,7 @@ type Pod struct { // +optional TailscaleContainer *Container `json:"tailscaleContainer,omitempty"` // Configuration for the proxy init container that enables forwarding. + // Not valid to apply to ProxyGroups of type "kube-apiserver". // +optional TailscaleInitContainer *Container `json:"tailscaleInitContainer,omitempty"` // Proxy Pod's security context. @@ -364,12 +365,21 @@ type Container struct { // the future. // +optional Env []Env `json:"env,omitempty"` - // Container image name. By default images are pulled from - // docker.io/tailscale/tailscale, but the official images are also - // available at ghcr.io/tailscale/tailscale. Specifying image name here - // will override any proxy image values specified via the Kubernetes - // operator's Helm chart values or PROXY_IMAGE env var in the operator - // Deployment. + // Container image name. By default images are pulled from docker.io/tailscale, + // but the official images are also available at ghcr.io/tailscale. + // + // For all uses except on ProxyGroups of type "kube-apiserver", this image must + // be either tailscale/tailscale, or an equivalent mirror of that image. + // To apply to ProxyGroups of type "kube-apiserver", this image must be + // tailscale/k8s-proxy or a mirror of that image. + // + // For "tailscale/tailscale"-based proxies, specifying image name here will + // override any proxy image values specified via the Kubernetes operator's + // Helm chart values or PROXY_IMAGE env var in the operator Deployment. + // For "tailscale/k8s-proxy"-based proxies, there is currently no way to + // configure your own default, and this field is the only way to use a + // custom image. + // // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image // +optional Image string `json:"image,omitempty"` diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index 5edb47f0da6c3..ad5b113612bbf 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -49,7 +49,7 @@ type ProxyGroupList struct { } type ProxyGroupSpec struct { - // Type of the ProxyGroup proxies. Supported types are egress and ingress. + // Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. // Type is immutable once a ProxyGroup is created. // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup type is immutable" Type ProxyGroupType `json:"type"` @@ -84,6 +84,11 @@ type ProxyGroupSpec struct { // configuration. // +optional ProxyClass string `json:"proxyClass,omitempty"` + + // KubeAPIServer contains configuration specific to the kube-apiserver + // ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + // +optional + KubeAPIServer *KubeAPIServerConfig `json:"kubeAPIServer,omitempty"` } type ProxyGroupStatus struct { @@ -122,14 +127,34 @@ type TailnetDevice struct { } // +kubebuilder:validation:Type=string -// +kubebuilder:validation:Enum=egress;ingress +// +kubebuilder:validation:Enum=egress;ingress;kube-apiserver type ProxyGroupType string const ( - ProxyGroupTypeEgress ProxyGroupType = "egress" - ProxyGroupTypeIngress ProxyGroupType = "ingress" + ProxyGroupTypeEgress ProxyGroupType = "egress" + ProxyGroupTypeIngress ProxyGroupType = "ingress" + ProxyGroupTypeKubernetesAPIServer ProxyGroupType = "kube-apiserver" +) + +// +kubebuilder:validation:Type=string +// +kubebuilder:validation:Enum=auth;noauth +type APIServerProxyMode string + +const ( + APIServerProxyModeAuth APIServerProxyMode = "auth" + APIServerProxyModeNoAuth APIServerProxyMode = "noauth" ) // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9-]{0,61}$` type HostnamePrefix string + +// KubeAPIServerConfig contains configuration specific to the kube-apiserver ProxyGroup type. +type KubeAPIServerConfig struct { + // Mode to run the API server proxy in. Supported modes are auth and noauth. + // In auth mode, requests from the tailnet proxied over to the Kubernetes + // API server are additionally impersonated using the sender's tailnet identity. + // If not specified, defaults to auth mode. + // +optional + Mode *APIServerProxyMode `json:"mode,omitempty"` +} diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index ffc04d3b9dde3..32adbd6804ed0 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -316,6 +316,26 @@ func (in *Env) DeepCopy() *Env { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { + *out = *in + if in.Mode != nil { + in, out := &in.Mode, &out.Mode + *out = new(APIServerProxyMode) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeAPIServerConfig. +func (in *KubeAPIServerConfig) DeepCopy() *KubeAPIServerConfig { + if in == nil { + return nil + } + out := new(KubeAPIServerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in Labels) DeepCopyInto(out *Labels) { { @@ -731,6 +751,11 @@ func (in *ProxyGroupSpec) DeepCopyInto(out *ProxyGroupSpec) { *out = new(int32) **out = **in } + if in.KubeAPIServer != nil { + in, out := &in.KubeAPIServer, &out.KubeAPIServer + *out = new(KubeAPIServerConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyGroupSpec. diff --git a/kube/k8s-proxy/conf/conf.go b/kube/k8s-proxy/conf/conf.go new file mode 100644 index 0000000000000..6b0e853c5c21c --- /dev/null +++ b/kube/k8s-proxy/conf/conf.go @@ -0,0 +1,101 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package conf contains code to load, manipulate, and access config file +// settings for k8s-proxy. +package conf + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/tailscale/hujson" + "tailscale.com/types/opt" +) + +const v1Alpha1 = "v1alpha1" + +// Config describes a config file. +type Config struct { + Path string // disk path of HuJSON + Raw []byte // raw bytes from disk, in HuJSON form + Std []byte // standardized JSON form + Version string // "v1alpha1" + + // Parsed is the parsed config, converted from its on-disk version to the + // latest known format. + Parsed ConfigV1Alpha1 +} + +// VersionedConfig allows specifying config at the root of the object, or in +// a versioned sub-object. +// e.g. {"version": "v1alpha1", "authKey": "abc123"} +// or {"version": "v1beta1", "a-beta-config": "a-beta-value", "v1alpha1": {"authKey": "abc123"}} +type VersionedConfig struct { + Version string `json:",omitempty"` // "v1alpha1" + + // Latest version of the config. + *ConfigV1Alpha1 + + // Backwards compatibility version(s) of the config. Fields and sub-fields + // from here should only be added to, never changed in place. + V1Alpha1 *ConfigV1Alpha1 `json:",omitempty"` + // V1Beta1 *ConfigV1Beta1 `json:",omitempty"` // Not yet used. +} + +type ConfigV1Alpha1 struct { + AuthKey *string `json:",omitempty"` // Tailscale auth key to use. + Hostname *string `json:",omitempty"` // Tailscale device hostname. + State *string `json:",omitempty"` // Path to the Tailscale state. + LogLevel *string `json:",omitempty"` // "debug", "info". Defaults to "info". + App *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer + KubeAPIServer *KubeAPIServer `json:",omitempty"` // Config specific to the API Server proxy. +} + +type KubeAPIServer struct { + AuthMode opt.Bool `json:",omitempty"` +} + +// Load reads and parses the config file at the provided path on disk. +func Load(path string) (c Config, err error) { + c.Path = path + + c.Raw, err = os.ReadFile(path) + if err != nil { + return c, fmt.Errorf("error reading config file %q: %w", path, err) + } + c.Std, err = hujson.Standardize(c.Raw) + if err != nil { + return c, fmt.Errorf("error parsing config file %q HuJSON/JSON: %w", path, err) + } + var ver VersionedConfig + if err := json.Unmarshal(c.Std, &ver); err != nil { + return c, fmt.Errorf("error parsing config file %q: %w", path, err) + } + rootV1Alpha1 := (ver.Version == v1Alpha1) + backCompatV1Alpha1 := (ver.V1Alpha1 != nil) + switch { + case ver.Version == "": + return c, fmt.Errorf("error parsing config file %q: no \"version\" field provided", path) + case rootV1Alpha1 && backCompatV1Alpha1: + // Exactly one of these should be set. + return c, fmt.Errorf("error parsing config file %q: both root and v1alpha1 config provided", path) + case rootV1Alpha1 != backCompatV1Alpha1: + c.Version = v1Alpha1 + switch { + case rootV1Alpha1 && ver.ConfigV1Alpha1 != nil: + c.Parsed = *ver.ConfigV1Alpha1 + case backCompatV1Alpha1: + c.Parsed = *ver.V1Alpha1 + default: + c.Parsed = ConfigV1Alpha1{} + } + default: + return c, fmt.Errorf("error parsing config file %q: unsupported \"version\" value %q; want \"%s\"", path, ver.Version, v1Alpha1) + } + + return c, nil +} diff --git a/kube/k8s-proxy/conf/conf_test.go b/kube/k8s-proxy/conf/conf_test.go new file mode 100644 index 0000000000000..a47391dc90ade --- /dev/null +++ b/kube/k8s-proxy/conf/conf_test.go @@ -0,0 +1,86 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package conf + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "tailscale.com/types/ptr" +) + +// Test that the config file can be at the root of the object, or in a versioned sub-object. +// or {"version": "v1beta1", "a-beta-config": "a-beta-value", "v1alpha1": {"authKey": "abc123"}} +func TestVersionedConfig(t *testing.T) { + testCases := map[string]struct { + inputConfig string + expectedConfig ConfigV1Alpha1 + expectedError string + }{ + "root_config_v1alpha1": { + inputConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")}, + }, + "backwards_compat_v1alpha1_config": { + // Client doesn't know about v1beta1, so it should read in v1alpha1. + inputConfig: `{"version": "v1beta1", "beta-key": "beta-value", "authKey": "def456", "v1alpha1": {"authKey": "abc123"}}`, + expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")}, + }, + "unknown_key_allowed": { + // Adding new keys to the config doesn't require a version bump. + inputConfig: `{"version": "v1alpha1", "unknown-key": "unknown-value", "authKey": "abc123"}`, + expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")}, + }, + "version_only_no_authkey": { + inputConfig: `{"version": "v1alpha1"}`, + expectedConfig: ConfigV1Alpha1{}, + }, + "both_config_v1alpha1": { + inputConfig: `{"version": "v1alpha1", "authKey": "abc123", "v1alpha1": {"authKey": "def456"}}`, + expectedError: "both root and v1alpha1 config provided", + }, + "empty_config": { + inputConfig: `{}`, + expectedError: `no "version" field provided`, + }, + "v1beta1_without_backwards_compat": { + inputConfig: `{"version": "v1beta1", "beta-key": "beta-value", "authKey": "def456"}`, + expectedError: `unsupported "version" value "v1beta1"; want "v1alpha1"`, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.json") + if err := os.WriteFile(path, []byte(tc.inputConfig), 0644); err != nil { + t.Fatalf("failed to write config file: %v", err) + } + cfg, err := Load(path) + switch { + case tc.expectedError == "" && err != nil: + t.Fatalf("unexpected error: %v", err) + case tc.expectedError != "": + if err == nil { + t.Fatalf("expected error %q, got nil", tc.expectedError) + } else if !strings.Contains(err.Error(), tc.expectedError) { + t.Fatalf("expected error %q, got %q", tc.expectedError, err.Error()) + } + return + } + if cfg.Version != "v1alpha1" { + t.Fatalf("expected version %q, got %q", "v1alpha1", cfg.Version) + } + // Diff actual vs expected config. + if diff := cmp.Diff(cfg.Parsed, tc.expectedConfig); diff != "" { + t.Fatalf("Unexpected parsed config (-got +want):\n%s", diff) + } + }) + } +} diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index 6f96875dddd0f..20b0050143c93 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -5,14 +5,15 @@ package kubetypes const ( // Hostinfo App values for the Tailscale Kubernetes Operator components. - AppOperator = "k8s-operator" - AppAPIServerProxy = "k8s-operator-proxy" - AppIngressProxy = "k8s-operator-ingress-proxy" - AppIngressResource = "k8s-operator-ingress-resource" - AppEgressProxy = "k8s-operator-egress-proxy" - AppConnector = "k8s-operator-connector-resource" - AppProxyGroupEgress = "k8s-operator-proxygroup-egress" - AppProxyGroupIngress = "k8s-operator-proxygroup-ingress" + AppOperator = "k8s-operator" + AppInProcessAPIServerProxy = "k8s-operator-proxy" + AppIngressProxy = "k8s-operator-ingress-proxy" + AppIngressResource = "k8s-operator-ingress-resource" + AppEgressProxy = "k8s-operator-egress-proxy" + AppConnector = "k8s-operator-connector-resource" + AppProxyGroupEgress = "k8s-operator-proxygroup-egress" + AppProxyGroupIngress = "k8s-operator-proxygroup-ingress" + AppProxyGroupKubeAPIServer = "k8s-operator-proxygroup-kube-apiserver" // Clientmetrics for Tailscale Kubernetes Operator components MetricIngressProxyCount = "k8s_ingress_proxies" // L3 @@ -29,6 +30,7 @@ const ( MetricEgressServiceCount = "k8s_egress_service_resources" MetricProxyGroupEgressCount = "k8s_proxygroup_egress_resources" MetricProxyGroupIngressCount = "k8s_proxygroup_ingress_resources" + MetricProxyGroupAPIServerCount = "k8s_proxygroup_kube_apiserver_resources" // Keys that containerboot writes to state file that can be used to determine its state. // fields set in Tailscale state Secret. These are mostly used by the Tailscale Kubernetes operator to determine diff --git a/kube/state/state.go b/kube/state/state.go new file mode 100644 index 0000000000000..4831a5f5b367a --- /dev/null +++ b/kube/state/state.go @@ -0,0 +1,97 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package state updates state keys for tailnet client devices managed by the +// operator. These keys are used to signal readiness, metadata, and current +// configuration state to the operator. Client packages deployed by the operator +// include containerboot, tsrecorder, and k8s-proxy, but currently containerboot +// has its own implementation to manage the same keys. +package state + +import ( + "encoding/json" + "fmt" + + "tailscale.com/ipn" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" + "tailscale.com/util/deephash" +) + +const ( + keyPodUID = ipn.StateKey(kubetypes.KeyPodUID) + keyCapVer = ipn.StateKey(kubetypes.KeyCapVer) + keyDeviceID = ipn.StateKey(kubetypes.KeyDeviceID) + keyDeviceIPs = ipn.StateKey(kubetypes.KeyDeviceIPs) + keyDeviceFQDN = ipn.StateKey(kubetypes.KeyDeviceFQDN) +) + +// SetInitialKeys sets Pod UID and cap ver and clears tailnet device state +// keys to help stop the operator using stale tailnet device state. +func SetInitialKeys(store ipn.StateStore, podUID string) error { + // Clear device state keys first so the operator knows if the pod UID + // matches, the other values are definitely not stale. + for _, key := range []ipn.StateKey{keyDeviceID, keyDeviceFQDN, keyDeviceIPs} { + if _, err := store.ReadState(key); err == nil { + if err := store.WriteState(key, nil); err != nil { + return fmt.Errorf("error writing %q to state store: %w", key, err) + } + } + } + + if err := store.WriteState(keyPodUID, []byte(podUID)); err != nil { + return fmt.Errorf("error writing pod UID to state store: %w", err) + } + if err := store.WriteState(keyCapVer, fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion)); err != nil { + return fmt.Errorf("error writing capability version to state store: %w", err) + } + + return nil +} + +// KeepKeysUpdated sets state store keys consistent with containerboot to +// signal proxy readiness to the operator. It runs until its context is +// cancelled or it hits an error. The passed in next function is expected to be +// from a local.IPNBusWatcher that is at least subscribed to +// ipn.NotifyInitialNetMap. +func KeepKeysUpdated(store ipn.StateStore, next func() (ipn.Notify, error)) error { + var currentDeviceID, currentDeviceIPs, currentDeviceFQDN deephash.Sum + + for { + n, err := next() // Blocks on a streaming LocalAPI HTTP call. + if err != nil { + return err + } + if n.NetMap == nil { + continue + } + + if deviceID := n.NetMap.SelfNode.StableID(); deephash.Update(¤tDeviceID, &deviceID) { + if err := store.WriteState(keyDeviceID, []byte(deviceID)); err != nil { + return fmt.Errorf("failed to store device ID in state: %w", err) + } + } + + if fqdn := n.NetMap.SelfNode.Name(); deephash.Update(¤tDeviceFQDN, &fqdn) { + if err := store.WriteState(keyDeviceFQDN, []byte(fqdn)); err != nil { + return fmt.Errorf("failed to store device FQDN in state: %w", err) + } + } + + if addrs := n.NetMap.SelfNode.Addresses(); deephash.Update(¤tDeviceIPs, &addrs) { + var deviceIPs []string + for _, addr := range addrs.AsSlice() { + deviceIPs = append(deviceIPs, addr.Addr().String()) + } + deviceIPsValue, err := json.Marshal(deviceIPs) + if err != nil { + return err + } + if err := store.WriteState(keyDeviceIPs, deviceIPsValue); err != nil { + return fmt.Errorf("failed to store device IPs in state: %w", err) + } + } + } +} diff --git a/kube/state/state_test.go b/kube/state/state_test.go new file mode 100644 index 0000000000000..0375b1c01d91a --- /dev/null +++ b/kube/state/state_test.go @@ -0,0 +1,203 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package state + +import ( + "bytes" + "fmt" + "net/netip" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "tailscale.com/ipn" + "tailscale.com/ipn/store" + "tailscale.com/tailcfg" + "tailscale.com/types/logger" + "tailscale.com/types/netmap" +) + +func TestSetInitialStateKeys(t *testing.T) { + var ( + podUID = []byte("test-pod-uid") + expectedCapVer = fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion) + ) + for name, tc := range map[string]struct { + initial map[ipn.StateKey][]byte + expected map[ipn.StateKey][]byte + }{ + "empty_initial": { + initial: map[ipn.StateKey][]byte{}, + expected: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + }, + }, + "existing_pod_uid_and_capver": { + initial: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + }, + expected: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + }, + }, + "all_keys_preexisting": { + initial: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + keyDeviceID: []byte("existing-device-id"), + keyDeviceFQDN: []byte("existing-device-fqdn"), + keyDeviceIPs: []byte(`["1.2.3.4"]`), + }, + expected: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + keyDeviceID: nil, + keyDeviceFQDN: nil, + keyDeviceIPs: nil, + }, + }, + } { + t.Run(name, func(t *testing.T) { + store, err := store.New(logger.Discard, "mem:") + if err != nil { + t.Fatalf("error creating in-memory store: %v", err) + } + + for key, value := range tc.initial { + if err := store.WriteState(key, value); err != nil { + t.Fatalf("error writing initial state key %q: %v", key, err) + } + } + + if err := SetInitialKeys(store, string(podUID)); err != nil { + t.Fatalf("setInitialStateKeys failed: %v", err) + } + + actual := make(map[ipn.StateKey][]byte) + for expectedKey, expectedValue := range tc.expected { + actualValue, err := store.ReadState(expectedKey) + if err != nil { + t.Errorf("error reading state key %q: %v", expectedKey, err) + continue + } + + actual[expectedKey] = actualValue + if !bytes.Equal(actualValue, expectedValue) { + t.Errorf("state key %q mismatch: expected %q, got %q", expectedKey, expectedValue, actualValue) + } + } + if diff := cmp.Diff(actual, tc.expected); diff != "" { + t.Errorf("state keys mismatch (-got +want):\n%s", diff) + } + }) + } +} + +func TestKeepStateKeysUpdated(t *testing.T) { + store, err := store.New(logger.Discard, "mem:") + if err != nil { + t.Fatalf("error creating in-memory store: %v", err) + } + + nextWaiting := make(chan struct{}) + go func() { + <-nextWaiting // Acknowledge the initial signal. + }() + notifyCh := make(chan ipn.Notify) + next := func() (ipn.Notify, error) { + nextWaiting <- struct{}{} // Send signal to test that state is consistent. + return <-notifyCh, nil // Wait for test input. + } + + errs := make(chan error, 1) + go func() { + err := KeepKeysUpdated(store, next) + if err != nil { + errs <- fmt.Errorf("keepStateKeysUpdated returned with error: %w", err) + } + }() + + for _, tc := range []struct { + name string + notify ipn.Notify + expected map[ipn.StateKey][]byte + }{ + { + name: "initial_not_authed", + notify: ipn.Notify{}, + expected: map[ipn.StateKey][]byte{ + keyDeviceID: nil, + keyDeviceFQDN: nil, + keyDeviceIPs: nil, + }, + }, + { + name: "authed", + notify: ipn.Notify{ + NetMap: &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{ + StableID: "TESTCTRL00000001", + Name: "test-node.test.ts.net", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32"), netip.MustParsePrefix("fd7a:115c:a1e0:ab12:4843:cd96:0:1/128")}, + }).View(), + }, + }, + expected: map[ipn.StateKey][]byte{ + keyDeviceID: []byte("TESTCTRL00000001"), + keyDeviceFQDN: []byte("test-node.test.ts.net"), + keyDeviceIPs: []byte(`["100.64.0.1","fd7a:115c:a1e0:ab12:4843:cd96:0:1"]`), + }, + }, + { + name: "updated_fields", + notify: ipn.Notify{ + NetMap: &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{ + StableID: "TESTCTRL00000001", + Name: "updated.test.ts.net", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.250/32")}, + }).View(), + }, + }, + expected: map[ipn.StateKey][]byte{ + keyDeviceID: []byte("TESTCTRL00000001"), + keyDeviceFQDN: []byte("updated.test.ts.net"), + keyDeviceIPs: []byte(`["100.64.0.250"]`), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + // Send test input. + select { + case notifyCh <- tc.notify: + case <-errs: + t.Fatal("keepStateKeysUpdated returned before test input") + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for next() to be called again") + } + + // Wait for next() to be called again so we know the goroutine has + // processed the event. + select { + case <-nextWaiting: + case <-errs: + t.Fatal("keepStateKeysUpdated returned before test input") + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for next() to be called again") + } + + for key, value := range tc.expected { + got, _ := store.ReadState(key) + if !bytes.Equal(got, value) { + t.Errorf("state key %q mismatch: expected %q, got %q", key, value, got) + } + } + }) + } +} From 27fa2ad868f0e1bf48614dd97b7fde9cd00fa93d Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 9 Jul 2025 09:37:45 +0100 Subject: [PATCH 179/263] cmd/k8s-operator: don't require generation for Available condition (#16497) The observed generation was set to always 0 in #16429, but this had the knock-on effect of other controllers considering ProxyGroups never ready because the observed generation is never up to date in proxyGroupCondition. Make sure the ProxyGroupAvailable function does not requires the observed generation to be up to date, and add testing coverage to catch regressions. Updates #16327 Change-Id: I42f50ad47dd81cc2d3c3ce2cd7b252160bb58e40 Signed-off-by: Tom Proctor --- cmd/k8s-operator/proxygroup_test.go | 30 +++++++++++++++++++++++------ k8s-operator/conditions.go | 13 +++++++------ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index c58e427aa06b6..6f143c0566dff 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "tailscale.com/client/tailscale" "tailscale.com/ipn" + kube "tailscale.com/k8s-operator" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -815,6 +816,7 @@ func TestProxyGroup(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test", Finalizers: []string{"tailscale.com/finalizer"}, + Generation: 1, }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeEgress, @@ -856,9 +858,12 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 1, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, false, pc) + if kube.ProxyGroupAvailable(pg) { + t.Fatal("expected ProxyGroup to not be available") + } }) t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) { @@ -874,13 +879,19 @@ func TestProxyGroup(t *testing.T) { if err := fc.Status().Update(t.Context(), pc); err != nil { t.Fatal(err) } - + pg.ObjectMeta.Generation = 2 + mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + p.ObjectMeta.Generation = pg.ObjectMeta.Generation + }) expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 2, cl, zl.Sugar()) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) + if kube.ProxyGroupAvailable(pg) { + t.Fatal("expected ProxyGroup to not be available") + } if expected := 1; reconciler.egressProxyGroups.Len() != expected { t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) } @@ -902,6 +913,10 @@ func TestProxyGroup(t *testing.T) { t.Run("simulate_successful_device_auth", func(t *testing.T) { addNodeIDToStateSecrets(t, fc, pg) + pg.ObjectMeta.Generation = 3 + mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + p.ObjectMeta.Generation = pg.ObjectMeta.Generation + }) expectReconciled(t, reconciler, "", pg.Name) pg.Status.Devices = []tsapi.TailnetDevice{ @@ -914,10 +929,13 @@ func TestProxyGroup(t *testing.T) { TailnetIPs: []string{"1.2.3.4", "::1"}, }, } - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar()) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) + if !kube.ProxyGroupAvailable(pg) { + t.Fatal("expected ProxyGroup to be available") + } }) t.Run("scale_up_to_3", func(t *testing.T) { @@ -926,14 +944,14 @@ func TestProxyGroup(t *testing.T) { p.Spec = pg.Spec }) expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 3, cl, zl.Sugar()) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) expectProxyGroupResources(t, fc, pg, true, pc) addNodeIDToStateSecrets(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar()) tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar()) pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{ Hostname: "hostname-nodeid-2", diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go index 1d30f352c3603..f6858c0059162 100644 --- a/k8s-operator/conditions.go +++ b/k8s-operator/conditions.go @@ -137,22 +137,23 @@ func ProxyClassIsReady(pc *tsapi.ProxyClass) bool { } func ProxyGroupIsReady(pg *tsapi.ProxyGroup) bool { - return proxyGroupCondition(pg, tsapi.ProxyGroupReady) + cond := proxyGroupCondition(pg, tsapi.ProxyGroupReady) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation } func ProxyGroupAvailable(pg *tsapi.ProxyGroup) bool { - return proxyGroupCondition(pg, tsapi.ProxyGroupAvailable) + cond := proxyGroupCondition(pg, tsapi.ProxyGroupAvailable) + return cond != nil && cond.Status == metav1.ConditionTrue } -func proxyGroupCondition(pg *tsapi.ProxyGroup, condType tsapi.ConditionType) bool { +func proxyGroupCondition(pg *tsapi.ProxyGroup, condType tsapi.ConditionType) *metav1.Condition { idx := xslices.IndexFunc(pg.Status.Conditions, func(cond metav1.Condition) bool { return cond.Type == string(condType) }) if idx == -1 { - return false + return nil } - cond := pg.Status.Conditions[idx] - return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation + return &pg.Status.Conditions[idx] } func DNSCfgIsReady(cfg *tsapi.DNSConfig) bool { From 008a238acddcf1cb73c544eee41f689392b74494 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 9 Jul 2025 09:16:29 -0700 Subject: [PATCH 180/263] wgengine/magicsock: support self as candidate peer relay (#16499) Updates tailscale/corp#30247 Signed-off-by: Jordan Whited --- wgengine/magicsock/magicsock.go | 51 ++++++------ wgengine/magicsock/magicsock_test.go | 114 ++++++++++++++++++--------- 2 files changed, 102 insertions(+), 63 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index ab7c2102fe1ec..1978867fa5e1c 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2618,8 +2618,8 @@ func (c *Conn) onFilterUpdate(f FilterUpdate) { c.updateRelayServersSet(f.Filter, self, peers) } -// updateRelayServersSet iterates all peers, evaluating filt for each one in -// order to determine which peers are relay server candidates. filt, self, and +// updateRelayServersSet iterates all peers and self, evaluating filt for each +// one in order to determine which are relay server candidates. filt, self, and // peers are passed as args (vs c.mu-guarded fields) to enable callers to // release c.mu before calling as this is O(m * n) (we iterate all cap rules 'm' // in filt for every peer 'n'). @@ -2631,8 +2631,9 @@ func (c *Conn) onFilterUpdate(f FilterUpdate) { // the computed result over the eventbus instead. func (c *Conn) updateRelayServersSet(filt *filter.Filter, self tailcfg.NodeView, peers views.Slice[tailcfg.NodeView]) { relayServers := make(set.Set[netip.AddrPort]) - for _, peer := range peers.All() { - peerAPI := peerAPIIfCandidateRelayServer(filt, self, peer) + nodes := append(peers.AsSlice(), self) + for _, maybeCandidate := range nodes { + peerAPI := peerAPIIfCandidateRelayServer(filt, self, maybeCandidate) if peerAPI.IsValid() { relayServers.Add(peerAPI) } @@ -2640,33 +2641,34 @@ func (c *Conn) updateRelayServersSet(filt *filter.Filter, self tailcfg.NodeView, c.relayManager.handleRelayServersSet(relayServers) } -// peerAPIIfCandidateRelayServer returns the peer API address of peer if it -// is considered to be a candidate relay server upon evaluation against filt and -// self, otherwise it returns a zero value. -func peerAPIIfCandidateRelayServer(filt *filter.Filter, self, peer tailcfg.NodeView) netip.AddrPort { +// peerAPIIfCandidateRelayServer returns the peer API address of maybeCandidate +// if it is considered to be a candidate relay server upon evaluation against +// filt and self, otherwise it returns a zero value. self and maybeCandidate +// may be equal. +func peerAPIIfCandidateRelayServer(filt *filter.Filter, self, maybeCandidate tailcfg.NodeView) netip.AddrPort { if filt == nil || !self.Valid() || - !peer.Valid() || - !capVerIsRelayServerCapable(peer.Cap()) || - !peer.Hostinfo().Valid() { + !maybeCandidate.Valid() || + !capVerIsRelayServerCapable(maybeCandidate.Cap()) || + !maybeCandidate.Hostinfo().Valid() { return netip.AddrPort{} } - for _, peerPrefix := range peer.Addresses().All() { - if !peerPrefix.IsSingleIP() { + for _, maybeCandidatePrefix := range maybeCandidate.Addresses().All() { + if !maybeCandidatePrefix.IsSingleIP() { continue } - peerAddr := peerPrefix.Addr() + maybeCandidateAddr := maybeCandidatePrefix.Addr() for _, selfPrefix := range self.Addresses().All() { if !selfPrefix.IsSingleIP() { continue } selfAddr := selfPrefix.Addr() - if selfAddr.BitLen() == peerAddr.BitLen() { // same address family - if filt.CapsWithValues(peerAddr, selfAddr).HasCapability(tailcfg.PeerCapabilityRelayTarget) { - for _, s := range peer.Hostinfo().Services().All() { - if peerAddr.Is4() && s.Proto == tailcfg.PeerAPI4 || - peerAddr.Is6() && s.Proto == tailcfg.PeerAPI6 { - return netip.AddrPortFrom(peerAddr, s.Port) + if selfAddr.BitLen() == maybeCandidateAddr.BitLen() { // same address family + if filt.CapsWithValues(maybeCandidateAddr, selfAddr).HasCapability(tailcfg.PeerCapabilityRelayTarget) { + for _, s := range maybeCandidate.Hostinfo().Services().All() { + if maybeCandidateAddr.Is4() && s.Proto == tailcfg.PeerAPI4 || + maybeCandidateAddr.Is6() && s.Proto == tailcfg.PeerAPI6 { + return netip.AddrPortFrom(maybeCandidateAddr, s.Port) } } return netip.AddrPort{} // no peerAPI @@ -2674,10 +2676,11 @@ func peerAPIIfCandidateRelayServer(filt *filter.Filter, self, peer tailcfg.NodeV // [nodeBackend.peerCapsLocked] only returns/considers the // [tailcfg.PeerCapMap] between the passed src and the // _first_ host (/32 or /128) address for self. We are - // consistent with that behavior here. If self and peer - // host addresses are of the same address family they either - // have the capability or not. We do not check against - // additional host addresses of the same address family. + // consistent with that behavior here. If self and + // maybeCandidate host addresses are of the same address + // family they either have the capability or not. We do not + // check against additional host addresses of the same + // address family. return netip.AddrPort{} } } diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index c388e9ed15d00..aea2de17dc223 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -3385,16 +3385,7 @@ func Test_virtualNetworkID(t *testing.T) { } func Test_peerAPIIfCandidateRelayServer(t *testing.T) { - selfOnlyIPv4 := &tailcfg.Node{ - Cap: math.MinInt32, - Addresses: []netip.Prefix{ - netip.MustParsePrefix("1.1.1.1/32"), - }, - } - selfOnlyIPv6 := selfOnlyIPv4.Clone() - selfOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::1/128") - - peerHostinfo := &tailcfg.Hostinfo{ + hostInfo := &tailcfg.Hostinfo{ Services: []tailcfg.Service{ { Proto: tailcfg.PeerAPI4, @@ -3406,12 +3397,23 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, } + + selfOnlyIPv4 := &tailcfg.Node{ + Cap: math.MinInt32, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("1.1.1.1/32"), + }, + Hostinfo: hostInfo.View(), + } + selfOnlyIPv6 := selfOnlyIPv4.Clone() + selfOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::1/128") + peerOnlyIPv4 := &tailcfg.Node{ Cap: math.MinInt32, Addresses: []netip.Prefix{ netip.MustParsePrefix("2.2.2.2/32"), }, - Hostinfo: peerHostinfo.View(), + Hostinfo: hostInfo.View(), } peerOnlyIPv6 := peerOnlyIPv4.Clone() @@ -3424,11 +3426,11 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { peerOnlyIPv4NilHostinfo.Hostinfo = tailcfg.HostinfoView{} tests := []struct { - name string - filt *filter.Filter - self tailcfg.NodeView - peer tailcfg.NodeView - want netip.AddrPort + name string + filt *filter.Filter + self tailcfg.NodeView + maybeCandidate tailcfg.NodeView + want netip.AddrPort }{ { name: "match v4", @@ -3443,9 +3445,26 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv4.View(), - peer: peerOnlyIPv4.View(), - want: netip.MustParseAddrPort("2.2.2.2:4"), + self: selfOnlyIPv4.View(), + maybeCandidate: peerOnlyIPv4.View(), + want: netip.MustParseAddrPort("2.2.2.2:4"), + }, + { + name: "match v4 self", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{selfOnlyIPv4.Addresses[0]}, + Caps: []filtertype.CapMatch{ + { + Dst: selfOnlyIPv4.Addresses[0], + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfOnlyIPv4.View(), + maybeCandidate: selfOnlyIPv4.View(), + want: netip.AddrPortFrom(selfOnlyIPv4.Addresses[0].Addr(), 4), }, { name: "match v6", @@ -3460,9 +3479,26 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv6.View(), - peer: peerOnlyIPv6.View(), - want: netip.MustParseAddrPort("[::2]:6"), + self: selfOnlyIPv6.View(), + maybeCandidate: peerOnlyIPv6.View(), + want: netip.MustParseAddrPort("[::2]:6"), + }, + { + name: "match v6 self", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{selfOnlyIPv6.Addresses[0]}, + Caps: []filtertype.CapMatch{ + { + Dst: selfOnlyIPv6.Addresses[0], + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfOnlyIPv6.View(), + maybeCandidate: selfOnlyIPv6.View(), + want: netip.AddrPortFrom(selfOnlyIPv6.Addresses[0].Addr(), 6), }, { name: "no match dst", @@ -3477,8 +3513,8 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv6.View(), - peer: peerOnlyIPv6.View(), + self: selfOnlyIPv6.View(), + maybeCandidate: peerOnlyIPv6.View(), }, { name: "no match peer cap", @@ -3493,8 +3529,8 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv6.View(), - peer: peerOnlyIPv6.View(), + self: selfOnlyIPv6.View(), + maybeCandidate: peerOnlyIPv6.View(), }, { name: "cap ver not relay capable", @@ -3509,14 +3545,14 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: peerOnlyIPv4.View(), - peer: peerOnlyIPv4ZeroCapVer.View(), + self: peerOnlyIPv4.View(), + maybeCandidate: peerOnlyIPv4ZeroCapVer.View(), }, { - name: "nil filt", - filt: nil, - self: selfOnlyIPv4.View(), - peer: peerOnlyIPv4.View(), + name: "nil filt", + filt: nil, + self: selfOnlyIPv4.View(), + maybeCandidate: peerOnlyIPv4.View(), }, { name: "nil self", @@ -3531,8 +3567,8 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: tailcfg.NodeView{}, - peer: peerOnlyIPv4.View(), + self: tailcfg.NodeView{}, + maybeCandidate: peerOnlyIPv4.View(), }, { name: "nil peer", @@ -3547,8 +3583,8 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv4.View(), - peer: tailcfg.NodeView{}, + self: selfOnlyIPv4.View(), + maybeCandidate: tailcfg.NodeView{}, }, { name: "nil peer hostinfo", @@ -3563,13 +3599,13 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv4.View(), - peer: peerOnlyIPv4NilHostinfo.View(), + self: selfOnlyIPv4.View(), + maybeCandidate: peerOnlyIPv4NilHostinfo.View(), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := peerAPIIfCandidateRelayServer(tt.filt, tt.self, tt.peer); !reflect.DeepEqual(got, tt.want) { + if got := peerAPIIfCandidateRelayServer(tt.filt, tt.self, tt.maybeCandidate); !reflect.DeepEqual(got, tt.want) { t.Errorf("peerAPIIfCandidateRelayServer() = %v, want %v", got, tt.want) } }) From cc2f4ac921106ad46691b9271008f0bf43aeb970 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Wed, 9 Jul 2025 11:59:57 -0500 Subject: [PATCH 181/263] ipn: move ParseAutoExitNodeID from ipn/ipnlocal to ipn So it can be used from the CLI without importing ipnlocal. While there, also remove isAutoExitNodeID, a wrapper around parseAutoExitNodeID that's no longer used. Updates tailscale/corp#29969 Updates #cleanup Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 33 ++---------- ipn/ipnlocal/local_test.go | 104 ------------------------------------- ipn/prefs.go | 22 ++++++++ ipn/prefs_test.go | 59 +++++++++++++++++++++ 4 files changed, 84 insertions(+), 134 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index c54cb32d3125c..048bb1219c2f9 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1890,7 +1890,7 @@ func (b *LocalBackend) applyExitNodeSysPolicyLocked(prefs *ipn.Prefs) (anyChange // and update prefs if it differs from the current one. // This includes cases where it was previously an expression but no longer is, // or where it wasn't before but now is. - autoExitNode, useAutoExitNode := parseAutoExitNodeID(exitNodeID) + autoExitNode, useAutoExitNode := ipn.ParseAutoExitNodeString(exitNodeID) if prefs.AutoExitNode != autoExitNode { prefs.AutoExitNode = autoExitNode anyChange = true @@ -4292,7 +4292,7 @@ func (b *LocalBackend) SetUseExitNodeEnabled(actor ipnauth.Actor, v bool) (ipn.P if v { mp.ExitNodeIDSet = true mp.ExitNodeID = p0.InternalExitNodePrior() - if expr, ok := parseAutoExitNodeID(mp.ExitNodeID); ok { + if expr, ok := ipn.ParseAutoExitNodeString(mp.ExitNodeID); ok { mp.AutoExitNodeSet = true mp.AutoExitNode = expr mp.ExitNodeID = unresolvedExitNodeID @@ -4304,7 +4304,7 @@ func (b *LocalBackend) SetUseExitNodeEnabled(actor ipnauth.Actor, v bool) (ipn.P mp.AutoExitNode = "" mp.InternalExitNodePriorSet = true if p0.AutoExitNode().IsSet() { - mp.InternalExitNodePrior = tailcfg.StableNodeID(autoExitNodePrefix + p0.AutoExitNode()) + mp.InternalExitNodePrior = tailcfg.StableNodeID(ipn.AutoExitNodePrefix + p0.AutoExitNode()) } else { mp.InternalExitNodePrior = p0.ExitNodeID() } @@ -7933,10 +7933,6 @@ func longLatDistance(fromLat, fromLong, toLat, toLong float64) float64 { } const ( - // autoExitNodePrefix is the prefix used in [syspolicy.ExitNodeID] values - // to indicate that the string following the prefix is an [ipn.ExitNodeExpression]. - autoExitNodePrefix = "auto:" - // unresolvedExitNodeID is a special [tailcfg.StableNodeID] value // used as an exit node ID to install a blackhole route, preventing // accidental non-exit-node usage until the [ipn.ExitNodeExpression] @@ -7947,29 +7943,6 @@ const ( unresolvedExitNodeID tailcfg.StableNodeID = "auto:any" ) -// isAutoExitNodeID reports whether the given [tailcfg.StableNodeID] is -// actually an "auto:"-prefixed [ipn.ExitNodeExpression]. -func isAutoExitNodeID(id tailcfg.StableNodeID) bool { - _, ok := parseAutoExitNodeID(id) - return ok -} - -// parseAutoExitNodeID attempts to parse the given [tailcfg.StableNodeID] -// as an [ExitNodeExpression]. -// -// It returns the parsed expression and true on success, -// or an empty string and false if the input does not appear to be -// an [ExitNodeExpression] (i.e., it doesn't start with "auto:"). -// -// It is mainly used to parse the [syspolicy.ExitNodeID] value -// when it is set to "auto:" (e.g., auto:any). -func parseAutoExitNodeID(id tailcfg.StableNodeID) (_ ipn.ExitNodeExpression, ok bool) { - if expr, ok := strings.CutPrefix(string(id), autoExitNodePrefix); ok && expr != "" { - return ipn.ExitNodeExpression(expr), true - } - return "", false -} - func isAllowedAutoExitNodeID(exitNodeID tailcfg.StableNodeID) bool { if exitNodeID == "" { return false // an exit node is required diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 8bc84b081c016..73bae7ede8720 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -4873,110 +4873,6 @@ func TestMinLatencyDERPregion(t *testing.T) { } } -func TestShouldAutoExitNode(t *testing.T) { - tests := []struct { - name string - exitNodeIDPolicyValue string - expectedBool bool - }{ - { - name: "auto:any", - exitNodeIDPolicyValue: "auto:any", - expectedBool: true, - }, - { - name: "no auto prefix", - exitNodeIDPolicyValue: "foo", - expectedBool: false, - }, - { - name: "auto prefix but empty suffix", - exitNodeIDPolicyValue: "auto:", - expectedBool: false, - }, - { - name: "auto prefix no colon", - exitNodeIDPolicyValue: "auto", - expectedBool: false, - }, - { - name: "auto prefix unknown suffix", - exitNodeIDPolicyValue: "auto:foo", - expectedBool: true, // "auto:{unknown}" is treated as "auto:any" - }, - } - - syspolicy.RegisterWellKnownSettingsForTest(t) - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := isAutoExitNodeID(tailcfg.StableNodeID(tt.exitNodeIDPolicyValue)) - if got != tt.expectedBool { - t.Fatalf("expected %v got %v for %v policy value", tt.expectedBool, got, tt.exitNodeIDPolicyValue) - } - }) - } -} - -func TestParseAutoExitNodeID(t *testing.T) { - tests := []struct { - name string - exitNodeID string - wantOk bool - wantExpr ipn.ExitNodeExpression - }{ - { - name: "empty expr", - exitNodeID: "", - wantOk: false, - wantExpr: "", - }, - { - name: "no auto prefix", - exitNodeID: "foo", - wantOk: false, - wantExpr: "", - }, - { - name: "auto:any", - exitNodeID: "auto:any", - wantOk: true, - wantExpr: ipn.AnyExitNode, - }, - { - name: "auto:foo", - exitNodeID: "auto:foo", - wantOk: true, - wantExpr: "foo", - }, - { - name: "auto prefix but empty suffix", - exitNodeID: "auto:", - wantOk: false, - wantExpr: "", - }, - { - name: "auto prefix no colon", - exitNodeID: "auto", - wantOk: false, - wantExpr: "", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotExpr, gotOk := parseAutoExitNodeID(tailcfg.StableNodeID(tt.exitNodeID)) - if gotOk != tt.wantOk || gotExpr != tt.wantExpr { - if tt.wantOk { - t.Fatalf("got %v (%q); want %v (%q)", gotOk, gotExpr, tt.wantOk, tt.wantExpr) - } else { - t.Fatalf("got %v (%q); want false", gotOk, gotExpr) - } - } - }) - } -} - func TestEnableAutoUpdates(t *testing.T) { lb := newTestLocalBackend(t) diff --git a/ipn/prefs.go b/ipn/prefs.go index 77cea0493af16..71a80b1828760 100644 --- a/ipn/prefs.go +++ b/ipn/prefs.go @@ -1088,3 +1088,25 @@ const AnyExitNode ExitNodeExpression = "any" func (e ExitNodeExpression) IsSet() bool { return e != "" } + +const ( + // AutoExitNodePrefix is the prefix used in [syspolicy.ExitNodeID] values and CLI + // to indicate that the string following the prefix is an [ipn.ExitNodeExpression]. + AutoExitNodePrefix = "auto:" +) + +// ParseAutoExitNodeString attempts to parse the given string +// as an [ExitNodeExpression]. +// +// It returns the parsed expression and true on success, +// or an empty string and false if the input does not appear to be +// an [ExitNodeExpression] (i.e., it doesn't start with "auto:"). +// +// It is mainly used to parse the [syspolicy.ExitNodeID] value +// when it is set to "auto:" (e.g., auto:any). +func ParseAutoExitNodeString[T ~string](s T) (_ ExitNodeExpression, ok bool) { + if expr, ok := strings.CutPrefix(string(s), AutoExitNodePrefix); ok && expr != "" { + return ExitNodeExpression(expr), true + } + return "", false +} diff --git a/ipn/prefs_test.go b/ipn/prefs_test.go index 268ea206c137f..43e360c6af0c2 100644 --- a/ipn/prefs_test.go +++ b/ipn/prefs_test.go @@ -1129,3 +1129,62 @@ func TestPrefsDowngrade(t *testing.T) { t.Fatal("AllowSingleHosts should be true") } } + +func TestParseAutoExitNodeString(t *testing.T) { + tests := []struct { + name string + exitNodeID string + wantOk bool + wantExpr ExitNodeExpression + }{ + { + name: "empty expr", + exitNodeID: "", + wantOk: false, + wantExpr: "", + }, + { + name: "no auto prefix", + exitNodeID: "foo", + wantOk: false, + wantExpr: "", + }, + { + name: "auto:any", + exitNodeID: "auto:any", + wantOk: true, + wantExpr: AnyExitNode, + }, + { + name: "auto:foo", + exitNodeID: "auto:foo", + wantOk: true, + wantExpr: "foo", + }, + { + name: "auto prefix but empty suffix", + exitNodeID: "auto:", + wantOk: false, + wantExpr: "", + }, + { + name: "auto prefix no colon", + exitNodeID: "auto", + wantOk: false, + wantExpr: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotExpr, gotOk := ParseAutoExitNodeString(tt.exitNodeID) + if gotOk != tt.wantOk || gotExpr != tt.wantExpr { + if tt.wantOk { + t.Fatalf("got %v (%q); want %v (%q)", gotOk, gotExpr, tt.wantOk, tt.wantExpr) + } else { + t.Fatalf("got %v (%q); want false", gotOk, gotExpr) + } + } + }) + } +} From c5fdf9e1db149d5c205ec971ffe9ae6d487833a4 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Wed, 9 Jul 2025 12:07:44 -0500 Subject: [PATCH 182/263] cmd/tailscale/cli: add support for tailscale {up,set} --exit-node=auto:any If the specified exit node string starts with "auto:" (i.e., can be parsed as an ipn.ExitNodeExpression), we update ipn.Prefs.AutoExitNode instead of ipn.Prefs.ExitNodeID. Fixes #16459 Signed-off-by: Nick Khyl --- cmd/tailscale/cli/cli_test.go | 24 ++++++++++++++++++++++-- cmd/tailscale/cli/set.go | 7 +++++-- cmd/tailscale/cli/up.go | 9 +++++++-- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/cmd/tailscale/cli/cli_test.go b/cmd/tailscale/cli/cli_test.go index 48121c7d912d9..5dd4fa2340360 100644 --- a/cmd/tailscale/cli/cli_test.go +++ b/cmd/tailscale/cli/cli_test.go @@ -972,8 +972,7 @@ func TestPrefFlagMapping(t *testing.T) { // No CLI flag for this. continue case "AutoExitNode": - // TODO(nickkhyl): should be handled by tailscale {set,up} --exit-node. - // See tailscale/tailscale#16459. + // Handled by tailscale {set,up} --exit-node=auto:any. continue } t.Errorf("unexpected new ipn.Pref field %q is not handled by up.go (see addPrefFlagMapping and checkForAccidentalSettingReverts)", prefName) @@ -1338,6 +1337,27 @@ func TestUpdatePrefs(t *testing.T) { } }, }, + { + name: "auto_exit_node", + flags: []string{"--exit-node=auto:any"}, + curPrefs: &ipn.Prefs{ + ControlURL: ipn.DefaultControlURL, + CorpDNS: true, // enabled by [ipn.NewPrefs] by default + NetfilterMode: preftype.NetfilterOn, // enabled by [ipn.NewPrefs] by default + }, + wantJustEditMP: &ipn.MaskedPrefs{ + WantRunningSet: true, // enabled by default for tailscale up + AutoExitNodeSet: true, + ExitNodeIDSet: true, // we want ExitNodeID cleared + ExitNodeIPSet: true, // same for ExitNodeIP + }, + env: upCheckEnv{backendState: "Running"}, + checkUpdatePrefsMutations: func(t *testing.T, newPrefs *ipn.Prefs) { + if newPrefs.AutoExitNode != ipn.AnyExitNode { + t.Errorf("AutoExitNode: got %q; want %q", newPrefs.AutoExitNode, ipn.AnyExitNode) + } + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/tailscale/cli/set.go b/cmd/tailscale/cli/set.go index 66e74d77ff5a5..f1b21995ec388 100644 --- a/cmd/tailscale/cli/set.go +++ b/cmd/tailscale/cli/set.go @@ -73,7 +73,7 @@ func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet { setf.StringVar(&setArgs.profileName, "nickname", "", "nickname for the current account") setf.BoolVar(&setArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes") setf.BoolVar(&setArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel") - setf.StringVar(&setArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP or base name) for internet traffic, or empty string to not use an exit node") + setf.StringVar(&setArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP, base name, or auto:any) for internet traffic, or empty string to not use an exit node") setf.BoolVar(&setArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node") setf.BoolVar(&setArgs.shieldsUp, "shields-up", false, "don't allow incoming connections") setf.BoolVar(&setArgs.runSSH, "ssh", false, "run an SSH server, permitting access per tailnet admin's declared policy") @@ -173,7 +173,10 @@ func runSet(ctx context.Context, args []string) (retErr error) { } if setArgs.exitNodeIP != "" { - if err := maskedPrefs.Prefs.SetExitNodeIP(setArgs.exitNodeIP, st); err != nil { + if expr, useAutoExitNode := ipn.ParseAutoExitNodeString(setArgs.exitNodeIP); useAutoExitNode { + maskedPrefs.AutoExitNode = expr + maskedPrefs.AutoExitNodeSet = true + } else if err := maskedPrefs.Prefs.SetExitNodeIP(setArgs.exitNodeIP, st); err != nil { var e ipn.ExitNodeLocalIPError if errors.As(err, &e) { return fmt.Errorf("%w; did you mean --advertise-exit-node?", err) diff --git a/cmd/tailscale/cli/up.go b/cmd/tailscale/cli/up.go index 37cdab754db18..1863957d3f143 100644 --- a/cmd/tailscale/cli/up.go +++ b/cmd/tailscale/cli/up.go @@ -100,7 +100,7 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet { upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes") upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel") upf.Var(notFalseVar{}, "host-routes", hidden+"install host routes to other Tailscale nodes (must be true as of Tailscale 1.67+)") - upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP or base name) for internet traffic, or empty string to not use an exit node") + upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP, base name, or auto:any) for internet traffic, or empty string to not use an exit node") upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node") upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections") upf.BoolVar(&upArgs.runSSH, "ssh", false, "run an SSH server, permitting access per tailnet admin's declared policy") @@ -278,7 +278,9 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo prefs.NetfilterMode = preftype.NetfilterOff } if upArgs.exitNodeIP != "" { - if err := prefs.SetExitNodeIP(upArgs.exitNodeIP, st); err != nil { + if expr, useAutoExitNode := ipn.ParseAutoExitNodeString(upArgs.exitNodeIP); useAutoExitNode { + prefs.AutoExitNode = expr + } else if err := prefs.SetExitNodeIP(upArgs.exitNodeIP, st); err != nil { var e ipn.ExitNodeLocalIPError if errors.As(err, &e) { return nil, fmt.Errorf("%w; did you mean --advertise-exit-node?", err) @@ -408,6 +410,9 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus if env.upArgs.reset { visitFlags = env.flagSet.VisitAll } + if prefs.AutoExitNode.IsSet() { + justEditMP.AutoExitNodeSet = true + } visitFlags(func(f *flag.Flag) { updateMaskedPrefsFromUpOrSetFlag(justEditMP, f.Name) }) From 21a4058ec71878373d68ef6c983e81dda690e441 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Tue, 8 Jul 2025 18:35:32 -0500 Subject: [PATCH 183/263] ipn/ipnlocal: add test to verify handling of unknown auto exit node expressions We already check this for cases where ipn.Prefs.AutoExitNode is configured via syspolicy. Configuring it directly through EditPrefs should behave the same, so we add a test for that as well. Additionally, we clarify the implementation and future extensibility in (*LocalBackend).resolveAutoExitNodeLocked, where the AutoExitNode is actually enforced. Updates tailscale/corp#29969 Updates #cleanup Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 8 ++++++++ ipn/ipnlocal/local_test.go | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 048bb1219c2f9..55730489e0d4a 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -2071,6 +2071,14 @@ func mutationsAreWorthyOfTellingIPNBus(muts []netmap.NodeMutation) bool { // // b.mu must be held. func (b *LocalBackend) resolveAutoExitNodeLocked(prefs *ipn.Prefs) (prefsChanged bool) { + // As of 2025-07-08, the only supported auto exit node expression is [ipn.AnyExitNode]. + // + // However, to maintain forward compatibility with future auto exit node expressions, + // we treat any non-empty AutoExitNode as [ipn.AnyExitNode]. + // + // If and when we support additional auto exit node expressions, this method should be updated + // to handle them appropriately, while still falling back to [ipn.AnyExitNode] or a more appropriate + // default for unknown (or partially supported) expressions. if !prefs.AutoExitNode.IsSet() { return false } diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 73bae7ede8720..e70e5ad2afe58 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -1002,6 +1002,23 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "foo", }, }, + { + name: "auto-foo-via-edit-prefs", // set auto exit node via EditPrefs with an unknown/unsupported expression + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "foo"}, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" + AutoExitNode: "foo", + }, + }, { name: "auto-any-via-policy/toggle-off", // cannot toggle off the exit node if it was set via syspolicy prefs: ipn.Prefs{ From ff1803158a60c37128557f40c643f3839bc5609a Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Wed, 9 Jul 2025 13:01:32 -0500 Subject: [PATCH 184/263] ipn/ipnlocal: change order of exit node refresh and netmap update so that clients receive the new netmap first If the GUI receives a new exit node ID before the new netmap, it may treat the node as offline or invalid if the previous netmap didn't include the peer at all, or if the peer was offline or not advertised as an exit node. This may result in briefly issuing and dismissing a warning, or a similar issue, which isn't ideal. In this PR, we change the operation order to send the new netmap to clients first before selecting the new exit node and notifying them of the Exit Node change. Updates tailscale/corp#30252 (an old issue discovered during testing this) Signed-off-by: Nick Khyl --- ipn/ipnlocal/local.go | 13 ++++++--- ipn/ipnlocal/local_test.go | 54 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 55730489e0d4a..48eceb36c1ab4 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -1709,9 +1709,6 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control // Now complete the lock-free parts of what we started while locked. if st.NetMap != nil { - // Check and update the exit node if needed, now that we have a new netmap. - b.RefreshExitNode() - if envknob.NoLogsNoSupport() && st.NetMap.HasCap(tailcfg.CapabilityDataPlaneAuditLogs) { msg := "tailnet requires logging to be enabled. Remove --no-logs-no-support from tailscaled command line." b.health.SetLocalLogConfigHealth(errors.New(msg)) @@ -1751,6 +1748,16 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control b.health.SetDERPMap(st.NetMap.DERPMap) b.send(ipn.Notify{NetMap: st.NetMap}) + + // Check and update the exit node if needed, now that we have a new netmap. + // + // This must happen after the netmap change is sent via [ipn.Notify], + // so the GUI can correctly display the exit node if it has changed + // since the last netmap was sent. + // + // Otherwise, it might briefly show the exit node as offline and display a warning, + // if the node wasn't online or wasn't advertising default routes in the previous netmap. + b.RefreshExitNode() } if st.URL != "" { b.logf("Received auth URL: %.20v...", st.URL) diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index e70e5ad2afe58..bb7f433c02cbe 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -1407,6 +1407,60 @@ func TestPrefsChangeDisablesExitNode(t *testing.T) { } } +func TestExitNodeNotifyOrder(t *testing.T) { + const controlURL = "https://localhost:1/" + + report := &netcheck.Report{ + RegionLatency: map[int]time.Duration{ + 1: 5 * time.Millisecond, + 2: 10 * time.Millisecond, + }, + PreferredDERP: 1, + } + + exitNode1 := makeExitNode(1, withName("node-1"), withDERP(1), withAddresses(netip.MustParsePrefix("100.64.1.1/32"))) + exitNode2 := makeExitNode(2, withName("node-2"), withDERP(2), withAddresses(netip.MustParsePrefix("100.64.1.2/32"))) + selfNode := makeExitNode(3, withName("node-3"), withDERP(1), withAddresses(netip.MustParsePrefix("100.64.1.3/32"))) + clientNetmap := buildNetmapWithPeers(selfNode, exitNode1, exitNode2) + + lb := newTestLocalBackend(t) + lb.sys.MagicSock.Get().SetLastNetcheckReportForTest(lb.ctx, report) + lb.SetPrefsForTest(&ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: ipn.AnyExitNode, + }) + + nw := newNotificationWatcher(t, lb, ipnauth.Self) + + // Updating the netmap should trigger both a netmap notification + // and an exit node ID notification (since an exit node is selected). + // The netmap notification should be sent first. + nw.watch(0, []wantedNotification{ + wantNetmapNotify(clientNetmap), + wantExitNodeIDNotify(exitNode1.StableID()), + }) + lb.SetControlClientStatus(lb.cc, controlclient.Status{NetMap: clientNetmap}) + nw.check() +} + +func wantNetmapNotify(want *netmap.NetworkMap) wantedNotification { + return wantedNotification{ + name: "Netmap", + cond: func(t testing.TB, _ ipnauth.Actor, n *ipn.Notify) bool { + return n.NetMap == want + }, + } +} + +func wantExitNodeIDNotify(want tailcfg.StableNodeID) wantedNotification { + return wantedNotification{ + name: fmt.Sprintf("ExitNodeID-%s", want), + cond: func(_ testing.TB, _ ipnauth.Actor, n *ipn.Notify) bool { + return n.Prefs != nil && n.Prefs.Valid() && n.Prefs.ExitNodeID() == want + }, + } +} + func TestInternalAndExternalInterfaces(t *testing.T) { type interfacePrefix struct { i netmon.Interface From d40b25326ccdb111ce4e99893164dd6742328a52 Mon Sep 17 00:00:00 2001 From: Dylan Bargatze Date: Wed, 9 Jul 2025 18:06:58 -0400 Subject: [PATCH 185/263] tailcfg, wgengine/magicsock: disable all UDP relay usage if disable-relay-client is set (#16492) If the NodeAttrDisableRelayClient node attribute is set, ensures that a node cannot allocate endpoints on a UDP relay server itself, and cannot use newly-discovered paths (via disco/CallMeMaybeVia) that traverse a UDP relay server. Fixes tailscale/corp#30180 Signed-off-by: Dylan Bargatze --- tailcfg/tailcfg.go | 18 ++++++++++-------- wgengine/magicsock/magicsock.go | 10 +++++++++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index d97f60a8acb84..6c88217de01f4 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2607,14 +2607,16 @@ const ( // only needs to be present in [NodeCapMap] to take effect. NodeAttrDisableRelayServer NodeCapability = "disable-relay-server" - // NodeAttrDisableRelayClient prevents the node from allocating UDP relay - // server endpoints itself; the node may still bind into and relay traffic - // using endpoints allocated by its peers. This attribute can be added to - // the node dynamically; if added while the node is already running, the - // node will be unable to allocate UDP relay server endpoints after it next - // updates its network map. There are no expected values for this key in - // [NodeCapMap]; the key only needs to be present in [NodeCapMap] to take - // effect. + // NodeAttrDisableRelayClient prevents the node from both allocating UDP + // relay server endpoints itself, and from using endpoints allocated by + // its peers. This attribute can be added to the node dynamically; if added + // while the node is already running, the node will be unable to allocate + // endpoints after it next updates its network map, and will be immediately + // unable to use new paths via a UDP relay server. Setting this attribute + // dynamically does not remove any existing paths, including paths that + // traverse a UDP relay server. There are no expected values for this key + // in [NodeCapMap]; the key only needs to be present in [NodeCapMap] to + // take effect. NodeAttrDisableRelayClient NodeCapability = "disable-relay-client" // NodeAttrMagicDNSPeerAAAA is a capability that tells the node's MagicDNS diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 1978867fa5e1c..582e74c8b3120 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -355,7 +355,7 @@ type Conn struct { self tailcfg.NodeView // from last onNodeViewsUpdate peers views.Slice[tailcfg.NodeView] // from last onNodeViewsUpdate, sorted by Node.ID; Note: [netmap.NodeMutation]'s rx'd in onNodeMutationsUpdate are never applied filt *filter.Filter // from last onFilterUpdate - relayClientEnabled bool // whether we can allocate UDP relay endpoints on UDP relay servers + relayClientEnabled bool // whether we can allocate UDP relay endpoints on UDP relay servers or receive CallMeMaybeVia messages from peers lastFlags debugFlags // at time of last onNodeViewsUpdate privateKey key.NodePrivate // WireGuard private key for this node everHadKey bool // whether we ever had a non-zero private key @@ -2149,6 +2149,14 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake c.logf("magicsock: disco: ignoring %s from %v; %v is unknown", msgType, sender.ShortString(), derpNodeSrc.ShortString()) return } + // If the "disable-relay-client" node attr is set for this node, it + // can't be a UDP relay client, so drop any CallMeMaybeVia messages it + // receives. + if isVia && !c.relayClientEnabled { + c.logf("magicsock: disco: ignoring %s from %v; disable-relay-client node attr is set", msgType, sender.ShortString()) + return + } + ep.mu.Lock() relayCapable := ep.relayCapable lastBest := ep.bestAddr From ae8641735df2844b4d5d0abcd25c074d297a013d Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 9 Jul 2025 15:17:51 -0700 Subject: [PATCH 186/263] cmd/tailscale/cli,ipn/ipnstate,wgengine/magicsock: label peer-relay (#16510) Updates tailscale/corp#30033 Signed-off-by: Jordan Whited --- cmd/tailscale/cli/status.go | 4 +++- ipn/ipnstate/ipnstate.go | 10 +++++++--- wgengine/magicsock/endpoint.go | 9 +++++---- wgengine/magicsock/magicsock.go | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/cmd/tailscale/cli/status.go b/cmd/tailscale/cli/status.go index e4dccc247fd54..85679a7decbc1 100644 --- a/cmd/tailscale/cli/status.go +++ b/cmd/tailscale/cli/status.go @@ -183,10 +183,12 @@ func runStatus(ctx context.Context, args []string) error { } else if ps.ExitNodeOption { f("offers exit node; ") } - if relay != "" && ps.CurAddr == "" { + if relay != "" && ps.CurAddr == "" && ps.PeerRelay == "" { f("relay %q", relay) } else if ps.CurAddr != "" { f("direct %s", ps.CurAddr) + } else if ps.PeerRelay != "" { + f("peer-relay %s", ps.PeerRelay) } if !ps.Online { f("; offline") diff --git a/ipn/ipnstate/ipnstate.go b/ipn/ipnstate/ipnstate.go index 89c6d7e24dbc5..fdfd4e3346958 100644 --- a/ipn/ipnstate/ipnstate.go +++ b/ipn/ipnstate/ipnstate.go @@ -251,9 +251,10 @@ type PeerStatus struct { PrimaryRoutes *views.Slice[netip.Prefix] `json:",omitempty"` // Endpoints: - Addrs []string - CurAddr string // one of Addrs, or unique if roaming - Relay string // DERP region + Addrs []string + CurAddr string // one of Addrs, or unique if roaming + Relay string // DERP region + PeerRelay string // peer relay address (ip:port:vni) RxBytes int64 TxBytes int64 @@ -451,6 +452,9 @@ func (sb *StatusBuilder) AddPeer(peer key.NodePublic, st *PeerStatus) { if v := st.Relay; v != "" { e.Relay = v } + if v := st.PeerRelay; v != "" { + e.PeerRelay = v + } if v := st.UserID; v != 0 { e.UserID = v } diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 4780c7f37a781..bfafec5ab6297 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1961,10 +1961,11 @@ func (de *endpoint) populatePeerStatus(ps *ipnstate.PeerStatus) { ps.Active = now.Sub(de.lastSendExt) < sessionActiveTimeout if udpAddr, derpAddr, _ := de.addrForSendLocked(now); udpAddr.ap.IsValid() && !derpAddr.IsValid() { - // TODO(jwhited): if udpAddr.vni.isSet() we are using a Tailscale client - // as a UDP relay; update PeerStatus and its interpretation by - // "tailscale status" to make this clear. - ps.CurAddr = udpAddr.String() + if udpAddr.vni.isSet() { + ps.PeerRelay = udpAddr.String() + } else { + ps.CurAddr = udpAddr.String() + } } } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 582e74c8b3120..8743fe9cc574d 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3437,7 +3437,7 @@ func (c *Conn) onNodeMutationsUpdate(update NodeMutationsUpdate) { } } -// UpdateStatus implements the interface nede by ipnstate.StatusBuilder. +// UpdateStatus implements the interface needed by ipnstate.StatusBuilder. // // This method adds in the magicsock-specific information only. Most // of the status is otherwise populated by LocalBackend. From 6a0fad1e1044f567cb4d00608d1b1f00cef954c1 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 9 Jul 2025 20:02:00 -0700 Subject: [PATCH 187/263] wgengine/magicsock: don't peer relay if NodeAttrOnlyTCP443 is set (#16517) Updates tailscale/corp#30138 Signed-off-by: Jordan Whited --- wgengine/magicsock/magicsock.go | 1 + 1 file changed, 1 insertion(+) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 8743fe9cc574d..18a6bbceb4788 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2704,6 +2704,7 @@ func (c *Conn) onNodeViewsUpdate(update NodeViewsUpdate) { relayClientEnabled := update.SelfNode.Valid() && !update.SelfNode.HasCap(tailcfg.NodeAttrDisableRelayClient) && + !update.SelfNode.HasCap(tailcfg.NodeAttrOnlyTCP443) && envknob.UseWIPCode() c.mu.Lock() From fbc4c34cf7377f4ddb1d95163085e2b27c845018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus=20Lensb=C3=B8l?= Date: Thu, 10 Jul 2025 03:04:29 -0400 Subject: [PATCH 188/263] ipn/localapi: do not break client on event marshalling errors (#16503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Errors were mashalled without the correct newlines. Also, they could generally be mashalled with more data, so an intermediate was introduced to make them slightly nicer to look at. Updates #15160 Signed-off-by: Claus Lensbøl --- ipn/localapi/localapi.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index 60ed89b3b2ad3..cd59c54e05489 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -919,6 +919,11 @@ func (h *Handler) serveDebugPortmap(w http.ResponseWriter, r *http.Request) { } } +// EventError provides the JSON encoding of internal errors from event processing. +type EventError struct { + Error string +} + // serveDebugBusEvents taps into the tailscaled/utils/eventbus and streams // events to the client. func (h *Handler) serveDebugBusEvents(w http.ResponseWriter, r *http.Request) { @@ -971,7 +976,16 @@ func (h *Handler) serveDebugBusEvents(w http.ResponseWriter, r *http.Request) { } if msg, err := json.Marshal(data); err != nil { - fmt.Fprintf(w, `{"Event":"[ERROR] failed to marshal JSON for %T"}\n`, event.Event) + data.Event = EventError{Error: fmt.Sprintf( + "failed to marshal JSON for %T", event.Event, + )} + if errMsg, err := json.Marshal(data); err != nil { + fmt.Fprintf(w, + `{"Count": %d, "Event":"[ERROR] failed to marshal JSON for %T\n"}`, + i, event.Event) + } else { + w.Write(errMsg) + } } else { w.Write(msg) } From cf0460b9da23f70fb8442baa0a1bca1df32ba2c1 Mon Sep 17 00:00:00 2001 From: David Bond Date: Thu, 10 Jul 2025 14:33:13 +0100 Subject: [PATCH 189/263] cmd/k8s-operator: allow letsencrypt staging on k8s proxies (#16521) This commit modifies the operator to detect the usage of k8s-apiserver type proxy groups that wish to use the letsencrypt staging directory and apply the appropriate environment variable to the statefulset it produces. Updates #13358 Signed-off-by: David Bond --- cmd/k8s-operator/sts.go | 23 +++++++++++++++-------- cmd/k8s-operator/sts_test.go | 5 +++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index fbb271800390a..df12554e0feca 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -761,14 +761,21 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, enableEndpoints(ss, metricsEnabled, debugEnabled) } } - if pc.Spec.UseLetsEncryptStagingEnvironment && (stsCfg.proxyType == proxyTypeIngressResource || stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress)) { - for i, c := range ss.Spec.Template.Spec.Containers { - if isMainContainer(&c) { - ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ - Name: "TS_DEBUG_ACME_DIRECTORY_URL", - Value: letsEncryptStagingEndpoint, - }) - break + + if stsCfg != nil { + usesLetsEncrypt := stsCfg.proxyType == proxyTypeIngressResource || + stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress) || + stsCfg.proxyType == string(tsapi.ProxyGroupTypeKubernetesAPIServer) + + if pc.Spec.UseLetsEncryptStagingEnvironment && usesLetsEncrypt { + for i, c := range ss.Spec.Template.Spec.Containers { + if isMainContainer(&c) { + ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ + Name: "TS_DEBUG_ACME_DIRECTORY_URL", + Value: letsEncryptStagingEndpoint, + }) + break + } } } } diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index 35c512c8cd05b..afa791ccc7904 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -61,6 +61,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { // Setup proxyClassAllOpts := &tsapi.ProxyClass{ Spec: tsapi.ProxyClassSpec{ + UseLetsEncryptStagingEnvironment: true, StatefulSet: &tsapi.StatefulSet{ Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"foo.io/bar": "foo"}, @@ -292,6 +293,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff) } + + // 8. A Kubernetes API proxy with letsencrypt staging enabled + gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), &tailscaleSTSConfig{proxyType: string(tsapi.ProxyGroupTypeKubernetesAPIServer)}, zl.Sugar()) + verifyEnvVar(t, gotSS, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint) } func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { From 2b665c370c50a85f65edf4b9cb15c41bc45a8008 Mon Sep 17 00:00:00 2001 From: David Bond Date: Thu, 10 Jul 2025 14:33:30 +0100 Subject: [PATCH 190/263] cmd/{k8s-operator,k8s-proxy}: allow setting login server url (#16504) This commit modifies the k8s proxy application configuration to include a new field named `ServerURL` which, when set, modifies the tailscale coordination server used by the proxy. This works in the same way as the operator and the proxies it deploys. If unset, the default coordination server is used. Updates https://github.com/tailscale/tailscale/issues/13358 Signed-off-by: David Bond --- cmd/k8s-operator/proxygroup.go | 5 +++++ cmd/k8s-proxy/k8s-proxy.go | 5 +++++ kube/k8s-proxy/conf/conf.go | 1 + 3 files changed, 11 insertions(+) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 3dfb004f1dd36..7b8a0754e6d0b 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -815,6 +815,11 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p }, }, } + + if r.loginServer != "" { + cfg.ServerURL = &r.loginServer + } + cfgB, err := json.Marshal(cfg) if err != nil { return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err) diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go index 6e7eadb7303b5..81a5a8483af26 100644 --- a/cmd/k8s-proxy/k8s-proxy.go +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -91,6 +91,11 @@ func run(logger *zap.SugaredLogger) error { Store: st, AuthKey: authKey, } + + if cfg.Parsed.ServerURL != nil { + ts.ControlURL = *cfg.Parsed.ServerURL + } + if cfg.Parsed.Hostname != nil { ts.Hostname = *cfg.Parsed.Hostname } diff --git a/kube/k8s-proxy/conf/conf.go b/kube/k8s-proxy/conf/conf.go index 6b0e853c5c21c..2901e7b44852e 100644 --- a/kube/k8s-proxy/conf/conf.go +++ b/kube/k8s-proxy/conf/conf.go @@ -53,6 +53,7 @@ type ConfigV1Alpha1 struct { LogLevel *string `json:",omitempty"` // "debug", "info". Defaults to "info". App *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer KubeAPIServer *KubeAPIServer `json:",omitempty"` // Config specific to the API Server proxy. + ServerURL *string `json:",omitempty"` // URL of the Tailscale coordination server. } type KubeAPIServer struct { From d0cafc0a6776397d9a346dde60962c679062a21c Mon Sep 17 00:00:00 2001 From: David Bond Date: Thu, 10 Jul 2025 15:53:01 +0100 Subject: [PATCH 191/263] cmd/{k8s-operator,k8s-proxy}: apply accept-routes configuration to k8s-proxy (#16522) This commit modifies the k8s-operator and k8s-proxy to support passing down the accept-routes configuration from the proxy class as a configuration value read and used by the k8s-proxy when ran as a distinct container managed by the operator. Updates #13358 Signed-off-by: David Bond --- cmd/k8s-operator/proxygroup.go | 4 ++++ cmd/k8s-proxy/k8s-proxy.go | 19 +++++++++++++++---- kube/k8s-proxy/conf/conf.go | 1 + 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 7b8a0754e6d0b..66b6c96e3c25c 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -820,6 +820,10 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p cfg.ServerURL = &r.loginServer } + if proxyClass != nil && proxyClass.Spec.TailscaleConfig != nil { + cfg.AcceptRoutes = &proxyClass.Spec.TailscaleConfig.AcceptRoutes + } + cfgB, err := json.Marshal(cfg) if err != nil { return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err) diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go index 81a5a8483af26..7dcf6c2ab5809 100644 --- a/cmd/k8s-proxy/k8s-proxy.go +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -114,12 +114,13 @@ func run(logger *zap.SugaredLogger) error { group, groupCtx := errgroup.WithContext(ctx) + lc, err := ts.LocalClient() + if err != nil { + return fmt.Errorf("error getting local client: %w", err) + } + // Setup for updating state keys. if podUID != "" { - lc, err := ts.LocalClient() - if err != nil { - return fmt.Errorf("error getting local client: %w", err) - } w, err := lc.WatchIPNBus(groupCtx, ipn.NotifyInitialNetMap) if err != nil { return fmt.Errorf("error watching IPN bus: %w", err) @@ -135,6 +136,16 @@ func run(logger *zap.SugaredLogger) error { }) } + if cfg.Parsed.AcceptRoutes != nil { + _, err = lc.EditPrefs(groupCtx, &ipn.MaskedPrefs{ + RouteAllSet: true, + Prefs: ipn.Prefs{RouteAll: *cfg.Parsed.AcceptRoutes}, + }) + if err != nil { + return fmt.Errorf("error editing prefs: %w", err) + } + } + // Setup for the API server proxy. restConfig, err := getRestConfig(logger) if err != nil { diff --git a/kube/k8s-proxy/conf/conf.go b/kube/k8s-proxy/conf/conf.go index 2901e7b44852e..fba4a39a420a1 100644 --- a/kube/k8s-proxy/conf/conf.go +++ b/kube/k8s-proxy/conf/conf.go @@ -54,6 +54,7 @@ type ConfigV1Alpha1 struct { App *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer KubeAPIServer *KubeAPIServer `json:",omitempty"` // Config specific to the API Server proxy. ServerURL *string `json:",omitempty"` // URL of the Tailscale coordination server. + AcceptRoutes *bool `json:",omitempty"` // Accepts routes advertised by other Tailscale nodes. } type KubeAPIServer struct { From f9bfd8118ae85b5782a29b442acb9ec3764caacb Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 10 Jul 2025 12:41:14 -0700 Subject: [PATCH 192/263] wgengine/magicsock: resolve epAddr collisions across peer relay conns (#16526) Updates tailscale/corp#30042 Updates tailscale/corp#29422 Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 9 +++-- wgengine/magicsock/magicsock.go | 57 +++++++++++++++++++++++----- wgengine/magicsock/magicsock_test.go | 40 +++++++++++++++++++ 3 files changed, 94 insertions(+), 12 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index bfafec5ab6297..c4ca812969bf9 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -499,8 +499,9 @@ func (de *endpoint) initFakeUDPAddr() { } // noteRecvActivity records receive activity on de, and invokes -// Conn.noteRecvActivity no more than once every 10s. -func (de *endpoint) noteRecvActivity(src epAddr, now mono.Time) { +// Conn.noteRecvActivity no more than once every 10s, returning true if it +// was called, otherwise false. +func (de *endpoint) noteRecvActivity(src epAddr, now mono.Time) bool { if de.isWireguardOnly { de.mu.Lock() de.bestAddr.ap = src.ap @@ -524,10 +525,12 @@ func (de *endpoint) noteRecvActivity(src epAddr, now mono.Time) { de.lastRecvWG.StoreAtomic(now) if de.c.noteRecvActivity == nil { - return + return false } de.c.noteRecvActivity(de.publicKey) + return true } + return false } func (de *endpoint) discoShort() string { diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 18a6bbceb4788..6ce91902d7425 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -27,6 +27,7 @@ import ( "time" "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/device" "go4.org/mem" "golang.org/x/net/ipv6" @@ -1632,6 +1633,16 @@ func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFu } } +// looksLikeInitiationMsg returns true if b looks like a WireGuard initiation +// message, otherwise it returns false. +func looksLikeInitiationMsg(b []byte) bool { + if len(b) == device.MessageInitiationSize && + binary.BigEndian.Uint32(b) == device.MessageInitiationType { + return true + } + return false +} + // receiveIP is the shared bits of ReceiveIPv4 and ReceiveIPv6. // // size is the length of 'b' to report up to wireguard-go (only relevant if @@ -1717,10 +1728,18 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach } now := mono.Now() ep.lastRecvUDPAny.StoreAtomic(now) - ep.noteRecvActivity(src, now) + connNoted := ep.noteRecvActivity(src, now) if stats := c.stats.Load(); stats != nil { stats.UpdateRxPhysical(ep.nodeAddr, ipp, 1, len(b)) } + if src.vni.isSet() && (connNoted || looksLikeInitiationMsg(b)) { + // connNoted is periodic, but we also want to verify if the peer is who + // we believe for all initiation messages, otherwise we could get + // unlucky and fail to JIT configure the "correct" peer. + // TODO(jwhited): relax this to include direct connections + // See http://go/corp/29422 & http://go/corp/30042 + return &lazyEndpoint{c: c, maybeEP: ep, src: src}, size, true + } return ep, size, true } @@ -3787,11 +3806,19 @@ func (c *Conn) SetLastNetcheckReportForTest(ctx context.Context, report *netchec // decrypts it. So we implement the [conn.InitiationAwareEndpoint] and // [conn.PeerAwareEndpoint] interfaces, to allow WireGuard to tell us who it is // later, just-in-time to configure the peer, and set the associated [epAddr] -// in the [peerMap]. Future receives on the associated [epAddr] will then be -// resolvable directly to an [*endpoint]. +// in the [peerMap]. Future receives on the associated [epAddr] will then +// resolve directly to an [*endpoint]. +// +// We also sometimes (see [Conn.receiveIP]) return a [*lazyEndpoint] to +// wireguard-go to verify an [epAddr] resolves to the [*endpoint] (maybeEP) we +// believe it to be, to resolve [epAddr] collisions across peers. [epAddr] +// collisions have a higher chance of occurrence for packets received over peer +// relays versus direct connections, as peer relay connections do not upsert +// into [peerMap] around disco packet reception, but direct connections do. type lazyEndpoint struct { - c *Conn - src epAddr + c *Conn + maybeEP *endpoint // or nil if unknown + src epAddr } var _ conn.InitiationAwareEndpoint = (*lazyEndpoint)(nil) @@ -3812,6 +3839,9 @@ var _ conn.Endpoint = (*lazyEndpoint)(nil) // wireguard-go peer (de)configuration. func (le *lazyEndpoint) InitiationMessagePublicKey(peerPublicKey [32]byte) { pubKey := key.NodePublicFromRaw32(mem.B(peerPublicKey[:])) + if le.maybeEP != nil && pubKey.Compare(le.maybeEP.publicKey) == 0 { + return + } le.c.mu.Lock() defer le.c.mu.Unlock() ep, ok := le.c.peerMap.endpointForNodeKey(pubKey) @@ -3821,6 +3851,11 @@ func (le *lazyEndpoint) InitiationMessagePublicKey(peerPublicKey [32]byte) { now := mono.Now() ep.lastRecvUDPAny.StoreAtomic(now) ep.noteRecvActivity(le.src, now) + // [ep.noteRecvActivity] may end up JIT configuring the peer, but we don't + // update [peerMap] as wireguard-go hasn't decrypted the initiation + // message yet. wireguard-go will call us below in [lazyEndpoint.FromPeer] + // if it successfully decrypts the message, at which point it's safe to + // insert le.src into the [peerMap] for ep. } func (le *lazyEndpoint) ClearSrc() {} @@ -3845,12 +3880,16 @@ func (le *lazyEndpoint) DstToBytes() []byte { } // FromPeer implements [conn.PeerAwareEndpoint]. We return a [*lazyEndpoint] in -// our [conn.ReceiveFunc]s when we are unable to identify the peer at WireGuard -// packet reception time, pre-decryption. If wireguard-go successfully decrypts -// the packet it calls us here, and we update our [peerMap] in order to -// associate le.src with peerPublicKey. +// [Conn.receiveIP] when we are unable to identify the peer at WireGuard +// packet reception time, pre-decryption, or we want wireguard-go to verify who +// we believe it to be (le.maybeEP). If wireguard-go successfully decrypts the +// packet it calls us here, and we update our [peerMap] to associate le.src with +// peerPublicKey. func (le *lazyEndpoint) FromPeer(peerPublicKey [32]byte) { pubKey := key.NodePublicFromRaw32(mem.B(peerPublicKey[:])) + if le.maybeEP != nil && pubKey.Compare(le.maybeEP.publicKey) == 0 { + return + } le.c.mu.Lock() defer le.c.mu.Unlock() ep, ok := le.c.peerMap.endpointForNodeKey(pubKey) diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index aea2de17dc223..0515162c72b9f 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -3611,3 +3611,43 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }) } } + +func Test_looksLikeInitiationMsg(t *testing.T) { + initMsg := make([]byte, device.MessageInitiationSize) + binary.BigEndian.PutUint32(initMsg, device.MessageInitiationType) + initMsgSizeTransportType := make([]byte, device.MessageInitiationSize) + binary.BigEndian.PutUint32(initMsgSizeTransportType, device.MessageTransportType) + tests := []struct { + name string + b []byte + want bool + }{ + { + name: "valid initiation", + b: initMsg, + want: true, + }, + { + name: "invalid message type field", + b: initMsgSizeTransportType, + want: false, + }, + { + name: "too small", + b: initMsg[:device.MessageInitiationSize-1], + want: false, + }, + { + name: "too big", + b: append(initMsg, 0), + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := looksLikeInitiationMsg(tt.b); got != tt.want { + t.Errorf("looksLikeInitiationMsg() = %v, want %v", got, tt.want) + } + }) + } +} From bebc796e6c124e090d01c4651fe79cac771a0b30 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Thu, 10 Jul 2025 12:45:05 -0700 Subject: [PATCH 193/263] ipn/ipnlocal: add traffic-steering nodecap (#16529) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To signal when a tailnet has the `traffic-steering` feature flag, Control will send a `traffic-steering` NodeCapability in netmap’s AllCaps. This patch adds `tailcfg.NodeAttrTrafficSteering` so that it can be used in the control plane. Future patches will implement the actual steering mechanisms. Updates tailscale/corp#29966 Signed-off-by: Simon Law --- tailcfg/tailcfg.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 6c88217de01f4..e55389f182f56 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2622,6 +2622,10 @@ const ( // NodeAttrMagicDNSPeerAAAA is a capability that tells the node's MagicDNS // server to answer AAAA queries about its peers. See tailscale/tailscale#1152. NodeAttrMagicDNSPeerAAAA NodeCapability = "magicdns-aaaa" + + // NodeAttrTrafficSteering configures the node to use the traffic + // steering subsystem for via routes. See tailscale/corp#29966. + NodeAttrTrafficSteering NodeCapability = "traffic-steering" ) // SetDNSRequest is a request to add a DNS record. From fbc6a9ec5a797d9a551e74a90bc96947825b7719 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 10 Jul 2025 11:14:08 -0700 Subject: [PATCH 194/263] all: detect JetKVM and specialize a handful of things for it Updates #16524 Change-Id: I183428de8c65d7155d82979d2d33f031c22e3331 Signed-off-by: Brad Fitzpatrick --- cmd/tailscaled/tailscaled.go | 15 +++++++-------- logpolicy/logpolicy.go | 3 +++ net/dns/manager_linux.go | 5 +++++ net/tstun/tun.go | 20 +++++++++++++++++++- net/tstun/tun_linux.go | 10 +++++++++- paths/paths.go | 32 ++++++++++++++++++++++++++++++++ paths/paths_unix.go | 3 +++ util/linuxfw/detector.go | 5 +++++ util/linuxfw/iptables_runner.go | 5 +++-- version/distro/distro.go | 3 +++ 10 files changed, 89 insertions(+), 12 deletions(-) diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index 3987b0c26927f..ab1590132ece6 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -257,14 +257,13 @@ func main() { // Only apply a default statepath when neither have been provided, so that a // user may specify only --statedir if they wish. if args.statepath == "" && args.statedir == "" { - if runtime.GOOS == "plan9" { - home, err := os.UserHomeDir() - if err != nil { - log.Fatalf("failed to get home directory: %v", err) - } - args.statedir = filepath.Join(home, "tailscale-state") - if err := os.MkdirAll(args.statedir, 0700); err != nil { - log.Fatalf("failed to create state directory: %v", err) + if paths.MakeAutomaticStateDir() { + d := paths.DefaultTailscaledStateDir() + if d != "" { + args.statedir = d + if err := os.MkdirAll(d, 0700); err != nil { + log.Fatalf("failed to create state directory: %v", err) + } } } else { args.statepath = paths.DefaultTailscaledStateFile() diff --git a/logpolicy/logpolicy.go b/logpolicy/logpolicy.go index b84528d7b76bd..f5c475712afe3 100644 --- a/logpolicy/logpolicy.go +++ b/logpolicy/logpolicy.go @@ -224,6 +224,9 @@ func LogsDir(logf logger.Logf) string { logf("logpolicy: using LocalAppData dir %v", dir) return dir case "linux": + if distro.Get() == distro.JetKVM { + return "/userdata/tailscale/var" + } // STATE_DIRECTORY is set by systemd 240+ but we support older // systems-d. For example, Ubuntu 18.04 (Bionic Beaver) is 237. systemdStateDir := os.Getenv("STATE_DIRECTORY") diff --git a/net/dns/manager_linux.go b/net/dns/manager_linux.go index 6bd368f50e330..643cc280af1e3 100644 --- a/net/dns/manager_linux.go +++ b/net/dns/manager_linux.go @@ -22,6 +22,7 @@ import ( "tailscale.com/types/logger" "tailscale.com/util/clientmetric" "tailscale.com/util/cmpver" + "tailscale.com/version/distro" ) type kv struct { @@ -38,6 +39,10 @@ var publishOnce sync.Once // // The health tracker may be nil; the knobs may be nil and are ignored on this platform. func NewOSConfigurator(logf logger.Logf, health *health.Tracker, _ *controlknobs.Knobs, interfaceName string) (ret OSConfigurator, err error) { + if distro.Get() == distro.JetKVM { + return NewNoopManager() + } + env := newOSConfigEnv{ fs: directFS{}, dbusPing: dbusPing, diff --git a/net/tstun/tun.go b/net/tstun/tun.go index 88679daa24b6c..bfdaddf58b283 100644 --- a/net/tstun/tun.go +++ b/net/tstun/tun.go @@ -24,6 +24,9 @@ import ( // CrateTAP is the hook set by feature/tap. var CreateTAP feature.Hook[func(logf logger.Logf, tapName, bridgeName string) (tun.Device, error)] +// modprobeTunHook is a Linux-specific hook to run "/sbin/modprobe tun". +var modprobeTunHook feature.Hook[func() error] + // New returns a tun.Device for the requested device name, along with // the OS-dependent name that was allocated to the device. func New(logf logger.Logf, tunName string) (tun.Device, string, error) { @@ -51,7 +54,22 @@ func New(logf logger.Logf, tunName string) (tun.Device, string, error) { if runtime.GOOS == "plan9" { cleanUpPlan9Interfaces() } - dev, err = tun.CreateTUN(tunName, int(DefaultTUNMTU())) + // Try to create the TUN device up to two times. If it fails + // the first time and we're on Linux, try a desperate + // "modprobe tun" to load the tun module and try again. + for try := range 2 { + dev, err = tun.CreateTUN(tunName, int(DefaultTUNMTU())) + if err == nil || !modprobeTunHook.IsSet() { + if try > 0 { + logf("created TUN device %q after doing `modprobe tun`", tunName) + } + break + } + if modprobeTunHook.Get()() != nil { + // modprobe failed; no point trying again. + break + } + } } if err != nil { return nil, "", err diff --git a/net/tstun/tun_linux.go b/net/tstun/tun_linux.go index 9600ceb77328f..05cf58c17df8a 100644 --- a/net/tstun/tun_linux.go +++ b/net/tstun/tun_linux.go @@ -17,6 +17,14 @@ import ( func init() { tunDiagnoseFailure = diagnoseLinuxTUNFailure + modprobeTunHook.Set(func() error { + _, err := modprobeTun() + return err + }) +} + +func modprobeTun() ([]byte, error) { + return exec.Command("/sbin/modprobe", "tun").CombinedOutput() } func diagnoseLinuxTUNFailure(tunName string, logf logger.Logf, createErr error) { @@ -36,7 +44,7 @@ func diagnoseLinuxTUNFailure(tunName string, logf logger.Logf, createErr error) kernel := utsReleaseField(&un) logf("Linux kernel version: %s", kernel) - modprobeOut, err := exec.Command("/sbin/modprobe", "tun").CombinedOutput() + modprobeOut, err := modprobeTun() if err == nil { logf("'modprobe tun' successful") // Either tun is currently loaded, or it's statically diff --git a/paths/paths.go b/paths/paths.go index 28c3be02a9c86..6c9c3fa6c9dea 100644 --- a/paths/paths.go +++ b/paths/paths.go @@ -6,6 +6,7 @@ package paths import ( + "log" "os" "path/filepath" "runtime" @@ -70,6 +71,37 @@ func DefaultTailscaledStateFile() string { return "" } +// DefaultTailscaledStateDir returns the default state directory +// to use for tailscaled, for use when the user provided neither +// a state directory or state file path to use. +// +// It returns the empty string if there's no reasonable default. +func DefaultTailscaledStateDir() string { + if runtime.GOOS == "plan9" { + home, err := os.UserHomeDir() + if err != nil { + log.Fatalf("failed to get home directory: %v", err) + } + return filepath.Join(home, "tailscale-state") + } + return filepath.Dir(DefaultTailscaledStateFile()) +} + +// MakeAutomaticStateDir reports whether the platform +// automatically creates the state directory for tailscaled +// when it's absent. +func MakeAutomaticStateDir() bool { + switch runtime.GOOS { + case "plan9": + return true + case "linux": + if distro.Get() == distro.JetKVM { + return true + } + } + return false +} + // MkStateDir ensures that dirPath, the daemon's configuration directory // containing machine keys etc, both exists and has the correct permissions. // We want it to only be accessible to the user the daemon is running under. diff --git a/paths/paths_unix.go b/paths/paths_unix.go index 50a8b7ca502f7..d317921d59cd9 100644 --- a/paths/paths_unix.go +++ b/paths/paths_unix.go @@ -21,6 +21,9 @@ func init() { } func statePath() string { + if runtime.GOOS == "linux" && distro.Get() == distro.JetKVM { + return "/userdata/tailscale/var/tailscaled.state" + } switch runtime.GOOS { case "linux", "illumos", "solaris": return "/var/lib/tailscale/tailscaled.state" diff --git a/util/linuxfw/detector.go b/util/linuxfw/detector.go index f3ee4aa0b84f0..fffa523afdcf4 100644 --- a/util/linuxfw/detector.go +++ b/util/linuxfw/detector.go @@ -23,6 +23,11 @@ func detectFirewallMode(logf logger.Logf, prefHint string) FirewallMode { hostinfo.SetFirewallMode("nft-gokrazy") return FirewallModeNfTables } + if distro.Get() == distro.JetKVM { + // JetKVM doesn't have iptables. + hostinfo.SetFirewallMode("nft-jetkvm") + return FirewallModeNfTables + } mode := envknob.String("TS_DEBUG_FIREWALL_MODE") // If the envknob isn't set, fall back to the pref suggested by c2n or diff --git a/util/linuxfw/iptables_runner.go b/util/linuxfw/iptables_runner.go index 9a6fc02248e62..78844065a4edd 100644 --- a/util/linuxfw/iptables_runner.go +++ b/util/linuxfw/iptables_runner.go @@ -688,8 +688,9 @@ func (i *iptablesRunner) DelMagicsockPortRule(port uint16, network string) error // IPTablesCleanUp removes all Tailscale added iptables rules. // Any errors that occur are logged to the provided logf. func IPTablesCleanUp(logf logger.Logf) { - if distro.Get() == distro.Gokrazy { - // Gokrazy uses nftables and doesn't have the "iptables" command. + switch distro.Get() { + case distro.Gokrazy, distro.JetKVM: + // These use nftables and don't have the "iptables" command. // Avoid log spam on cleanup. (#12277) return } diff --git a/version/distro/distro.go b/version/distro/distro.go index f7997e1d9f81b..dd5e0b21b2eb1 100644 --- a/version/distro/distro.go +++ b/version/distro/distro.go @@ -31,6 +31,7 @@ const ( Unraid = Distro("unraid") Alpine = Distro("alpine") UBNT = Distro("ubnt") // Ubiquiti Networks + JetKVM = Distro("jetkvm") ) var distro lazy.SyncValue[Distro] @@ -102,6 +103,8 @@ func linuxDistro() Distro { return Unraid case have("/etc/alpine-release"): return Alpine + case haveDir("/userdata/jetkvm") && haveDir("/sys/kernel/config/usb_gadget/jetkvm"): + return JetKVM } return "" } From fed72e2aa9d55620655ab1790036523cbae9956f Mon Sep 17 00:00:00 2001 From: Dylan Bargatze Date: Thu, 10 Jul 2025 18:22:25 -0400 Subject: [PATCH 195/263] cmd/tailscale, ipn/ipnstate, wgengine/magicsock: update ping output for peer relay (#16515) Updates the output for "tailscale ping" to indicate if a peer relay was traversed, just like the output for DERP or direct connections. Fixes tailscale/corp#30034 Signed-off-by: Dylan Bargatze --- cmd/tailscale/cli/ping.go | 4 +++- ipn/ipnstate/ipnstate.go | 12 ++++++++++-- tailcfg/tailcfg.go | 8 ++++++-- wgengine/magicsock/magicsock.go | 9 +++++---- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/cmd/tailscale/cli/ping.go b/cmd/tailscale/cli/ping.go index 3a909f30dee86..d438cb2286d4c 100644 --- a/cmd/tailscale/cli/ping.go +++ b/cmd/tailscale/cli/ping.go @@ -152,7 +152,9 @@ func runPing(ctx context.Context, args []string) error { } latency := time.Duration(pr.LatencySeconds * float64(time.Second)).Round(time.Millisecond) via := pr.Endpoint - if pr.DERPRegionID != 0 { + if pr.PeerRelay != "" { + via = fmt.Sprintf("peer-relay(%s)", pr.PeerRelay) + } else if pr.DERPRegionID != 0 { via = fmt.Sprintf("DERP(%s)", pr.DERPRegionCode) } if via == "" { diff --git a/ipn/ipnstate/ipnstate.go b/ipn/ipnstate/ipnstate.go index fdfd4e3346958..e7ae2d62bd6b2 100644 --- a/ipn/ipnstate/ipnstate.go +++ b/ipn/ipnstate/ipnstate.go @@ -701,10 +701,17 @@ type PingResult struct { Err string LatencySeconds float64 - // Endpoint is the ip:port if direct UDP was used. - // It is not currently set for TSMP pings. + // Endpoint is a string of the form "{ip}:{port}" if direct UDP was used. It + // is not currently set for TSMP. Endpoint string + // PeerRelay is a string of the form "{ip}:{port}:vni:{vni}" if a peer + // relay was used. It is not currently set for TSMP. Note that this field + // is not omitted during JSON encoding if it contains a zero value. This is + // done for consistency with the Endpoint field; this structure is exposed + // externally via localAPI, so we want to maintain the existing convention. + PeerRelay string + // DERPRegionID is non-zero DERP region ID if DERP was used. // It is not currently set for TSMP pings. DERPRegionID int @@ -739,6 +746,7 @@ func (pr *PingResult) ToPingResponse(pingType tailcfg.PingType) *tailcfg.PingRes Err: pr.Err, LatencySeconds: pr.LatencySeconds, Endpoint: pr.Endpoint, + PeerRelay: pr.PeerRelay, DERPRegionID: pr.DERPRegionID, DERPRegionCode: pr.DERPRegionCode, PeerAPIPort: pr.PeerAPIPort, diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index e55389f182f56..ab8add5b83a3e 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -1854,10 +1854,14 @@ type PingResponse struct { // omitted, Err should contain information as to the cause. LatencySeconds float64 `json:",omitempty"` - // Endpoint is the ip:port if direct UDP was used. - // It is not currently set for TSMP pings. + // Endpoint is a string of the form "{ip}:{port}" if direct UDP was used. It + // is not currently set for TSMP. Endpoint string `json:",omitempty"` + // PeerRelay is a string of the form "{ip}:{port}:vni:{vni}" if a peer + // relay was used. It is not currently set for TSMP. + PeerRelay string `json:",omitempty"` + // DERPRegionID is non-zero DERP region ID if DERP was used. // It is not currently set for TSMP pings. DERPRegionID int `json:",omitempty"` diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 6ce91902d7425..b5087b02e530e 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -1106,10 +1106,11 @@ func (c *Conn) Ping(peer tailcfg.NodeView, res *ipnstate.PingResult, size int, c func (c *Conn) populateCLIPingResponseLocked(res *ipnstate.PingResult, latency time.Duration, ep epAddr) { res.LatencySeconds = latency.Seconds() if ep.ap.Addr() != tailcfg.DerpMagicIPAddr { - // TODO(jwhited): if ep.vni.isSet() we are using a Tailscale client - // as a UDP relay; update PingResult and its interpretation by - // "tailscale ping" to make this clear. - res.Endpoint = ep.String() + if ep.vni.isSet() { + res.PeerRelay = ep.String() + } else { + res.Endpoint = ep.String() + } return } regionID := int(ep.ap.Port()) From 5f678b9becfbd13b3a5ec57c48fc4bd78d8353db Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Wed, 9 Jul 2025 17:41:55 -0500 Subject: [PATCH 196/263] docs/windows/policy: add ExitNode.AllowOverride as an option to ExitNodeID policy In this PR, we make ExitNode.AllowOverride configurable as part of the Exit Node ADMX policy setting, similarly to Always On w/ "Disconnect with reason" option. Updates tailscale/corp#29969 Signed-off-by: Nick Khyl --- docs/windows/policy/en-US/tailscale.adml | 4 +++- docs/windows/policy/tailscale.admx | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/windows/policy/en-US/tailscale.adml b/docs/windows/policy/en-US/tailscale.adml index c09d847bc7c0d..2e143d49c9c6c 100644 --- a/docs/windows/policy/en-US/tailscale.adml +++ b/docs/windows/policy/en-US/tailscale.adml @@ -23,6 +23,7 @@ Tailscale UI customization Settings + Allowed Allowed (with audit) Not Allowed Require using a specific Tailscale coordination server @@ -69,7 +70,7 @@ See https://tailscale.com/kb/1315/mdm-keys#set-an-auth-key for more details.]]>< Require using a specific Exit Node + User override:           Registration mode: diff --git a/docs/windows/policy/tailscale.admx b/docs/windows/policy/tailscale.admx index 0a8aa1a75eb50..0da8aef42ded6 100644 --- a/docs/windows/policy/tailscale.admx +++ b/docs/windows/policy/tailscale.admx @@ -115,6 +115,18 @@ + + + + + + + + + + + + From bd29a1c8c1000d620b26dcb31363c7b678463c2d Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 10 Jul 2025 18:52:01 -0700 Subject: [PATCH 197/263] feature/relayserver,wgengine/magicsock: remove WIP gating of peer relay (#16533) Updates tailscale/corp#30051 Signed-off-by: Jordan Whited --- feature/relayserver/relayserver.go | 4 ---- wgengine/magicsock/magicsock.go | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/feature/relayserver/relayserver.go b/feature/relayserver/relayserver.go index f4a533193999e..d0ad27624f09f 100644 --- a/feature/relayserver/relayserver.go +++ b/feature/relayserver/relayserver.go @@ -14,7 +14,6 @@ import ( "sync" "time" - "tailscale.com/envknob" "tailscale.com/feature" "tailscale.com/ipn" "tailscale.com/ipn/ipnext" @@ -133,9 +132,6 @@ func (e *extension) relayServerOrInit() (relayServer, error) { if e.hasNodeAttrDisableRelayServer { return nil, errors.New("disable-relay-server node attribute is present") } - if !envknob.UseWIPCode() { - return nil, errors.New("TAILSCALE_USE_WIP_CODE envvar is not set") - } var err error e.server, err = udprelay.NewServer(e.logf, *e.port, nil) if err != nil { diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index b5087b02e530e..14feed32b5929 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -2724,8 +2724,7 @@ func (c *Conn) onNodeViewsUpdate(update NodeViewsUpdate) { relayClientEnabled := update.SelfNode.Valid() && !update.SelfNode.HasCap(tailcfg.NodeAttrDisableRelayClient) && - !update.SelfNode.HasCap(tailcfg.NodeAttrOnlyTCP443) && - envknob.UseWIPCode() + !update.SelfNode.HasCap(tailcfg.NodeAttrOnlyTCP443) c.mu.Lock() relayClientChanged := c.relayClientEnabled != relayClientEnabled From c18ba4470b452112b83975f042705e950ef7d232 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Thu, 10 Jul 2025 22:15:55 -0700 Subject: [PATCH 198/263] ipn/ipnlocal: add traffic steering support to exit-node suggestions (#16527) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When `tailscale exit-node suggest` contacts the LocalAPI for a suggested exit node, the client consults its netmap for peers that contain the `suggest-exit-node` peercap. It currently uses a series of heuristics to determine the exit node to suggest. When the `traffic-steering` feature flag is enabled on its tailnet, the client will defer to Control’s priority scores for a particular peer. These scores, in `tailcfg.Hostinfo.Location.Priority`, were historically only used for Mullvad exit nodes, but they have now been extended to score any peer that could host a redundant resource. Client capability version 119 is the earliest client that understands these traffic steering scores. Control tells the client to switch to rely on these scores by adding `tailcfg.NodeAttrTrafficSteering` to its `AllCaps`. Updates tailscale/corp#29966 Signed-off-by: Simon Law --- ipn/ipnlocal/local.go | 134 +++++++++++- ipn/ipnlocal/local_test.go | 417 +++++++++++++++++++++++++++++++++++++ tailcfg/tailcfg.go | 3 +- 3 files changed, 546 insertions(+), 8 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 48eceb36c1ab4..4ed012f2e46f8 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -7675,13 +7675,10 @@ func allowedAutoRoute(ipp netip.Prefix) bool { var ErrNoPreferredDERP = errors.New("no preferred DERP, try again later") -// suggestExitNodeLocked computes a suggestion based on the current netmap and last netcheck report. If -// there are multiple equally good options, one is selected at random, so the result is not stable. To be -// eligible for consideration, the peer must have NodeAttrSuggestExitNode in its CapMap. -// -// Currently, peers with a DERP home are preferred over those without (typically this means Mullvad). -// Peers are selected based on having a DERP home that is the lowest latency to this device. For peers -// without a DERP home, we look for geographic proximity to this device's DERP home. +// suggestExitNodeLocked computes a suggestion based on the current netmap and +// other optional factors. If there are multiple equally good options, one may +// be selected at random, so the result is not stable. To be eligible for +// consideration, the peer must have NodeAttrSuggestExitNode in its CapMap. // // b.mu.lock() must be held. func (b *LocalBackend) suggestExitNodeLocked() (response apitype.ExitNodeSuggestionResponse, err error) { @@ -7743,7 +7740,32 @@ func fillAllowedSuggestions() set.Set[tailcfg.StableNodeID] { return s } +// suggestExitNode returns a suggestion for reasonably good exit node based on +// the current netmap and the previous suggestion. func suggestExitNode(report *netcheck.Report, nb *nodeBackend, prevSuggestion tailcfg.StableNodeID, selectRegion selectRegionFunc, selectNode selectNodeFunc, allowList set.Set[tailcfg.StableNodeID]) (res apitype.ExitNodeSuggestionResponse, err error) { + switch { + case nb.SelfHasCap(tailcfg.NodeAttrTrafficSteering): + // The traffic-steering feature flag is enabled on this tailnet. + return suggestExitNodeUsingTrafficSteering(nb, prevSuggestion, allowList) + default: + return suggestExitNodeUsingDERP(report, nb, prevSuggestion, selectRegion, selectNode, allowList) + } +} + +// suggestExitNodeUsingDERP is the classic algorithm used to suggest exit nodes, +// before traffic steering was implemented. This handles the plain failover +// case, in addition to the optional Regional Routing. +// +// It computes a suggestion based on the current netmap and last netcheck +// report. If there are multiple equally good options, one is selected at +// random, so the result is not stable. To be eligible for consideration, the +// peer must have NodeAttrSuggestExitNode in its CapMap. +// +// Currently, peers with a DERP home are preferred over those without (typically +// this means Mullvad). Peers are selected based on having a DERP home that is +// the lowest latency to this device. For peers without a DERP home, we look for +// geographic proximity to this device's DERP home. +func suggestExitNodeUsingDERP(report *netcheck.Report, nb *nodeBackend, prevSuggestion tailcfg.StableNodeID, selectRegion selectRegionFunc, selectNode selectNodeFunc, allowList set.Set[tailcfg.StableNodeID]) (res apitype.ExitNodeSuggestionResponse, err error) { netMap := nb.NetMap() if report == nil || report.PreferredDERP == 0 || netMap == nil || netMap.DERPMap == nil { return res, ErrNoPreferredDERP @@ -7864,6 +7886,104 @@ func suggestExitNode(report *netcheck.Report, nb *nodeBackend, prevSuggestion ta return res, nil } +var ErrNoNetMap = errors.New("no network map, try again later") + +// suggestExitNodeUsingTrafficSteering uses traffic steering priority scores to +// pick one of the best exit nodes. These priorities are provided by Control in +// the node’s [tailcfg.Location]. To be eligible for consideration, the node +// must have NodeAttrSuggestExitNode in its CapMap. +func suggestExitNodeUsingTrafficSteering(nb *nodeBackend, prev tailcfg.StableNodeID, allowed set.Set[tailcfg.StableNodeID]) (apitype.ExitNodeSuggestionResponse, error) { + nm := nb.NetMap() + if nm == nil { + return apitype.ExitNodeSuggestionResponse{}, ErrNoNetMap + } + + if !nb.SelfHasCap(tailcfg.NodeAttrTrafficSteering) { + panic("missing traffic-steering capability") + } + + peers := nm.Peers + nodes := make([]tailcfg.NodeView, 0, len(peers)) + + for _, p := range peers { + if !p.Valid() { + continue + } + if allowed != nil && !allowed.Contains(p.StableID()) { + continue + } + if !p.CapMap().Contains(tailcfg.NodeAttrSuggestExitNode) { + continue + } + if !tsaddr.ContainsExitRoutes(p.AllowedIPs()) { + continue + } + if p.StableID() == prev { + // Prevent flapping: since prev is a valid suggestion, + // force prev to be the only valid pick. + nodes = []tailcfg.NodeView{p} + break + } + nodes = append(nodes, p) + } + + var pick tailcfg.NodeView + + scores := make(map[tailcfg.NodeID]int, len(nodes)) + score := func(n tailcfg.NodeView) int { + id := n.ID() + s, ok := scores[id] + if !ok { + s = 0 // score of zero means incomparable + if hi := n.Hostinfo(); hi.Valid() { + if loc := hi.Location(); loc.Valid() { + s = loc.Priority() + } + } + scores[id] = s + } + return s + } + + if len(nodes) > 0 { + // Find the highest scoring exit nodes. + slices.SortFunc(nodes, func(a, b tailcfg.NodeView) int { + return cmp.Compare(score(b), score(a)) // reverse sort + }) + + // Find the top exit nodes, which all have the same score. + topI := len(nodes) + ts := score(nodes[0]) + for i, n := range nodes[1:] { + if score(n) < ts { + // n is the first node with a lower score. + // Make nodes[:topI] to slice the top exit nodes. + topI = i + 1 + break + } + } + + // TODO(sfllaw): add a temperature knob so that this client has + // a chance of picking the next best option. + randSeed := uint64(nm.SelfNode.ID()) + pick = nodes[rands.IntN(randSeed, topI)] + } + + if !pick.Valid() { + return apitype.ExitNodeSuggestionResponse{}, nil + } + res := apitype.ExitNodeSuggestionResponse{ + ID: pick.StableID(), + Name: pick.Name(), + } + if hi := pick.Hostinfo(); hi.Valid() { + if loc := hi.Location(); loc.Valid() { + res.Location = loc + } + } + return res, nil +} + // pickWeighted chooses the node with highest priority given a list of mullvad nodes. func pickWeighted(candidates []tailcfg.NodeView) []tailcfg.NodeView { maxWeight := 0 diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index bb7f433c02cbe..0b39c45c28f7d 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -4229,6 +4229,23 @@ func withLocation(loc tailcfg.LocationView) peerOptFunc { } } +func withLocationPriority(pri int) peerOptFunc { + return func(n *tailcfg.Node) { + var hi *tailcfg.Hostinfo + if n.Hostinfo.Valid() { + hi = n.Hostinfo.AsStruct() + } else { + hi = new(tailcfg.Hostinfo) + } + if hi.Location == nil { + hi.Location = new(tailcfg.Location) + } + hi.Location.Priority = pri + + n.Hostinfo = hi.View() + } +} + func withExitRoutes() peerOptFunc { return func(n *tailcfg.Node) { n.AllowedIPs = append(n.AllowedIPs, tsaddr.ExitRoutes()...) @@ -4895,6 +4912,406 @@ func TestSuggestExitNodeLongLatDistance(t *testing.T) { } } +func TestSuggestExitNodeTrafficSteering(t *testing.T) { + city := &tailcfg.Location{ + Country: "Canada", + CountryCode: "CA", + City: "Montreal", + CityCode: "MTR", + Latitude: 45.5053, + Longitude: -73.5525, + } + noLatLng := &tailcfg.Location{ + Country: "Canada", + CountryCode: "CA", + City: "Montreal", + CityCode: "MTR", + } + + selfNode := tailcfg.Node{ + ID: 0, // randomness is seeded off NetMap.SelfNode.ID + Addresses: []netip.Prefix{ + netip.MustParsePrefix("100.64.1.1/32"), + netip.MustParsePrefix("fe70::1/128"), + }, + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrTrafficSteering: []tailcfg.RawMessage{}, + }, + } + + for _, tt := range []struct { + name string + + netMap *netmap.NetworkMap + lastExit tailcfg.StableNodeID + allowPolicy []tailcfg.StableNodeID + + wantID tailcfg.StableNodeID + wantName string + wantLoc *tailcfg.Location + wantPri int + + wantErr error + }{ + { + name: "no-netmap", + netMap: nil, + wantErr: ErrNoNetMap, + }, + { + name: "no-nodes", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{}, + }, + wantID: "", + }, + { + name: "no-exit-nodes", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1), + }, + }, + wantID: "", + }, + { + name: "exit-node-without-suggestion", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes()), + }, + }, + wantID: "", + }, + { + name: "suggested-exit-node-without-routes", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withSuggest()), + }, + }, + wantID: "", + }, + { + name: "suggested-exit-node", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest()), + }, + }, + wantID: "stable1", + wantName: "peer1", + }, + { + name: "many-suggested-exit-nodes", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest()), + makePeer(2, + withExitRoutes(), + withSuggest()), + makePeer(3, + withExitRoutes(), + withSuggest()), + makePeer(4, + withExitRoutes(), + withSuggest()), + }, + }, + wantID: "stable3", + wantName: "peer3", + }, + { + name: "suggested-exit-node-was-last-suggested", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest()), + makePeer(2, + withExitRoutes(), + withSuggest()), + makePeer(3, + withExitRoutes(), + withSuggest()), + makePeer(4, + withExitRoutes(), + withSuggest()), + }, + }, + lastExit: "stable2", // overrides many-suggested-exit-nodes + wantID: "stable2", + wantName: "peer2", + }, + { + name: "suggested-exit-node-was-never-suggested", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest()), + makePeer(2, + withExitRoutes(), + withSuggest()), + makePeer(3, + withExitRoutes(), + withSuggest()), + makePeer(4, + withExitRoutes(), + withSuggest()), + }, + }, + lastExit: "stable10", + wantID: "stable3", // matches many-suggested-exit-nodes + wantName: "peer3", + }, + { + name: "exit-nodes-with-and-without-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocationPriority(1)), + makePeer(2, + withExitRoutes(), + withSuggest()), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantPri: 1, + }, + { + name: "exit-nodes-without-and-with-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest()), + makePeer(2, + withExitRoutes(), + withSuggest(), + withLocationPriority(1)), + }, + }, + wantID: "stable2", + wantName: "peer2", + wantPri: 1, + }, + { + name: "exit-nodes-with-negative-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocationPriority(-1)), + makePeer(2, + withExitRoutes(), + withSuggest(), + withLocationPriority(-2)), + makePeer(3, + withExitRoutes(), + withSuggest(), + withLocationPriority(-3)), + makePeer(4, + withExitRoutes(), + withSuggest(), + withLocationPriority(-4)), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantPri: -1, + }, + { + name: "exit-nodes-no-priority-beats-negative-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocationPriority(-1)), + makePeer(2, + withExitRoutes(), + withSuggest(), + withLocationPriority(-2)), + makePeer(3, + withExitRoutes(), + withSuggest()), + }, + }, + wantID: "stable3", + wantName: "peer3", + }, + { + name: "exit-nodes-same-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocationPriority(1)), + makePeer(2, + withExitRoutes(), + withSuggest(), + withLocationPriority(2)), // top + makePeer(3, + withExitRoutes(), + withSuggest(), + withLocationPriority(1)), + makePeer(4, + withExitRoutes(), + withSuggest(), + withLocationPriority(2)), // top + makePeer(5, + withExitRoutes(), + withSuggest(), + withLocationPriority(2)), // top + makePeer(6, + withExitRoutes(), + withSuggest()), + makePeer(7, + withExitRoutes(), + withSuggest(), + withLocationPriority(2)), // top + }, + }, + wantID: "stable5", + wantName: "peer5", + wantPri: 2, + }, + { + name: "suggested-exit-node-with-city", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocation(city.View())), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantLoc: city, + }, + { + name: "suggested-exit-node-with-city-and-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocation(city.View()), + withLocationPriority(1)), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantLoc: city, + wantPri: 1, + }, + { + name: "suggested-exit-node-without-latlng", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocation(noLatLng.View())), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantLoc: noLatLng, + }, + { + name: "suggested-exit-node-without-latlng-with-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocation(noLatLng.View()), + withLocationPriority(1)), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantLoc: noLatLng, + wantPri: 1, + }, + } { + t.Run(tt.name, func(t *testing.T) { + var allowList set.Set[tailcfg.StableNodeID] + if tt.allowPolicy != nil { + allowList = set.SetOf(tt.allowPolicy) + } + + // HACK: NetMap.AllCaps is populated by Control: + if tt.netMap != nil { + caps := maps.Keys(tt.netMap.SelfNode.CapMap().AsMap()) + tt.netMap.AllCaps = set.SetOf(slices.Collect(caps)) + } + + nb := newNodeBackend(t.Context(), eventbus.New()) + defer nb.shutdown(errShutdown) + nb.SetNetMap(tt.netMap) + + got, err := suggestExitNodeUsingTrafficSteering(nb, tt.lastExit, allowList) + if tt.wantErr == nil && err != nil { + t.Fatalf("err=%v, want nil", err) + } + if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { + t.Fatalf("err=%v, want %v", err, tt.wantErr) + } + + if got.Name != tt.wantName { + t.Errorf("name=%q, want %q", got.Name, tt.wantName) + } + + if got.ID != tt.wantID { + t.Errorf("ID=%q, want %q", got.ID, tt.wantID) + } + + wantLoc := tt.wantLoc + if tt.wantPri != 0 { + if wantLoc == nil { + wantLoc = new(tailcfg.Location) + } + wantLoc.Priority = tt.wantPri + } + if diff := cmp.Diff(got.Location.AsStruct(), wantLoc); diff != "" { + t.Errorf("location mismatch (+want -got)\n%s", diff) + } + }) + } +} + func TestMinLatencyDERPregion(t *testing.T) { tests := []struct { name string diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index ab8add5b83a3e..53c4683c1b000 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -163,7 +163,8 @@ type CapabilityVersion int // - 116: 2025-05-05: Client serves MagicDNS "AAAA" if NodeAttrMagicDNSPeerAAAA set on self node // - 117: 2025-05-28: Client understands DisplayMessages (structured health messages), but not necessarily PrimaryAction. // - 118: 2025-07-01: Client sends Hostinfo.StateEncrypted to report whether the state file is encrypted at rest (#15830) -const CurrentCapabilityVersion CapabilityVersion = 118 +// - 119: 2025-07-10: Client uses Hostinfo.Location.Priority to prioritize one route over another. +const CurrentCapabilityVersion CapabilityVersion = 119 // ID is an integer ID for a user, node, or login allocated by the // control plane. From 04e8d21b0bcaab54f1906fb6a0ebc507ed7114ea Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 10 Jul 2025 22:21:08 -0700 Subject: [PATCH 199/263] go.mod: bump wg-go to fix keepalive detection (#16535) Updates tailscale/corp#30364 Signed-off-by: Jordan Whited --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e89a383a62726..f040d7799768d 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/tailscale/setec v0.0.0-20250205144240-8898a29c3fbb github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 - github.com/tailscale/wireguard-go v0.0.0-20250707220504-1f398ae148a8 + github.com/tailscale/wireguard-go v0.0.0-20250711050509-4064566ecaf9 github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e github.com/tc-hib/winres v0.2.1 github.com/tcnksm/go-httpstat v0.2.0 diff --git a/go.sum b/go.sum index 062af66622b85..ea17b11821392 100644 --- a/go.sum +++ b/go.sum @@ -975,8 +975,8 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 h1:U github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y= -github.com/tailscale/wireguard-go v0.0.0-20250707220504-1f398ae148a8 h1:Yjg/+1VVRcdY3DL9fs8g+QnZ1aizotU0pp0VSOSCuTQ= -github.com/tailscale/wireguard-go v0.0.0-20250707220504-1f398ae148a8/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= +github.com/tailscale/wireguard-go v0.0.0-20250711050509-4064566ecaf9 h1:kSzi/ugdekAxhcVdCxH6er7OjoNc2oDRcimWJDvnRFM= +github.com/tailscale/wireguard-go v0.0.0-20250711050509-4064566ecaf9/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg= github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA= From 30da2e1c3206b7e45b42fd3fddfe1d9081c6982d Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 11 Jul 2025 08:51:02 -0700 Subject: [PATCH 200/263] cmd/tailscale/cli: add "configure jetkvm" subcommand To write the init script. And fix the JetKVM detection to work during early boot while the filesystem and modules are still being loaded; it wasn't being detected on early boot and then tailscaled was failing to start because it didn't know it was on JetKVM and didn't modprobe tun. Updates #16524 Change-Id: I0524ca3abd7ace68a69af96aab4175d32c07e116 Signed-off-by: Brad Fitzpatrick --- cmd/tailscale/cli/configure-jetkvm.go | 81 +++++++++++++++++++++++++++ cmd/tailscale/cli/configure.go | 3 + version/distro/distro.go | 11 +++- 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 cmd/tailscale/cli/configure-jetkvm.go diff --git a/cmd/tailscale/cli/configure-jetkvm.go b/cmd/tailscale/cli/configure-jetkvm.go new file mode 100644 index 0000000000000..a8e0a7cb542ef --- /dev/null +++ b/cmd/tailscale/cli/configure-jetkvm.go @@ -0,0 +1,81 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux && !android && arm + +package cli + +import ( + "bytes" + "context" + "errors" + "flag" + "os" + "runtime" + "strings" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/version/distro" +) + +func init() { + maybeJetKVMConfigureCmd = jetKVMConfigureCmd +} + +func jetKVMConfigureCmd() *ffcli.Command { + if runtime.GOOS != "linux" || distro.Get() != distro.JetKVM { + return nil + } + return &ffcli.Command{ + Name: "jetkvm", + Exec: runConfigureJetKVM, + ShortUsage: "tailscale configure jetkvm", + ShortHelp: "Configure JetKVM to run tailscaled at boot", + LongHelp: strings.TrimSpace(` +This command configures the JetKVM host to run tailscaled at boot. +`), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("jetkvm") + return fs + })(), + } +} + +func runConfigureJetKVM(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("unknown arguments") + } + if runtime.GOOS != "linux" || distro.Get() != distro.JetKVM { + return errors.New("only implemented on JetKVM") + } + err := os.WriteFile("/etc/init.d/S22tailscale", bytes.TrimLeft([]byte(` +#!/bin/sh +# /etc/init.d/S22tailscale +# Start/stop tailscaled + +case "$1" in + start) + /userdata/tailscale/tailscaled > /dev/null 2>&1 & + ;; + stop) + killall tailscaled + ;; + *) + echo "Usage: $0 {start|stop}" + exit 1 + ;; +esac +`), "\n"), 0755) + if err != nil { + return err + } + + if err := os.Symlink("/userdata/tailscale/tailscale", "/bin/tailscale"); err != nil { + if !os.IsExist(err) { + return err + } + } + + printf("Done. Now restart your JetKVM.\n") + return nil +} diff --git a/cmd/tailscale/cli/configure.go b/cmd/tailscale/cli/configure.go index acb416755a586..da6278ce24330 100644 --- a/cmd/tailscale/cli/configure.go +++ b/cmd/tailscale/cli/configure.go @@ -10,6 +10,8 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" ) +var maybeJetKVMConfigureCmd func() *ffcli.Command // non-nil only on Linux/arm for JetKVM + func configureCmd() *ffcli.Command { return &ffcli.Command{ Name: "configure", @@ -29,6 +31,7 @@ services on the host to use Tailscale in more ways. synologyConfigureCertCmd(), ccall(maybeSysExtCmd), ccall(maybeVPNConfigCmd), + ccall(maybeJetKVMConfigureCmd), ), } } diff --git a/version/distro/distro.go b/version/distro/distro.go index dd5e0b21b2eb1..0e88bdd2fa297 100644 --- a/version/distro/distro.go +++ b/version/distro/distro.go @@ -9,6 +9,7 @@ import ( "os" "runtime" "strconv" + "strings" "tailscale.com/types/lazy" "tailscale.com/util/lineiter" @@ -103,12 +104,20 @@ func linuxDistro() Distro { return Unraid case have("/etc/alpine-release"): return Alpine - case haveDir("/userdata/jetkvm") && haveDir("/sys/kernel/config/usb_gadget/jetkvm"): + case runtime.GOARCH == "arm" && isDeviceModel("JetKVM"): return JetKVM } return "" } +func isDeviceModel(want string) bool { + if runtime.GOOS != "linux" { + return false + } + v, _ := os.ReadFile("/sys/firmware/devicetree/base/model") + return want == strings.Trim(string(v), "\x00\r\n\t ") +} + func freebsdDistro() Distro { switch { case have("/etc/pfSense-rc"): From 39bf84d1c70d1b31384acbf37dd9f8d36db47404 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Fri, 11 Jul 2025 16:01:15 -0700 Subject: [PATCH 201/263] cmd/tsidp: set hostinfo.App in tsnet mode (#16544) This makes it easier to track how widely tsidp is used in practice. Updates #cleanup Signed-off-by: Andrew Lytvynov --- cmd/tsidp/tsidp.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/tsidp/tsidp.go b/cmd/tsidp/tsidp.go index 43020eaf73e63..6a0c2d89e685e 100644 --- a/cmd/tsidp/tsidp.go +++ b/cmd/tsidp/tsidp.go @@ -39,6 +39,7 @@ import ( "tailscale.com/client/local" "tailscale.com/client/tailscale/apitype" "tailscale.com/envknob" + "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" @@ -121,6 +122,7 @@ func main() { } defer cleanup() } else { + hostinfo.SetApp("tsidp") ts := &tsnet.Server{ Hostname: *flagHostname, Dir: *flagDir, From 24062e33d13a4859b7d08f2bcfc518827517784e Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Fri, 11 Jul 2025 17:12:23 -0700 Subject: [PATCH 202/263] net/udprelay: fix peer relay server deadlock (#16542) Fixes tailscale/corp#30381 Signed-off-by: Jordan Whited --- net/udprelay/server.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/net/udprelay/server.go b/net/udprelay/server.go index 979ccf71765ed..e2652ae99637f 100644 --- a/net/udprelay/server.go +++ b/net/udprelay/server.go @@ -488,14 +488,17 @@ func (s *Server) listenOn(port int) error { // Close closes the server. func (s *Server) Close() error { s.closeOnce.Do(func() { - s.mu.Lock() - defer s.mu.Unlock() s.uc4.Close() if s.uc6 != nil { s.uc6.Close() } close(s.closeCh) s.wg.Wait() + // s.mu must not be held while s.wg.Wait'ing, otherwise we can + // deadlock. The goroutines we are waiting on to return can also + // acquire s.mu. + s.mu.Lock() + defer s.mu.Unlock() clear(s.byVNI) clear(s.byDisco) s.vniPool = nil @@ -564,6 +567,12 @@ func (s *Server) handlePacket(from netip.AddrPort, b []byte, rxSocket, otherAFSo func (s *Server) packetReadLoop(readFromSocket, otherSocket *net.UDPConn) { defer func() { + // We intentionally close the [Server] if we encounter a socket read + // error below, at least until socket "re-binding" is implemented as + // part of http://go/corp/30118. + // + // Decrementing this [sync.WaitGroup] _before_ calling [Server.Close] is + // intentional as [Server.Close] waits on it. s.wg.Done() s.Close() }() From f23e4279c42aec766eb6a89562c1fed3a1b97e09 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Sun, 13 Jul 2025 05:47:56 -0700 Subject: [PATCH 203/263] types/lazy: add lazy.GMap: a map of lazily computed GValues (#16532) Fixes tailscale/corp#30360 Signed-off-by: Simon Law --- cmd/stund/depaware.txt | 2 +- types/lazy/map.go | 62 +++++++++++++++++++++++++++ types/lazy/map_test.go | 95 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 types/lazy/map.go create mode 100644 types/lazy/map_test.go diff --git a/cmd/stund/depaware.txt b/cmd/stund/depaware.txt index da768039431fe..81544b7505dc7 100644 --- a/cmd/stund/depaware.txt +++ b/cmd/stund/depaware.txt @@ -76,7 +76,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar L 💣 tailscale.com/util/dirwalk from tailscale.com/metrics tailscale.com/util/dnsname from tailscale.com/tailcfg tailscale.com/util/lineiter from tailscale.com/version/distro - tailscale.com/util/mak from tailscale.com/syncs + tailscale.com/util/mak from tailscale.com/syncs+ tailscale.com/util/nocasemaps from tailscale.com/types/ipproto tailscale.com/util/rands from tailscale.com/tsweb tailscale.com/util/slicesx from tailscale.com/tailcfg diff --git a/types/lazy/map.go b/types/lazy/map.go new file mode 100644 index 0000000000000..75a1dd739d3bc --- /dev/null +++ b/types/lazy/map.go @@ -0,0 +1,62 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package lazy + +import "tailscale.com/util/mak" + +// GMap is a map of lazily computed [GValue] pointers, keyed by a comparable +// type. +// +// Use either Get or GetErr, depending on whether your fill function returns an +// error. +// +// GMap is not safe for concurrent use. +type GMap[K comparable, V any] struct { + store map[K]*GValue[V] +} + +// Len returns the number of entries in the map. +func (s *GMap[K, V]) Len() int { + return len(s.store) +} + +// Set attempts to set the value of k to v, and reports whether it succeeded. +// Set only succeeds if k has never been called with Get/GetErr/Set before. +func (s *GMap[K, V]) Set(k K, v V) bool { + z, ok := s.store[k] + if !ok { + z = new(GValue[V]) + mak.Set(&s.store, k, z) + } + return z.Set(v) +} + +// MustSet sets the value of k to v, or panics if k already has a value. +func (s *GMap[K, V]) MustSet(k K, v V) { + if !s.Set(k, v) { + panic("Set after already filled") + } +} + +// Get returns the value for k, computing it with fill if it's not already +// present. +func (s *GMap[K, V]) Get(k K, fill func() V) V { + z, ok := s.store[k] + if !ok { + z = new(GValue[V]) + mak.Set(&s.store, k, z) + } + return z.Get(fill) +} + +// GetErr returns the value for k, computing it with fill if it's not already +// present. +func (s *GMap[K, V]) GetErr(k K, fill func() (V, error)) (V, error) { + z, ok := s.store[k] + if !ok { + z = new(GValue[V]) + mak.Set(&s.store, k, z) + } + return z.GetErr(fill) +} diff --git a/types/lazy/map_test.go b/types/lazy/map_test.go new file mode 100644 index 0000000000000..ec1152b0b802c --- /dev/null +++ b/types/lazy/map_test.go @@ -0,0 +1,95 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package lazy + +import ( + "errors" + "testing" +) + +func TestGMap(t *testing.T) { + var gm GMap[string, int] + n := int(testing.AllocsPerRun(1000, func() { + got := gm.Get("42", fortyTwo) + if got != 42 { + t.Fatalf("got %v; want 42", got) + } + })) + if n != 0 { + t.Errorf("allocs = %v; want 0", n) + } +} + +func TestGMapErr(t *testing.T) { + var gm GMap[string, int] + n := int(testing.AllocsPerRun(1000, func() { + got, err := gm.GetErr("42", func() (int, error) { + return 42, nil + }) + if got != 42 || err != nil { + t.Fatalf("got %v, %v; want 42, nil", got, err) + } + })) + if n != 0 { + t.Errorf("allocs = %v; want 0", n) + } + + var gmErr GMap[string, int] + wantErr := errors.New("test error") + n = int(testing.AllocsPerRun(1000, func() { + got, err := gmErr.GetErr("42", func() (int, error) { + return 0, wantErr + }) + if got != 0 || err != wantErr { + t.Fatalf("got %v, %v; want 0, %v", got, err, wantErr) + } + })) + if n != 0 { + t.Errorf("allocs = %v; want 0", n) + } +} + +func TestGMapSet(t *testing.T) { + var gm GMap[string, int] + if !gm.Set("42", 42) { + t.Fatalf("Set failed") + } + if gm.Set("42", 43) { + t.Fatalf("Set succeeded after first Set") + } + n := int(testing.AllocsPerRun(1000, func() { + got := gm.Get("42", fortyTwo) + if got != 42 { + t.Fatalf("got %v; want 42", got) + } + })) + if n != 0 { + t.Errorf("allocs = %v; want 0", n) + } +} + +func TestGMapMustSet(t *testing.T) { + var gm GMap[string, int] + gm.MustSet("42", 42) + defer func() { + if e := recover(); e == nil { + t.Errorf("unexpected success; want panic") + } + }() + gm.MustSet("42", 43) +} + +func TestGMapRecursivePanic(t *testing.T) { + defer func() { + if e := recover(); e != nil { + t.Logf("got panic, as expected") + } else { + t.Errorf("unexpected success; want panic") + } + }() + gm := GMap[string, int]{} + gm.Get("42", func() int { + return gm.Get("42", func() int { return 42 }) + }) +} From bcaea4f24597d840d8a0fd94cafbb2dc0ff7a774 Mon Sep 17 00:00:00 2001 From: Tom Meadows Date: Mon, 14 Jul 2025 15:17:20 +0100 Subject: [PATCH 204/263] k8s-operator,sessionrecording: fixing race condition between resize (#16454) messages and cast headers when recording `kubectl attach` sessions Updates #16490 Signed-off-by: chaosinthecrd --- k8s-operator/api-proxy/proxy.go | 51 ++++--- k8s-operator/sessionrecording/fakes/fakes.go | 12 +- k8s-operator/sessionrecording/hijacker.go | 63 +++++--- .../sessionrecording/hijacker_test.go | 2 +- k8s-operator/sessionrecording/spdy/conn.go | 98 +++++++----- .../sessionrecording/spdy/conn_test.go | 98 ++++++------ .../sessionrecording/tsrecorder/tsrecorder.go | 1 + k8s-operator/sessionrecording/ws/conn.go | 115 ++++++++------ k8s-operator/sessionrecording/ws/conn_test.go | 144 ++++++++++-------- sessionrecording/header.go | 10 +- 10 files changed, 351 insertions(+), 243 deletions(-) diff --git a/k8s-operator/api-proxy/proxy.go b/k8s-operator/api-proxy/proxy.go index c3c13e7846915..d33c088de78db 100644 --- a/k8s-operator/api-proxy/proxy.go +++ b/k8s-operator/api-proxy/proxy.go @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/transport" "tailscale.com/client/local" "tailscale.com/client/tailscale/apitype" + "tailscale.com/k8s-operator/sessionrecording" ksr "tailscale.com/k8s-operator/sessionrecording" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" @@ -49,6 +50,7 @@ func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsn if !authMode { restConfig = rest.AnonymousClientConfig(restConfig) } + cfg, err := restConfig.TransportConfig() if err != nil { return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err) @@ -111,6 +113,8 @@ func (ap *APIServerProxy) Run(ctx context.Context) error { mux.HandleFunc("/", ap.serveDefault) mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY) mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS) + mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/attach", ap.serveAttachSPDY) + mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/attach", ap.serveAttachWS) ap.hs = &http.Server{ // Kubernetes uses SPDY for exec and port-forward, however SPDY is @@ -165,19 +169,31 @@ func (ap *APIServerProxy) serveDefault(w http.ResponseWriter, r *http.Request) { ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who))) } -// serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY, +// serveExecSPDY serves '/exec' requests for sessions streamed over SPDY, // optionally configuring the kubectl exec sessions to be recorded. func (ap *APIServerProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) { - ap.execForProto(w, r, ksr.SPDYProtocol) + ap.sessionForProto(w, r, ksr.ExecSessionType, ksr.SPDYProtocol) } -// serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket, +// serveExecWS serves '/exec' requests for sessions streamed over WebSocket, // optionally configuring the kubectl exec sessions to be recorded. func (ap *APIServerProxy) serveExecWS(w http.ResponseWriter, r *http.Request) { - ap.execForProto(w, r, ksr.WSProtocol) + ap.sessionForProto(w, r, ksr.ExecSessionType, ksr.WSProtocol) +} + +// serveExecSPDY serves '/attach' requests for sessions streamed over SPDY, +// optionally configuring the kubectl exec sessions to be recorded. +func (ap *APIServerProxy) serveAttachSPDY(w http.ResponseWriter, r *http.Request) { + ap.sessionForProto(w, r, ksr.AttachSessionType, ksr.SPDYProtocol) +} + +// serveExecWS serves '/attach' requests for sessions streamed over WebSocket, +// optionally configuring the kubectl exec sessions to be recorded. +func (ap *APIServerProxy) serveAttachWS(w http.ResponseWriter, r *http.Request) { + ap.sessionForProto(w, r, ksr.AttachSessionType, ksr.WSProtocol) } -func (ap *APIServerProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) { +func (ap *APIServerProxy) sessionForProto(w http.ResponseWriter, r *http.Request, sessionType sessionrecording.SessionType, proto ksr.Protocol) { const ( podNameKey = "pod" namespaceNameKey = "namespace" @@ -192,7 +208,7 @@ func (ap *APIServerProxy) execForProto(w http.ResponseWriter, r *http.Request, p counterNumRequestsProxied.Add(1) failOpen, addrs, err := determineRecorderConfig(who) if err != nil { - ap.log.Errorf("error trying to determine whether the 'kubectl exec' session needs to be recorded: %v", err) + ap.log.Errorf("error trying to determine whether the 'kubectl %s' session needs to be recorded: %v", sessionType, err) return } if failOpen && len(addrs) == 0 { // will not record @@ -201,7 +217,7 @@ func (ap *APIServerProxy) execForProto(w http.ResponseWriter, r *http.Request, p } ksr.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded if !failOpen && len(addrs) == 0 { - msg := "forbidden: 'kubectl exec' session must be recorded, but no recorders are available." + msg := fmt.Sprintf("forbidden: 'kubectl %s' session must be recorded, but no recorders are available.", sessionType) ap.log.Error(msg) http.Error(w, msg, http.StatusForbidden) return @@ -223,16 +239,17 @@ func (ap *APIServerProxy) execForProto(w http.ResponseWriter, r *http.Request, p } opts := ksr.HijackerOpts{ - Req: r, - W: w, - Proto: proto, - TS: ap.ts, - Who: who, - Addrs: addrs, - FailOpen: failOpen, - Pod: r.PathValue(podNameKey), - Namespace: r.PathValue(namespaceNameKey), - Log: ap.log, + Req: r, + W: w, + Proto: proto, + SessionType: sessionType, + TS: ap.ts, + Who: who, + Addrs: addrs, + FailOpen: failOpen, + Pod: r.PathValue(podNameKey), + Namespace: r.PathValue(namespaceNameKey), + Log: ap.log, } h := ksr.New(opts) diff --git a/k8s-operator/sessionrecording/fakes/fakes.go b/k8s-operator/sessionrecording/fakes/fakes.go index 9eb1047e4242f..94853df195f7c 100644 --- a/k8s-operator/sessionrecording/fakes/fakes.go +++ b/k8s-operator/sessionrecording/fakes/fakes.go @@ -10,13 +10,13 @@ package fakes import ( "bytes" "encoding/json" + "fmt" + "math/rand" "net" "sync" "testing" "time" - "math/rand" - "tailscale.com/sessionrecording" "tailscale.com/tstime" ) @@ -107,7 +107,13 @@ func CastLine(t *testing.T, p []byte, clock tstime.Clock) []byte { return append(j, '\n') } -func AsciinemaResizeMsg(t *testing.T, width, height int) []byte { +func AsciinemaCastResizeMsg(t *testing.T, width, height int) []byte { + msg := fmt.Sprintf(`[0,"r","%dx%d"]`, height, width) + + return append([]byte(msg), '\n') +} + +func AsciinemaCastHeaderMsg(t *testing.T, width, height int) []byte { t.Helper() ch := sessionrecording.CastHeader{ Width: width, diff --git a/k8s-operator/sessionrecording/hijacker.go b/k8s-operator/sessionrecording/hijacker.go index a9ed658964787..e8c534afc9319 100644 --- a/k8s-operator/sessionrecording/hijacker.go +++ b/k8s-operator/sessionrecording/hijacker.go @@ -4,7 +4,7 @@ //go:build !plan9 // Package sessionrecording contains functionality for recording Kubernetes API -// server proxy 'kubectl exec' sessions. +// server proxy 'kubectl exec/attach' sessions. package sessionrecording import ( @@ -35,14 +35,20 @@ import ( ) const ( - SPDYProtocol Protocol = "SPDY" - WSProtocol Protocol = "WebSocket" + SPDYProtocol Protocol = "SPDY" + WSProtocol Protocol = "WebSocket" + ExecSessionType SessionType = "exec" + AttachSessionType SessionType = "attach" ) // Protocol is the streaming protocol of the hijacked session. Supported // protocols are SPDY and WebSocket. type Protocol string +// SessionType is the type of session initiated with `kubectl` +// (`exec` or `attach`) +type SessionType string + var ( // CounterSessionRecordingsAttempted counts the number of session recording attempts. CounterSessionRecordingsAttempted = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_attempted") @@ -63,25 +69,27 @@ func New(opts HijackerOpts) *Hijacker { failOpen: opts.FailOpen, proto: opts.Proto, log: opts.Log, + sessionType: opts.SessionType, connectToRecorder: sessionrecording.ConnectToRecorder, } } type HijackerOpts struct { - TS *tsnet.Server - Req *http.Request - W http.ResponseWriter - Who *apitype.WhoIsResponse - Addrs []netip.AddrPort - Log *zap.SugaredLogger - Pod string - Namespace string - FailOpen bool - Proto Protocol + TS *tsnet.Server + Req *http.Request + W http.ResponseWriter + Who *apitype.WhoIsResponse + Addrs []netip.AddrPort + Log *zap.SugaredLogger + Pod string + Namespace string + FailOpen bool + Proto Protocol + SessionType SessionType } // Hijacker implements [net/http.Hijacker] interface. -// It must be configured with an http request for a 'kubectl exec' session that +// It must be configured with an http request for a 'kubectl exec/attach' session that // needs to be recorded. It knows how to hijack the connection and configure for // the session contents to be sent to a tsrecorder instance. type Hijacker struct { @@ -90,12 +98,13 @@ type Hijacker struct { req *http.Request who *apitype.WhoIsResponse log *zap.SugaredLogger - pod string // pod being exec-d - ns string // namespace of the pod being exec-d + pod string // pod being exec/attach-d + ns string // namespace of the pod being exec/attach-d addrs []netip.AddrPort // tsrecorder addresses failOpen bool // whether to fail open if recording fails connectToRecorder RecorderDialFn - proto Protocol // streaming protocol + proto Protocol // streaming protocol + sessionType SessionType // subcommand, e.g., "exec, attach" } // RecorderDialFn dials the specified netip.AddrPorts that should be tsrecorder @@ -105,7 +114,7 @@ type Hijacker struct { // after having been established, an error is sent down the channel. type RecorderDialFn func(context.Context, []netip.AddrPort, netx.DialFunc) (io.WriteCloser, []*tailcfg.SSHRecordingAttempt, <-chan error, error) -// Hijack hijacks a 'kubectl exec' session and configures for the session +// Hijack hijacks a 'kubectl exec/attach' session and configures for the session // contents to be sent to a recorder. func (h *Hijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) { h.log.Infof("recorder addrs: %v, failOpen: %v", h.addrs, h.failOpen) @@ -114,7 +123,7 @@ func (h *Hijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) { return nil, nil, fmt.Errorf("error hijacking connection: %w", err) } - conn, err := h.setUpRecording(context.Background(), reqConn) + conn, err := h.setUpRecording(h.req.Context(), reqConn) if err != nil { return nil, nil, fmt.Errorf("error setting up session recording: %w", err) } @@ -138,7 +147,7 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, err error errChan <-chan error ) - h.log.Infof("kubectl exec session will be recorded, recorders: %v, fail open policy: %t", h.addrs, h.failOpen) + h.log.Infof("kubectl %s session will be recorded, recorders: %v, fail open policy: %t", h.sessionType, h.addrs, h.failOpen) qp := h.req.URL.Query() container := strings.Join(qp[containerKey], "") var recorderAddr net.Addr @@ -161,7 +170,7 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, } return nil, errors.New(msg) } else { - h.log.Infof("exec session to container %q in Pod %q namespace %q will be recorded, the recording will be sent to a tsrecorder instance at %q", container, h.pod, h.ns, recorderAddr) + h.log.Infof("%s session to container %q in Pod %q namespace %q will be recorded, the recording will be sent to a tsrecorder instance at %q", h.sessionType, container, h.pod, h.ns, recorderAddr) } cl := tstime.DefaultClock{} @@ -190,9 +199,15 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, var lc net.Conn switch h.proto { case SPDYProtocol: - lc = spdy.New(conn, rec, ch, hasTerm, h.log) + lc, err = spdy.New(ctx, conn, rec, ch, hasTerm, h.log) + if err != nil { + return nil, fmt.Errorf("failed to initialize spdy connection: %w", err) + } case WSProtocol: - lc = ws.New(conn, rec, ch, hasTerm, h.log) + lc, err = ws.New(ctx, conn, rec, ch, hasTerm, h.log) + if err != nil { + return nil, fmt.Errorf("failed to initialize websocket connection: %w", err) + } default: return nil, fmt.Errorf("unknown protocol: %s", h.proto) } @@ -209,7 +224,7 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, h.log.Info("finished uploading the recording") return } - msg := fmt.Sprintf("connection to the session recorder errorred: %v;", err) + msg := fmt.Sprintf("connection to the session recorder errored: %v;", err) if h.failOpen { msg += msg + "; failure mode is 'fail open'; continuing session without recording." h.log.Info(msg) diff --git a/k8s-operator/sessionrecording/hijacker_test.go b/k8s-operator/sessionrecording/hijacker_test.go index 880015b22c2d0..cac6f55c7c7d7 100644 --- a/k8s-operator/sessionrecording/hijacker_test.go +++ b/k8s-operator/sessionrecording/hijacker_test.go @@ -91,7 +91,7 @@ func Test_Hijacker(t *testing.T) { who: &apitype.WhoIsResponse{Node: &tailcfg.Node{}, UserProfile: &tailcfg.UserProfile{}}, log: zl.Sugar(), ts: &tsnet.Server{}, - req: &http.Request{URL: &url.URL{}}, + req: &http.Request{URL: &url.URL{RawQuery: "tty=true"}}, proto: tt.proto, } ctx := context.Background() diff --git a/k8s-operator/sessionrecording/spdy/conn.go b/k8s-operator/sessionrecording/spdy/conn.go index 455c2225ad921..9fefca11fc2b8 100644 --- a/k8s-operator/sessionrecording/spdy/conn.go +++ b/k8s-operator/sessionrecording/spdy/conn.go @@ -4,11 +4,12 @@ //go:build !plan9 // Package spdy contains functionality for parsing SPDY streaming sessions. This -// is used for 'kubectl exec' session recording. +// is used for 'kubectl exec/attach' session recording. package spdy import ( "bytes" + "context" "encoding/binary" "encoding/json" "fmt" @@ -24,29 +25,50 @@ import ( ) // New wraps the provided network connection and returns a connection whose reads and writes will get triggered as data is received on the hijacked connection. -// The connection must be a hijacked connection for a 'kubectl exec' session using SPDY. +// The connection must be a hijacked connection for a 'kubectl exec/attach' session using SPDY. // The hijacked connection is used to transmit SPDY streams between Kubernetes client ('kubectl') and the destination container. // Data read from the underlying network connection is data sent via one of the SPDY streams from the client to the container. // Data written to the underlying connection is data sent from the container to the client. // We parse the data and send everything for the stdout/stderr streams to the configured tsrecorder as an asciinema recording with the provided header. // https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/4006-transition-spdy-to-websockets#background-remotecommand-subprotocol -func New(nc net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, hasTerm bool, log *zap.SugaredLogger) net.Conn { - return &conn{ - Conn: nc, - rec: rec, - ch: ch, - log: log, - hasTerm: hasTerm, - initialTermSizeSet: make(chan struct{}), +func New(ctx context.Context, nc net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, hasTerm bool, log *zap.SugaredLogger) (net.Conn, error) { + lc := &conn{ + Conn: nc, + ctx: ctx, + rec: rec, + ch: ch, + log: log, + hasTerm: hasTerm, + initialCastHeaderSent: make(chan struct{}, 1), } + + // if there is no term, we don't need to wait for a resize message + if !hasTerm { + var err error + lc.writeCastHeaderOnce.Do(func() { + // If this is a session with a terminal attached, + // we must wait for the terminal width and + // height to be parsed from a resize message + // before sending CastHeader, else tsrecorder + // will not be able to play this recording. + err = lc.rec.WriteCastHeader(ch) + close(lc.initialCastHeaderSent) + }) + if err != nil { + return nil, fmt.Errorf("error writing CastHeader: %w", err) + } + } + + return lc, nil } // conn is a wrapper around net.Conn. It reads the bytestream for a 'kubectl -// exec' session streamed using SPDY protocol, sends session recording data to +// exec/attach' session streamed using SPDY protocol, sends session recording data to // the configured recorder and forwards the raw bytes to the original // destination. type conn struct { net.Conn + ctx context.Context // rec knows how to send data written to it to a tsrecorder instance. rec *tsrecorder.Client @@ -63,7 +85,7 @@ type conn struct { // CastHeader must be sent before any payload. If the session has a // terminal attached, the CastHeader must have '.Width' and '.Height' // fields set for the tsrecorder UI to be able to play the recording. - // For 'kubectl exec' sessions, terminal width and height are sent as a + // For 'kubectl exec/attach' sessions, terminal width and height are sent as a // resize message on resize stream from the client when the session // starts as well as at any time the client detects a terminal change. // We can intercept the resize message on Read calls. As there is no @@ -79,15 +101,10 @@ type conn struct { // writeCastHeaderOnce is used to ensure CastHeader gets sent to tsrecorder once. writeCastHeaderOnce sync.Once hasTerm bool // whether the session had TTY attached - // initialTermSizeSet channel gets sent a value once, when the Read has - // received a resize message and set the initial terminal size. It must - // be set to a buffered channel to prevent Reads being blocked on the - // first stdout/stderr write reading from the channel. - initialTermSizeSet chan struct{} - // sendInitialTermSizeSetOnce is used to ensure that a value is sent to - // initialTermSizeSet channel only once, when the initial resize message - // is received. - sendinitialTermSizeSetOnce sync.Once + // initialCastHeaderSent is a channel to ensure that the cast + // header is the first thing that is streamed to the session recorder. + // Otherwise the stream will fail. + initialCastHeaderSent chan struct{} zlibReqReader zlibReader // writeBuf is used to store data written to the connection that has not @@ -124,7 +141,7 @@ func (c *conn) Read(b []byte) (int, error) { } c.readBuf.Next(len(sf.Raw)) // advance buffer past the parsed frame - if !sf.Ctrl { // data frame + if !sf.Ctrl && c.hasTerm { // data frame switch sf.StreamID { case c.resizeStreamID.Load(): @@ -140,10 +157,19 @@ func (c *conn) Read(b []byte) (int, error) { // subsequent resize message, we need to send asciinema // resize message. var isInitialResize bool - c.sendinitialTermSizeSetOnce.Do(func() { + c.writeCastHeaderOnce.Do(func() { isInitialResize = true - close(c.initialTermSizeSet) // unblock sending of CastHeader + // If this is a session with a terminal attached, + // we must wait for the terminal width and + // height to be parsed from a resize message + // before sending CastHeader, else tsrecorder + // will not be able to play this recording. + err = c.rec.WriteCastHeader(c.ch) + close(c.initialCastHeaderSent) }) + if err != nil { + return 0, fmt.Errorf("error writing CastHeader: %w", err) + } if !isInitialResize { if err := c.rec.WriteResize(c.ch.Height, c.ch.Width); err != nil { return 0, fmt.Errorf("error writing resize message: %w", err) @@ -190,24 +216,14 @@ func (c *conn) Write(b []byte) (int, error) { if !sf.Ctrl { switch sf.StreamID { case c.stdoutStreamID.Load(), c.stderrStreamID.Load(): - var err error - c.writeCastHeaderOnce.Do(func() { - // If this is a session with a terminal attached, - // we must wait for the terminal width and - // height to be parsed from a resize message - // before sending CastHeader, else tsrecorder - // will not be able to play this recording. - if c.hasTerm { - c.log.Debugf("write: waiting for the initial terminal size to be set before proceeding with sending the first payload") - <-c.initialTermSizeSet + // we must wait for confirmation that the initial cast header was sent before proceeding with any more writes + select { + case <-c.ctx.Done(): + return 0, c.ctx.Err() + case <-c.initialCastHeaderSent: + if err := c.rec.WriteOutput(sf.Payload); err != nil { + return 0, fmt.Errorf("error sending payload to session recorder: %w", err) } - err = c.rec.WriteCastHeader(c.ch) - }) - if err != nil { - return 0, fmt.Errorf("error writing CastHeader: %w", err) - } - if err := c.rec.WriteOutput(sf.Payload); err != nil { - return 0, fmt.Errorf("error sending payload to session recorder: %w", err) } } } diff --git a/k8s-operator/sessionrecording/spdy/conn_test.go b/k8s-operator/sessionrecording/spdy/conn_test.go index 3485d61c4f454..3c1cb8427d822 100644 --- a/k8s-operator/sessionrecording/spdy/conn_test.go +++ b/k8s-operator/sessionrecording/spdy/conn_test.go @@ -6,10 +6,12 @@ package spdy import ( + "context" "encoding/json" "fmt" "reflect" "testing" + "time" "go.uber.org/zap" "tailscale.com/k8s-operator/sessionrecording/fakes" @@ -29,15 +31,11 @@ func Test_Writes(t *testing.T) { } cl := tstest.NewClock(tstest.ClockOpts{}) tests := []struct { - name string - inputs [][]byte - wantForwarded []byte - wantRecorded []byte - firstWrite bool - width int - height int - sendInitialResize bool - hasTerm bool + name string + inputs [][]byte + wantForwarded []byte + wantRecorded []byte + hasTerm bool }{ { name: "single_write_control_frame_with_payload", @@ -78,24 +76,17 @@ func Test_Writes(t *testing.T) { wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl), }, { - name: "single_first_write_stdout_data_frame_with_payload_sess_has_terminal", - inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}}, - wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}, - wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...), - width: 10, - height: 20, - hasTerm: true, - firstWrite: true, - sendInitialResize: true, + name: "single_first_write_stdout_data_frame_with_payload_sess_has_terminal", + inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}}, + wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}, + wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl), + hasTerm: true, }, { name: "single_first_write_stdout_data_frame_with_payload_sess_does_not_have_terminal", inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}}, wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}, - wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...), - width: 10, - height: 20, - firstWrite: true, + wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl), }, } for _, tt := range tests { @@ -104,29 +95,25 @@ func Test_Writes(t *testing.T) { sr := &fakes.TestSessionRecorder{} rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar()) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() c := &conn{ - Conn: tc, - log: zl.Sugar(), - rec: rec, - ch: sessionrecording.CastHeader{ - Width: tt.width, - Height: tt.height, - }, - initialTermSizeSet: make(chan struct{}), - hasTerm: tt.hasTerm, - } - if !tt.firstWrite { - // this test case does not intend to test that cast header gets written once - c.writeCastHeaderOnce.Do(func() {}) - } - if tt.sendInitialResize { - close(c.initialTermSizeSet) + ctx: ctx, + Conn: tc, + log: zl.Sugar(), + rec: rec, + ch: sessionrecording.CastHeader{}, + initialCastHeaderSent: make(chan struct{}), + hasTerm: tt.hasTerm, } + c.writeCastHeaderOnce.Do(func() { + close(c.initialCastHeaderSent) + }) + c.stdoutStreamID.Store(stdoutStreamID) c.stderrStreamID.Store(stderrStreamID) for i, input := range tt.inputs { - c.hasTerm = tt.hasTerm if _, err := c.Write(input); err != nil { t.Errorf("[%d] spdyRemoteConnRecorder.Write() unexpected error %v", i, err) } @@ -171,11 +158,25 @@ func Test_Reads(t *testing.T) { wantResizeStreamID uint32 wantWidth int wantHeight int + wantRecorded []byte resizeStreamIDBeforeRead uint32 }{ { name: "resize_data_frame_single_read", inputs: [][]byte{append([]byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg...)}, + wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), + resizeStreamIDBeforeRead: 1, + wantWidth: 10, + wantHeight: 20, + }, + { + name: "resize_data_frame_many", + inputs: [][]byte{ + append([]byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg...), + append([]byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg...), + }, + wantRecorded: append(fakes.AsciinemaCastHeaderMsg(t, 10, 20), fakes.AsciinemaCastResizeMsg(t, 10, 20)...), + resizeStreamIDBeforeRead: 1, wantWidth: 10, wantHeight: 20, @@ -183,6 +184,7 @@ func Test_Reads(t *testing.T) { { name: "resize_data_frame_two_reads", inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg}, + wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), resizeStreamIDBeforeRead: 1, wantWidth: 10, wantHeight: 20, @@ -215,11 +217,15 @@ func Test_Reads(t *testing.T) { tc := &fakes.TestConn{} sr := &fakes.TestSessionRecorder{} rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar()) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() c := &conn{ - Conn: tc, - log: zl.Sugar(), - rec: rec, - initialTermSizeSet: make(chan struct{}), + ctx: ctx, + Conn: tc, + log: zl.Sugar(), + rec: rec, + initialCastHeaderSent: make(chan struct{}), + hasTerm: true, } c.resizeStreamID.Store(tt.resizeStreamIDBeforeRead) @@ -251,6 +257,12 @@ func Test_Reads(t *testing.T) { t.Errorf("want height: %v, got %v", tt.wantHeight, c.ch.Height) } } + + // Assert that the expected bytes have been forwarded to the session recorder. + gotRecorded := sr.Bytes() + if !reflect.DeepEqual(gotRecorded, tt.wantRecorded) { + t.Errorf("expected bytes not recorded, wants\n%v\ngot\n%v", tt.wantRecorded, gotRecorded) + } }) } } diff --git a/k8s-operator/sessionrecording/tsrecorder/tsrecorder.go b/k8s-operator/sessionrecording/tsrecorder/tsrecorder.go index af5fcb8da641a..a5bdf7ddddeeb 100644 --- a/k8s-operator/sessionrecording/tsrecorder/tsrecorder.go +++ b/k8s-operator/sessionrecording/tsrecorder/tsrecorder.go @@ -25,6 +25,7 @@ func New(conn io.WriteCloser, clock tstime.Clock, start time.Time, failOpen bool clock: clock, conn: conn, failOpen: failOpen, + logger: logger, } } diff --git a/k8s-operator/sessionrecording/ws/conn.go b/k8s-operator/sessionrecording/ws/conn.go index 86029f67b1f13..0d8aefaace52e 100644 --- a/k8s-operator/sessionrecording/ws/conn.go +++ b/k8s-operator/sessionrecording/ws/conn.go @@ -3,12 +3,13 @@ //go:build !plan9 -// package ws has functionality to parse 'kubectl exec' sessions streamed using +// package ws has functionality to parse 'kubectl exec/attach' sessions streamed using // WebSocket protocol. package ws import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -24,31 +25,53 @@ import ( ) // New wraps the provided network connection and returns a connection whose reads and writes will get triggered as data is received on the hijacked connection. -// The connection must be a hijacked connection for a 'kubectl exec' session using WebSocket protocol and a *.channel.k8s.io subprotocol. +// The connection must be a hijacked connection for a 'kubectl exec/attach' session using WebSocket protocol and a *.channel.k8s.io subprotocol. // The hijacked connection is used to transmit *.channel.k8s.io streams between Kubernetes client ('kubectl') and the destination proxy controlled by Kubernetes. // Data read from the underlying network connection is data sent via one of the streams from the client to the container. // Data written to the underlying connection is data sent from the container to the client. // We parse the data and send everything for the stdout/stderr streams to the configured tsrecorder as an asciinema recording with the provided header. // https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/4006-transition-spdy-to-websockets#proposal-new-remotecommand-sub-protocol-version---v5channelk8sio -func New(c net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, hasTerm bool, log *zap.SugaredLogger) net.Conn { - return &conn{ - Conn: c, - rec: rec, - ch: ch, - hasTerm: hasTerm, - log: log, - initialTermSizeSet: make(chan struct{}, 1), +func New(ctx context.Context, c net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, hasTerm bool, log *zap.SugaredLogger) (net.Conn, error) { + lc := &conn{ + Conn: c, + ctx: ctx, + rec: rec, + ch: ch, + hasTerm: hasTerm, + log: log, + initialCastHeaderSent: make(chan struct{}, 1), } + + // if there is no term, we don't need to wait for a resize message + if !hasTerm { + var err error + lc.writeCastHeaderOnce.Do(func() { + // If this is a session with a terminal attached, + // we must wait for the terminal width and + // height to be parsed from a resize message + // before sending CastHeader, else tsrecorder + // will not be able to play this recording. + err = lc.rec.WriteCastHeader(ch) + close(lc.initialCastHeaderSent) + }) + if err != nil { + return nil, fmt.Errorf("error writing CastHeader: %w", err) + } + } + + return lc, nil } // conn is a wrapper around net.Conn. It reads the bytestream -// for a 'kubectl exec' session, sends session recording data to the configured +// for a 'kubectl exec/attach' session, sends session recording data to the configured // recorder and forwards the raw bytes to the original destination. // A new conn is created per session. -// conn only knows to how to read a 'kubectl exec' session that is streamed using WebSocket protocol. +// conn only knows to how to read a 'kubectl exec/attach' session that is streamed using WebSocket protocol. // https://www.rfc-editor.org/rfc/rfc6455 type conn struct { net.Conn + + ctx context.Context // rec knows how to send data to a tsrecorder instance. rec *tsrecorder.Client @@ -56,7 +79,7 @@ type conn struct { // CastHeader must be sent before any payload. If the session has a // terminal attached, the CastHeader must have '.Width' and '.Height' // fields set for the tsrecorder UI to be able to play the recording. - // For 'kubectl exec' sessions, terminal width and height are sent as a + // For 'kubectl exec/attach' sessions, terminal width and height are sent as a // resize message on resize stream from the client when the session // starts as well as at any time the client detects a terminal change. // We can intercept the resize message on Read calls. As there is no @@ -72,15 +95,10 @@ type conn struct { // writeCastHeaderOnce is used to ensure CastHeader gets sent to tsrecorder once. writeCastHeaderOnce sync.Once hasTerm bool // whether the session has TTY attached - // initialTermSizeSet channel gets sent a value once, when the Read has - // received a resize message and set the initial terminal size. It must - // be set to a buffered channel to prevent Reads being blocked on the - // first stdout/stderr write reading from the channel. - initialTermSizeSet chan struct{} - // sendInitialTermSizeSetOnce is used to ensure that a value is sent to - // initialTermSizeSet channel only once, when the initial resize message - // is received. - sendInitialTermSizeSetOnce sync.Once + // initialCastHeaderSent is a boolean that is set to ensure that the cast + // header is the first thing that is streamed to the session recorder. + // Otherwise the stream will fail. + initialCastHeaderSent chan struct{} log *zap.SugaredLogger @@ -171,9 +189,10 @@ func (c *conn) Read(b []byte) (int, error) { c.readBuf.Next(len(readMsg.raw)) if readMsg.isFinalized && !c.readMsgIsIncomplete() { + // we want to send stream resize messages for terminal sessions // Stream IDs for websocket streams are static. // https://github.com/kubernetes/client-go/blob/v0.30.0-rc.1/tools/remotecommand/websocket.go#L218 - if readMsg.streamID.Load() == remotecommand.StreamResize { + if readMsg.streamID.Load() == remotecommand.StreamResize && c.hasTerm { var msg tsrecorder.ResizeMsg if err = json.Unmarshal(readMsg.payload, &msg); err != nil { return 0, fmt.Errorf("error umarshalling resize message: %w", err) @@ -182,22 +201,29 @@ func (c *conn) Read(b []byte) (int, error) { c.ch.Width = msg.Width c.ch.Height = msg.Height - // If this is initial resize message, the width and - // height will be sent in the CastHeader. If this is a - // subsequent resize message, we need to send asciinema - // resize message. var isInitialResize bool - c.sendInitialTermSizeSetOnce.Do(func() { + c.writeCastHeaderOnce.Do(func() { isInitialResize = true - close(c.initialTermSizeSet) // unblock sending of CastHeader + // If this is a session with a terminal attached, + // we must wait for the terminal width and + // height to be parsed from a resize message + // before sending CastHeader, else tsrecorder + // will not be able to play this recording. + err = c.rec.WriteCastHeader(c.ch) + close(c.initialCastHeaderSent) }) + if err != nil { + return 0, fmt.Errorf("error writing CastHeader: %w", err) + } + if !isInitialResize { - if err := c.rec.WriteResize(c.ch.Height, c.ch.Width); err != nil { + if err := c.rec.WriteResize(msg.Height, msg.Width); err != nil { return 0, fmt.Errorf("error writing resize message: %w", err) } } } } + c.currentReadMsg = readMsg return n, nil } @@ -244,39 +270,33 @@ func (c *conn) Write(b []byte) (int, error) { c.log.Errorf("write: parsing a message errored: %v", err) return 0, fmt.Errorf("write: error parsing message: %v", err) } + c.currentWriteMsg = writeMsg if !ok { // incomplete fragment return len(b), nil } + c.writeBuf.Next(len(writeMsg.raw)) // advance frame if len(writeMsg.payload) != 0 && writeMsg.isFinalized { if writeMsg.streamID.Load() == remotecommand.StreamStdOut || writeMsg.streamID.Load() == remotecommand.StreamStdErr { - var err error - c.writeCastHeaderOnce.Do(func() { - // If this is a session with a terminal attached, - // we must wait for the terminal width and - // height to be parsed from a resize message - // before sending CastHeader, else tsrecorder - // will not be able to play this recording. - if c.hasTerm { - c.log.Debug("waiting for terminal size to be set before starting to send recorded data") - <-c.initialTermSizeSet + // we must wait for confirmation that the initial cast header was sent before proceeding with any more writes + select { + case <-c.ctx.Done(): + return 0, c.ctx.Err() + case <-c.initialCastHeaderSent: + if err := c.rec.WriteOutput(writeMsg.payload); err != nil { + return 0, fmt.Errorf("error writing message to recorder: %w", err) } - err = c.rec.WriteCastHeader(c.ch) - }) - if err != nil { - return 0, fmt.Errorf("error writing CastHeader: %w", err) - } - if err := c.rec.WriteOutput(writeMsg.payload); err != nil { - return 0, fmt.Errorf("error writing message to recorder: %v", err) } } } + _, err = c.Conn.Write(c.currentWriteMsg.raw) if err != nil { c.log.Errorf("write: error writing to conn: %v", err) } + return len(b), nil } @@ -321,6 +341,7 @@ func (c *conn) writeMsgIsIncomplete() bool { func (c *conn) readMsgIsIncomplete() bool { return c.currentReadMsg != nil && !c.currentReadMsg.isFinalized } + func (c *conn) curReadMsgType() (messageType, error) { if c.currentReadMsg != nil { return c.currentReadMsg.typ, nil diff --git a/k8s-operator/sessionrecording/ws/conn_test.go b/k8s-operator/sessionrecording/ws/conn_test.go index 11174480ba605..f29154c622602 100644 --- a/k8s-operator/sessionrecording/ws/conn_test.go +++ b/k8s-operator/sessionrecording/ws/conn_test.go @@ -6,9 +6,11 @@ package ws import ( + "context" "fmt" "reflect" "testing" + "time" "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/remotecommand" @@ -26,46 +28,69 @@ func Test_conn_Read(t *testing.T) { // Resize stream ID + {"width": 10, "height": 20} testResizeMsg := []byte{byte(remotecommand.StreamResize), 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x3a, 0x32, 0x30, 0x7d} lenResizeMsgPayload := byte(len(testResizeMsg)) - + cl := tstest.NewClock(tstest.ClockOpts{}) tests := []struct { - name string - inputs [][]byte - wantWidth int - wantHeight int + name string + inputs [][]byte + wantCastHeaderWidth int + wantCastHeaderHeight int + wantRecorded []byte }{ { name: "single_read_control_message", inputs: [][]byte{{0x88, 0x0}}, }, { - name: "single_read_resize_message", - inputs: [][]byte{append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...)}, - wantWidth: 10, - wantHeight: 20, + name: "single_read_resize_message", + inputs: [][]byte{append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...)}, + wantCastHeaderWidth: 10, + wantCastHeaderHeight: 20, + wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), }, { - name: "two_reads_resize_message", - inputs: [][]byte{{0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x3a, 0x32, 0x30, 0x7d}}, - wantWidth: 10, - wantHeight: 20, + name: "resize_data_frame_many", + inputs: [][]byte{ + append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...), + append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...), + }, + wantRecorded: append(fakes.AsciinemaCastHeaderMsg(t, 10, 20), fakes.AsciinemaCastResizeMsg(t, 10, 20)...), + wantCastHeaderWidth: 10, + wantCastHeaderHeight: 20, }, { - name: "three_reads_resize_message_with_split_fragment", - inputs: [][]byte{{0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74}, {0x22, 0x3a, 0x32, 0x30, 0x7d}}, - wantWidth: 10, - wantHeight: 20, + name: "two_reads_resize_message", + inputs: [][]byte{{0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x3a, 0x32, 0x30, 0x7d}}, + wantCastHeaderWidth: 10, + wantCastHeaderHeight: 20, + wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), + }, + { + name: "three_reads_resize_message_with_split_fragment", + inputs: [][]byte{{0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74}, {0x22, 0x3a, 0x32, 0x30, 0x7d}}, + wantCastHeaderWidth: 10, + wantCastHeaderHeight: 20, + wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + l := zl.Sugar() tc := &fakes.TestConn{} + sr := &fakes.TestSessionRecorder{} + rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar()) tc.ResetReadBuf() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() c := &conn{ - Conn: tc, - log: zl.Sugar(), + ctx: ctx, + Conn: tc, + log: l, + hasTerm: true, + initialCastHeaderSent: make(chan struct{}), + rec: rec, } for i, input := range tt.inputs { - c.initialTermSizeSet = make(chan struct{}) if err := tc.WriteReadBufBytes(input); err != nil { t.Fatalf("writing bytes to test conn: %v", err) } @@ -75,14 +100,20 @@ func Test_conn_Read(t *testing.T) { return } } - if tt.wantHeight != 0 || tt.wantWidth != 0 { - if tt.wantWidth != c.ch.Width { - t.Errorf("wants width: %v, got %v", tt.wantWidth, c.ch.Width) + + if tt.wantCastHeaderHeight != 0 || tt.wantCastHeaderWidth != 0 { + if tt.wantCastHeaderWidth != c.ch.Width { + t.Errorf("wants width: %v, got %v", tt.wantCastHeaderWidth, c.ch.Width) } - if tt.wantHeight != c.ch.Height { - t.Errorf("want height: %v, got %v", tt.wantHeight, c.ch.Height) + if tt.wantCastHeaderHeight != c.ch.Height { + t.Errorf("want height: %v, got %v", tt.wantCastHeaderHeight, c.ch.Height) } } + + gotRecorded := sr.Bytes() + if !reflect.DeepEqual(gotRecorded, tt.wantRecorded) { + t.Errorf("expected bytes not recorded, wants\n%v\ngot\n%v", string(tt.wantRecorded), string(gotRecorded)) + } }) } } @@ -94,15 +125,11 @@ func Test_conn_Write(t *testing.T) { } cl := tstest.NewClock(tstest.ClockOpts{}) tests := []struct { - name string - inputs [][]byte - wantForwarded []byte - wantRecorded []byte - firstWrite bool - width int - height int - hasTerm bool - sendInitialResize bool + name string + inputs [][]byte + wantForwarded []byte + wantRecorded []byte + hasTerm bool }{ { name: "single_write_control_frame", @@ -130,10 +157,7 @@ func Test_conn_Write(t *testing.T) { name: "single_write_stdout_data_message_with_cast_header", inputs: [][]byte{{0x82, 0x3, 0x1, 0x7, 0x8}}, wantForwarded: []byte{0x82, 0x3, 0x1, 0x7, 0x8}, - wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x7, 0x8}, cl)...), - width: 10, - height: 20, - firstWrite: true, + wantRecorded: fakes.CastLine(t, []byte{0x7, 0x8}, cl), }, { name: "two_writes_stdout_data_message", @@ -148,15 +172,11 @@ func Test_conn_Write(t *testing.T) { wantRecorded: fakes.CastLine(t, []byte{0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5}, cl), }, { - name: "three_writes_stdout_data_message_with_split_fragment_cast_header_with_terminal", - inputs: [][]byte{{0x2, 0x3, 0x1, 0x7, 0x8}, {0x80, 0x6, 0x1, 0x1, 0x2, 0x3}, {0x4, 0x5}}, - wantForwarded: []byte{0x2, 0x3, 0x1, 0x7, 0x8, 0x80, 0x6, 0x1, 0x1, 0x2, 0x3, 0x4, 0x5}, - wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5}, cl)...), - height: 20, - width: 10, - hasTerm: true, - firstWrite: true, - sendInitialResize: true, + name: "three_writes_stdout_data_message_with_split_fragment_cast_header_with_terminal", + inputs: [][]byte{{0x2, 0x3, 0x1, 0x7, 0x8}, {0x80, 0x6, 0x1, 0x1, 0x2, 0x3}, {0x4, 0x5}}, + wantForwarded: []byte{0x2, 0x3, 0x1, 0x7, 0x8, 0x80, 0x6, 0x1, 0x1, 0x2, 0x3, 0x4, 0x5}, + wantRecorded: fakes.CastLine(t, []byte{0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5}, cl), + hasTerm: true, }, } for _, tt := range tests { @@ -164,24 +184,22 @@ func Test_conn_Write(t *testing.T) { tc := &fakes.TestConn{} sr := &fakes.TestSessionRecorder{} rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar()) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() c := &conn{ - Conn: tc, - log: zl.Sugar(), - ch: sessionrecording.CastHeader{ - Width: tt.width, - Height: tt.height, - }, - rec: rec, - initialTermSizeSet: make(chan struct{}), - hasTerm: tt.hasTerm, - } - if !tt.firstWrite { - // This test case does not intend to test that cast header gets written once. - c.writeCastHeaderOnce.Do(func() {}) - } - if tt.sendInitialResize { - close(c.initialTermSizeSet) + Conn: tc, + ctx: ctx, + log: zl.Sugar(), + ch: sessionrecording.CastHeader{}, + rec: rec, + initialCastHeaderSent: make(chan struct{}), + hasTerm: tt.hasTerm, } + + c.writeCastHeaderOnce.Do(func() { + close(c.initialCastHeaderSent) + }) + for i, input := range tt.inputs { _, err := c.Write(input) if err != nil { diff --git a/sessionrecording/header.go b/sessionrecording/header.go index 4806f6585f976..545bf06bd5984 100644 --- a/sessionrecording/header.go +++ b/sessionrecording/header.go @@ -66,13 +66,15 @@ type CastHeader struct { Kubernetes *Kubernetes `json:"kubernetes,omitempty"` } -// Kubernetes contains 'kubectl exec' session specific information for +// Kubernetes contains 'kubectl exec/attach' session specific information for // tsrecorder. type Kubernetes struct { - // PodName is the name of the Pod being exec-ed. + // PodName is the name of the Pod the session was recorded for. PodName string - // Namespace is the namespace in which is the Pod that is being exec-ed. + // Namespace is the namespace in which the Pod the session was recorded for exists in. Namespace string - // Container is the container being exec-ed. + // Container is the container the session was recorded for. Container string + // SessionType is the type of session that was executed (e.g., exec, attach) + SessionType string } From fe46f33885f5abb797f7289fa00d5b49a59d8468 Mon Sep 17 00:00:00 2001 From: Tom Meadows Date: Mon, 14 Jul 2025 15:39:39 +0100 Subject: [PATCH 205/263] cmd/{k8s-operator,k8s-proxy},kube/k8s-proxy: add static endpoints for kube-apiserver type ProxyGroups (#16523) Updates #13358 Signed-off-by: chaosinthecrd --- cmd/k8s-operator/proxygroup.go | 4 ++ cmd/k8s-operator/proxygroup_specs.go | 83 ++++++++++++++++------------ cmd/k8s-proxy/k8s-proxy.go | 15 +++++ kube/k8s-proxy/conf/conf.go | 4 ++ 4 files changed, 70 insertions(+), 36 deletions(-) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 66b6c96e3c25c..1fdc076f94cad 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -824,6 +824,10 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p cfg.AcceptRoutes = &proxyClass.Spec.TailscaleConfig.AcceptRoutes } + if len(endpoints[nodePortSvcName]) > 0 { + cfg.StaticEndpoints = endpoints[nodePortSvcName] + } + cfgB, err := json.Marshal(cfg) if err != nil { return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err) diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 5d6d0b8ef9626..71398d0d54c2f 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -66,7 +66,7 @@ func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *cor // applied over the top after. func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { - return kubeAPIServerStatefulSet(pg, namespace, image) + return kubeAPIServerStatefulSet(pg, namespace, image, port) } ss := new(appsv1.StatefulSet) if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { @@ -276,7 +276,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string return ss, nil } -func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string) (*appsv1.StatefulSet, error) { +func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, port *uint16) (*appsv1.StatefulSet, error) { sts := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: pg.Name, @@ -302,48 +302,59 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string) (*a { Name: mainContainerName, Image: image, - Env: []corev1.EnvVar{ - { - // Used as default hostname and in Secret names. - Name: "POD_NAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.name", + Env: func() []corev1.EnvVar { + envs := []corev1.EnvVar{ + { + // Used as default hostname and in Secret names. + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, }, }, - }, - { - // Used by kubeclient to post Events about the Pod's lifecycle. - Name: "POD_UID", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.uid", + { + // Used by kubeclient to post Events about the Pod's lifecycle. + Name: "POD_UID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.uid", + }, }, }, - }, - { - // Used in an interpolated env var if metrics enabled. - Name: "POD_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", + { + // Used in an interpolated env var if metrics enabled. + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, }, }, - }, - { - // Included for completeness with POD_IP and easier backwards compatibility in future. - Name: "POD_IPS", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIPs", + { + // Included for completeness with POD_IP and easier backwards compatibility in future. + Name: "POD_IPS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, }, }, - }, - { - Name: "TS_K8S_PROXY_CONFIG", - Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile), - }, - }, + { + Name: "TS_K8S_PROXY_CONFIG", + Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile), + }, + } + + if port != nil { + envs = append(envs, corev1.EnvVar{ + Name: "PORT", + Value: strconv.Itoa(int(*port)), + }) + } + + return envs + }(), VolumeMounts: func() []corev1.VolumeMount { var mounts []corev1.VolumeMount diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go index 7dcf6c2ab5809..b7f3d9535a071 100644 --- a/cmd/k8s-proxy/k8s-proxy.go +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -14,6 +14,7 @@ import ( "fmt" "os" "os/signal" + "strings" "syscall" "time" @@ -63,6 +64,20 @@ func run(logger *zap.SugaredLogger) error { logger = logger.WithOptions(zap.IncreaseLevel(level)) } + // TODO:(ChaosInTheCRD) This is a temporary workaround until we can set static endpoints using prefs + if se := cfg.Parsed.StaticEndpoints; len(se) > 0 { + logger.Debugf("setting static endpoints '%v' via TS_DEBUG_PRETENDPOINT environment variable", cfg.Parsed.StaticEndpoints) + ses := make([]string, len(se)) + for i, e := range se { + ses[i] = e.String() + } + + err := os.Setenv("TS_DEBUG_PRETENDPOINT", strings.Join(ses, ",")) + if err != nil { + return err + } + } + if cfg.Parsed.App != nil { hostinfo.SetApp(*cfg.Parsed.App) } diff --git a/kube/k8s-proxy/conf/conf.go b/kube/k8s-proxy/conf/conf.go index fba4a39a420a1..8882360c5ea21 100644 --- a/kube/k8s-proxy/conf/conf.go +++ b/kube/k8s-proxy/conf/conf.go @@ -10,6 +10,7 @@ package conf import ( "encoding/json" "fmt" + "net/netip" "os" "github.com/tailscale/hujson" @@ -55,6 +56,9 @@ type ConfigV1Alpha1 struct { KubeAPIServer *KubeAPIServer `json:",omitempty"` // Config specific to the API Server proxy. ServerURL *string `json:",omitempty"` // URL of the Tailscale coordination server. AcceptRoutes *bool `json:",omitempty"` // Accepts routes advertised by other Tailscale nodes. + // StaticEndpoints are additional, user-defined endpoints that this node + // should advertise amongst its wireguard endpoints. + StaticEndpoints []netip.AddrPort `json:",omitempty"` } type KubeAPIServer struct { From fc5050048ee9c71dcdeb232d3a38f068072f489f Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 14 Jul 2025 10:42:56 -0700 Subject: [PATCH 206/263] wgengine/magicsock: don't acquire Conn.mu in udpRelayEndpointReady (#16557) udpRelayEndpointReady used to write into the peerMap, which required holding Conn.mu, but this changed in f9e7131. Updates #cleanup Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index c4ca812969bf9..d8d1e6ee338d3 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -104,8 +104,6 @@ type endpoint struct { // be installed as de.bestAddr. It is only called by [relayManager] once it has // determined maybeBest is functional via [disco.Pong] reception. func (de *endpoint) udpRelayEndpointReady(maybeBest addrQuality) { - de.c.mu.Lock() - defer de.c.mu.Unlock() de.mu.Lock() defer de.mu.Unlock() From f338c4074d4bb67acfbabdddf2974b23274236d9 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Mon, 14 Jul 2025 11:57:54 -1000 Subject: [PATCH 207/263] util/jsonutil: remove unused package (#16563) This package promises more performance, but was never used. The intent of the package is somewhat moot as "encoding/json" in Go 1.25 (while under GOEXPERIMENT=jsonv2) has been completely re-implemented using "encoding/json/v2" such that unmarshal is dramatically faster. Updates #cleanup Updates tailscale/corp#791 Signed-off-by: Joe Tsai --- util/jsonutil/types.go | 16 ------ util/jsonutil/unmarshal.go | 89 --------------------------------- util/jsonutil/unmarshal_test.go | 64 ------------------------ 3 files changed, 169 deletions(-) delete mode 100644 util/jsonutil/types.go delete mode 100644 util/jsonutil/unmarshal.go delete mode 100644 util/jsonutil/unmarshal_test.go diff --git a/util/jsonutil/types.go b/util/jsonutil/types.go deleted file mode 100644 index 057473249f258..0000000000000 --- a/util/jsonutil/types.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package jsonutil - -// Bytes is a byte slice in a json-encoded struct. -// encoding/json assumes that []byte fields are hex-encoded. -// Bytes are not hex-encoded; they are treated the same as strings. -// This can avoid unnecessary allocations due to a round trip through strings. -type Bytes []byte - -func (b *Bytes) UnmarshalText(text []byte) error { - // Copy the contexts of text. - *b = append(*b, text...) - return nil -} diff --git a/util/jsonutil/unmarshal.go b/util/jsonutil/unmarshal.go deleted file mode 100644 index b1eb4ea873e67..0000000000000 --- a/util/jsonutil/unmarshal.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -// Package jsonutil provides utilities to improve JSON performance. -// It includes an Unmarshal wrapper that amortizes allocated garbage over subsequent runs -// and a Bytes type to reduce allocations when unmarshalling a non-hex-encoded string into a []byte. -package jsonutil - -import ( - "bytes" - "encoding/json" - "sync" -) - -// decoder is a re-usable json decoder. -type decoder struct { - dec *json.Decoder - r *bytes.Reader -} - -var readerPool = sync.Pool{ - New: func() any { - return bytes.NewReader(nil) - }, -} - -var decoderPool = sync.Pool{ - New: func() any { - var d decoder - d.r = readerPool.Get().(*bytes.Reader) - d.dec = json.NewDecoder(d.r) - return &d - }, -} - -// Unmarshal is similar to encoding/json.Unmarshal. -// There are three major differences: -// -// On error, encoding/json.Unmarshal zeros v. -// This Unmarshal may leave partial data in v. -// Always check the error before using v! -// (Future improvements may remove this bug.) -// -// The errors they return don't always match perfectly. -// If you do error matching more precise than err != nil, -// don't use this Unmarshal. -// -// This Unmarshal allocates considerably less memory. -func Unmarshal(b []byte, v any) error { - d := decoderPool.Get().(*decoder) - d.r.Reset(b) - off := d.dec.InputOffset() - err := d.dec.Decode(v) - d.r.Reset(nil) // don't keep a reference to b - // In case of error, report the offset in this byte slice, - // instead of in the totality of all bytes this decoder has processed. - // It is not possible to make all errors match json.Unmarshal exactly, - // but we can at least try. - switch jsonerr := err.(type) { - case *json.SyntaxError: - jsonerr.Offset -= off - case *json.UnmarshalTypeError: - jsonerr.Offset -= off - case nil: - // json.Unmarshal fails if there's any extra junk in the input. - // json.Decoder does not; see https://github.com/golang/go/issues/36225. - // We need to check for anything left over in the buffer. - if d.dec.More() { - // TODO: Provide a better error message. - // Unfortunately, we can't set the msg field. - // The offset doesn't perfectly match json: - // Ours is at the end of the valid data, - // and theirs is at the beginning of the extra data after whitespace. - // Close enough, though. - err = &json.SyntaxError{Offset: d.dec.InputOffset() - off} - - // TODO: zero v. This is hard; see encoding/json.indirect. - } - } - if err == nil { - decoderPool.Put(d) - } else { - // There might be junk left in the decoder's buffer. - // There's no way to flush it, no Reset method. - // Abandoned the decoder but reuse the reader. - readerPool.Put(d.r) - } - return err -} diff --git a/util/jsonutil/unmarshal_test.go b/util/jsonutil/unmarshal_test.go deleted file mode 100644 index 32f8402f02e58..0000000000000 --- a/util/jsonutil/unmarshal_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package jsonutil - -import ( - "encoding/json" - "reflect" - "testing" -) - -func TestCompareToStd(t *testing.T) { - tests := []string{ - `{}`, - `{"a": 1}`, - `{]`, - `"abc"`, - `5`, - `{"a": 1} `, - `{"a": 1} {}`, - `{} bad data`, - `{"a": 1} "hello"`, - `[]`, - ` {"x": {"t": [3,4,5]}}`, - } - - for _, test := range tests { - b := []byte(test) - var ourV, stdV any - ourErr := Unmarshal(b, &ourV) - stdErr := json.Unmarshal(b, &stdV) - if (ourErr == nil) != (stdErr == nil) { - t.Errorf("Unmarshal(%q): our err = %#[2]v (%[2]T), std err = %#[3]v (%[3]T)", test, ourErr, stdErr) - } - // if !reflect.DeepEqual(ourErr, stdErr) { - // t.Logf("Unmarshal(%q): our err = %#[2]v (%[2]T), std err = %#[3]v (%[3]T)", test, ourErr, stdErr) - // } - if ourErr != nil { - // TODO: if we zero ourV on error, remove this continue. - continue - } - if !reflect.DeepEqual(ourV, stdV) { - t.Errorf("Unmarshal(%q): our val = %v, std val = %v", test, ourV, stdV) - } - } -} - -func BenchmarkUnmarshal(b *testing.B) { - var m any - j := []byte("5") - b.ReportAllocs() - for range b.N { - Unmarshal(j, &m) - } -} - -func BenchmarkStdUnmarshal(b *testing.B) { - var m any - j := []byte("5") - b.ReportAllocs() - for range b.N { - json.Unmarshal(j, &m) - } -} From b63f8a457dbb14700a7c6bdb96e4df95a5c258b3 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 14 Jul 2025 15:09:31 -0700 Subject: [PATCH 208/263] wgengine/magicsock: prioritize trusted peer relay paths over untrusted (#16559) A trusted peer relay path is always better than an untrusted direct or peer relay path. Updates tailscale/corp#30412 Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 35 ++++++------ wgengine/magicsock/endpoint_test.go | 89 +++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 16 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index d8d1e6ee338d3..385c9245ec4e7 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -106,24 +106,27 @@ type endpoint struct { func (de *endpoint) udpRelayEndpointReady(maybeBest addrQuality) { de.mu.Lock() defer de.mu.Unlock() - - if maybeBest.relayServerDisco.Compare(de.bestAddr.relayServerDisco) == 0 { - // TODO(jwhited): add some observability for this case, e.g. did we - // flip transports during a de.bestAddr transition from untrusted to - // trusted? + now := mono.Now() + curBestAddrTrusted := now.Before(de.trustBestAddrUntil) + sameRelayServer := de.bestAddr.vni.isSet() && maybeBest.relayServerDisco.Compare(de.bestAddr.relayServerDisco) == 0 + + if !curBestAddrTrusted || + sameRelayServer || + betterAddr(maybeBest, de.bestAddr) { + // We must set maybeBest as de.bestAddr if: + // 1. de.bestAddr is untrusted. betterAddr does not consider + // time-based trust. + // 2. maybeBest & de.bestAddr are on the same relay. If the maybeBest + // handshake happened to use a different source address/transport, + // the relay will drop packets from the 'old' de.bestAddr's. + // 3. maybeBest is a 'betterAddr'. // - // If these are equal we must set maybeBest as bestAddr, otherwise we - // could leave a stale bestAddr if it goes over a different - // address family or src. - } else if !betterAddr(maybeBest, de.bestAddr) { - return + // TODO(jwhited): add observability around !curBestAddrTrusted and sameRelayServer + // TODO(jwhited): collapse path change logging with endpoint.handlePongConnLocked() + de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v", de.publicKey.ShortString(), de.discoShort(), maybeBest.epAddr, maybeBest.wireMTU) + de.setBestAddrLocked(maybeBest) + de.trustBestAddrUntil = now.Add(trustUDPAddrDuration) } - - // Promote maybeBest to bestAddr. - // TODO(jwhited): collapse path change logging with endpoint.handlePongConnLocked() - de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v", de.publicKey.ShortString(), de.discoShort(), maybeBest.epAddr, maybeBest.wireMTU) - de.setBestAddrLocked(maybeBest) - de.trustBestAddrUntil = mono.Now().Add(trustUDPAddrDuration) } func (de *endpoint) setBestAddrLocked(v addrQuality) { diff --git a/wgengine/magicsock/endpoint_test.go b/wgengine/magicsock/endpoint_test.go index 3a1e55b8b9728..92f4ef1d3aac1 100644 --- a/wgengine/magicsock/endpoint_test.go +++ b/wgengine/magicsock/endpoint_test.go @@ -9,6 +9,7 @@ import ( "time" "tailscale.com/tailcfg" + "tailscale.com/tstime/mono" "tailscale.com/types/key" ) @@ -365,3 +366,91 @@ func Test_epAddr_isDirectUDP(t *testing.T) { }) } } + +func Test_endpoint_udpRelayEndpointReady(t *testing.T) { + directAddrQuality := addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.1:7")}} + peerRelayAddrQuality := addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.2:77")}, latency: time.Second} + peerRelayAddrQuality.vni.set(1) + peerRelayAddrQualityHigherLatencySameServer := addrQuality{ + epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.3:77"), vni: peerRelayAddrQuality.vni}, + latency: peerRelayAddrQuality.latency * 10, + } + peerRelayAddrQualityHigherLatencyDiffServer := addrQuality{ + epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.3:77"), vni: peerRelayAddrQuality.vni}, + latency: peerRelayAddrQuality.latency * 10, + relayServerDisco: key.NewDisco().Public(), + } + peerRelayAddrQualityLowerLatencyDiffServer := addrQuality{ + epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.4:77"), vni: peerRelayAddrQuality.vni}, + latency: peerRelayAddrQuality.latency / 10, + relayServerDisco: key.NewDisco().Public(), + } + peerRelayAddrQualityEqualLatencyDiffServer := addrQuality{ + epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.4:77"), vni: peerRelayAddrQuality.vni}, + latency: peerRelayAddrQuality.latency, + relayServerDisco: key.NewDisco().Public(), + } + tests := []struct { + name string + curBestAddr addrQuality + trustBestAddrUntil mono.Time + maybeBest addrQuality + wantBestAddr addrQuality + }{ + { + name: "bestAddr trusted direct", + curBestAddr: directAddrQuality, + trustBestAddrUntil: mono.Now().Add(1 * time.Hour), + maybeBest: peerRelayAddrQuality, + wantBestAddr: directAddrQuality, + }, + { + name: "bestAddr untrusted direct", + curBestAddr: directAddrQuality, + trustBestAddrUntil: mono.Now().Add(-1 * time.Hour), + maybeBest: peerRelayAddrQuality, + wantBestAddr: peerRelayAddrQuality, + }, + { + name: "maybeBest same relay server higher latency bestAddr trusted", + curBestAddr: peerRelayAddrQuality, + trustBestAddrUntil: mono.Now().Add(1 * time.Hour), + maybeBest: peerRelayAddrQualityHigherLatencySameServer, + wantBestAddr: peerRelayAddrQualityHigherLatencySameServer, + }, + { + name: "maybeBest diff relay server higher latency bestAddr trusted", + curBestAddr: peerRelayAddrQuality, + trustBestAddrUntil: mono.Now().Add(1 * time.Hour), + maybeBest: peerRelayAddrQualityHigherLatencyDiffServer, + wantBestAddr: peerRelayAddrQuality, + }, + { + name: "maybeBest diff relay server lower latency bestAddr trusted", + curBestAddr: peerRelayAddrQuality, + trustBestAddrUntil: mono.Now().Add(1 * time.Hour), + maybeBest: peerRelayAddrQualityLowerLatencyDiffServer, + wantBestAddr: peerRelayAddrQualityLowerLatencyDiffServer, + }, + { + name: "maybeBest diff relay server equal latency bestAddr trusted", + curBestAddr: peerRelayAddrQuality, + trustBestAddrUntil: mono.Now().Add(1 * time.Hour), + maybeBest: peerRelayAddrQualityEqualLatencyDiffServer, + wantBestAddr: peerRelayAddrQuality, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + de := &endpoint{ + c: &Conn{logf: func(msg string, args ...any) { return }}, + bestAddr: tt.curBestAddr, + trustBestAddrUntil: tt.trustBestAddrUntil, + } + de.udpRelayEndpointReady(tt.maybeBest) + if de.bestAddr != tt.wantBestAddr { + t.Errorf("de.bestAddr = %v, want %v", de.bestAddr, tt.wantBestAddr) + } + }) + } +} From bfb344905f5d12648031b0aaec27393ae4173e12 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Mon, 14 Jul 2025 18:51:55 -0700 Subject: [PATCH 209/263] ipn/ipnlocal: modernize nm.Peers with AppendMatchingPeers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks to @nickkhyl for pointing out that NetMap.Peers doesn’t get incremental updates since the last full NetMap update. Instead, he recommends using ipn/ipnlocal.nodeBackend.AppendMatchingPeers. Updates #cleanup Signed-off-by: Simon Law --- ipn/ipnlocal/local.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 4ed012f2e46f8..cd1654eb159b8 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -7902,33 +7902,32 @@ func suggestExitNodeUsingTrafficSteering(nb *nodeBackend, prev tailcfg.StableNod panic("missing traffic-steering capability") } - peers := nm.Peers - nodes := make([]tailcfg.NodeView, 0, len(peers)) - - for _, p := range peers { + var force tailcfg.NodeView + nodes := nb.AppendMatchingPeers(nil, func(p tailcfg.NodeView) bool { if !p.Valid() { - continue + return false } if allowed != nil && !allowed.Contains(p.StableID()) { - continue + return false } if !p.CapMap().Contains(tailcfg.NodeAttrSuggestExitNode) { - continue + return false } if !tsaddr.ContainsExitRoutes(p.AllowedIPs()) { - continue + return false } if p.StableID() == prev { // Prevent flapping: since prev is a valid suggestion, // force prev to be the only valid pick. - nodes = []tailcfg.NodeView{p} - break + force = p + return false } - nodes = append(nodes, p) + return true + }) + if force.Valid() { + nodes = append(nodes[:0], force) } - var pick tailcfg.NodeView - scores := make(map[tailcfg.NodeID]int, len(nodes)) score := func(n tailcfg.NodeView) int { id := n.ID() @@ -7945,7 +7944,11 @@ func suggestExitNodeUsingTrafficSteering(nb *nodeBackend, prev tailcfg.StableNod return s } - if len(nodes) > 0 { + var pick tailcfg.NodeView + if len(nodes) == 1 { + pick = nodes[0] + } + if len(nodes) > 1 { // Find the highest scoring exit nodes. slices.SortFunc(nodes, func(a, b tailcfg.NodeView) int { return cmp.Compare(score(b), score(a)) // reverse sort From 205f822372d203f32b3fb3c7562347770a927181 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Mon, 14 Jul 2025 19:01:02 -0700 Subject: [PATCH 210/263] ipn/ipnlocal: check if suggested exit node is online @nickkyl added an peer.Online check to suggestExitNodeUsingDERP, so it should also check when running suggestExitNodeUsingTrafficSteering. Updates tailscale/corp#29966 Signed-off-by: Simon Law --- ipn/ipnlocal/local.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index cd1654eb159b8..9b9bd82b5e9e5 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -7907,6 +7907,9 @@ func suggestExitNodeUsingTrafficSteering(nb *nodeBackend, prev tailcfg.StableNod if !p.Valid() { return false } + if !p.Online().Get() { + return false + } if allowed != nil && !allowed.Contains(p.StableID()) { return false } From 7a3221177e0e323d89b5e6389a4a4274065eb725 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 08:33:22 -0600 Subject: [PATCH 211/263] .github: Bump slackapi/slack-github-action from 2.1.0 to 2.1.1 (#16553) Bumps [slackapi/slack-github-action](https://github.com/slackapi/slack-github-action) from 2.1.0 to 2.1.1. - [Release notes](https://github.com/slackapi/slack-github-action/releases) - [Commits](https://github.com/slackapi/slack-github-action/compare/b0fa283ad8fea605de13dc3f449259339835fc52...91efab103c0de0a537f72a35f6b8cda0ee76bf0a) --- updated-dependencies: - dependency-name: slackapi/slack-github-action dependency-version: 2.1.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/govulncheck.yml | 2 +- .github/workflows/installer.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index 36ed1fe9bf603..c7560983abeb6 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -24,7 +24,7 @@ jobs: - name: Post to slack if: failure() && github.event_name == 'schedule' - uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 with: method: chat.postMessage token: ${{ secrets.GOVULNCHECK_BOT_TOKEN }} diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml index 0ca16ae9fa6c1..6144864fd53b8 100644 --- a/.github/workflows/installer.yml +++ b/.github/workflows/installer.yml @@ -108,7 +108,7 @@ jobs: steps: - name: Notify Slack of failure on scheduled runs if: failure() && github.event_name == 'schedule' - uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL }} webhook-type: incoming-webhook diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e80b44dcc4d3..d5b09a9e6cc07 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -831,7 +831,7 @@ jobs: # By having the job always run, but skipping its only step as needed, we # let the CI output collapse nicely in PRs. if: failure() && github.event_name == 'push' - uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL }} webhook-type: incoming-webhook From e0fcd596bf50556243c488f916d5128dccba6638 Mon Sep 17 00:00:00 2001 From: James Sanderson Date: Mon, 14 Jul 2025 17:54:56 +0100 Subject: [PATCH 212/263] tailcfg: send health update if DisplayMessage URL changes Updates tailscale/corp#27759 Signed-off-by: James Sanderson --- health/health_test.go | 160 +++++++++++++++++++--------------------- tailcfg/tailcfg.go | 5 +- tailcfg/tailcfg_test.go | 113 ++++++++++++++++++++-------- 3 files changed, 162 insertions(+), 116 deletions(-) diff --git a/health/health_test.go b/health/health_test.go index 0f1140f621bb4..53f012ecffd55 100644 --- a/health/health_test.go +++ b/health/health_test.go @@ -555,98 +555,88 @@ func TestControlHealth(t *testing.T) { }) } -func TestControlHealthNotifiesOnSet(t *testing.T) { - ht := Tracker{} - ht.SetIPNState("NeedsLogin", true) - ht.GotStreamedMapResponse() - - gotNotified := false - ht.registerSyncWatcher(func(_ Change) { - gotNotified = true - }) - - ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ - "test": {}, - }) - - if !gotNotified { - t.Errorf("watcher did not get called, want it to be called") - } -} - -func TestControlHealthNotifiesOnChange(t *testing.T) { - ht := Tracker{} - ht.SetIPNState("NeedsLogin", true) - ht.GotStreamedMapResponse() - - ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ - "test-1": {}, - }) - - gotNotified := false - ht.registerSyncWatcher(func(_ Change) { - gotNotified = true - }) - - ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ - "test-2": {}, - }) - - if !gotNotified { - t.Errorf("watcher did not get called, want it to be called") - } -} - -func TestControlHealthNotifiesOnDetailsChange(t *testing.T) { - ht := Tracker{} - ht.SetIPNState("NeedsLogin", true) - ht.GotStreamedMapResponse() - - ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ - "test-1": { - Title: "Title", +func TestControlHealthNotifies(t *testing.T) { + type test struct { + name string + initialState map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage + newState map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage + wantNotify bool + } + tests := []test{ + { + name: "no-change", + initialState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": {}, + }, + newState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": {}, + }, + wantNotify: false, }, - }) - - gotNotified := false - ht.registerSyncWatcher(func(_ Change) { - gotNotified = true - }) - - ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ - "test-1": { - Title: "Updated title", + { + name: "on-set", + initialState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{}, + newState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": {}, + }, + wantNotify: true, + }, + { + name: "details-change", + initialState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": { + Title: "Title", + }, + }, + newState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": { + Title: "Updated title", + }, + }, + wantNotify: true, + }, + { + name: "action-changes", + initialState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": { + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "http://www.example.com/a/123456", + Label: "Sign in", + }, + }, + }, + newState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": { + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "http://www.example.com/a/abcdefg", + Label: "Sign in", + }, + }, + }, + wantNotify: true, }, - }) - - if !gotNotified { - t.Errorf("watcher did not get called, want it to be called") } -} - -func TestControlHealthNoNotifyOnUnchanged(t *testing.T) { - ht := Tracker{} - ht.SetIPNState("NeedsLogin", true) - ht.GotStreamedMapResponse() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ht := Tracker{} + ht.SetIPNState("NeedsLogin", true) + ht.GotStreamedMapResponse() - // Set up an existing control health issue - ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ - "test": {}, - }) + if len(test.initialState) != 0 { + ht.SetControlHealth(test.initialState) + } - // Now register our watcher - gotNotified := false - ht.registerSyncWatcher(func(_ Change) { - gotNotified = true - }) + gotNotified := false + ht.registerSyncWatcher(func(_ Change) { + gotNotified = true + }) - // Send the same control health message again - should not notify - ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ - "test": {}, - }) + ht.SetControlHealth(test.newState) - if gotNotified { - t.Errorf("watcher got called, want it to not be called") + if gotNotified != test.wantNotify { + t.Errorf("notified: got %v, want %v", gotNotified, test.wantNotify) + } + }) } } diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 53c4683c1b000..0f13c725ebbfa 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -2171,7 +2171,10 @@ func (m DisplayMessage) Equal(o DisplayMessage) bool { return m.Title == o.Title && m.Text == o.Text && m.Severity == o.Severity && - m.ImpactsConnectivity == o.ImpactsConnectivity + m.ImpactsConnectivity == o.ImpactsConnectivity && + (m.PrimaryAction == nil) == (o.PrimaryAction == nil) && + (m.PrimaryAction == nil || (m.PrimaryAction.URL == o.PrimaryAction.URL && + m.PrimaryAction.Label == o.PrimaryAction.Label)) } // DisplayMessageSeverity represents how serious a [DisplayMessage] is. Analogous diff --git a/tailcfg/tailcfg_test.go b/tailcfg/tailcfg_test.go index e8e86cdb139bd..833314df8fd6d 100644 --- a/tailcfg/tailcfg_test.go +++ b/tailcfg/tailcfg_test.go @@ -881,76 +881,129 @@ func TestCheckTag(t *testing.T) { } func TestDisplayMessageEqual(t *testing.T) { - base := DisplayMessage{ - Title: "title", - Text: "text", - Severity: SeverityHigh, - ImpactsConnectivity: false, - } - type test struct { name string - value DisplayMessage + value1 DisplayMessage + value2 DisplayMessage wantEqual bool } for _, test := range []test{ { name: "same", - value: DisplayMessage{ + value1: DisplayMessage{ + Title: "title", + Text: "text", + Severity: SeverityHigh, + ImpactsConnectivity: false, + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Open", + }, + }, + value2: DisplayMessage{ Title: "title", Text: "text", Severity: SeverityHigh, ImpactsConnectivity: false, + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Open", + }, }, wantEqual: true, }, { name: "different-title", - value: DisplayMessage{ - Title: "different title", - Text: "text", - Severity: SeverityHigh, - ImpactsConnectivity: false, + value1: DisplayMessage{ + Title: "title", + }, + value2: DisplayMessage{ + Title: "different title", }, wantEqual: false, }, { name: "different-text", - value: DisplayMessage{ - Title: "title", - Text: "different text", - Severity: SeverityHigh, - ImpactsConnectivity: false, + value1: DisplayMessage{ + Text: "some text", + }, + value2: DisplayMessage{ + Text: "different text", }, wantEqual: false, }, { name: "different-severity", - value: DisplayMessage{ - Title: "title", - Text: "text", - Severity: SeverityMedium, - ImpactsConnectivity: false, + value1: DisplayMessage{ + Severity: SeverityHigh, + }, + value2: DisplayMessage{ + Severity: SeverityMedium, }, wantEqual: false, }, { name: "different-impactsConnectivity", - value: DisplayMessage{ - Title: "title", - Text: "text", - Severity: SeverityHigh, + value1: DisplayMessage{ ImpactsConnectivity: true, }, + value2: DisplayMessage{ + ImpactsConnectivity: false, + }, + wantEqual: false, + }, + { + name: "different-primaryAction-nil-non-nil", + value1: DisplayMessage{}, + value2: DisplayMessage{ + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Open", + }, + }, + wantEqual: false, + }, + { + name: "different-primaryAction-url", + value1: DisplayMessage{ + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Open", + }, + }, + value2: DisplayMessage{ + PrimaryAction: &DisplayMessageAction{ + URL: "https://zombo.com", + Label: "Open", + }, + }, + wantEqual: false, + }, + { + name: "different-primaryAction-label", + value1: DisplayMessage{ + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Open", + }, + }, + value2: DisplayMessage{ + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Learn more", + }, + }, wantEqual: false, }, } { t.Run(test.name, func(t *testing.T) { - got := base.Equal(test.value) + got := test.value1.Equal(test.value2) if got != test.wantEqual { - t.Errorf("Equal: got %t, want %t", got, test.wantEqual) + value1 := must.Get(json.MarshalIndent(test.value1, "", " ")) + value2 := must.Get(json.MarshalIndent(test.value2, "", " ")) + t.Errorf("value1.Equal(value2): got %t, want %t\nvalue1:\n%s\nvalue2:\n%s", got, test.wantEqual, value1, value2) } }) } From ffe8cc9442335ffb76c0e7555c67493a1975181c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 09:54:00 -0600 Subject: [PATCH 213/263] .github: Bump github/codeql-action from 3.29.1 to 3.29.2 (#16480) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.29.1 to 3.29.2. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/39edc492dbe16b1465b0cafca41432d857bdb31a...181d5eefc20863364f96762470ba6f862bdef56b) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 3.29.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 610b93b610ea3..4e129b8471ea5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -55,7 +55,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@39edc492dbe16b1465b0cafca41432d857bdb31a # v3.29.1 + uses: github/codeql-action/init@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,7 +66,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@39edc492dbe16b1465b0cafca41432d857bdb31a # v3.29.1 + uses: github/codeql-action/autobuild@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -80,4 +80,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@39edc492dbe16b1465b0cafca41432d857bdb31a # v3.29.1 + uses: github/codeql-action/analyze@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2 From d65c0fd2d04a49fb11964cf0457df499a0e6e366 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 15 Jul 2025 12:29:07 -0700 Subject: [PATCH 214/263] tailcfg,wgengine/magicsock: set peer relay CapVer (#16531) Updates tailscale/corp#27502 Updates tailscale/corp#30051 Signed-off-by: Jordan Whited --- tailcfg/tailcfg.go | 3 ++- wgengine/magicsock/debugknobs.go | 6 ------ wgengine/magicsock/debugknobs_stubs.go | 1 - wgengine/magicsock/magicsock.go | 19 ++++++++--------- wgengine/magicsock/magicsock_test.go | 28 ++++++++++++++++++++++++-- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 0f13c725ebbfa..636e2434de276 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -164,7 +164,8 @@ type CapabilityVersion int // - 117: 2025-05-28: Client understands DisplayMessages (structured health messages), but not necessarily PrimaryAction. // - 118: 2025-07-01: Client sends Hostinfo.StateEncrypted to report whether the state file is encrypted at rest (#15830) // - 119: 2025-07-10: Client uses Hostinfo.Location.Priority to prioritize one route over another. -const CurrentCapabilityVersion CapabilityVersion = 119 +// - 120: 2025-07-15: Client understands peer relay disco messages, and implements peer client and relay server functions +const CurrentCapabilityVersion CapabilityVersion = 120 // ID is an integer ID for a user, node, or login allocated by the // control plane. diff --git a/wgengine/magicsock/debugknobs.go b/wgengine/magicsock/debugknobs.go index 0558953887ae0..f8fd9f0407d44 100644 --- a/wgengine/magicsock/debugknobs.go +++ b/wgengine/magicsock/debugknobs.go @@ -62,12 +62,6 @@ var ( // //lint:ignore U1000 used on Linux/Darwin only debugPMTUD = envknob.RegisterBool("TS_DEBUG_PMTUD") - // debugAssumeUDPRelayCapable forces magicsock to assume that all peers are - // UDP relay capable clients and servers. This will eventually be replaced - // by a [tailcfg.CapabilityVersion] comparison. It enables early testing of - // the UDP relay feature before we have established related - // [tailcfg.CapabilityVersion]'s. - debugAssumeUDPRelayCapable = envknob.RegisterBool("TS_DEBUG_ASSUME_UDP_RELAY_CAPABLE") // Hey you! Adding a new debugknob? Make sure to stub it out in the // debugknobs_stubs.go file too. ) diff --git a/wgengine/magicsock/debugknobs_stubs.go b/wgengine/magicsock/debugknobs_stubs.go index 3d23b1f8e8f01..336d7baa19645 100644 --- a/wgengine/magicsock/debugknobs_stubs.go +++ b/wgengine/magicsock/debugknobs_stubs.go @@ -31,4 +31,3 @@ func debugRingBufferMaxSizeBytes() int { return 0 } func inTest() bool { return false } func debugPeerMap() bool { return false } func pretendpoints() []netip.AddrPort { return []netip.AddrPort{} } -func debugAssumeUDPRelayCapable() bool { return false } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 14feed32b5929..a8b1c8f15032d 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -14,7 +14,6 @@ import ( "expvar" "fmt" "io" - "math" "net" "net/netip" "reflect" @@ -2616,14 +2615,10 @@ func (c *Conn) SetProbeUDPLifetime(v bool) { }) } +// capVerIsRelayCapable returns true if version is relay client and server +// capable, otherwise it returns false. func capVerIsRelayCapable(version tailcfg.CapabilityVersion) bool { - // TODO(jwhited): implement once capVer is bumped - return version == math.MinInt32 || debugAssumeUDPRelayCapable() -} - -func capVerIsRelayServerCapable(version tailcfg.CapabilityVersion) bool { - // TODO(jwhited): implement once capVer is bumped & update Test_peerAPIIfCandidateRelayServer - return version == math.MinInt32 || debugAssumeUDPRelayCapable() + return version >= 120 } // onFilterUpdate is called when a [FilterUpdate] is received over the @@ -2677,10 +2672,16 @@ func peerAPIIfCandidateRelayServer(filt *filter.Filter, self, maybeCandidate tai if filt == nil || !self.Valid() || !maybeCandidate.Valid() || - !capVerIsRelayServerCapable(maybeCandidate.Cap()) || !maybeCandidate.Hostinfo().Valid() { return netip.AddrPort{} } + if maybeCandidate.ID() != self.ID() && !capVerIsRelayCapable(maybeCandidate.Cap()) { + // If maybeCandidate's [tailcfg.CapabilityVersion] is not relay-capable, + // we skip it. If maybeCandidate happens to be self, then this check is + // unnecessary as self is always capable from this point (the statically + // compiled [tailcfg.CurrentCapabilityVersion]) forward. + return netip.AddrPort{} + } for _, maybeCandidatePrefix := range maybeCandidate.Addresses().All() { if !maybeCandidatePrefix.IsSingleIP() { continue diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 0515162c72b9f..1d76e6c595a47 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -3399,7 +3399,11 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { } selfOnlyIPv4 := &tailcfg.Node{ - Cap: math.MinInt32, + ID: 1, + // Intentionally set a value < 120 to verify the statically compiled + // [tailcfg.CurrentCapabilityVersion] is used when self is + // maybeCandidate. + Cap: 119, Addresses: []netip.Prefix{ netip.MustParsePrefix("1.1.1.1/32"), }, @@ -3409,13 +3413,17 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { selfOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::1/128") peerOnlyIPv4 := &tailcfg.Node{ - Cap: math.MinInt32, + ID: 2, + Cap: 120, Addresses: []netip.Prefix{ netip.MustParsePrefix("2.2.2.2/32"), }, Hostinfo: hostInfo.View(), } + peerOnlyIPv4NotCapable := peerOnlyIPv4.Clone() + peerOnlyIPv4NotCapable.Cap = 119 + peerOnlyIPv6 := peerOnlyIPv4.Clone() peerOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::2/128") @@ -3500,6 +3508,22 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { maybeCandidate: selfOnlyIPv6.View(), want: netip.AddrPortFrom(selfOnlyIPv6.Addresses[0].Addr(), 6), }, + { + name: "peer incapable", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("1.1.1.1/32"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfOnlyIPv4.View(), + maybeCandidate: peerOnlyIPv4NotCapable.View(), + }, { name: "no match dst", filt: filter.New([]filtertype.Match{ From cb7a0b1dca91cef710f61cd4f3694bafa27bb7a0 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 15 Jul 2025 15:23:47 -0700 Subject: [PATCH 215/263] net/udprelay: log socket read errors (#16573) Socket read errors currently close the server, so we need to understand when and why they occur. Updates tailscale/corp#27502 Updates tailscale/corp#30118 Signed-off-by: Jordan Whited --- net/udprelay/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/net/udprelay/server.go b/net/udprelay/server.go index e2652ae99637f..7651bf295a233 100644 --- a/net/udprelay/server.go +++ b/net/udprelay/server.go @@ -581,6 +581,7 @@ func (s *Server) packetReadLoop(readFromSocket, otherSocket *net.UDPConn) { // TODO: extract laddr from IP_PKTINFO for use in reply n, from, err := readFromSocket.ReadFromUDPAddrPort(b) if err != nil { + s.logf("error reading from socket(%v): %v", readFromSocket.LocalAddr(), err) return } s.handlePacket(from, b[:n], readFromSocket, otherSocket) From 67514f5eb2f9737e7d819f43f007be970e17f293 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Wed, 16 Jul 2025 08:08:59 -0700 Subject: [PATCH 216/263] ssh/tailssh: fix path of "true" on Darwin (#16569) This is a follow-up to #15351, which fixed the test for Linux but not for Darwin, which stores its "true" executable in /usr/bin instead of /bin. Try both paths when not running on Windows. In addition, disable CGo in the integration test build, which was causing the linker to fail. These tests do not need CGo, and it appears we had some version skew with the base image on the runners. In addition, in error cases the recover step of the permissions check was spuriously panicking and masking the "real" failure reason. Don't do that check when a command was not produced. Updates #15350 Change-Id: Icd91517f45c90f7554310ebf1c888cdfd109f43a Signed-off-by: M. J. Fromberger --- Makefile | 4 ++-- ssh/tailssh/incubator.go | 25 ++++++++++++++----------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index f5fc205891191..55e55f209575c 100644 --- a/Makefile +++ b/Makefile @@ -126,8 +126,8 @@ publishdevproxy: check-image-repo ## Build and publish k8s-proxy image to locati .PHONY: sshintegrationtest sshintegrationtest: ## Run the SSH integration tests in various Docker containers - @GOOS=linux GOARCH=amd64 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ - GOOS=linux GOARCH=amd64 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \ + @GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \ echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \ echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \ echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \ diff --git a/ssh/tailssh/incubator.go b/ssh/tailssh/incubator.go index 9e1a9ea94e424..dd280143e36e3 100644 --- a/ssh/tailssh/incubator.go +++ b/ssh/tailssh/incubator.go @@ -51,6 +51,7 @@ const ( darwin = "darwin" freebsd = "freebsd" openbsd = "openbsd" + windows = "windows" ) func init() { @@ -80,20 +81,22 @@ func tryExecInDir(ctx context.Context, dir string) error { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() + run := func(path string) error { + cmd := exec.CommandContext(ctx, path) + cmd.Dir = dir + return cmd.Run() + } + // Assume that the following executables exist, are executable, and // immediately return. - var name string - switch runtime.GOOS { - case "windows": + if runtime.GOOS == windows { windir := os.Getenv("windir") - name = filepath.Join(windir, "system32", "doskey.exe") - default: - name = "/bin/true" + return run(filepath.Join(windir, "system32", "doskey.exe")) } - - cmd := exec.CommandContext(ctx, name) - cmd.Dir = dir - return cmd.Run() + if err := run("/bin/true"); !errors.Is(err, exec.ErrNotFound) { // including nil + return err + } + return run("/usr/bin/true") } // newIncubatorCommand returns a new exec.Cmd configured with @@ -107,7 +110,7 @@ func tryExecInDir(ctx context.Context, dir string) error { // The returned Cmd.Env is guaranteed to be nil; the caller populates it. func (ss *sshSession) newIncubatorCommand(logf logger.Logf) (cmd *exec.Cmd, err error) { defer func() { - if cmd.Env != nil { + if cmd != nil && cmd.Env != nil { panic("internal error") } }() From 3c6d17e6f114a2dc166e62b84789154b176e07c6 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 16 Jul 2025 10:03:05 -0700 Subject: [PATCH 217/263] cmd/tailscale/cli,ipn/ipnlocal,wgengine/magicsock: implement tailscale debug peer-relay-servers (#16577) Updates tailscale/corp#30036 Signed-off-by: Jordan Whited --- cmd/tailscale/cli/debug.go | 20 ++++++++++++++++++++ ipn/ipnlocal/local.go | 4 ++++ ipn/localapi/localapi.go | 6 ++++++ wgengine/magicsock/magicsock.go | 5 +++++ wgengine/magicsock/relaymanager.go | 21 +++++++++++++++++++++ wgengine/magicsock/relaymanager_test.go | 15 +++++++++++++++ 6 files changed, 71 insertions(+) diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index ec8a0700dec19..8473c4a1707fa 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -356,6 +356,12 @@ func debugCmd() *ffcli.Command { ShortHelp: "Print Go's runtime/debug.BuildInfo", Exec: runGoBuildInfo, }, + { + Name: "peer-relay-servers", + ShortUsage: "tailscale debug peer-relay-servers", + ShortHelp: "Print the current set of candidate peer relay servers", + Exec: runPeerRelayServers, + }, }...), } } @@ -1327,3 +1333,17 @@ func runDebugResolve(ctx context.Context, args []string) error { } return nil } + +func runPeerRelayServers(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("unexpected arguments") + } + v, err := localClient.DebugResultJSON(ctx, "peer-relay-servers") + if err != nil { + return err + } + e := json.NewEncoder(os.Stdout) + e.SetIndent("", " ") + e.Encode(v) + return nil +} diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 9b9bd82b5e9e5..62ab6d9047338 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -6956,6 +6956,10 @@ func (b *LocalBackend) DebugReSTUN() error { return nil } +func (b *LocalBackend) DebugPeerRelayServers() set.Set[netip.AddrPort] { + return b.MagicConn().PeerRelays() +} + // ControlKnobs returns the node's control knobs. func (b *LocalBackend) ControlKnobs() *controlknobs.Knobs { return b.sys.ControlKnobs() diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index cd59c54e05489..fb024039ba52a 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -696,6 +696,12 @@ func (h *Handler) serveDebug(w http.ResponseWriter, r *http.Request) { break } h.b.DebugForcePreferDERP(n) + case "peer-relay-servers": + servers := h.b.DebugPeerRelayServers() + err = json.NewEncoder(w).Encode(servers) + if err == nil { + return + } case "": err = fmt.Errorf("missing parameter 'action'") default: diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index a8b1c8f15032d..24a4fc073f1f1 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3907,3 +3907,8 @@ func (le *lazyEndpoint) FromPeer(peerPublicKey [32]byte) { le.c.peerMap.setNodeKeyForEpAddr(le.src, pubKey) le.c.logf("magicsock: lazyEndpoint.FromPeer(%v) setting epAddr(%v) in peerMap for node(%v)", pubKey.ShortString(), le.src, ep.nodeAddr) } + +// PeerRelays returns the current set of candidate peer relays. +func (c *Conn) PeerRelays() set.Set[netip.AddrPort] { + return c.relayManager.getServers() +} diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index c8c9ed41b7b82..d7acf80b51a58 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -57,6 +57,7 @@ type relayManager struct { newServerEndpointCh chan newRelayServerEndpointEvent rxHandshakeDiscoMsgCh chan relayHandshakeDiscoMsgEvent serversCh chan set.Set[netip.AddrPort] + getServersCh chan chan set.Set[netip.AddrPort] discoInfoMu sync.Mutex // guards the following field discoInfoByServerDisco map[key.DiscoPublic]*relayHandshakeDiscoInfo @@ -185,10 +186,29 @@ func (r *relayManager) runLoop() { if !r.hasActiveWorkRunLoop() { return } + case getServersCh := <-r.getServersCh: + r.handleGetServersRunLoop(getServersCh) + if !r.hasActiveWorkRunLoop() { + return + } } } } +func (r *relayManager) handleGetServersRunLoop(getServersCh chan set.Set[netip.AddrPort]) { + servers := make(set.Set[netip.AddrPort], len(r.serversByAddrPort)) + for server := range r.serversByAddrPort { + servers.Add(server) + } + getServersCh <- servers +} + +func (r *relayManager) getServers() set.Set[netip.AddrPort] { + ch := make(chan set.Set[netip.AddrPort]) + relayManagerInputEvent(r, nil, &r.getServersCh, ch) + return <-ch +} + func (r *relayManager) handleServersUpdateRunLoop(update set.Set[netip.AddrPort]) { for k, v := range r.serversByAddrPort { if !update.Contains(k) { @@ -244,6 +264,7 @@ func (r *relayManager) init() { r.newServerEndpointCh = make(chan newRelayServerEndpointEvent) r.rxHandshakeDiscoMsgCh = make(chan relayHandshakeDiscoMsgEvent) r.serversCh = make(chan set.Set[netip.AddrPort]) + r.getServersCh = make(chan chan set.Set[netip.AddrPort]) r.runLoopStoppedCh = make(chan struct{}, 1) r.runLoopStoppedCh <- struct{}{} }) diff --git a/wgengine/magicsock/relaymanager_test.go b/wgengine/magicsock/relaymanager_test.go index 8f92360122d0e..01f9258ad7521 100644 --- a/wgengine/magicsock/relaymanager_test.go +++ b/wgengine/magicsock/relaymanager_test.go @@ -32,4 +32,19 @@ func TestRelayManagerInitAndIdle(t *testing.T) { rm = relayManager{} rm.handleRelayServersSet(make(set.Set[netip.AddrPort])) <-rm.runLoopStoppedCh + + rm = relayManager{} + rm.getServers() + <-rm.runLoopStoppedCh +} + +func TestRelayManagerGetServers(t *testing.T) { + rm := relayManager{} + servers := make(set.Set[netip.AddrPort], 1) + servers.Add(netip.MustParseAddrPort("192.0.2.1:7")) + rm.handleRelayServersSet(servers) + got := rm.getServers() + if !servers.Equal(got) { + t.Errorf("got %v != want %v", got, servers) + } } From 097c2bcf6700e5dc074187bbe0c05ae4cd8b3c26 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 16 Jul 2025 11:04:32 -0700 Subject: [PATCH 218/263] go.mod: bump wireguard-go (#16578) So that conn.PeerAwareEndpoint is always evaluated per-packet, rather than at least once per packet batch. Updates tailscale/corp#30042 Signed-off-by: Jordan Whited --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f040d7799768d..3d7514158f069 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/tailscale/setec v0.0.0-20250205144240-8898a29c3fbb github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 - github.com/tailscale/wireguard-go v0.0.0-20250711050509-4064566ecaf9 + github.com/tailscale/wireguard-go v0.0.0-20250716170648-1d0488a3d7da github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e github.com/tc-hib/winres v0.2.1 github.com/tcnksm/go-httpstat v0.2.0 diff --git a/go.sum b/go.sum index ea17b11821392..995b930100ff9 100644 --- a/go.sum +++ b/go.sum @@ -975,8 +975,8 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 h1:U github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y= -github.com/tailscale/wireguard-go v0.0.0-20250711050509-4064566ecaf9 h1:kSzi/ugdekAxhcVdCxH6er7OjoNc2oDRcimWJDvnRFM= -github.com/tailscale/wireguard-go v0.0.0-20250711050509-4064566ecaf9/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= +github.com/tailscale/wireguard-go v0.0.0-20250716170648-1d0488a3d7da h1:jVRUZPRs9sqyKlYHHzHjAqKN+6e/Vog6NpHYeNPJqOw= +github.com/tailscale/wireguard-go v0.0.0-20250716170648-1d0488a3d7da/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg= github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA= From 17c5116d469f79d5fba20e50fc414932f3ce681d Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 16 Jul 2025 11:19:21 -0700 Subject: [PATCH 219/263] ipn/ipnlocal: sort tailscale debug peer-relay-servers slice (#16579) Updates tailscale/corp#30036 Signed-off-by: Jordan Whited --- ipn/localapi/localapi.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index fb024039ba52a..d7c64b917ead4 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -697,7 +697,10 @@ func (h *Handler) serveDebug(w http.ResponseWriter, r *http.Request) { } h.b.DebugForcePreferDERP(n) case "peer-relay-servers": - servers := h.b.DebugPeerRelayServers() + servers := h.b.DebugPeerRelayServers().Slice() + slices.SortFunc(servers, func(a, b netip.AddrPort) int { + return a.Compare(b) + }) err = json.NewEncoder(w).Encode(servers) if err == nil { return From e84e58c56733072b15fb92c10e4ff702d8fa84d4 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Wed, 16 Jul 2025 11:50:13 -0700 Subject: [PATCH 220/263] ipn/ipnlocal: use rendezvous hashing to traffic-steer exit nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With auto exit nodes enabled, the client picks exit nodes from the ones advertised in the network map. Usually, it picks the one with the highest priority score, but when the top spot is tied, it used to pick randomly. Then, once it made a selection, it would strongly prefer to stick with that exit node. It wouldn’t even consider another exit node unless the client was shutdown or the exit node went offline. This is to prevent flapping, where a client constantly chooses a different random exit node. The major problem with this algorithm is that new exit nodes don’t get selected as often as they should. In fact, they wouldn’t even move over if a higher scoring exit node appeared. Let’s say that you have an exit node and it’s overloaded. So you spin up a new exit node, right beside your existing one, in the hopes that the traffic will be split across them. But since the client had this strong affinity, they stick with the exit node they know and love. Using rendezvous hashing, we can have different clients spread their selections equally across their top scoring exit nodes. When an exit node shuts down, its clients will spread themselves evenly to their other equal options. When an exit node starts, a proportional number of clients will migrate to their new best option. Read more: https://en.wikipedia.org/wiki/Rendezvous_hashing The trade-off is that starting up a new exit node may cause some clients to move over, interrupting their existing network connections. So this change is only enabled for tailnets with `traffic-steering` enabled. Updates tailscale/corp#29966 Fixes #16551 Signed-off-by: Simon Law --- ipn/ipnlocal/local.go | 53 +++++++++++++++++++++----------------- ipn/ipnlocal/local_test.go | 51 +++--------------------------------- 2 files changed, 33 insertions(+), 71 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 62ab6d9047338..d3754e5409c1a 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -11,6 +11,7 @@ import ( "context" "crypto/sha256" "encoding/base64" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -7750,7 +7751,7 @@ func suggestExitNode(report *netcheck.Report, nb *nodeBackend, prevSuggestion ta switch { case nb.SelfHasCap(tailcfg.NodeAttrTrafficSteering): // The traffic-steering feature flag is enabled on this tailnet. - return suggestExitNodeUsingTrafficSteering(nb, prevSuggestion, allowList) + return suggestExitNodeUsingTrafficSteering(nb, allowList) default: return suggestExitNodeUsingDERP(report, nb, prevSuggestion, selectRegion, selectNode, allowList) } @@ -7896,12 +7897,17 @@ var ErrNoNetMap = errors.New("no network map, try again later") // pick one of the best exit nodes. These priorities are provided by Control in // the node’s [tailcfg.Location]. To be eligible for consideration, the node // must have NodeAttrSuggestExitNode in its CapMap. -func suggestExitNodeUsingTrafficSteering(nb *nodeBackend, prev tailcfg.StableNodeID, allowed set.Set[tailcfg.StableNodeID]) (apitype.ExitNodeSuggestionResponse, error) { +func suggestExitNodeUsingTrafficSteering(nb *nodeBackend, allowed set.Set[tailcfg.StableNodeID]) (apitype.ExitNodeSuggestionResponse, error) { nm := nb.NetMap() if nm == nil { return apitype.ExitNodeSuggestionResponse{}, ErrNoNetMap } + self := nb.Self() + if !self.Valid() { + return apitype.ExitNodeSuggestionResponse{}, ErrNoNetMap + } + if !nb.SelfHasCap(tailcfg.NodeAttrTrafficSteering) { panic("missing traffic-steering capability") } @@ -7923,12 +7929,6 @@ func suggestExitNodeUsingTrafficSteering(nb *nodeBackend, prev tailcfg.StableNod if !tsaddr.ContainsExitRoutes(p.AllowedIPs()) { return false } - if p.StableID() == prev { - // Prevent flapping: since prev is a valid suggestion, - // force prev to be the only valid pick. - force = p - return false - } return true }) if force.Valid() { @@ -7950,6 +7950,7 @@ func suggestExitNodeUsingTrafficSteering(nb *nodeBackend, prev tailcfg.StableNod } return s } + rdvHash := makeRendezvousHasher(self.ID()) var pick tailcfg.NodeView if len(nodes) == 1 { @@ -7958,25 +7959,18 @@ func suggestExitNodeUsingTrafficSteering(nb *nodeBackend, prev tailcfg.StableNod if len(nodes) > 1 { // Find the highest scoring exit nodes. slices.SortFunc(nodes, func(a, b tailcfg.NodeView) int { - return cmp.Compare(score(b), score(a)) // reverse sort - }) - - // Find the top exit nodes, which all have the same score. - topI := len(nodes) - ts := score(nodes[0]) - for i, n := range nodes[1:] { - if score(n) < ts { - // n is the first node with a lower score. - // Make nodes[:topI] to slice the top exit nodes. - topI = i + 1 - break + c := cmp.Compare(score(b), score(a)) // Highest score first. + if c == 0 { + // Rendezvous hashing for reliably picking the + // same node from a list: tailscale/tailscale#16551. + return cmp.Compare(rdvHash(b.ID()), rdvHash(a.ID())) } - } + return c + }) // TODO(sfllaw): add a temperature knob so that this client has // a chance of picking the next best option. - randSeed := uint64(nm.SelfNode.ID()) - pick = nodes[rands.IntN(randSeed, topI)] + pick = nodes[0] } if !pick.Valid() { @@ -8077,6 +8071,19 @@ func longLatDistance(fromLat, fromLong, toLat, toLong float64) float64 { return earthRadiusMeters * c } +// makeRendezvousHasher returns a function that hashes a node ID to a uint64. +// https://en.wikipedia.org/wiki/Rendezvous_hashing +func makeRendezvousHasher(seed tailcfg.NodeID) func(tailcfg.NodeID) uint64 { + en := binary.BigEndian + return func(n tailcfg.NodeID) uint64 { + var b [16]byte + en.PutUint64(b[:], uint64(seed)) + en.PutUint64(b[8:], uint64(n)) + v := sha256.Sum256(b[:]) + return en.Uint64(v[:]) + } +} + const ( // unresolvedExitNodeID is a special [tailcfg.StableNodeID] value // used as an exit node ID to install a blackhole route, preventing diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 0b39c45c28f7d..13681fc0430ea 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -5012,7 +5012,7 @@ func TestSuggestExitNodeTrafficSteering(t *testing.T) { wantName: "peer1", }, { - name: "many-suggested-exit-nodes", + name: "suggest-exit-node-stable-pick", netMap: &netmap.NetworkMap{ SelfNode: selfNode.View(), Peers: []tailcfg.NodeView{ @@ -5030,55 +5030,10 @@ func TestSuggestExitNodeTrafficSteering(t *testing.T) { withSuggest()), }, }, + // Change this, if the hashing function changes. wantID: "stable3", wantName: "peer3", }, - { - name: "suggested-exit-node-was-last-suggested", - netMap: &netmap.NetworkMap{ - SelfNode: selfNode.View(), - Peers: []tailcfg.NodeView{ - makePeer(1, - withExitRoutes(), - withSuggest()), - makePeer(2, - withExitRoutes(), - withSuggest()), - makePeer(3, - withExitRoutes(), - withSuggest()), - makePeer(4, - withExitRoutes(), - withSuggest()), - }, - }, - lastExit: "stable2", // overrides many-suggested-exit-nodes - wantID: "stable2", - wantName: "peer2", - }, - { - name: "suggested-exit-node-was-never-suggested", - netMap: &netmap.NetworkMap{ - SelfNode: selfNode.View(), - Peers: []tailcfg.NodeView{ - makePeer(1, - withExitRoutes(), - withSuggest()), - makePeer(2, - withExitRoutes(), - withSuggest()), - makePeer(3, - withExitRoutes(), - withSuggest()), - makePeer(4, - withExitRoutes(), - withSuggest()), - }, - }, - lastExit: "stable10", - wantID: "stable3", // matches many-suggested-exit-nodes - wantName: "peer3", - }, { name: "exit-nodes-with-and-without-priority", netMap: &netmap.NetworkMap{ @@ -5282,7 +5237,7 @@ func TestSuggestExitNodeTrafficSteering(t *testing.T) { defer nb.shutdown(errShutdown) nb.SetNetMap(tt.netMap) - got, err := suggestExitNodeUsingTrafficSteering(nb, tt.lastExit, allowList) + got, err := suggestExitNodeUsingTrafficSteering(nb, allowList) if tt.wantErr == nil && err != nil { t.Fatalf("err=%v, want nil", err) } From 36aeacb297ae97f5b21358cfe6ddc814d3920d59 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 16 Jul 2025 14:34:05 -0700 Subject: [PATCH 221/263] wgengine/magicsock: add peer relay metrics (#16582) Updates tailscale/corp#30040 Signed-off-by: Jordan Whited --- wgengine/magicsock/endpoint.go | 21 +++- wgengine/magicsock/magicsock.go | 194 ++++++++++++++++++++++---------- 2 files changed, 151 insertions(+), 64 deletions(-) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 385c9245ec4e7..48d5ef5a11338 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -1064,11 +1064,21 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { switch { case udpAddr.ap.Addr().Is4(): - de.c.metrics.outboundPacketsIPv4Total.Add(int64(len(buffs))) - de.c.metrics.outboundBytesIPv4Total.Add(int64(txBytes)) + if udpAddr.vni.isSet() { + de.c.metrics.outboundPacketsPeerRelayIPv4Total.Add(int64(len(buffs))) + de.c.metrics.outboundBytesPeerRelayIPv4Total.Add(int64(txBytes)) + } else { + de.c.metrics.outboundPacketsIPv4Total.Add(int64(len(buffs))) + de.c.metrics.outboundBytesIPv4Total.Add(int64(txBytes)) + } case udpAddr.ap.Addr().Is6(): - de.c.metrics.outboundPacketsIPv6Total.Add(int64(len(buffs))) - de.c.metrics.outboundBytesIPv6Total.Add(int64(txBytes)) + if udpAddr.vni.isSet() { + de.c.metrics.outboundPacketsPeerRelayIPv6Total.Add(int64(len(buffs))) + de.c.metrics.outboundBytesPeerRelayIPv6Total.Add(int64(txBytes)) + } else { + de.c.metrics.outboundPacketsIPv6Total.Add(int64(len(buffs))) + de.c.metrics.outboundBytesIPv6Total.Add(int64(txBytes)) + } } // TODO(raggi): needs updating for accuracy, as in error conditions we may have partial sends. @@ -1082,7 +1092,8 @@ func (de *endpoint) send(buffs [][]byte, offset int) error { for _, buff := range buffs { buff = buff[offset:] const isDisco = false - ok, _ := de.c.sendAddr(derpAddr, de.publicKey, buff, isDisco) + const isGeneveEncap = false + ok, _ := de.c.sendAddr(derpAddr, de.publicKey, buff, isDisco, isGeneveEncap) txBytes += len(buff) if !ok { allOk = false diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 24a4fc073f1f1..ad07003f72fbb 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -87,9 +87,11 @@ const ( type Path string const ( - PathDirectIPv4 Path = "direct_ipv4" - PathDirectIPv6 Path = "direct_ipv6" - PathDERP Path = "derp" + PathDirectIPv4 Path = "direct_ipv4" + PathDirectIPv6 Path = "direct_ipv6" + PathDERP Path = "derp" + PathPeerRelayIPv4 Path = "peer_relay_ipv4" + PathPeerRelayIPv6 Path = "peer_relay_ipv6" ) type pathLabel struct { @@ -97,6 +99,8 @@ type pathLabel struct { // - direct_ipv4 // - direct_ipv6 // - derp + // - peer_relay_ipv4 + // - peer_relay_ipv6 Path Path } @@ -108,27 +112,35 @@ type pathLabel struct { type metrics struct { // inboundPacketsTotal is the total number of inbound packets received, // labeled by the path the packet took. - inboundPacketsIPv4Total expvar.Int - inboundPacketsIPv6Total expvar.Int - inboundPacketsDERPTotal expvar.Int + inboundPacketsIPv4Total expvar.Int + inboundPacketsIPv6Total expvar.Int + inboundPacketsDERPTotal expvar.Int + inboundPacketsPeerRelayIPv4Total expvar.Int + inboundPacketsPeerRelayIPv6Total expvar.Int // inboundBytesTotal is the total number of inbound bytes received, // labeled by the path the packet took. - inboundBytesIPv4Total expvar.Int - inboundBytesIPv6Total expvar.Int - inboundBytesDERPTotal expvar.Int + inboundBytesIPv4Total expvar.Int + inboundBytesIPv6Total expvar.Int + inboundBytesDERPTotal expvar.Int + inboundBytesPeerRelayIPv4Total expvar.Int + inboundBytesPeerRelayIPv6Total expvar.Int // outboundPacketsTotal is the total number of outbound packets sent, // labeled by the path the packet took. - outboundPacketsIPv4Total expvar.Int - outboundPacketsIPv6Total expvar.Int - outboundPacketsDERPTotal expvar.Int + outboundPacketsIPv4Total expvar.Int + outboundPacketsIPv6Total expvar.Int + outboundPacketsDERPTotal expvar.Int + outboundPacketsPeerRelayIPv4Total expvar.Int + outboundPacketsPeerRelayIPv6Total expvar.Int // outboundBytesTotal is the total number of outbound bytes sent, // labeled by the path the packet took. - outboundBytesIPv4Total expvar.Int - outboundBytesIPv6Total expvar.Int - outboundBytesDERPTotal expvar.Int + outboundBytesIPv4Total expvar.Int + outboundBytesIPv6Total expvar.Int + outboundBytesDERPTotal expvar.Int + outboundBytesPeerRelayIPv4Total expvar.Int + outboundBytesPeerRelayIPv6Total expvar.Int // outboundPacketsDroppedErrors is the total number of outbound packets // dropped due to errors. @@ -723,6 +735,8 @@ func registerMetrics(reg *usermetric.Registry) *metrics { pathDirectV4 := pathLabel{Path: PathDirectIPv4} pathDirectV6 := pathLabel{Path: PathDirectIPv6} pathDERP := pathLabel{Path: PathDERP} + pathPeerRelayV4 := pathLabel{Path: PathPeerRelayIPv4} + pathPeerRelayV6 := pathLabel{Path: PathPeerRelayIPv6} inboundPacketsTotal := usermetric.NewMultiLabelMapWithRegistry[pathLabel]( reg, "tailscaled_inbound_packets_total", @@ -755,25 +769,37 @@ func registerMetrics(reg *usermetric.Registry) *metrics { metricRecvDataPacketsIPv4.Register(&m.inboundPacketsIPv4Total) metricRecvDataPacketsIPv6.Register(&m.inboundPacketsIPv6Total) metricRecvDataPacketsDERP.Register(&m.inboundPacketsDERPTotal) + metricRecvDataPacketsPeerRelayIPv4.Register(&m.inboundPacketsPeerRelayIPv4Total) + metricRecvDataPacketsPeerRelayIPv6.Register(&m.inboundPacketsPeerRelayIPv6Total) metricSendUDP.Register(&m.outboundPacketsIPv4Total) metricSendUDP.Register(&m.outboundPacketsIPv6Total) metricSendDERP.Register(&m.outboundPacketsDERPTotal) + metricSendPeerRelay.Register(&m.outboundPacketsPeerRelayIPv4Total) + metricSendPeerRelay.Register(&m.outboundPacketsPeerRelayIPv6Total) inboundPacketsTotal.Set(pathDirectV4, &m.inboundPacketsIPv4Total) inboundPacketsTotal.Set(pathDirectV6, &m.inboundPacketsIPv6Total) inboundPacketsTotal.Set(pathDERP, &m.inboundPacketsDERPTotal) + inboundPacketsTotal.Set(pathPeerRelayV4, &m.inboundPacketsPeerRelayIPv4Total) + inboundPacketsTotal.Set(pathPeerRelayV6, &m.inboundPacketsPeerRelayIPv6Total) inboundBytesTotal.Set(pathDirectV4, &m.inboundBytesIPv4Total) inboundBytesTotal.Set(pathDirectV6, &m.inboundBytesIPv6Total) inboundBytesTotal.Set(pathDERP, &m.inboundBytesDERPTotal) + inboundBytesTotal.Set(pathPeerRelayV4, &m.inboundBytesPeerRelayIPv4Total) + inboundBytesTotal.Set(pathPeerRelayV6, &m.inboundBytesPeerRelayIPv6Total) outboundPacketsTotal.Set(pathDirectV4, &m.outboundPacketsIPv4Total) outboundPacketsTotal.Set(pathDirectV6, &m.outboundPacketsIPv6Total) outboundPacketsTotal.Set(pathDERP, &m.outboundPacketsDERPTotal) + outboundPacketsTotal.Set(pathPeerRelayV4, &m.outboundPacketsPeerRelayIPv4Total) + outboundPacketsTotal.Set(pathPeerRelayV6, &m.outboundPacketsPeerRelayIPv6Total) outboundBytesTotal.Set(pathDirectV4, &m.outboundBytesIPv4Total) outboundBytesTotal.Set(pathDirectV6, &m.outboundBytesIPv6Total) outboundBytesTotal.Set(pathDERP, &m.outboundBytesDERPTotal) + outboundBytesTotal.Set(pathPeerRelayV4, &m.outboundBytesPeerRelayIPv4Total) + outboundBytesTotal.Set(pathPeerRelayV6, &m.outboundBytesPeerRelayIPv6Total) outboundPacketsDroppedErrors.Set(usermetric.DropLabels{Reason: usermetric.ReasonError}, &m.outboundPacketsDroppedErrors) @@ -786,8 +812,11 @@ func deregisterMetrics(m *metrics) { metricRecvDataPacketsIPv4.UnregisterAll() metricRecvDataPacketsIPv6.UnregisterAll() metricRecvDataPacketsDERP.UnregisterAll() + metricRecvDataPacketsPeerRelayIPv4.UnregisterAll() + metricRecvDataPacketsPeerRelayIPv6.UnregisterAll() metricSendUDP.UnregisterAll() metricSendDERP.UnregisterAll() + metricSendPeerRelay.UnregisterAll() } // InstallCaptureHook installs a callback which is called to @@ -1415,23 +1444,37 @@ func (c *Conn) sendUDPBatch(addr epAddr, buffs [][]byte, offset int) (sent bool, // sendUDP sends UDP packet b to ipp. // See sendAddr's docs on the return value meanings. -func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool) (sent bool, err error) { +func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool, isGeneveEncap bool) (sent bool, err error) { if runtime.GOOS == "js" { return false, errNoUDP } sent, err = c.sendUDPStd(ipp, b) if err != nil { - metricSendUDPError.Add(1) + if isGeneveEncap { + metricSendPeerRelayError.Add(1) + } else { + metricSendUDPError.Add(1) + } c.maybeRebindOnError(err) } else { if sent && !isDisco { switch { case ipp.Addr().Is4(): - c.metrics.outboundPacketsIPv4Total.Add(1) - c.metrics.outboundBytesIPv4Total.Add(int64(len(b))) + if isGeneveEncap { + c.metrics.outboundPacketsPeerRelayIPv4Total.Add(1) + c.metrics.outboundBytesPeerRelayIPv4Total.Add(int64(len(b))) + } else { + c.metrics.outboundPacketsIPv4Total.Add(1) + c.metrics.outboundBytesIPv4Total.Add(int64(len(b))) + } case ipp.Addr().Is6(): - c.metrics.outboundPacketsIPv6Total.Add(1) - c.metrics.outboundBytesIPv6Total.Add(int64(len(b))) + if isGeneveEncap { + c.metrics.outboundPacketsPeerRelayIPv6Total.Add(1) + c.metrics.outboundBytesPeerRelayIPv6Total.Add(int64(len(b))) + } else { + c.metrics.outboundPacketsIPv6Total.Add(1) + c.metrics.outboundBytesIPv6Total.Add(int64(len(b))) + } } } } @@ -1506,9 +1549,9 @@ func (c *Conn) sendUDPStd(addr netip.AddrPort, b []byte) (sent bool, err error) // An example of when they might be different: sending to an // IPv6 address when the local machine doesn't have IPv6 support // returns (false, nil); it's not an error, but nothing was sent. -func (c *Conn) sendAddr(addr netip.AddrPort, pubKey key.NodePublic, b []byte, isDisco bool) (sent bool, err error) { +func (c *Conn) sendAddr(addr netip.AddrPort, pubKey key.NodePublic, b []byte, isDisco bool, isGeneveEncap bool) (sent bool, err error) { if addr.Addr() != tailcfg.DerpMagicIPAddr { - return c.sendUDP(addr, b, isDisco) + return c.sendUDP(addr, b, isDisco, isGeneveEncap) } regionID := int(addr.Port()) @@ -1562,7 +1605,9 @@ func (c *Conn) putReceiveBatch(batch *receiveBatch) { func (c *Conn) receiveIPv4() conn.ReceiveFunc { return c.mkReceiveFunc(&c.pconn4, c.health.ReceiveFuncStats(health.ReceiveIPv4), &c.metrics.inboundPacketsIPv4Total, + &c.metrics.inboundPacketsPeerRelayIPv4Total, &c.metrics.inboundBytesIPv4Total, + &c.metrics.inboundBytesPeerRelayIPv4Total, ) } @@ -1570,13 +1615,15 @@ func (c *Conn) receiveIPv4() conn.ReceiveFunc { func (c *Conn) receiveIPv6() conn.ReceiveFunc { return c.mkReceiveFunc(&c.pconn6, c.health.ReceiveFuncStats(health.ReceiveIPv6), &c.metrics.inboundPacketsIPv6Total, + &c.metrics.inboundPacketsPeerRelayIPv6Total, &c.metrics.inboundBytesIPv6Total, + &c.metrics.inboundBytesPeerRelayIPv6Total, ) } // mkReceiveFunc creates a ReceiveFunc reading from ruc. // The provided healthItem and metrics are updated if non-nil. -func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFuncStats, packetMetric, bytesMetric *expvar.Int) conn.ReceiveFunc { +func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFuncStats, directPacketMetric, peerRelayPacketMetric, directBytesMetric, peerRelayBytesMetric *expvar.Int) conn.ReceiveFunc { // epCache caches an epAddr->endpoint for hot flows. var epCache epAddrEndpointCache @@ -1612,12 +1659,21 @@ func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFu continue } ipp := msg.Addr.(*net.UDPAddr).AddrPort() - if ep, size, ok := c.receiveIP(msg.Buffers[0][:msg.N], ipp, &epCache); ok { - if packetMetric != nil { - packetMetric.Add(1) - } - if bytesMetric != nil { - bytesMetric.Add(int64(msg.N)) + if ep, size, isGeneveEncap, ok := c.receiveIP(msg.Buffers[0][:msg.N], ipp, &epCache); ok { + if isGeneveEncap { + if peerRelayPacketMetric != nil { + peerRelayPacketMetric.Add(1) + } + if peerRelayBytesMetric != nil { + peerRelayBytesMetric.Add(int64(msg.N)) + } + } else { + if directPacketMetric != nil { + directPacketMetric.Add(1) + } + if directBytesMetric != nil { + directBytesMetric.Add(int64(msg.N)) + } } eps[i] = ep sizes[i] = size @@ -1646,11 +1702,14 @@ func looksLikeInitiationMsg(b []byte) bool { // receiveIP is the shared bits of ReceiveIPv4 and ReceiveIPv6. // // size is the length of 'b' to report up to wireguard-go (only relevant if -// 'ok' is true) +// 'ok' is true). +// +// isGeneveEncap is whether 'b' is encapsulated by a Geneve header (only +// relevant if 'ok' is true). // // ok is whether this read should be reported up to wireguard-go (our // caller). -func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCache) (_ conn.Endpoint, size int, ok bool) { +func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCache) (_ conn.Endpoint, size int, isGeneveEncap bool, ok bool) { var ep *endpoint size = len(b) @@ -1663,7 +1722,7 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach // Decode only returns an error when 'b' is too short, and // 'isGeneveEncap' indicates it's a sufficient length. c.logf("[unexpected] geneve header decoding error: %v", err) - return nil, 0, false + return nil, 0, false, false } src.vni.set(geneve.VNI) } @@ -1678,10 +1737,10 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach // [disco.MessageType], but we assert it should be handshake-related. shouldByRelayHandshakeMsg := geneve.Control == true c.handleDiscoMessage(b, src, shouldByRelayHandshakeMsg, key.NodePublic{}, discoRXPathUDP) - return nil, 0, false + return nil, 0, false, false case packetLooksLikeSTUNBinding: c.netChecker.ReceiveSTUNPacket(b, ipp) - return nil, 0, false + return nil, 0, false, false default: // Fall through for all other packet types as they are assumed to // be potentially WireGuard. @@ -1691,7 +1750,7 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach // If we have no private key, we're logged out or // stopped. Don't try to pass these wireguard packets // up to wireguard-go; it'll just complain (issue 1167). - return nil, 0, false + return nil, 0, false, false } if src.vni.isSet() { @@ -1715,11 +1774,11 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach // Note: UDP relay is dependent on cryptorouting enablement. We // only update Geneve-encapsulated [epAddr]s in the [peerMap] // via [lazyEndpoint]. - return nil, 0, false + return nil, 0, false, false } // TODO(jwhited): reuse [lazyEndpoint] across calls to receiveIP() // for the same batch & [epAddr] src. - return &lazyEndpoint{c: c, src: src}, size, true + return &lazyEndpoint{c: c, src: src}, size, isGeneveEncap, true } cache.epAddr = src cache.de = de @@ -1738,9 +1797,9 @@ func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCach // unlucky and fail to JIT configure the "correct" peer. // TODO(jwhited): relax this to include direct connections // See http://go/corp/29422 & http://go/corp/30042 - return &lazyEndpoint{c: c, maybeEP: ep, src: src}, size, true + return &lazyEndpoint{c: c, maybeEP: ep, src: src}, size, isGeneveEncap, true } - return ep, size, true + return ep, size, isGeneveEncap, true } // discoLogLevel controls the verbosity of discovery log messages. @@ -1861,7 +1920,7 @@ func (c *Conn) sendDiscoMessage(dst epAddr, dstKey key.NodePublic, dstDisco key. box := di.sharedKey.Seal(m.AppendMarshal(nil)) pkt = append(pkt, box...) const isDisco = true - sent, err = c.sendAddr(dst.ap, dstKey, pkt, isDisco) + sent, err = c.sendAddr(dst.ap, dstKey, pkt, isDisco, dst.vni.isSet()) if sent { if logLevel == discoLog || (logLevel == discoVerboseLog && debugDisco()) { node := "?" @@ -2149,13 +2208,15 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake isVia := false msgType := "CallMeMaybe" cmm, ok := dm.(*disco.CallMeMaybe) - if !ok { + if ok { + metricRecvDiscoCallMeMaybe.Add(1) + } else { + metricRecvDiscoCallMeMaybeVia.Add(1) via = dm.(*disco.CallMeMaybeVia) msgType = "CallMeMaybeVia" isVia = true } - metricRecvDiscoCallMeMaybe.Add(1) if !isDERP || derpNodeSrc.IsZero() { // CallMeMaybe{Via} messages should only come via DERP. c.logf("[unexpected] %s packets should only come via DERP", msgType) @@ -2164,7 +2225,11 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake nodeKey := derpNodeSrc ep, ok := c.peerMap.endpointForNodeKey(nodeKey) if !ok { - metricRecvDiscoCallMeMaybeBadNode.Add(1) + if isVia { + metricRecvDiscoCallMeMaybeViaBadNode.Add(1) + } else { + metricRecvDiscoCallMeMaybeBadNode.Add(1) + } c.logf("magicsock: disco: ignoring %s from %v; %v is unknown", msgType, sender.ShortString(), derpNodeSrc.ShortString()) return } @@ -2190,7 +2255,11 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake return } if epDisco.key != di.discoKey { - metricRecvDiscoCallMeMaybeBadDisco.Add(1) + if isVia { + metricRecvDiscoCallMeMaybeViaBadDisco.Add(1) + } else { + metricRecvDiscoCallMeMaybeBadDisco.Add(1) + } c.logf("[unexpected] %s from peer via DERP whose netmap discokey != disco source", msgType) return } @@ -3695,15 +3764,19 @@ var ( metricSendDERPErrorQueue = clientmetric.NewCounter("magicsock_send_derp_error_queue") metricSendUDP = clientmetric.NewAggregateCounter("magicsock_send_udp") metricSendUDPError = clientmetric.NewCounter("magicsock_send_udp_error") + metricSendPeerRelay = clientmetric.NewAggregateCounter("magicsock_send_peer_relay") + metricSendPeerRelayError = clientmetric.NewCounter("magicsock_send_peer_relay_error") metricSendDERP = clientmetric.NewAggregateCounter("magicsock_send_derp") metricSendDERPError = clientmetric.NewCounter("magicsock_send_derp_error") // Data packets (non-disco) - metricSendData = clientmetric.NewCounter("magicsock_send_data") - metricSendDataNetworkDown = clientmetric.NewCounter("magicsock_send_data_network_down") - metricRecvDataPacketsDERP = clientmetric.NewAggregateCounter("magicsock_recv_data_derp") - metricRecvDataPacketsIPv4 = clientmetric.NewAggregateCounter("magicsock_recv_data_ipv4") - metricRecvDataPacketsIPv6 = clientmetric.NewAggregateCounter("magicsock_recv_data_ipv6") + metricSendData = clientmetric.NewCounter("magicsock_send_data") + metricSendDataNetworkDown = clientmetric.NewCounter("magicsock_send_data_network_down") + metricRecvDataPacketsDERP = clientmetric.NewAggregateCounter("magicsock_recv_data_derp") + metricRecvDataPacketsIPv4 = clientmetric.NewAggregateCounter("magicsock_recv_data_ipv4") + metricRecvDataPacketsIPv6 = clientmetric.NewAggregateCounter("magicsock_recv_data_ipv6") + metricRecvDataPacketsPeerRelayIPv4 = clientmetric.NewAggregateCounter("magicsock_recv_data_peer_relay_ipv4") + metricRecvDataPacketsPeerRelayIPv6 = clientmetric.NewAggregateCounter("magicsock_recv_data_peer_relay_ipv6") // Disco packets metricSendDiscoUDP = clientmetric.NewCounter("magicsock_disco_send_udp") @@ -3719,15 +3792,18 @@ var ( metricRecvDiscoBadKey = clientmetric.NewCounter("magicsock_disco_recv_bad_key") metricRecvDiscoBadParse = clientmetric.NewCounter("magicsock_disco_recv_bad_parse") - metricRecvDiscoUDP = clientmetric.NewCounter("magicsock_disco_recv_udp") - metricRecvDiscoDERP = clientmetric.NewCounter("magicsock_disco_recv_derp") - metricRecvDiscoPing = clientmetric.NewCounter("magicsock_disco_recv_ping") - metricRecvDiscoPong = clientmetric.NewCounter("magicsock_disco_recv_pong") - metricRecvDiscoCallMeMaybe = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe") - metricRecvDiscoCallMeMaybeBadNode = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_node") - metricRecvDiscoCallMeMaybeBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_disco") - metricRecvDiscoDERPPeerNotHere = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_not_here") - metricRecvDiscoDERPPeerGoneUnknown = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_gone_unknown") + metricRecvDiscoUDP = clientmetric.NewCounter("magicsock_disco_recv_udp") + metricRecvDiscoDERP = clientmetric.NewCounter("magicsock_disco_recv_derp") + metricRecvDiscoPing = clientmetric.NewCounter("magicsock_disco_recv_ping") + metricRecvDiscoPong = clientmetric.NewCounter("magicsock_disco_recv_pong") + metricRecvDiscoCallMeMaybe = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe") + metricRecvDiscoCallMeMaybeVia = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia") + metricRecvDiscoCallMeMaybeBadNode = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_node") + metricRecvDiscoCallMeMaybeViaBadNode = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia_bad_node") + metricRecvDiscoCallMeMaybeBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_disco") + metricRecvDiscoCallMeMaybeViaBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia_bad_disco") + metricRecvDiscoDERPPeerNotHere = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_not_here") + metricRecvDiscoDERPPeerGoneUnknown = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_gone_unknown") // metricDERPHomeChange is how many times our DERP home region DI has // changed from non-zero to a different non-zero. metricDERPHomeChange = clientmetric.NewCounter("derp_home_change") From e7238efafa427c2a360540534bd08613a81ca7bc Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Wed, 16 Jul 2025 19:37:46 -0400 Subject: [PATCH 222/263] cmd/tailscale/cli: Add service flag to serve command (#16191) * cmd/tailscale/cli: Add service flag to serve command This commit adds the service flag to serve command which allows serving a service and add the service to the advertisedServices field in prefs (What advertise command does that will be removed later). When adding proxies, TCP proxies and WEB proxies work the same way as normal serve, just under a different DNSname. There is a services specific L3 serving mode called Tun, can be set via --tun flag. Serving a service is always in --bg mode. If --bg is explicitly set t o false, an error message will be sent out. The restriction on proxy target being localhost or 127.0.0.1 also applies to services. When removing proxies, TCP proxies can be removed with type and port flag and off argument. Web proxies can be removed with type, port, setPath flag and off argument. To align with normal serve, when setPath is not set, all handler under the hostport will be removed. When flags are not set but off argument was passed by user, it will be a noop. Removing all config for a service will be available later with a new subcommand clear. Updates tailscale/corp#22954 Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: fix ai comments and fix a test Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: Add a test for addServiceToPrefs Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: fix comment Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * add dnsName in error message Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * change the cli input flag variable type Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * replace FindServiceConfig with map lookup Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * some code simplification and add asServiceName This commit cotains code simplification for IsServingHTTPS, SetWebHandler, SetTCPForwarding Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * replace IsServiceName with tailcfg.AsServiceName Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * replace all assemble of host name for service with strings.Join Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: adjust parameter order and update output message This commit updates the parameter order for IsTCPForwardingOnPort and SetWebHandler. Also updated the message msgServiceIPNotAssigned to msgServiceWaitingApproval to adapt to latest terminologies around services. Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: flip bool condition This commit fixes a previous bug added that throws error when serve funnel without service. It should've been the opposite, which throws error when serve funnel with service. Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: change parameter of IsTCPForwardingOnPort This commit changes the dnsName string parameter for IsTCPForwardingOnPort to svcName tailcfg.ServiceName. This change is made to reduce ambiguity when a single service might have different dnsNames Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * ipn/ipnlocal: replace the key to webHandler for services This commit changes the way we get the webhandler for vipServices. It used to use the host name from request to find the webHandler, now everything targeting the vipService IP have the same set of handlers. This commit also stores service:port instead of FQDN:port as the key in serviceConfig for Web map. Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: Updated use of service name. This commit removes serviceName.IsEmpty and use direct comparison to instead. In legacy code, when an empty service name needs to be passed, a new constant noService is passed. Removed redundant code for checking service name validity and string method for serviceNameFlag. Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: Update bgBoolFlag This commit update field name, set and string method of bgBoolFlag to make code cleaner. Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: remove isDefaultService output from srvTypeAndPortFromFlags This commit removes the isDefaultService out put as it's no longer needed. Also deleted redundant code. Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: remove unnessesary variable declare in messageForPort Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * replace bool output for AsServiceName with err Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: Replace DNSName with NoService if DNSname only used to identify service This commit moves noService constant to tailcfg, updates AsServiceName to return tailcfg.NoService if the input is not a valid service name. This commit also removes using the local DNSName as scvName parameter. When a function is only using DNSName to identify if it's working with a service, the input in replaced with svcName and expect caller to pass tailcfg.NoService if it's a local serve. This commit also replaces some use of Sprintf with net.JoinHostPort for ipn.HostPort creation. Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: Remove the returned error for AsServiceName Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * apply suggested code and comment Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * replace local dnsName in test with tailcfg.NoService Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * cmd/tailscale/cli: move noService back and use else where The constant serves the purpose of provide readability for passing as a function parameter. It's more meaningful comparing to a . It can just be an empty string in other places. Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * ipn: Make WebHandlerExists and RemoveTCPForwarding accept svcName This commit replaces two functions' string input with svcName input since they only use the dnsName to identify service. Also did some minor cleanups Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --------- Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- cmd/tailscale/cli/serve_legacy.go | 42 +- cmd/tailscale/cli/serve_legacy_test.go | 16 + cmd/tailscale/cli/serve_v2.go | 395 +++++++++-- cmd/tailscale/cli/serve_v2_test.go | 925 ++++++++++++++++++++++++- cmd/tailscale/cli/status.go | 2 +- ipn/ipnlocal/serve.go | 4 +- ipn/serve.go | 208 ++++-- ipn/serve_test.go | 115 +++ tailcfg/tailcfg.go | 10 + 9 files changed, 1573 insertions(+), 144 deletions(-) diff --git a/cmd/tailscale/cli/serve_legacy.go b/cmd/tailscale/cli/serve_legacy.go index 96629b5ad45ef..7c79f7f7bc972 100644 --- a/cmd/tailscale/cli/serve_legacy.go +++ b/cmd/tailscale/cli/serve_legacy.go @@ -141,6 +141,8 @@ type localServeClient interface { QueryFeature(ctx context.Context, feature string) (*tailcfg.QueryFeatureResponse, error) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*tailscale.IPNBusWatcher, error) IncrementCounter(ctx context.Context, name string, delta int) error + GetPrefs(ctx context.Context) (*ipn.Prefs, error) + EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Prefs, error) } // serveEnv is the environment the serve command runs within. All I/O should be @@ -154,14 +156,16 @@ type serveEnv struct { json bool // output JSON (status only for now) // v2 specific flags - bg bool // background mode - setPath string // serve path - https uint // HTTP port - http uint // HTTP port - tcp uint // TCP port - tlsTerminatedTCP uint // a TLS terminated TCP port - subcmd serveMode // subcommand - yes bool // update without prompt + bg bgBoolFlag // background mode + setPath string // serve path + https uint // HTTP port + http uint // HTTP port + tcp uint // TCP port + tlsTerminatedTCP uint // a TLS terminated TCP port + subcmd serveMode // subcommand + yes bool // update without prompt + service tailcfg.ServiceName // service name + tun bool // redirect traffic to OS for service lc localServeClient // localClient interface, specific to serve @@ -354,7 +358,7 @@ func (e *serveEnv) handleWebServe(ctx context.Context, srvPort uint16, useTLS bo if err != nil { return err } - if sc.IsTCPForwardingOnPort(srvPort) { + if sc.IsTCPForwardingOnPort(srvPort, noService) { fmt.Fprintf(Stderr, "error: cannot serve web; already serving TCP\n") return errHelp } @@ -411,11 +415,11 @@ func (e *serveEnv) handleWebServeRemove(ctx context.Context, srvPort uint16, mou if err != nil { return err } - if sc.IsTCPForwardingOnPort(srvPort) { + if sc.IsTCPForwardingOnPort(srvPort, noService) { return errors.New("cannot remove web handler; currently serving TCP") } hp := ipn.HostPort(net.JoinHostPort(dnsName, strconv.Itoa(int(srvPort)))) - if !sc.WebHandlerExists(hp, mount) { + if !sc.WebHandlerExists(noService, hp, mount) { return errors.New("error: handler does not exist") } sc.RemoveWebHandler(dnsName, srvPort, []string{mount}, false) @@ -550,15 +554,15 @@ func (e *serveEnv) handleTCPServe(ctx context.Context, srcType string, srcPort u fwdAddr := "127.0.0.1:" + dstPortStr - if sc.IsServingWeb(srcPort) { - return fmt.Errorf("cannot serve TCP; already serving web on %d", srcPort) - } - dnsName, err := e.getSelfDNSName(ctx) if err != nil { return err } + if sc.IsServingWeb(srcPort, noService) { + return fmt.Errorf("cannot serve TCP; already serving web on %d", srcPort) + } + sc.SetTCPForwarding(srcPort, fwdAddr, terminateTLS, dnsName) if !reflect.DeepEqual(cursc, sc) { @@ -581,11 +585,11 @@ func (e *serveEnv) handleTCPServeRemove(ctx context.Context, src uint16) error { if sc == nil { sc = new(ipn.ServeConfig) } - if sc.IsServingWeb(src) { + if sc.IsServingWeb(src, noService) { return fmt.Errorf("unable to remove; serving web, not TCP forwarding on serve port %d", src) } - if ph := sc.GetTCPPortHandler(src); ph != nil { - sc.RemoveTCPForwarding(src) + if ph := sc.GetTCPPortHandler(src, noService); ph != nil { + sc.RemoveTCPForwarding(noService, src) return e.lc.SetServeConfig(ctx, sc) } return errors.New("error: serve config does not exist") @@ -682,7 +686,7 @@ func (e *serveEnv) printWebStatusTree(sc *ipn.ServeConfig, hp ipn.HostPort) erro } scheme := "https" - if sc.IsServingHTTP(port) { + if sc.IsServingHTTP(port, noService) { scheme = "http" } diff --git a/cmd/tailscale/cli/serve_legacy_test.go b/cmd/tailscale/cli/serve_legacy_test.go index df68b5edd32a1..6b053fbd774ba 100644 --- a/cmd/tailscale/cli/serve_legacy_test.go +++ b/cmd/tailscale/cli/serve_legacy_test.go @@ -859,6 +859,7 @@ type fakeLocalServeClient struct { config *ipn.ServeConfig setCount int // counts calls to SetServeConfig queryFeatureResponse *mockQueryFeatureResponse // mock response to QueryFeature calls + prefs *ipn.Prefs // fake preferences, used to test GetPrefs and SetPrefs } // fakeStatus is a fake ipnstate.Status value for tests. @@ -891,6 +892,21 @@ func (lc *fakeLocalServeClient) SetServeConfig(ctx context.Context, config *ipn. return nil } +func (lc *fakeLocalServeClient) GetPrefs(ctx context.Context) (*ipn.Prefs, error) { + if lc.prefs == nil { + lc.prefs = ipn.NewPrefs() + } + return lc.prefs, nil +} + +func (lc *fakeLocalServeClient) EditPrefs(ctx context.Context, prefs *ipn.MaskedPrefs) (*ipn.Prefs, error) { + if lc.prefs == nil { + lc.prefs = ipn.NewPrefs() + } + lc.prefs.ApplyEdits(prefs) + return lc.prefs, nil +} + type mockQueryFeatureResponse struct { resp *tailcfg.QueryFeatureResponse err error diff --git a/cmd/tailscale/cli/serve_v2.go b/cmd/tailscale/cli/serve_v2.go index bb51fb7d0e131..15de0609c72ad 100644 --- a/cmd/tailscale/cli/serve_v2.go +++ b/cmd/tailscale/cli/serve_v2.go @@ -18,6 +18,7 @@ import ( "os/signal" "path" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -41,6 +42,55 @@ type commandInfo struct { LongHelp string } +type serviceNameFlag struct { + Value *tailcfg.ServiceName +} + +func (s *serviceNameFlag) Set(sv string) error { + if sv == "" { + s.Value = new(tailcfg.ServiceName) + return nil + } + v := tailcfg.ServiceName(sv) + if err := v.Validate(); err != nil { + return fmt.Errorf("invalid service name: %q", sv) + } + *s.Value = v + return nil +} + +// String returns the string representation of service name. +func (s *serviceNameFlag) String() string { + return s.Value.String() +} + +type bgBoolFlag struct { + Value bool + IsSet bool // tracks if the flag was set by the user +} + +// Set sets the boolean flag and whether it's explicitly set by user based on the string value. +func (b *bgBoolFlag) Set(s string) error { + v, err := strconv.ParseBool(s) + if err != nil { + return err + } + b.Value = v + b.IsSet = true + return nil +} + +// This is a hack to make the flag package recognize that this is a boolean flag. +func (b *bgBoolFlag) IsBoolFlag() bool { return true } + +// String returns the string representation of the boolean flag. +func (b *bgBoolFlag) String() string { + if !b.IsSet { + return "default" + } + return strconv.FormatBool(b.Value) +} + var serveHelpCommon = strings.TrimSpace(` can be a file, directory, text, or most commonly the location to a service running on the local machine. The location to the location service can be expressed as a port number (e.g., 3000), @@ -73,8 +123,11 @@ const ( serveTypeHTTP serveTypeTCP serveTypeTLSTerminatedTCP + serveTypeTUN ) +const noService tailcfg.ServiceName = "" + var infoMap = map[serveMode]commandInfo{ serve: { Name: "serve", @@ -120,7 +173,7 @@ func newServeV2Command(e *serveEnv, subcmd serveMode) *ffcli.Command { Exec: e.runServeCombined(subcmd), FlagSet: e.newFlags("serve-set", func(fs *flag.FlagSet) { - fs.BoolVar(&e.bg, "bg", false, "Run the command as a background process (default false)") + fs.Var(&e.bg, "bg", "Run the command as a background process (default false, when --service is set defaults to true).") fs.StringVar(&e.setPath, "set-path", "", "Appends the specified path to the base URL for accessing the underlying service") fs.UintVar(&e.https, "https", 0, "Expose an HTTPS server at the specified port (default mode)") if subcmd == serve { @@ -128,7 +181,9 @@ func newServeV2Command(e *serveEnv, subcmd serveMode) *ffcli.Command { } fs.UintVar(&e.tcp, "tcp", 0, "Expose a TCP forwarder to forward raw TCP packets at the specified port") fs.UintVar(&e.tlsTerminatedTCP, "tls-terminated-tcp", 0, "Expose a TCP forwarder to forward TLS-terminated TCP packets at the specified port") + fs.Var(&serviceNameFlag{Value: &e.service}, "service", "Serve for a service with distinct virtual IP instead on node itself.") fs.BoolVar(&e.yes, "yes", false, "Update without interactive prompts (default false)") + fs.BoolVar(&e.tun, "tun", false, "Forward all traffic to the local machine (default false), only supported for services. Refer to docs for more information.") }), UsageFunc: usageFuncNoDefaultValues, Subcommands: []*ffcli.Command{ @@ -162,9 +217,16 @@ func (e *serveEnv) validateArgs(subcmd serveMode, args []string) error { fmt.Fprint(e.stderr(), "\nPlease see https://tailscale.com/kb/1242/tailscale-serve for more information.\n") return errHelpFunc(subcmd) } + if len(args) == 0 && e.tun { + return nil + } if len(args) == 0 { return flag.ErrHelp } + if e.tun && len(args) > 1 { + fmt.Fprintln(e.stderr(), "Error: invalid argument format") + return errHelpFunc(subcmd) + } if len(args) > 2 { fmt.Fprintf(e.stderr(), "Error: invalid number of arguments (%d)\n", len(args)) return errHelpFunc(subcmd) @@ -206,7 +268,16 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { ctx, cancel := signal.NotifyContext(ctx, os.Interrupt) defer cancel() + forService := e.service != "" + if !e.bg.IsSet { + e.bg.Value = forService + } + funnel := subcmd == funnel + if forService && funnel { + return errors.New("Error: --service flag is not supported with funnel") + } + if funnel { // verify node has funnel capabilities if err := e.verifyFunnelEnabled(ctx, 443); err != nil { @@ -214,6 +285,10 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } } + if forService && !e.bg.Value { + return errors.New("Error: --service flag is only compatible with background mode") + } + mount, err := cleanURLPath(e.setPath) if err != nil { return fmt.Errorf("failed to clean the mount point: %w", err) @@ -246,7 +321,7 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { // foreground or background. parentSC := sc - turnOff := "off" == args[len(args)-1] + turnOff := len(args) > 0 && "off" == args[len(args)-1] if !turnOff && srvType == serveTypeHTTPS { // Running serve with https requires that the tailnet has enabled // https cert provisioning. Send users through an interactive flow @@ -263,10 +338,19 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } var watcher *tailscale.IPNBusWatcher - wantFg := !e.bg && !turnOff + svcName := noService + + if forService { + svcName = e.service + dnsName = e.service.String() + } + if !forService && srvType == serveTypeTUN { + return errors.New("tun mode is only supported for services") + } + wantFg := !e.bg.Value && !turnOff if wantFg { // validate the config before creating a WatchIPNBus session - if err := e.validateConfig(parentSC, srvPort, srvType); err != nil { + if err := e.validateConfig(parentSC, srvPort, srvType, svcName); err != nil { return err } @@ -292,12 +376,20 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { var msg string if turnOff { - err = e.unsetServe(sc, dnsName, srvType, srvPort, mount) + // only unset serve when trying to unset with type and port flags. + err = e.unsetServe(sc, st, dnsName, srvType, srvPort, mount) } else { - if err := e.validateConfig(parentSC, srvPort, srvType); err != nil { + if err := e.validateConfig(parentSC, srvPort, srvType, svcName); err != nil { return err } - err = e.setServe(sc, st, dnsName, srvType, srvPort, mount, args[0], funnel) + if forService { + e.addServiceToPrefs(ctx, svcName.String()) + } + target := "" + if len(args) > 0 { + target = args[0] + } + err = e.setServe(sc, dnsName, srvType, srvPort, mount, target, funnel) msg = e.messageForPort(sc, st, dnsName, srvType, srvPort) } if err != nil { @@ -332,22 +424,66 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } } -const backgroundExistsMsg = "background configuration already exists, use `tailscale %s --%s=%d off` to remove the existing configuration" - -func (e *serveEnv) validateConfig(sc *ipn.ServeConfig, port uint16, wantServe serveType) error { - sc, isFg := sc.FindConfig(port) - if sc == nil { - return nil +func (e *serveEnv) addServiceToPrefs(ctx context.Context, serviceName string) error { + prefs, err := e.lc.GetPrefs(ctx) + if err != nil { + return fmt.Errorf("error getting prefs: %w", err) } - if isFg { - return errors.New("foreground already exists under this port") + advertisedServices := prefs.AdvertiseServices + if slices.Contains(advertisedServices, serviceName) { + return nil // already advertised } - if !e.bg { - return fmt.Errorf(backgroundExistsMsg, infoMap[e.subcmd].Name, wantServe.String(), port) + advertisedServices = append(advertisedServices, serviceName) + _, err = e.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: advertisedServices, + }, + }) + return err +} + +const backgroundExistsMsg = "background configuration already exists, use `tailscale %s --%s=%d off` to remove the existing configuration" + +// validateConfig checks if the serve config is valid to serve the type wanted on the port. +// dnsName is a FQDN or a serviceName (with `svc:` prefix). +func (e *serveEnv) validateConfig(sc *ipn.ServeConfig, port uint16, wantServe serveType, svcName tailcfg.ServiceName) error { + var tcpHandlerForPort *ipn.TCPPortHandler + if svcName != noService { + svc := sc.Services[svcName] + if svc == nil { + return nil + } + if wantServe == serveTypeTUN && (svc.TCP != nil || svc.Web != nil) { + return errors.New("service already has a TCP or Web handler, cannot serve in TUN mode") + } + if svc.Tun && wantServe != serveTypeTUN { + return errors.New("service is already being served in TUN mode") + } + if svc.TCP[port] == nil { + return nil + } + tcpHandlerForPort = svc.TCP[port] + } else { + sc, isFg := sc.FindConfig(port) + if sc == nil { + return nil + } + if isFg { + return errors.New("foreground already exists under this port") + } + if !e.bg.Value { + return fmt.Errorf(backgroundExistsMsg, infoMap[e.subcmd].Name, wantServe.String(), port) + } + tcpHandlerForPort = sc.TCP[port] } - existingServe := serveFromPortHandler(sc.TCP[port]) + existingServe := serveFromPortHandler(tcpHandlerForPort) if wantServe != existingServe { - return fmt.Errorf("want %q but port is already serving %q", wantServe, existingServe) + target := svcName + if target == noService { + target = "machine" + } + return fmt.Errorf("want to serve %q but port is already serving %q for %q", wantServe, existingServe, target) } return nil } @@ -367,7 +503,7 @@ func serveFromPortHandler(tcp *ipn.TCPPortHandler) serveType { } } -func (e *serveEnv) setServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName string, srvType serveType, srvPort uint16, mount string, target string, allowFunnel bool) error { +func (e *serveEnv) setServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string, target string, allowFunnel bool) error { // update serve config based on the type switch srvType { case serveTypeHTTPS, serveTypeHTTP: @@ -380,45 +516,61 @@ func (e *serveEnv) setServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName st if e.setPath != "" { return fmt.Errorf("cannot mount a path for TCP serve") } - err := e.applyTCPServe(sc, dnsName, srvType, srvPort, target) if err != nil { return fmt.Errorf("failed to apply TCP serve: %w", err) } + case serveTypeTUN: + // Caller checks that TUN mode is only supported for services. + svcName := tailcfg.ServiceName(dnsName) + if _, ok := sc.Services[svcName]; !ok { + mak.Set(&sc.Services, svcName, new(ipn.ServiceConfig)) + } + sc.Services[svcName].Tun = true default: return fmt.Errorf("invalid type %q", srvType) } // update the serve config based on if funnel is enabled - e.applyFunnel(sc, dnsName, srvPort, allowFunnel) - + // Since funnel is not supported for services, we only apply it for node's serve. + if svcName := tailcfg.AsServiceName(dnsName); svcName == noService { + e.applyFunnel(sc, dnsName, srvPort, allowFunnel) + } return nil } var ( - msgFunnelAvailable = "Available on the internet:" - msgServeAvailable = "Available within your tailnet:" - msgRunningInBackground = "%s started and running in the background." - msgDisableProxy = "To disable the proxy, run: tailscale %s --%s=%d off" - msgToExit = "Press Ctrl+C to exit." + msgFunnelAvailable = "Available on the internet:" + msgServeAvailable = "Available within your tailnet:" + msgServiceWaitingApproval = "This machine is configured as a service proxy for %s, but approval from an admin is required. Once approved, it will be available in your Tailnet as:" + msgRunningInBackground = "%s started and running in the background." + msgRunningTunService = "IPv4 and IPv6 traffic to %s is being routed to your operating system." + msgDisableProxy = "To disable the proxy, run: tailscale %s --%s=%d off" + msgDisableServiceProxy = "To disable the proxy, run: tailscale serve --service=%s --%s=%d off" + msgDisableServiceTun = "To disable the service in TUN mode, run: tailscale serve --service=%s --tun off" + msgDisableService = "To remove config for the service, run: tailscale serve clear --service=%s" + msgToExit = "Press Ctrl+C to exit." ) // messageForPort returns a message for the given port based on the // serve config and status. func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName string, srvType serveType, srvPort uint16) string { var output strings.Builder - - hp := ipn.HostPort(net.JoinHostPort(dnsName, strconv.Itoa(int(srvPort)))) - - if sc.AllowFunnel[hp] == true { - output.WriteString(msgFunnelAvailable) - } else { - output.WriteString(msgServeAvailable) + svcName := tailcfg.AsServiceName(dnsName) + forService := svcName != noService + var webConfig *ipn.WebServerConfig + var tcpHandler *ipn.TCPPortHandler + ips := st.TailscaleIPs + host := dnsName + displayedHost := dnsName + if forService { + displayedHost = strings.Join([]string{svcName.WithoutPrefix(), st.CurrentTailnet.MagicDNSSuffix}, ".") + host = svcName.WithoutPrefix() } - output.WriteString("\n\n") + hp := ipn.HostPort(net.JoinHostPort(host, strconv.Itoa(int(srvPort)))) scheme := "https" - if sc.IsServingHTTP(srvPort) { + if sc.IsServingHTTP(srvPort, svcName) { scheme = "http" } @@ -439,37 +591,68 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN } return "", "" } + if forService { + serviceIPMaps, err := tailcfg.UnmarshalNodeCapJSON[tailcfg.ServiceIPMappings](st.Self.CapMap, tailcfg.NodeAttrServiceHost) + if err != nil || len(serviceIPMaps) == 0 || serviceIPMaps[0][svcName] == nil { + // The capmap does not contain IPs for this service yet. Usually this means + // the service hasn't been added to prefs and sent to control yet. + output.WriteString(fmt.Sprintf(msgServiceWaitingApproval, svcName.String())) + ips = nil + } else { + output.WriteString(msgServeAvailable) + ips = serviceIPMaps[0][svcName] + } + output.WriteString("\n\n") + svc := sc.Services[svcName] + if srvType == serveTypeTUN && svc.Tun { + output.WriteString(fmt.Sprintf(msgRunningTunService, displayedHost)) + output.WriteString("\n") + output.WriteString(fmt.Sprintf(msgDisableServiceTun, dnsName)) + output.WriteString("\n") + output.WriteString(fmt.Sprintf(msgDisableService, dnsName)) + return output.String() + } + if svc != nil { + webConfig = svc.Web[hp] + tcpHandler = svc.TCP[srvPort] + } + } else { + if sc.AllowFunnel[hp] == true { + output.WriteString(msgFunnelAvailable) + } else { + output.WriteString(msgServeAvailable) + } + output.WriteString("\n\n") + webConfig = sc.Web[hp] + tcpHandler = sc.TCP[srvPort] + } - if sc.Web[hp] != nil { - mounts := slicesx.MapKeys(sc.Web[hp].Handlers) + if webConfig != nil { + mounts := slicesx.MapKeys(webConfig.Handlers) sort.Slice(mounts, func(i, j int) bool { return len(mounts[i]) < len(mounts[j]) }) - for _, m := range mounts { - h := sc.Web[hp].Handlers[m] - t, d := srvTypeAndDesc(h) - output.WriteString(fmt.Sprintf("%s://%s%s%s\n", scheme, dnsName, portPart, m)) + t, d := srvTypeAndDesc(webConfig.Handlers[m]) + output.WriteString(fmt.Sprintf("%s://%s%s%s\n", scheme, displayedHost, portPart, m)) output.WriteString(fmt.Sprintf("%s %-5s %s\n\n", "|--", t, d)) } - } else if sc.TCP[srvPort] != nil { - h := sc.TCP[srvPort] + } else if tcpHandler != nil { tlsStatus := "TLS over TCP" - if h.TerminateTLS != "" { + if tcpHandler.TerminateTLS != "" { tlsStatus = "TLS terminated" } - output.WriteString(fmt.Sprintf("%s://%s%s\n", scheme, dnsName, portPart)) - output.WriteString(fmt.Sprintf("|-- tcp://%s (%s)\n", hp, tlsStatus)) - for _, a := range st.TailscaleIPs { + output.WriteString(fmt.Sprintf("|-- tcp://%s:%d (%s)\n", displayedHost, srvPort, tlsStatus)) + for _, a := range ips { ipp := net.JoinHostPort(a.String(), strconv.Itoa(int(srvPort))) output.WriteString(fmt.Sprintf("|-- tcp://%s\n", ipp)) } - output.WriteString(fmt.Sprintf("|--> tcp://%s\n", h.TCPForward)) + output.WriteString(fmt.Sprintf("|--> tcp://%s\n\n", tcpHandler.TCPForward)) } - if !e.bg { + if !forService && !e.bg.Value { output.WriteString(msgToExit) return output.String() } @@ -479,14 +662,19 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN output.WriteString(fmt.Sprintf(msgRunningInBackground, subCmdUpper)) output.WriteString("\n") - output.WriteString(fmt.Sprintf(msgDisableProxy, subCmd, srvType.String(), srvPort)) + if forService { + output.WriteString(fmt.Sprintf(msgDisableServiceProxy, dnsName, srvType.String(), srvPort)) + output.WriteString("\n") + output.WriteString(fmt.Sprintf(msgDisableService, dnsName)) + } else { + output.WriteString(fmt.Sprintf(msgDisableProxy, subCmd, srvType.String(), srvPort)) + } return output.String() } func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, useTLS bool, mount, target string) error { h := new(ipn.HTTPHandler) - switch { case strings.HasPrefix(target, "text:"): text := strings.TrimPrefix(target, "text:") @@ -522,7 +710,8 @@ func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort ui } // TODO: validation needs to check nested foreground configs - if sc.IsTCPForwardingOnPort(srvPort) { + svcName := tailcfg.AsServiceName(dnsName) + if sc.IsTCPForwardingOnPort(srvPort, svcName) { return errors.New("cannot serve web; already serving TCP") } @@ -553,8 +742,9 @@ func (e *serveEnv) applyTCPServe(sc *ipn.ServeConfig, dnsName string, srcType se } // TODO: needs to account for multiple configs from foreground mode - if sc.IsServingWeb(srcPort) { - return fmt.Errorf("cannot serve TCP; already serving web on %d", srcPort) + svcName := tailcfg.AsServiceName(dnsName) + if sc.IsServingWeb(srcPort, svcName) { + return fmt.Errorf("cannot serve TCP; already serving web on %d for %s", srcPort, dnsName) } sc.SetTCPForwarding(srcPort, dstURL.Host, terminateTLS, dnsName) @@ -578,18 +768,24 @@ func (e *serveEnv) applyFunnel(sc *ipn.ServeConfig, dnsName string, srvPort uint } // unsetServe removes the serve config for the given serve port. -func (e *serveEnv) unsetServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string) error { +// dnsName is a FQDN or a serviceName (with `svc:` prefix). +func (e *serveEnv) unsetServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName string, srvType serveType, srvPort uint16, mount string) error { switch srvType { case serveTypeHTTPS, serveTypeHTTP: - err := e.removeWebServe(sc, dnsName, srvPort, mount) + err := e.removeWebServe(sc, st, dnsName, srvPort, mount) if err != nil { return fmt.Errorf("failed to remove web serve: %w", err) } case serveTypeTCP, serveTypeTLSTerminatedTCP: - err := e.removeTCPServe(sc, srvPort) + err := e.removeTCPServe(sc, dnsName, srvPort) if err != nil { return fmt.Errorf("failed to remove TCP serve: %w", err) } + case serveTypeTUN: + err := e.removeTunServe(sc, dnsName) + if err != nil { + return fmt.Errorf("failed to remove TUN serve: %w", err) + } default: return fmt.Errorf("invalid type %q", srvType) } @@ -620,11 +816,16 @@ func srvTypeAndPortFromFlags(e *serveEnv) (srvType serveType, srvPort uint16, er } } + if e.tun { + srcTypeCount++ + srvType = serveTypeTUN + } + if srcTypeCount > 1 { return 0, 0, fmt.Errorf("cannot serve multiple types for a single mount point") - } else if srcTypeCount == 0 { - srvType = serveTypeHTTPS - srvPort = 443 + } + if srcTypeCount == 0 { + return serveTypeHTTPS, 443, nil } return srvType, srvPort, nil @@ -728,32 +929,48 @@ func isLegacyInvocation(subcmd serveMode, args []string) (string, bool) { // and removes funnel if no remaining mounts exist for the serve port. // The srvPort argument is the serving port and the mount argument is // the mount point or registered path to remove. -func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, mount string) error { - if sc.IsTCPForwardingOnPort(srvPort) { - return errors.New("cannot remove web handler; currently serving TCP") +func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName string, srvPort uint16, mount string) error { + if sc == nil { + return nil } portStr := strconv.Itoa(int(srvPort)) - hp := ipn.HostPort(net.JoinHostPort(dnsName, portStr)) + hostName := dnsName + webServeMap := sc.Web + svcName := tailcfg.AsServiceName(dnsName) + forService := svcName != noService + if forService { + svc := sc.Services[svcName] + if svc == nil { + return errors.New("service does not exist") + } + hostName = svcName.WithoutPrefix() + webServeMap = svc.Web + } + + hp := ipn.HostPort(net.JoinHostPort(hostName, portStr)) + if sc.IsTCPForwardingOnPort(srvPort, svcName) { + return errors.New("cannot remove web handler; currently serving TCP") + } var targetExists bool var mounts []string // mount is deduced from e.setPath but it is ambiguous as // to whether the user explicitly passed "/" or it was defaulted to. if e.setPath == "" { - targetExists = sc.Web[hp] != nil && len(sc.Web[hp].Handlers) > 0 + targetExists = webServeMap[hp] != nil && len(webServeMap[hp].Handlers) > 0 if targetExists { - for mount := range sc.Web[hp].Handlers { + for mount := range webServeMap[hp].Handlers { mounts = append(mounts, mount) } } } else { - targetExists = sc.WebHandlerExists(hp, mount) + targetExists = sc.WebHandlerExists(svcName, hp, mount) mounts = []string{mount} } if !targetExists { - return errors.New("error: handler does not exist") + return errors.New("handler does not exist") } if len(mounts) > 1 { @@ -763,23 +980,47 @@ func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, dnsName string, srvPort u } } - sc.RemoveWebHandler(dnsName, srvPort, mounts, true) + if forService { + sc.RemoveServiceWebHandler(st, svcName, srvPort, mounts) + } else { + sc.RemoveWebHandler(dnsName, srvPort, mounts, true) + } return nil } // removeTCPServe removes the TCP forwarding configuration for the -// given srvPort, or serving port. -func (e *serveEnv) removeTCPServe(sc *ipn.ServeConfig, src uint16) error { +// given srvPort, or serving port for the given dnsName. +func (e *serveEnv) removeTCPServe(sc *ipn.ServeConfig, dnsName string, src uint16) error { if sc == nil { return nil } - if sc.GetTCPPortHandler(src) == nil { - return errors.New("error: serve config does not exist") + svcName := tailcfg.AsServiceName(dnsName) + if sc.GetTCPPortHandler(src, svcName) == nil { + return errors.New("serve config does not exist") } - if sc.IsServingWeb(src) { + if sc.IsServingWeb(src, svcName) { return fmt.Errorf("unable to remove; serving web, not TCP forwarding on serve port %d", src) } - sc.RemoveTCPForwarding(src) + sc.RemoveTCPForwarding(svcName, src) + return nil +} + +func (e *serveEnv) removeTunServe(sc *ipn.ServeConfig, dnsName string) error { + if sc == nil { + return nil + } + svcName := tailcfg.ServiceName(dnsName) + svc, ok := sc.Services[svcName] + if !ok || svc == nil { + return errors.New("service does not exist") + } + if !svc.Tun { + return errors.New("service is not being served in TUN mode") + } + delete(sc.Services, svcName) + if len(sc.Services) == 0 { + sc.Services = nil // clean up empty map + } return nil } diff --git a/cmd/tailscale/cli/serve_v2_test.go b/cmd/tailscale/cli/serve_v2_test.go index 5768127ad0421..b3e7ea773c698 100644 --- a/cmd/tailscale/cli/serve_v2_test.go +++ b/cmd/tailscale/cli/serve_v2_test.go @@ -8,9 +8,11 @@ import ( "context" "encoding/json" "fmt" + "net/netip" "os" "path/filepath" "reflect" + "slices" "strconv" "strings" "testing" @@ -19,6 +21,7 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" ) func TestServeDevConfigMutations(t *testing.T) { @@ -874,9 +877,10 @@ func TestValidateConfig(t *testing.T) { name string desc string cfg *ipn.ServeConfig + svc tailcfg.ServiceName servePort uint16 serveType serveType - bg bool + bg bgBoolFlag wantErr bool }{ { @@ -894,7 +898,7 @@ func TestValidateConfig(t *testing.T) { 443: {HTTPS: true}, }, }, - bg: true, + bg: bgBoolFlag{true, false}, servePort: 10000, serveType: serveTypeHTTPS, }, @@ -906,7 +910,7 @@ func TestValidateConfig(t *testing.T) { 443: {TCPForward: "http://localhost:4545"}, }, }, - bg: true, + bg: bgBoolFlag{true, false}, servePort: 443, serveType: serveTypeTCP, }, @@ -918,7 +922,7 @@ func TestValidateConfig(t *testing.T) { 443: {HTTPS: true}, }, }, - bg: true, + bg: bgBoolFlag{true, false}, servePort: 443, serveType: serveTypeHTTP, wantErr: true, @@ -957,12 +961,90 @@ func TestValidateConfig(t *testing.T) { serveType: serveTypeTCP, wantErr: true, }, + { + name: "new_service_tcp", + desc: "no error when adding a new service port", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + }, + }, + }, + svc: "svc:foo", + servePort: 8080, + serveType: serveTypeTCP, + }, + { + name: "override_service_tcp", + desc: "no error when overwriting a previous service port", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {TCPForward: "http://localhost:4545"}, + }, + }, + }, + }, + svc: "svc:foo", + servePort: 443, + serveType: serveTypeTCP, + }, + { + name: "override_service_tcp", + desc: "error when overwriting a previous service port with a different serve type", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {HTTPS: true}, + }, + }, + }, + }, + svc: "svc:foo", + servePort: 443, + serveType: serveTypeHTTP, + wantErr: true, + }, + { + name: "override_service_tcp", + desc: "error when setting previous tcp service to tun mode", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {TCPForward: "http://localhost:4545"}, + }, + }, + }, + }, + svc: "svc:foo", + serveType: serveTypeTUN, + wantErr: true, + }, + { + name: "override_service_tun", + desc: "error when setting previous tun service to tcp forwarder", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + Tun: true, + }, + }, + }, + svc: "svc:foo", + serveType: serveTypeTCP, + servePort: 443, + wantErr: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { se := serveEnv{bg: tc.bg} - err := se.validateConfig(tc.cfg, tc.servePort, tc.serveType) + err := se.validateConfig(tc.cfg, tc.servePort, tc.serveType, tc.svc) if err == nil && tc.wantErr { t.Fatal("expected an error but got nil") } @@ -1017,6 +1099,13 @@ func TestSrcTypeFromFlags(t *testing.T) { expectedPort: 443, expectedErr: false, }, + { + name: "defaults to https, port 443 for service", + env: &serveEnv{service: "svc:foo"}, + expectedType: serveTypeHTTPS, + expectedPort: 443, + expectedErr: false, + }, { name: "multiple types set", env: &serveEnv{http: 80, https: 443}, @@ -1075,12 +1164,70 @@ func TestCleanURLPath(t *testing.T) { } } +func TestAddServiceToPrefs(t *testing.T) { + tests := []struct { + name string + dnsName string + startServices []string + expected []string + }{ + { + name: "add service to empty prefs", + dnsName: "svc:foo", + expected: []string{"svc:foo"}, + }, + { + name: "add service to existing prefs", + dnsName: "svc:bar", + startServices: []string{"svc:foo"}, + expected: []string{"svc:foo", "svc:bar"}, + }, + { + name: "add existing service to prefs", + dnsName: "svc:foo", + startServices: []string{"svc:foo"}, + expected: []string{"svc:foo"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lc := &fakeLocalServeClient{} + ctx := t.Context() + lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: tt.startServices, + }, + }) + e := &serveEnv{lc: lc, bg: bgBoolFlag{true, false}} + err := e.addServiceToPrefs(ctx, tt.dnsName) + if err != nil { + t.Fatalf("addServiceToPrefs(%q) returned unexpected error: %v", tt.dnsName, err) + } + if !slices.Equal(lc.prefs.AdvertiseServices, tt.expected) { + t.Errorf("addServiceToPrefs(%q) = %v, want %v", tt.dnsName, lc.prefs.AdvertiseServices, tt.expected) + } + }) + } + +} + func TestMessageForPort(t *testing.T) { + svcIPMap := tailcfg.ServiceIPMappings{ + "svc:foo": []netip.Addr{ + netip.MustParseAddr("100.101.101.101"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12:4843:cd96:6565:6565"), + }, + } + svcIPMapJSON, _ := json.Marshal(svcIPMap) + svcIPMapJSONRawMSG := tailcfg.RawMessage(svcIPMapJSON) + tests := []struct { name string subcmd serveMode serveConfig *ipn.ServeConfig status *ipnstate.Status + prefs *ipn.Prefs dnsName string srvType serveType srvPort uint16 @@ -1147,10 +1294,206 @@ func TestMessageForPort(t *testing.T) { fmt.Sprintf(msgDisableProxy, "serve", "http", 80), }, "\n"), }, + { + name: "serve service http", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{ + AdvertiseServices: []string{"svc:foo"}, + }, + dnsName: "svc:foo", + srvType: serveTypeHTTP, + srvPort: 80, + expected: strings.Join([]string{ + msgServeAvailable, + "", + "http://foo.test.ts.net/", + "|-- proxy http://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:foo", "http", 80), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, + { + name: "serve service no capmap", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{ + AdvertiseServices: []string{"svc:bar"}, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + expected: strings.Join([]string{ + fmt.Sprintf(msgServiceWaitingApproval, "svc:bar"), + "", + "http://bar.test.ts.net/", + "|-- proxy http://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:bar", "http", 80), + fmt.Sprintf(msgDisableService, "svc:bar"), + }, "\n"), + }, + { + name: "serve service https non-default port", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 2200: {HTTPS: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo:2200": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{AdvertiseServices: []string{"svc:foo"}}, + dnsName: "svc:foo", + srvType: serveTypeHTTPS, + srvPort: 2200, + expected: strings.Join([]string{ + msgServeAvailable, + "", + "https://foo.test.ts.net:2200/", + "|-- proxy http://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:foo", "https", 2200), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, + { + name: "serve service TCPForward", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 2200: {TCPForward: "localhost:3000"}, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{AdvertiseServices: []string{"svc:foo"}}, + dnsName: "svc:foo", + srvType: serveTypeTCP, + srvPort: 2200, + expected: strings.Join([]string{ + msgServeAvailable, + "", + "|-- tcp://foo.test.ts.net:2200 (TLS over TCP)", + "|-- tcp://100.101.101.101:2200", + "|-- tcp://[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:2200", + "|--> tcp://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:foo", "tcp", 2200), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, + { + name: "serve service Tun", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + Tun: true, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{AdvertiseServices: []string{"svc:foo"}}, + dnsName: "svc:foo", + srvType: serveTypeTUN, + expected: strings.Join([]string{ + msgServeAvailable, + "", + fmt.Sprintf(msgRunningTunService, "foo.test.ts.net"), + fmt.Sprintf(msgDisableServiceTun, "svc:foo"), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, } for _, tt := range tests { - e := &serveEnv{bg: true, subcmd: tt.subcmd} + e := &serveEnv{bg: bgBoolFlag{true, false}, subcmd: tt.subcmd} t.Run(tt.name, func(t *testing.T) { actual := e.messageForPort(tt.serveConfig, tt.status, tt.dnsName, tt.srvType, tt.srvPort) @@ -1277,6 +1620,576 @@ func TestIsLegacyInvocation(t *testing.T) { } } +func TestSetServe(t *testing.T) { + e := &serveEnv{} + tests := []struct { + name string + desc string + cfg *ipn.ServeConfig + st *ipnstate.Status + dnsName string + srvType serveType + srvPort uint16 + mountPath string + target string + allowFunnel bool + expected *ipn.ServeConfig + expectErr bool + }{ + { + name: "add new handler", + desc: "add a new http handler to empty config", + cfg: &ipn.ServeConfig{}, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3000", + expected: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + { + name: "update http handler", + desc: "update an existing http handler on the same port to same type", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + { + name: "update TCP handler", + desc: "update an existing TCP handler on the same port to a http handler", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "http://localhost:3000"}}, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expectErr: true, + }, + { + name: "add new service handler", + desc: "add a new service TCP handler to empty config", + cfg: &ipn.ServeConfig{}, + + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + target: "3000", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3000"}}, + }, + }, + }, + }, + { + name: "update service handler", + desc: "update an existing service TCP handler on the same port to same type", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3000"}}, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + target: "3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3001"}}, + }, + }, + }, + }, + { + name: "update service handler", + desc: "update an existing service TCP handler on the same port to a http handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3000"}}, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expectErr: true, + }, + { + name: "add new service handler", + desc: "add a new service HTTP handler to empty config", + cfg: &ipn.ServeConfig{}, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3000", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "update existing service handler", + desc: "update an existing service HTTP handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "add new service handler", + desc: "add a new service HTTP handler to existing service config", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 88, + mountPath: "/", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + 88: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + "bar:88": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "add new service mount", + desc: "add a new service mount to existing service config", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/added", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + "/added": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "add new service handler", + desc: "add a new service handler in tun mode to empty config", + cfg: &ipn.ServeConfig{}, + dnsName: "svc:bar", + srvType: serveTypeTUN, + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + Tun: true, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := e.setServe(tt.cfg, tt.dnsName, tt.srvType, tt.srvPort, tt.mountPath, tt.target, tt.allowFunnel) + if err != nil && !tt.expectErr { + t.Fatalf("got error: %v; did not expect error.", err) + } + if err == nil && tt.expectErr { + t.Fatalf("got no error; expected error.") + } + if !tt.expectErr && !reflect.DeepEqual(tt.cfg, tt.expected) { + svcName := tailcfg.ServiceName(tt.dnsName) + t.Fatalf("got: %v; expected: %v", tt.cfg.Services[svcName], tt.expected.Services[svcName]) + } + }) + } +} + +func TestUnsetServe(t *testing.T) { + tests := []struct { + name string + desc string + cfg *ipn.ServeConfig + st *ipnstate.Status + dnsName string + srvType serveType + srvPort uint16 + mount string + setServeEnv bool + serveEnv *serveEnv // if set, use this instead of the default serveEnv + expected *ipn.ServeConfig + expectErr bool + }{ + { + name: "unset http handler", + desc: "remove an existing http handler", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/", + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset service handler", + desc: "remove an existing service TCP handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/", + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset service handler tun", + desc: "remove an existing service handler in tun mode", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + Tun: true, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeTUN, + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset service handler tcp", + desc: "remove an existing service TCP handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {TCPForward: "11.11.11.11:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset http handler not found", + desc: "try to remove a non-existing http handler", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "bar.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/abc", + expected: &ipn.ServeConfig{}, + expectErr: true, + }, + { + name: "unset service handler not found", + desc: "try to remove a non-existing service TCP handler", + + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/abc", + setServeEnv: true, + serveEnv: &serveEnv{setPath: "/abc"}, + expected: &ipn.ServeConfig{}, + expectErr: true, + }, + { + name: "unset service doesn't exist", + desc: "try to remove a non-existing service's handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {TCPForward: "11.11.11.11:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:foo", + srvType: serveTypeTCP, + srvPort: 80, + expectErr: true, + }, + { + name: "unset tcp while port is in use", + desc: "try to remove a TCP handler while the port is used for web", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeTCP, + srvPort: 80, + mount: "/", + expectErr: true, + }, + { + name: "unset service tcp while port is in use", + desc: "try to remove a service TCP handler while the port is used for web", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + mount: "/", + expectErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &serveEnv{} + if tt.setServeEnv { + e = tt.serveEnv + } + err := e.unsetServe(tt.cfg, tt.st, tt.dnsName, tt.srvType, tt.srvPort, tt.mount) + if err != nil && !tt.expectErr { + t.Fatalf("got error: %v; did not expect error.", err) + } + if err == nil && tt.expectErr { + t.Fatalf("got no error; expected error.") + } + if !tt.expectErr && !reflect.DeepEqual(tt.cfg, tt.expected) { + t.Fatalf("got: %v; expected: %v", tt.cfg, tt.expected) + } + }) + } +} + // exactErrMsg returns an error checker that wants exactly the provided want error. // If optName is non-empty, it's used in the error message. func exactErrMsg(want error) func(error) string { diff --git a/cmd/tailscale/cli/status.go b/cmd/tailscale/cli/status.go index 85679a7decbc1..39e6f9fbdfd8a 100644 --- a/cmd/tailscale/cli/status.go +++ b/cmd/tailscale/cli/status.go @@ -262,7 +262,7 @@ func printFunnelStatus(ctx context.Context) { } sni, portStr, _ := net.SplitHostPort(string(hp)) p, _ := strconv.ParseUint(portStr, 10, 16) - isTCP := sc.IsTCPForwardingOnPort(uint16(p)) + isTCP := sc.IsTCPForwardingOnPort(uint16(p), noService) url := "https://" if isTCP { url = "tcp://" diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index 44d63fe54a902..28262251c6880 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -1007,8 +1007,6 @@ func allNumeric(s string) bool { } func (b *LocalBackend) webServerConfig(hostname string, forVIPService tailcfg.ServiceName, port uint16) (c ipn.WebServerConfigView, ok bool) { - key := ipn.HostPort(fmt.Sprintf("%s:%v", hostname, port)) - b.mu.Lock() defer b.mu.Unlock() @@ -1016,8 +1014,10 @@ func (b *LocalBackend) webServerConfig(hostname string, forVIPService tailcfg.Se return c, false } if forVIPService != "" { + key := ipn.HostPort(net.JoinHostPort(forVIPService.WithoutPrefix(), fmt.Sprintf("%d", port))) return b.serveConfig.FindServiceWeb(forVIPService, key) } + key := ipn.HostPort(net.JoinHostPort(hostname, fmt.Sprintf("%d", port))) return b.serveConfig.FindWeb(key) } diff --git a/ipn/serve.go b/ipn/serve.go index ac92287bdc08f..fae0ad5d6568a 100644 --- a/ipn/serve.go +++ b/ipn/serve.go @@ -166,26 +166,44 @@ type HTTPHandler struct { // WebHandlerExists reports whether if the ServeConfig Web handler exists for // the given host:port and mount point. -func (sc *ServeConfig) WebHandlerExists(hp HostPort, mount string) bool { - h := sc.GetWebHandler(hp, mount) +func (sc *ServeConfig) WebHandlerExists(svcName tailcfg.ServiceName, hp HostPort, mount string) bool { + h := sc.GetWebHandler(svcName, hp, mount) return h != nil } // GetWebHandler returns the HTTPHandler for the given host:port and mount point. // Returns nil if the handler does not exist. -func (sc *ServeConfig) GetWebHandler(hp HostPort, mount string) *HTTPHandler { - if sc == nil || sc.Web[hp] == nil { +func (sc *ServeConfig) GetWebHandler(svcName tailcfg.ServiceName, hp HostPort, mount string) *HTTPHandler { + if sc == nil { + return nil + } + if svcName != "" { + if svc, ok := sc.Services[svcName]; ok && svc.Web != nil { + if webCfg, ok := svc.Web[hp]; ok { + return webCfg.Handlers[mount] + } + } + return nil + } + if sc.Web[hp] == nil { return nil } return sc.Web[hp].Handlers[mount] } -// GetTCPPortHandler returns the TCPPortHandler for the given port. -// If the port is not configured, nil is returned. -func (sc *ServeConfig) GetTCPPortHandler(port uint16) *TCPPortHandler { +// GetTCPPortHandler returns the TCPPortHandler for the given port. If the port +// is not configured, nil is returned. Parameter svcName can be tailcfg.NoService +// for local serve or a service name for a service hosted on node. +func (sc *ServeConfig) GetTCPPortHandler(port uint16, svcName tailcfg.ServiceName) *TCPPortHandler { if sc == nil { return nil } + if svcName != "" { + if svc, ok := sc.Services[svcName]; ok && svc != nil { + return svc.TCP[port] + } + return nil + } return sc.TCP[port] } @@ -227,34 +245,78 @@ func (sc *ServeConfig) IsTCPForwardingAny() bool { return false } -// IsTCPForwardingOnPort reports whether if ServeConfig is currently forwarding -// in TCPForward mode on the given port. This is exclusive of Web/HTTPS serving. -func (sc *ServeConfig) IsTCPForwardingOnPort(port uint16) bool { - if sc == nil || sc.TCP[port] == nil { +// IsTCPForwardingOnPort reports whether ServeConfig is currently forwarding +// in TCPForward mode on the given port for local or a service. svcName will +// either be noService (empty string) for local serve or a serviceName for service +// hosted on node. Notice TCPForwarding is exclusive with Web/HTTPS serving. +func (sc *ServeConfig) IsTCPForwardingOnPort(port uint16, svcName tailcfg.ServiceName) bool { + if sc == nil { + return false + } + + if svcName != "" { + svc, ok := sc.Services[svcName] + if !ok || svc == nil { + return false + } + if svc.TCP[port] == nil { + return false + } + } else if sc.TCP[port] == nil { return false } - return !sc.IsServingWeb(port) + return !sc.IsServingWeb(port, svcName) } -// IsServingWeb reports whether if ServeConfig is currently serving Web -// (HTTP/HTTPS) on the given port. This is exclusive of TCPForwarding. -func (sc *ServeConfig) IsServingWeb(port uint16) bool { - return sc.IsServingHTTP(port) || sc.IsServingHTTPS(port) +// IsServingWeb reports whether ServeConfig is currently serving Web (HTTP/HTTPS) +// on the given port for local or a service. svcName will be either tailcfg.NoService, +// or a serviceName for service hosted on node. This is exclusive with TCPForwarding. +func (sc *ServeConfig) IsServingWeb(port uint16, svcName tailcfg.ServiceName) bool { + return sc.IsServingHTTP(port, svcName) || sc.IsServingHTTPS(port, svcName) } -// IsServingHTTPS reports whether if ServeConfig is currently serving HTTPS on -// the given port. This is exclusive of HTTP and TCPForwarding. -func (sc *ServeConfig) IsServingHTTPS(port uint16) bool { - if sc == nil || sc.TCP[port] == nil { +// IsServingHTTPS reports whether ServeConfig is currently serving HTTPS on +// the given port for local or a service. svcName will be either tailcfg.NoService +// for local serve, or a serviceName for service hosted on node. This is exclusive +// with HTTP and TCPForwarding. +func (sc *ServeConfig) IsServingHTTPS(port uint16, svcName tailcfg.ServiceName) bool { + if sc == nil { + return false + } + var tcpHandlers map[uint16]*TCPPortHandler + if svcName != "" { + if svc := sc.Services[svcName]; svc != nil { + tcpHandlers = svc.TCP + } + } else { + tcpHandlers = sc.TCP + } + + th := tcpHandlers[port] + if th == nil { return false } - return sc.TCP[port].HTTPS + return th.HTTPS } -// IsServingHTTP reports whether if ServeConfig is currently serving HTTP on the -// given port. This is exclusive of HTTPS and TCPForwarding. -func (sc *ServeConfig) IsServingHTTP(port uint16) bool { - if sc == nil || sc.TCP[port] == nil { +// IsServingHTTP reports whether ServeConfig is currently serving HTTP on the +// given port for local or a service. svcName will be either tailcfg.NoService for +// local serve, or a serviceName for service hosted on node. This is exclusive +// with HTTPS and TCPForwarding. +func (sc *ServeConfig) IsServingHTTP(port uint16, svcName tailcfg.ServiceName) bool { + if sc == nil { + return false + } + if svcName != "" { + if svc := sc.Services[svcName]; svc != nil { + if svc.TCP[port] != nil { + return svc.TCP[port].HTTP + } + } + return false + } + + if sc.TCP[port] == nil { return false } return sc.TCP[port].HTTP @@ -280,21 +342,37 @@ func (sc *ServeConfig) FindConfig(port uint16) (*ServeConfig, bool) { // SetWebHandler sets the given HTTPHandler at the specified host, port, // and mount in the serve config. sc.TCP is also updated to reflect web -// serving usage of the given port. +// serving usage of the given port. The st argument is needed when setting +// a web handler for a service, otherwise it can be nil. func (sc *ServeConfig) SetWebHandler(handler *HTTPHandler, host string, port uint16, mount string, useTLS bool) { if sc == nil { sc = new(ServeConfig) } - mak.Set(&sc.TCP, port, &TCPPortHandler{HTTPS: useTLS, HTTP: !useTLS}) - hp := HostPort(net.JoinHostPort(host, strconv.Itoa(int(port)))) - if _, ok := sc.Web[hp]; !ok { - mak.Set(&sc.Web, hp, new(WebServerConfig)) + tcpMap := &sc.TCP + webServerMap := &sc.Web + hostName := host + if svcName := tailcfg.AsServiceName(host); svcName != "" { + hostName = svcName.WithoutPrefix() + svc, ok := sc.Services[svcName] + if !ok { + svc = new(ServiceConfig) + mak.Set(&sc.Services, svcName, svc) + } + tcpMap = &svc.TCP + webServerMap = &svc.Web } - mak.Set(&sc.Web[hp].Handlers, mount, handler) + mak.Set(tcpMap, port, &TCPPortHandler{HTTPS: useTLS, HTTP: !useTLS}) + hp := HostPort(net.JoinHostPort(hostName, strconv.Itoa(int(port)))) + webCfg, ok := (*webServerMap)[hp] + if !ok { + webCfg = new(WebServerConfig) + mak.Set(webServerMap, hp, webCfg) + } + mak.Set(&webCfg.Handlers, mount, handler) // TODO(tylersmalley): handle multiple web handlers from foreground mode - for k, v := range sc.Web[hp].Handlers { + for k, v := range webCfg.Handlers { if v == handler { continue } @@ -305,7 +383,7 @@ func (sc *ServeConfig) SetWebHandler(handler *HTTPHandler, host string, port uin m1 := strings.TrimSuffix(mount, "/") m2 := strings.TrimSuffix(k, "/") if m1 == m2 { - delete(sc.Web[hp].Handlers, k) + delete(webCfg.Handlers, k) } } } @@ -318,9 +396,19 @@ func (sc *ServeConfig) SetTCPForwarding(port uint16, fwdAddr string, terminateTL if sc == nil { sc = new(ServeConfig) } - mak.Set(&sc.TCP, port, &TCPPortHandler{TCPForward: fwdAddr}) + tcpPortHandler := &sc.TCP + if svcName := tailcfg.AsServiceName(host); svcName != "" { + svcConfig, ok := sc.Services[svcName] + if !ok { + svcConfig = new(ServiceConfig) + mak.Set(&sc.Services, svcName, svcConfig) + } + tcpPortHandler = &svcConfig.TCP + } + mak.Set(tcpPortHandler, port, &TCPPortHandler{TCPForward: fwdAddr}) + if terminateTLS { - sc.TCP[port].TerminateTLS = host + (*tcpPortHandler)[port].TerminateTLS = host } } @@ -344,9 +432,9 @@ func (sc *ServeConfig) SetFunnel(host string, port uint16, setOn bool) { } } -// RemoveWebHandler deletes the web handlers at all of the given mount points -// for the provided host and port in the serve config. If cleanupFunnel is -// true, this also removes the funnel value for this port if no handlers remain. +// RemoveWebHandler deletes the web handlers at all of the given mount points for the +// provided host and port in the serve config for the node (as opposed to a service). +// If cleanupFunnel is true, this also removes the funnel value for this port if no handlers remain. func (sc *ServeConfig) RemoveWebHandler(host string, port uint16, mounts []string, cleanupFunnel bool) { hp := HostPort(net.JoinHostPort(host, strconv.Itoa(int(port)))) @@ -374,9 +462,51 @@ func (sc *ServeConfig) RemoveWebHandler(host string, port uint16, mounts []strin } } +// RemoveServiceWebHandler deletes the web handlers at all of the given mount points +// for the provided host and port in the serve config for the given service. +func (sc *ServeConfig) RemoveServiceWebHandler(st *ipnstate.Status, svcName tailcfg.ServiceName, port uint16, mounts []string) { + hostName := svcName.WithoutPrefix() + hp := HostPort(net.JoinHostPort(hostName, strconv.Itoa(int(port)))) + + svc, ok := sc.Services[svcName] + if !ok || svc == nil { + return + } + + // Delete existing handler, then cascade delete if empty. + for _, m := range mounts { + delete(svc.Web[hp].Handlers, m) + } + if len(svc.Web[hp].Handlers) == 0 { + delete(svc.Web, hp) + delete(svc.TCP, port) + } + if len(svc.Web) == 0 && len(svc.TCP) == 0 { + delete(sc.Services, svcName) + } + if len(sc.Services) == 0 { + sc.Services = nil + } +} + // RemoveTCPForwarding deletes the TCP forwarding configuration for the given // port from the serve config. -func (sc *ServeConfig) RemoveTCPForwarding(port uint16) { +func (sc *ServeConfig) RemoveTCPForwarding(svcName tailcfg.ServiceName, port uint16) { + if svcName != "" { + if svc := sc.Services[svcName]; svc != nil { + delete(svc.TCP, port) + if len(svc.TCP) == 0 { + svc.TCP = nil + } + if len(svc.Web) == 0 && len(svc.TCP) == 0 { + delete(sc.Services, svcName) + } + if len(sc.Services) == 0 { + sc.Services = nil + } + } + return + } delete(sc.TCP, port) if len(sc.TCP) == 0 { sc.TCP = nil diff --git a/ipn/serve_test.go b/ipn/serve_test.go index ba0a26f8c0698..7028c1e17cd71 100644 --- a/ipn/serve_test.go +++ b/ipn/serve_test.go @@ -128,6 +128,121 @@ func TestHasPathHandler(t *testing.T) { } } +func TestIsTCPForwardingOnPort(t *testing.T) { + tests := []struct { + name string + cfg ServeConfig + svcName tailcfg.ServiceName + port uint16 + want bool + }{ + { + name: "empty-config", + cfg: ServeConfig{}, + svcName: "", + port: 80, + want: false, + }, + { + name: "node-tcp-config-match", + cfg: ServeConfig{ + TCP: map[uint16]*TCPPortHandler{80: {TCPForward: "10.0.0.123:3000"}}, + }, + svcName: "", + port: 80, + want: true, + }, + { + name: "node-tcp-config-no-match", + cfg: ServeConfig{ + TCP: map[uint16]*TCPPortHandler{80: {TCPForward: "10.0.0.123:3000"}}, + }, + svcName: "", + port: 443, + want: false, + }, + { + name: "node-tcp-config-no-match-with-service", + cfg: ServeConfig{ + TCP: map[uint16]*TCPPortHandler{80: {TCPForward: "10.0.0.123:3000"}}, + }, + svcName: "svc:bar", + port: 80, + want: false, + }, + { + name: "node-web-config-no-match", + cfg: ServeConfig{ + TCP: map[uint16]*TCPPortHandler{80: {HTTPS: true}}, + Web: map[HostPort]*WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*HTTPHandler{ + "/": {Text: "Hello, world!"}, + }, + }, + }, + }, + svcName: "", + port: 80, + want: false, + }, + { + name: "service-tcp-config-match", + cfg: ServeConfig{ + Services: map[tailcfg.ServiceName]*ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*TCPPortHandler{80: {TCPForward: "10.0.0.123:3000"}}, + }, + }, + }, + svcName: "svc:foo", + port: 80, + want: true, + }, + { + name: "service-tcp-config-no-match", + cfg: ServeConfig{ + Services: map[tailcfg.ServiceName]*ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*TCPPortHandler{80: {TCPForward: "10.0.0.123:3000"}}, + }, + }, + }, + svcName: "svc:bar", + port: 80, + want: false, + }, + { + name: "service-web-config-no-match", + cfg: ServeConfig{ + Services: map[tailcfg.ServiceName]*ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*TCPPortHandler{80: {HTTPS: true}}, + Web: map[HostPort]*WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*HTTPHandler{ + "/": {Text: "Hello, world!"}, + }, + }, + }, + }, + }, + }, + svcName: "svc:foo", + port: 80, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.cfg.IsTCPForwardingOnPort(tt.port, tt.svcName) + if tt.want != got { + t.Errorf("IsTCPForwardingOnPort() = %v, want %v", got, tt.want) + } + }) + } +} + func TestExpandProxyTargetDev(t *testing.T) { tests := []struct { name string diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 636e2434de276..398a2c8a2b93a 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -927,6 +927,16 @@ func (t *TPMInfo) Present() bool { return t != nil } // This is not related to the older [Service] used in [Hostinfo.Services]. type ServiceName string +// AsServiceName reports whether the given string is a valid service name. +// If so returns the name as a [tailcfg.ServiceName], otherwise returns "". +func AsServiceName(s string) ServiceName { + svcName := ServiceName(s) + if err := svcName.Validate(); err != nil { + return "" + } + return svcName +} + // Validate validates if the service name is formatted correctly. // We only allow valid DNS labels, since the expectation is that these will be // used as parts of domain names. All errors are [vizerror.Error]. From 93511be04483dfd9ab6fa3164b70dcae5ec366f9 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Thu, 17 Jul 2025 01:30:08 -0700 Subject: [PATCH 223/263] types/geo: add geo.Point and its associated units (#16583) Package geo provides functionality to represent and process geographical locations on a sphere. The main type, geo.Point, represents a pair of latitude and longitude coordinates. Updates tailscale/corp#29968 Signed-off-by: Simon Law --- types/geo/doc.go | 6 + types/geo/point.go | 279 +++++++++++++++++++ types/geo/point_test.go | 541 +++++++++++++++++++++++++++++++++++++ types/geo/quantize.go | 106 ++++++++ types/geo/quantize_test.go | 130 +++++++++ types/geo/units.go | 191 +++++++++++++ types/geo/units_test.go | 395 +++++++++++++++++++++++++++ 7 files changed, 1648 insertions(+) create mode 100644 types/geo/doc.go create mode 100644 types/geo/point.go create mode 100644 types/geo/point_test.go create mode 100644 types/geo/quantize.go create mode 100644 types/geo/quantize_test.go create mode 100644 types/geo/units.go create mode 100644 types/geo/units_test.go diff --git a/types/geo/doc.go b/types/geo/doc.go new file mode 100644 index 0000000000000..749c6308093f6 --- /dev/null +++ b/types/geo/doc.go @@ -0,0 +1,6 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package geo provides functionality to represent and process geographical +// locations on a spherical Earth. +package geo diff --git a/types/geo/point.go b/types/geo/point.go new file mode 100644 index 0000000000000..d7160ac593338 --- /dev/null +++ b/types/geo/point.go @@ -0,0 +1,279 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo + +import ( + "encoding/binary" + "errors" + "fmt" + "math" + "strconv" +) + +// ErrBadPoint indicates that the point is malformed. +var ErrBadPoint = errors.New("not a valid point") + +// Point represents a pair of latitude and longitude coordinates. +type Point struct { + lat Degrees + // lng180 is the longitude offset by +180° so the zero value is invalid + // and +0+0/ is Point{lat: +0.0, lng180: +180.0}. + lng180 Degrees +} + +// MakePoint returns a Point representing a given latitude and longitude on +// a WGS 84 ellipsoid. The Coordinate Reference System is EPSG:4326. +// Latitude is wrapped to [-90°, +90°] and longitude to (-180°, +180°]. +func MakePoint(latitude, longitude Degrees) Point { + lat, lng := float64(latitude), float64(longitude) + + switch { + case math.IsNaN(lat) || math.IsInf(lat, 0): + // don’t wrap + case lat < -90 || lat > 90: + // Latitude wraps by flipping the longitude + lat = math.Mod(lat, 360.0) + switch { + case lat == 0.0: + lat = 0.0 // -0.0 == 0.0, but -0° is not valid + case lat < -270.0: + lat = +360.0 + lat + case lat < -90.0: + lat = -180.0 - lat + lng += 180.0 + case lat > +270.0: + lat = -360.0 + lat + case lat > +90.0: + lat = +180.0 - lat + lng += 180.0 + } + } + + switch { + case lat == -90.0 || lat == +90.0: + // By convention, the north and south poles have longitude 0°. + lng = 0 + case math.IsNaN(lng) || math.IsInf(lng, 0): + // don’t wrap + case lng <= -180.0 || lng > 180.0: + // Longitude wraps around normally + lng = math.Mod(lng, 360.0) + switch { + case lng == 0.0: + lng = 0.0 // -0.0 == 0.0, but -0° is not valid + case lng <= -180.0: + lng = +360.0 + lng + case lng > +180.0: + lng = -360.0 + lng + } + } + + return Point{ + lat: Degrees(lat), + lng180: Degrees(lng + 180.0), + } +} + +// Valid reports if p is a valid point. +func (p Point) Valid() bool { + return !p.IsZero() +} + +// LatLng reports the latitude and longitude. +func (p Point) LatLng() (lat, lng Degrees, err error) { + if p.IsZero() { + return 0 * Degree, 0 * Degree, ErrBadPoint + } + return p.lat, p.lng180 - 180.0*Degree, nil +} + +// LatLng reports the latitude and longitude in float64. If err is nil, then lat +// and lng will never both be 0.0 to disambiguate between an empty struct and +// Null Island (0° 0°). +func (p Point) LatLngFloat64() (lat, lng float64, err error) { + dlat, dlng, err := p.LatLng() + if err != nil { + return 0.0, 0.0, err + } + if dlat == 0.0 && dlng == 0.0 { + // dlng must survive conversion to float32. + dlng = math.SmallestNonzeroFloat32 + } + return float64(dlat), float64(dlng), err +} + +// SphericalAngleTo returns the angular distance from p to q, calculated on a +// spherical Earth. +func (p Point) SphericalAngleTo(q Point) (Radians, error) { + pLat, pLng, pErr := p.LatLng() + qLat, qLng, qErr := q.LatLng() + switch { + case pErr != nil && qErr != nil: + return 0.0, fmt.Errorf("spherical distance from %v to %v: %w", p, q, errors.Join(pErr, qErr)) + case pErr != nil: + return 0.0, fmt.Errorf("spherical distance from %v: %w", p, pErr) + case qErr != nil: + return 0.0, fmt.Errorf("spherical distance to %v: %w", q, qErr) + } + // The spherical law of cosines is accurate enough for close points when + // using float64. + // + // The haversine formula is an alternative, but it is poorly behaved + // when points are on opposite sides of the sphere. + rLat, rLng := float64(pLat.Radians()), float64(pLng.Radians()) + sLat, sLng := float64(qLat.Radians()), float64(qLng.Radians()) + cosA := math.Sin(rLat)*math.Sin(sLat) + + math.Cos(rLat)*math.Cos(sLat)*math.Cos(rLng-sLng) + return Radians(math.Acos(cosA)), nil +} + +// DistanceTo reports the great-circle distance between p and q, in meters. +func (p Point) DistanceTo(q Point) (Distance, error) { + r, err := p.SphericalAngleTo(q) + if err != nil { + return 0, err + } + return DistanceOnEarth(r.Turns()), nil +} + +// String returns a space-separated pair of latitude and longitude, in decimal +// degrees. Positive latitudes are in the northern hemisphere, and positive +// longitudes are east of the prime meridian. If p was not initialized, this +// will return "nowhere". +func (p Point) String() string { + lat, lng, err := p.LatLng() + if err != nil { + if err == ErrBadPoint { + return "nowhere" + } + panic(err) + } + + return lat.String() + " " + lng.String() +} + +// AppendBinary implements [encoding.BinaryAppender]. The output consists of two +// float32s in big-endian byte order: latitude and longitude offset by 180°. +// If p is not a valid, the output will be an 8-byte zero value. +func (p Point) AppendBinary(b []byte) ([]byte, error) { + end := binary.BigEndian + b = end.AppendUint32(b, math.Float32bits(float32(p.lat))) + b = end.AppendUint32(b, math.Float32bits(float32(p.lng180))) + return b, nil +} + +// MarshalBinary implements [encoding.BinaryMarshaller]. The output matches that +// of calling [Point.AppendBinary]. +func (p Point) MarshalBinary() ([]byte, error) { + var b [8]byte + return p.AppendBinary(b[:0]) +} + +// UnmarshalBinary implements [encoding.BinaryUnmarshaler]. It expects input +// that was formatted by [Point.AppendBinary]: in big-endian byte order, a +// float32 representing latitude followed by a float32 representing longitude +// offset by 180°. If latitude and longitude fall outside valid ranges, then +// an error is returned. +func (p *Point) UnmarshalBinary(data []byte) error { + if len(data) < 8 { // Two uint32s are 8 bytes long + return fmt.Errorf("%w: not enough data: %q", ErrBadPoint, data) + } + + end := binary.BigEndian + lat := Degrees(math.Float32frombits(end.Uint32(data[0:]))) + if lat < -90*Degree || lat > 90*Degree { + return fmt.Errorf("%w: latitude outside [-90°, +90°]: %s", ErrBadPoint, lat) + } + lng180 := Degrees(math.Float32frombits(end.Uint32(data[4:]))) + if lng180 != 0 && (lng180 < 0*Degree || lng180 > 360*Degree) { + // lng180 == 0 is OK: the zero value represents invalid points. + lng := lng180 - 180*Degree + return fmt.Errorf("%w: longitude outside (-180°, +180°]: %s", ErrBadPoint, lng) + } + + p.lat = lat + p.lng180 = lng180 + return nil +} + +// AppendText implements [encoding.TextAppender]. The output is a point +// formatted as OGC Well-Known Text, as "POINT (longitude latitude)" where +// longitude and latitude are in decimal degrees. If p is not valid, the output +// will be "POINT EMPTY". +func (p Point) AppendText(b []byte) ([]byte, error) { + if p.IsZero() { + b = append(b, []byte("POINT EMPTY")...) + return b, nil + } + + lat, lng, err := p.LatLng() + if err != nil { + return b, err + } + + b = append(b, []byte("POINT (")...) + b = strconv.AppendFloat(b, float64(lng), 'f', -1, 64) + b = append(b, ' ') + b = strconv.AppendFloat(b, float64(lat), 'f', -1, 64) + b = append(b, ')') + return b, nil +} + +// MarshalText implements [encoding.TextMarshaller]. The output matches that +// of calling [Point.AppendText]. +func (p Point) MarshalText() ([]byte, error) { + var b [8]byte + return p.AppendText(b[:0]) +} + +// MarshalUint64 produces the same output as MashalBinary, encoded in a uint64. +func (p Point) MarshalUint64() (uint64, error) { + b, err := p.MarshalBinary() + return binary.NativeEndian.Uint64(b), err +} + +// UnmarshalUint64 expects input formatted by MarshalUint64. +func (p *Point) UnmarshalUint64(v uint64) error { + b := binary.NativeEndian.AppendUint64(nil, v) + return p.UnmarshalBinary(b) +} + +// IsZero reports if p is the zero value. +func (p Point) IsZero() bool { + return p == Point{} +} + +// EqualApprox reports if p and q are approximately equal: that is the absolute +// difference of both latitude and longitude are less than tol. If tol is +// negative, then tol defaults to a reasonably small number (10⁻⁵). If tol is +// zero, then p and q must be exactly equal. +func (p Point) EqualApprox(q Point, tol float64) bool { + if tol == 0 { + return p == q + } + + if p.IsZero() && q.IsZero() { + return true + } else if p.IsZero() || q.IsZero() { + return false + } + + plat, plng, err := p.LatLng() + if err != nil { + panic(err) + } + qlat, qlng, err := q.LatLng() + if err != nil { + panic(err) + } + + if tol < 0 { + tol = 1e-5 + } + + dlat := float64(plat) - float64(qlat) + dlng := float64(plng) - float64(qlng) + return ((dlat < 0 && -dlat < tol) || (dlat >= 0 && dlat < tol)) && + ((dlng < 0 && -dlng < tol) || (dlng >= 0 && dlng < tol)) +} diff --git a/types/geo/point_test.go b/types/geo/point_test.go new file mode 100644 index 0000000000000..308c1a1834377 --- /dev/null +++ b/types/geo/point_test.go @@ -0,0 +1,541 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo_test + +import ( + "fmt" + "math" + "testing" + "testing/quick" + + "tailscale.com/types/geo" +) + +func TestPointZero(t *testing.T) { + var zero geo.Point + + if got := zero.IsZero(); !got { + t.Errorf("IsZero() got %t", got) + } + + if got := zero.Valid(); got { + t.Errorf("Valid() got %t", got) + } + + wantErr := geo.ErrBadPoint.Error() + if _, _, err := zero.LatLng(); err.Error() != wantErr { + t.Errorf("LatLng() err %q, want %q", err, wantErr) + } + + wantStr := "nowhere" + if got := zero.String(); got != wantStr { + t.Errorf("String() got %q, want %q", got, wantStr) + } + + wantB := []byte{0, 0, 0, 0, 0, 0, 0, 0} + if b, err := zero.MarshalBinary(); err != nil { + t.Errorf("MarshalBinary() err %q, want nil", err) + } else if string(b) != string(wantB) { + t.Errorf("MarshalBinary got %q, want %q", b, wantB) + } + + wantI := uint64(0x00000000) + if i, err := zero.MarshalUint64(); err != nil { + t.Errorf("MarshalUint64() err %q, want nil", err) + } else if i != wantI { + t.Errorf("MarshalUint64 got %v, want %v", i, wantI) + } +} + +func TestPoint(t *testing.T) { + for _, tt := range []struct { + name string + lat geo.Degrees + lng geo.Degrees + wantLat geo.Degrees + wantLng geo.Degrees + wantString string + wantText string + }{ + { + name: "null-island", + lat: +0.0, + lng: +0.0, + wantLat: +0.0, + wantLng: +0.0, + wantString: "+0° +0°", + wantText: "POINT (0 0)", + }, + { + name: "north-pole", + lat: +90.0, + lng: +0.0, + wantLat: +90.0, + wantLng: +0.0, + wantString: "+90° +0°", + wantText: "POINT (0 90)", + }, + { + name: "south-pole", + lat: -90.0, + lng: +0.0, + wantLat: -90.0, + wantLng: +0.0, + wantString: "-90° +0°", + wantText: "POINT (0 -90)", + }, + { + name: "north-pole-weird-longitude", + lat: +90.0, + lng: +1.0, + wantLat: +90.0, + wantLng: +0.0, + wantString: "+90° +0°", + wantText: "POINT (0 90)", + }, + { + name: "south-pole-weird-longitude", + lat: -90.0, + lng: +1.0, + wantLat: -90.0, + wantLng: +0.0, + wantString: "-90° +0°", + wantText: "POINT (0 -90)", + }, + { + name: "almost-north", + lat: +89.0, + lng: +0.0, + wantLat: +89.0, + wantLng: +0.0, + wantString: "+89° +0°", + wantText: "POINT (0 89)", + }, + { + name: "past-north", + lat: +91.0, + lng: +0.0, + wantLat: +89.0, + wantLng: +180.0, + wantString: "+89° +180°", + wantText: "POINT (180 89)", + }, + { + name: "almost-south", + lat: -89.0, + lng: +0.0, + wantLat: -89.0, + wantLng: +0.0, + wantString: "-89° +0°", + wantText: "POINT (0 -89)", + }, + { + name: "past-south", + lat: -91.0, + lng: +0.0, + wantLat: -89.0, + wantLng: +180.0, + wantString: "-89° +180°", + wantText: "POINT (180 -89)", + }, + { + name: "antimeridian-north", + lat: +180.0, + lng: +0.0, + wantLat: +0.0, + wantLng: +180.0, + wantString: "+0° +180°", + wantText: "POINT (180 0)", + }, + { + name: "antimeridian-south", + lat: -180.0, + lng: +0.0, + wantLat: +0.0, + wantLng: +180.0, + wantString: "+0° +180°", + wantText: "POINT (180 0)", + }, + { + name: "almost-antimeridian-north", + lat: +179.0, + lng: +0.0, + wantLat: +1.0, + wantLng: +180.0, + wantString: "+1° +180°", + wantText: "POINT (180 1)", + }, + { + name: "past-antimeridian-north", + lat: +181.0, + lng: +0.0, + wantLat: -1.0, + wantLng: +180.0, + wantString: "-1° +180°", + wantText: "POINT (180 -1)", + }, + { + name: "almost-antimeridian-south", + lat: -179.0, + lng: +0.0, + wantLat: -1.0, + wantLng: +180.0, + wantString: "-1° +180°", + wantText: "POINT (180 -1)", + }, + { + name: "past-antimeridian-south", + lat: -181.0, + lng: +0.0, + wantLat: +1.0, + wantLng: +180.0, + wantString: "+1° +180°", + wantText: "POINT (180 1)", + }, + { + name: "circumnavigate-north", + lat: +360.0, + lng: +1.0, + wantLat: +0.0, + wantLng: +1.0, + wantString: "+0° +1°", + wantText: "POINT (1 0)", + }, + { + name: "circumnavigate-south", + lat: -360.0, + lng: +1.0, + wantLat: +0.0, + wantLng: +1.0, + wantString: "+0° +1°", + wantText: "POINT (1 0)", + }, + { + name: "almost-circumnavigate-north", + lat: +359.0, + lng: +1.0, + wantLat: -1.0, + wantLng: +1.0, + wantString: "-1° +1°", + wantText: "POINT (1 -1)", + }, + { + name: "past-circumnavigate-north", + lat: +361.0, + lng: +1.0, + wantLat: +1.0, + wantLng: +1.0, + wantString: "+1° +1°", + wantText: "POINT (1 1)", + }, + { + name: "almost-circumnavigate-south", + lat: -359.0, + lng: +1.0, + wantLat: +1.0, + wantLng: +1.0, + wantString: "+1° +1°", + wantText: "POINT (1 1)", + }, + { + name: "past-circumnavigate-south", + lat: -361.0, + lng: +1.0, + wantLat: -1.0, + wantLng: +1.0, + wantString: "-1° +1°", + wantText: "POINT (1 -1)", + }, + { + name: "antimeridian-east", + lat: +0.0, + lng: +180.0, + wantLat: +0.0, + wantLng: +180.0, + wantString: "+0° +180°", + wantText: "POINT (180 0)", + }, + { + name: "antimeridian-west", + lat: +0.0, + lng: -180.0, + wantLat: +0.0, + wantLng: +180.0, + wantString: "+0° +180°", + wantText: "POINT (180 0)", + }, + { + name: "almost-antimeridian-east", + lat: +0.0, + lng: +179.0, + wantLat: +0.0, + wantLng: +179.0, + wantString: "+0° +179°", + wantText: "POINT (179 0)", + }, + { + name: "past-antimeridian-east", + lat: +0.0, + lng: +181.0, + wantLat: +0.0, + wantLng: -179.0, + wantString: "+0° -179°", + wantText: "POINT (-179 0)", + }, + { + name: "almost-antimeridian-west", + lat: +0.0, + lng: -179.0, + wantLat: +0.0, + wantLng: -179.0, + wantString: "+0° -179°", + wantText: "POINT (-179 0)", + }, + { + name: "past-antimeridian-west", + lat: +0.0, + lng: -181.0, + wantLat: +0.0, + wantLng: +179.0, + wantString: "+0° +179°", + wantText: "POINT (179 0)", + }, + { + name: "montreal", + lat: +45.508888, + lng: -73.561668, + wantLat: +45.508888, + wantLng: -73.561668, + wantString: "+45.508888° -73.561668°", + wantText: "POINT (-73.561668 45.508888)", + }, + { + name: "canada", + lat: 57.550480044655636, + lng: -98.41680517868062, + wantLat: 57.550480044655636, + wantLng: -98.41680517868062, + wantString: "+57.550480044655636° -98.41680517868062°", + wantText: "POINT (-98.41680517868062 57.550480044655636)", + }, + } { + t.Run(tt.name, func(t *testing.T) { + p := geo.MakePoint(tt.lat, tt.lng) + + lat, lng, err := p.LatLng() + if !approx(lat, tt.wantLat) { + t.Errorf("MakePoint: lat %v, want %v", lat, tt.wantLat) + } + if !approx(lng, tt.wantLng) { + t.Errorf("MakePoint: lng %v, want %v", lng, tt.wantLng) + } + if err != nil { + t.Fatalf("LatLng: err %q, expected nil", err) + } + + if got := p.String(); got != tt.wantString { + t.Errorf("String: got %q, wantString %q", got, tt.wantString) + } + + txt, err := p.MarshalText() + if err != nil { + t.Errorf("Text: err %q, expected nil", err) + } else if string(txt) != tt.wantText { + t.Errorf("Text: got %q, wantText %q", txt, tt.wantText) + } + + b, err := p.MarshalBinary() + if err != nil { + t.Fatalf("MarshalBinary: err %q, expected nil", err) + } + + var q geo.Point + if err := q.UnmarshalBinary(b); err != nil { + t.Fatalf("UnmarshalBinary: err %q, expected nil", err) + } + if !q.EqualApprox(p, -1) { + t.Errorf("UnmarshalBinary: roundtrip failed: %#v != %#v", q, p) + } + + i, err := p.MarshalUint64() + if err != nil { + t.Fatalf("MarshalUint64: err %q, expected nil", err) + } + + var r geo.Point + if err := r.UnmarshalUint64(i); err != nil { + t.Fatalf("UnmarshalUint64: err %r, expected nil", err) + } + if !q.EqualApprox(r, -1) { + t.Errorf("UnmarshalUint64: roundtrip failed: %#v != %#v", r, p) + } + }) + } +} + +func TestPointMarshalBinary(t *testing.T) { + roundtrip := func(p geo.Point) error { + b, err := p.MarshalBinary() + if err != nil { + return fmt.Errorf("marshal: %v", err) + } + var q geo.Point + if err := q.UnmarshalBinary(b); err != nil { + return fmt.Errorf("unmarshal: %v", err) + } + if q != p { + return fmt.Errorf("%#v != %#v", q, p) + } + return nil + } + + t.Run("nowhere", func(t *testing.T) { + var nowhere geo.Point + if err := roundtrip(nowhere); err != nil { + t.Errorf("roundtrip: %v", err) + } + }) + + t.Run("quick-check", func(t *testing.T) { + f := func(lat geo.Degrees, lng geo.Degrees) (ok bool) { + pt := geo.MakePoint(lat, lng) + if err := roundtrip(pt); err != nil { + t.Errorf("roundtrip: %v", err) + } + return !t.Failed() + } + if err := quick.Check(f, nil); err != nil { + t.Error(err) + } + }) +} + +func TestPointMarshalUint64(t *testing.T) { + t.Skip("skip") + roundtrip := func(p geo.Point) error { + i, err := p.MarshalUint64() + if err != nil { + return fmt.Errorf("marshal: %v", err) + } + var q geo.Point + if err := q.UnmarshalUint64(i); err != nil { + return fmt.Errorf("unmarshal: %v", err) + } + if q != p { + return fmt.Errorf("%#v != %#v", q, p) + } + return nil + } + + t.Run("nowhere", func(t *testing.T) { + var nowhere geo.Point + if err := roundtrip(nowhere); err != nil { + t.Errorf("roundtrip: %v", err) + } + }) + + t.Run("quick-check", func(t *testing.T) { + f := func(lat geo.Degrees, lng geo.Degrees) (ok bool) { + if err := roundtrip(geo.MakePoint(lat, lng)); err != nil { + t.Errorf("roundtrip: %v", err) + } + return !t.Failed() + } + if err := quick.Check(f, nil); err != nil { + t.Error(err) + } + }) +} + +func TestPointSphericalAngleTo(t *testing.T) { + const earthRadius = 6371.000 // volumetric mean radius (km) + const kmToRad = 1 / earthRadius + for _, tt := range []struct { + name string + x geo.Point + y geo.Point + want geo.Radians + wantErr string + }{ + { + name: "same-point-null-island", + x: geo.MakePoint(0, 0), + y: geo.MakePoint(0, 0), + want: 0.0 * geo.Radian, + }, + { + name: "same-point-north-pole", + x: geo.MakePoint(+90, 0), + y: geo.MakePoint(+90, +90), + want: 0.0 * geo.Radian, + }, + { + name: "same-point-south-pole", + x: geo.MakePoint(-90, 0), + y: geo.MakePoint(-90, -90), + want: 0.0 * geo.Radian, + }, + { + name: "north-pole-to-south-pole", + x: geo.MakePoint(+90, 0), + y: geo.MakePoint(-90, -90), + want: math.Pi * geo.Radian, + }, + { + name: "toronto-to-montreal", + x: geo.MakePoint(+43.6532, -79.3832), + y: geo.MakePoint(+45.5019, -73.5674), + want: 504.26 * kmToRad * geo.Radian, + }, + { + name: "sydney-to-san-francisco", + x: geo.MakePoint(-33.8727, +151.2057), + y: geo.MakePoint(+37.7749, -122.4194), + want: 11948.18 * kmToRad * geo.Radian, + }, + { + name: "new-york-to-paris", + x: geo.MakePoint(+40.7128, -74.0060), + y: geo.MakePoint(+48.8575, +2.3514), + want: 5837.15 * kmToRad * geo.Radian, + }, + { + name: "seattle-to-tokyo", + x: geo.MakePoint(+47.6061, -122.3328), + y: geo.MakePoint(+35.6764, +139.6500), + want: 7700.00 * kmToRad * geo.Radian, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.x.SphericalAngleTo(tt.y) + if tt.wantErr == "" && err != nil { + t.Fatalf("err %q, expected nil", err) + } + if tt.wantErr != "" && (err == nil || err.Error() != tt.wantErr) { + t.Fatalf("err %q, expected %q", err, tt.wantErr) + } + if tt.wantErr != "" { + return + } + + if !approx(got, tt.want) { + t.Errorf("x to y: got %v, want %v", got, tt.want) + } + + // Distance should be commutative + got, err = tt.y.SphericalAngleTo(tt.x) + if err != nil { + t.Fatalf("err %q, expected nil", err) + } + if !approx(got, tt.want) { + t.Errorf("y to x: got %v, want %v", got, tt.want) + } + t.Logf("x to y: %v km", got/kmToRad) + }) + } +} + +func approx[T ~float64](x, y T) bool { + return math.Abs(float64(x)-float64(y)) <= 1e-5 +} diff --git a/types/geo/quantize.go b/types/geo/quantize.go new file mode 100644 index 0000000000000..18ec11f9f119c --- /dev/null +++ b/types/geo/quantize.go @@ -0,0 +1,106 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo + +import ( + "math" + "sync" +) + +// MinSeparation is the minimum separation between two points after quantizing. +// [Point.Quantize] guarantees that two points will either be snapped to exactly +// the same point, which conflates multiple positions together, or that the two +// points will be far enough apart that successfully performing most reverse +// lookups would be highly improbable. +const MinSeparation = 50_000 * Meter + +// Latitude +var ( + // numSepsEquatorToPole is the number of separations between a point on + // the equator to a point on a pole, that satisfies [minPointSep]. In + // other words, the number of separations between 0° and +90° degrees + // latitude. + numSepsEquatorToPole = int(math.Floor(float64( + earthPolarCircumference / MinSeparation / 4))) + + // latSep is the number of degrees between two adjacent latitudinal + // points. In other words, the next point going straight north of + // 0° would be latSep°. + latSep = Degrees(90.0 / float64(numSepsEquatorToPole)) +) + +// snapToLat returns the number of the nearest latitudinal separation to +// lat. A positive result is north of the equator, a negative result is south, +// and zero is the equator itself. For example, a result of -1 would mean a +// point that is [latSep] south of the equator. +func snapToLat(lat Degrees) int { + return int(math.Round(float64(lat / latSep))) +} + +// lngSep is a lookup table for the number of degrees between two adjacent +// longitudinal separations. where the index corresponds to the absolute value +// of the latitude separation. The first value corresponds to the equator and +// the last value corresponds to the separation before the pole. There is no +// value for the pole itself, because longitude has no meaning there. +// +// [lngSep] is calculated on init, which is so quick and will be used so often +// that the startup cost is negligible. +var lngSep = sync.OnceValue(func() []Degrees { + lut := make([]Degrees, numSepsEquatorToPole) + + // i ranges from the equator to a pole + for i := range len(lut) { + // lat ranges from [0°, 90°], because the southern hemisphere is + // a reflection of the northern one. + lat := Degrees(i) * latSep + ratio := math.Cos(float64(lat.Radians())) + circ := Distance(ratio) * earthEquatorialCircumference + num := int(math.Floor(float64(circ / MinSeparation))) + // We define lut[0] as 0°, lut[len(lut)] to be the north pole, + // which means -lut[len(lut)] is the south pole. + lut[i] = Degrees(360.0 / float64(num)) + } + return lut +}) + +// snapToLatLng returns the number of the nearest latitudinal separation to lat, +// and the nearest longitudinal separation to lng. +func snapToLatLng(lat, lng Degrees) (Degrees, Degrees) { + latN := snapToLat(lat) + + // absolute index into lngSep + n := latN + if n < 0 { + n = -latN + } + + lngSep := lngSep() + if n < len(lngSep) { + sep := lngSep[n] + lngN := int(math.Round(float64(lng / sep))) + return Degrees(latN) * latSep, Degrees(lngN) * sep + } + if latN < 0 { // south pole + return -90 * Degree, 0 * Degree + } else { // north pole + return +90 * Degree, 0 * Degree + } +} + +// Quantize returns a new [Point] after throwing away enough location data in p +// so that it would be difficult to distinguish a node among all the other nodes +// in its general vicinity. One caveat is that if there’s only one point in an +// obscure location, someone could triangulate the node using additional data. +// +// This method is stable: given the same p, it will always return the same +// result. It is equivalent to snapping to points on Earth that are at least +// [MinSeparation] apart. +func (p Point) Quantize() Point { + if p.IsZero() { + return p + } + + lat, lng := snapToLatLng(p.lat, p.lng180-180) + return MakePoint(lat, lng) +} diff --git a/types/geo/quantize_test.go b/types/geo/quantize_test.go new file mode 100644 index 0000000000000..3c707e303c250 --- /dev/null +++ b/types/geo/quantize_test.go @@ -0,0 +1,130 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo_test + +import ( + "testing" + "testing/quick" + + "tailscale.com/types/geo" +) + +func TestPointAnonymize(t *testing.T) { + t.Run("nowhere", func(t *testing.T) { + var zero geo.Point + p := zero.Quantize() + want := zero.Valid() + if got := p.Valid(); got != want { + t.Fatalf("zero.Valid %t, want %t", got, want) + } + }) + + t.Run("separation", func(t *testing.T) { + // Walk from the south pole to the north pole and check that each + // point on the latitude is approximately MinSeparation apart. + const southPole = -90 * geo.Degree + const northPole = 90 * geo.Degree + const dateLine = 180 * geo.Degree + + llat := southPole + for lat := llat; lat <= northPole; lat += 0x1p-4 { + last := geo.MakePoint(llat, 0) + cur := geo.MakePoint(lat, 0) + anon := cur.Quantize() + switch l, g, err := anon.LatLng(); { + case err != nil: + t.Fatal(err) + case lat == southPole: + // initialize llng, to the first snapped longitude + llat = l + goto Lng + case g != 0: + t.Fatalf("%v is west or east of %v", anon, last) + case l < llat: + t.Fatalf("%v is south of %v", anon, last) + case l == llat: + continue + case l > llat: + switch dist, err := last.DistanceTo(anon); { + case err != nil: + t.Fatal(err) + case dist == 0.0: + continue + case dist < geo.MinSeparation: + t.Logf("lat=%v last=%v cur=%v anon=%v", lat, last, cur, anon) + t.Fatalf("%v is too close to %v", anon, last) + default: + llat = l + } + } + + Lng: + llng := dateLine + for lng := llng; lng <= dateLine && lng >= -dateLine; lng -= 0x1p-3 { + last := geo.MakePoint(llat, llng) + cur := geo.MakePoint(lat, lng) + anon := cur.Quantize() + switch l, g, err := anon.LatLng(); { + case err != nil: + t.Fatal(err) + case lng == dateLine: + // initialize llng, to the first snapped longitude + llng = g + continue + case l != llat: + t.Fatalf("%v is north or south of %v", anon, last) + case g != llng: + const tolerance = geo.MinSeparation * 0x1p-9 + switch dist, err := last.DistanceTo(anon); { + case err != nil: + t.Fatal(err) + case dist < tolerance: + continue + case dist < (geo.MinSeparation - tolerance): + t.Logf("lat=%v lng=%v last=%v cur=%v anon=%v", lat, lng, last, cur, anon) + t.Fatalf("%v is too close to %v: %v", anon, last, dist) + default: + llng = g + } + + } + } + } + if llat == southPole { + t.Fatal("llat never incremented") + } + }) + + t.Run("quick-check", func(t *testing.T) { + f := func(lat, lng geo.Degrees) bool { + p := geo.MakePoint(lat, lng) + q := p.Quantize() + t.Logf("quantize %v = %v", p, q) + + lat, lng, err := q.LatLng() + if err != nil { + t.Errorf("err %v, want nil", err) + return !t.Failed() + } + + if lat < -90*geo.Degree || lat > 90*geo.Degree { + t.Errorf("lat outside [-90°, +90°]: %v", lat) + } + if lng < -180*geo.Degree || lng > 180*geo.Degree { + t.Errorf("lng outside [-180°, +180°], %v", lng) + } + + if dist, err := p.DistanceTo(q); err != nil { + t.Error(err) + } else if dist > (geo.MinSeparation * 2) { + t.Errorf("moved too far: %v", dist) + } + + return !t.Failed() + } + if err := quick.Check(f, nil); err != nil { + t.Fatal(err) + } + }) +} diff --git a/types/geo/units.go b/types/geo/units.go new file mode 100644 index 0000000000000..76a4c02f79f34 --- /dev/null +++ b/types/geo/units.go @@ -0,0 +1,191 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo + +import ( + "math" + "strconv" + "strings" + "unicode" +) + +const ( + Degree Degrees = 1 + Radian Radians = 1 + Turn Turns = 1 + Meter Distance = 1 +) + +// Degrees represents a latitude or longitude, in decimal degrees. +type Degrees float64 + +// ParseDegrees parses s as decimal degrees. +func ParseDegrees(s string) (Degrees, error) { + s = strings.TrimSuffix(s, "°") + f, err := strconv.ParseFloat(s, 64) + return Degrees(f), err +} + +// MustParseDegrees parses s as decimal degrees, but panics on error. +func MustParseDegrees(s string) Degrees { + d, err := ParseDegrees(s) + if err != nil { + panic(err) + } + return d +} + +// String implements the [Stringer] interface. The output is formatted in +// decimal degrees, prefixed by either the appropriate + or - sign, and suffixed +// by a ° degree symbol. +func (d Degrees) String() string { + b, _ := d.AppendText(nil) + b = append(b, []byte("°")...) + return string(b) +} + +// AppendText implements [encoding.TextAppender]. The output is formatted in +// decimal degrees, prefixed by either the appropriate + or - sign. +func (d Degrees) AppendText(b []byte) ([]byte, error) { + b = d.AppendZeroPaddedText(b, 0) + return b, nil +} + +// AppendZeroPaddedText appends d formatted as decimal degrees to b. The number of +// integer digits will be zero-padded to nint. +func (d Degrees) AppendZeroPaddedText(b []byte, nint int) []byte { + n := float64(d) + + if math.IsInf(n, 0) || math.IsNaN(n) { + return strconv.AppendFloat(b, n, 'f', -1, 64) + } + + sign := byte('+') + if math.Signbit(n) { + sign = '-' + n = -n + } + b = append(b, sign) + + pad := nint - 1 + for nn := n / 10; nn >= 1 && pad > 0; nn /= 10 { + pad-- + } + for range pad { + b = append(b, '0') + } + return strconv.AppendFloat(b, n, 'f', -1, 64) +} + +// Radians converts d into radians. +func (d Degrees) Radians() Radians { + return Radians(d * math.Pi / 180.0) +} + +// Turns converts d into a number of turns. +func (d Degrees) Turns() Turns { + return Turns(d / 360.0) +} + +// Radians represents a latitude or longitude, in radians. +type Radians float64 + +// ParseRadians parses s as radians. +func ParseRadians(s string) (Radians, error) { + s = strings.TrimSuffix(s, "rad") + s = strings.TrimRightFunc(s, unicode.IsSpace) + f, err := strconv.ParseFloat(s, 64) + return Radians(f), err +} + +// MustParseRadians parses s as radians, but panics on error. +func MustParseRadians(s string) Radians { + r, err := ParseRadians(s) + if err != nil { + panic(err) + } + return r +} + +// String implements the [Stringer] interface. +func (r Radians) String() string { + return strconv.FormatFloat(float64(r), 'f', -1, 64) + " rad" +} + +// Degrees converts r into decimal degrees. +func (r Radians) Degrees() Degrees { + return Degrees(r * 180.0 / math.Pi) +} + +// Turns converts r into a number of turns. +func (r Radians) Turns() Turns { + return Turns(r / 2 / math.Pi) +} + +// Turns represents a number of complete revolutions around a sphere. +type Turns float64 + +// String implements the [Stringer] interface. +func (o Turns) String() string { + return strconv.FormatFloat(float64(o), 'f', -1, 64) +} + +// Degrees converts t into decimal degrees. +func (o Turns) Degrees() Degrees { + return Degrees(o * 360.0) +} + +// Radians converts t into radians. +func (o Turns) Radians() Radians { + return Radians(o * 2 * math.Pi) +} + +// Distance represents a great-circle distance in meters. +type Distance float64 + +// ParseDistance parses s as distance in meters. +func ParseDistance(s string) (Distance, error) { + s = strings.TrimSuffix(s, "m") + s = strings.TrimRightFunc(s, unicode.IsSpace) + f, err := strconv.ParseFloat(s, 64) + return Distance(f), err +} + +// MustParseDistance parses s as distance in meters, but panics on error. +func MustParseDistance(s string) Distance { + d, err := ParseDistance(s) + if err != nil { + panic(err) + } + return d +} + +// String implements the [Stringer] interface. +func (d Distance) String() string { + return strconv.FormatFloat(float64(d), 'f', -1, 64) + "m" +} + +// DistanceOnEarth converts t turns into the great-circle distance, in meters. +func DistanceOnEarth(t Turns) Distance { + return Distance(t) * EarthMeanCircumference +} + +// Earth Fact Sheet +// https://nssdc.gsfc.nasa.gov/planetary/factsheet/earthfact.html +const ( + // EarthMeanRadius is the volumetric mean radius of the Earth. + EarthMeanRadius = 6_371_000 * Meter + // EarthMeanCircumference is the volumetric mean circumference of the Earth. + EarthMeanCircumference = 2 * math.Pi * EarthMeanRadius + + // earthEquatorialRadius is the equatorial radius of the Earth. + earthEquatorialRadius = 6_378_137 * Meter + // earthEquatorialCircumference is the equatorial circumference of the Earth. + earthEquatorialCircumference = 2 * math.Pi * earthEquatorialRadius + + // earthPolarRadius is the polar radius of the Earth. + earthPolarRadius = 6_356_752 * Meter + // earthPolarCircumference is the polar circumference of the Earth. + earthPolarCircumference = 2 * math.Pi * earthPolarRadius +) diff --git a/types/geo/units_test.go b/types/geo/units_test.go new file mode 100644 index 0000000000000..b6f724ce0d9b3 --- /dev/null +++ b/types/geo/units_test.go @@ -0,0 +1,395 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo_test + +import ( + "math" + "strings" + "testing" + + "tailscale.com/types/geo" +) + +func TestDegrees(t *testing.T) { + for _, tt := range []struct { + name string + degs geo.Degrees + wantStr string + wantText string + wantPad string + wantRads geo.Radians + wantTurns geo.Turns + }{ + { + name: "zero", + degs: 0.0 * geo.Degree, + wantStr: "+0°", + wantText: "+0", + wantPad: "+000", + wantRads: 0.0 * geo.Radian, + wantTurns: 0 * geo.Turn, + }, + { + name: "quarter-turn", + degs: 90.0 * geo.Degree, + wantStr: "+90°", + wantText: "+90", + wantPad: "+090", + wantRads: 0.5 * math.Pi * geo.Radian, + wantTurns: 0.25 * geo.Turn, + }, + { + name: "half-turn", + degs: 180.0 * geo.Degree, + wantStr: "+180°", + wantText: "+180", + wantPad: "+180", + wantRads: 1.0 * math.Pi * geo.Radian, + wantTurns: 0.5 * geo.Turn, + }, + { + name: "full-turn", + degs: 360.0 * geo.Degree, + wantStr: "+360°", + wantText: "+360", + wantPad: "+360", + wantRads: 2.0 * math.Pi * geo.Radian, + wantTurns: 1.0 * geo.Turn, + }, + { + name: "negative-zero", + degs: geo.MustParseDegrees("-0.0"), + wantStr: "-0°", + wantText: "-0", + wantPad: "-000", + wantRads: 0 * geo.Radian * -1, + wantTurns: 0 * geo.Turn * -1, + }, + { + name: "small-degree", + degs: -1.2003 * geo.Degree, + wantStr: "-1.2003°", + wantText: "-1.2003", + wantPad: "-001.2003", + wantRads: -0.020949187011687936 * geo.Radian, + wantTurns: -0.0033341666666666663 * geo.Turn, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if got := tt.degs.String(); got != tt.wantStr { + t.Errorf("String got %q, want %q", got, tt.wantStr) + } + + d, err := geo.ParseDegrees(tt.wantStr) + if err != nil { + t.Fatalf("ParseDegrees err %q, want nil", err.Error()) + } + if d != tt.degs { + t.Errorf("ParseDegrees got %q, want %q", d, tt.degs) + } + + b, err := tt.degs.AppendText(nil) + if err != nil { + t.Fatalf("AppendText err %q, want nil", err.Error()) + } + if string(b) != tt.wantText { + t.Errorf("AppendText got %q, want %q", b, tt.wantText) + } + + b = tt.degs.AppendZeroPaddedText(nil, 3) + if string(b) != tt.wantPad { + t.Errorf("AppendZeroPaddedText got %q, want %q", b, tt.wantPad) + } + + r := tt.degs.Radians() + if r != tt.wantRads { + t.Errorf("Radian got %v, want %v", r, tt.wantRads) + } + if d := r.Degrees(); d != tt.degs { // Roundtrip + t.Errorf("Degrees got %v, want %v", d, tt.degs) + } + + o := tt.degs.Turns() + if o != tt.wantTurns { + t.Errorf("Turns got %v, want %v", o, tt.wantTurns) + } + }) + } +} + +func TestRadians(t *testing.T) { + for _, tt := range []struct { + name string + rads geo.Radians + wantStr string + wantText string + wantDegs geo.Degrees + wantTurns geo.Turns + }{ + { + name: "zero", + rads: 0.0 * geo.Radian, + wantStr: "0 rad", + wantDegs: 0.0 * geo.Degree, + wantTurns: 0 * geo.Turn, + }, + { + name: "quarter-turn", + rads: 0.5 * math.Pi * geo.Radian, + wantStr: "1.5707963267948966 rad", + wantDegs: 90.0 * geo.Degree, + wantTurns: 0.25 * geo.Turn, + }, + { + name: "half-turn", + rads: 1.0 * math.Pi * geo.Radian, + wantStr: "3.141592653589793 rad", + wantDegs: 180.0 * geo.Degree, + wantTurns: 0.5 * geo.Turn, + }, + { + name: "full-turn", + rads: 2.0 * math.Pi * geo.Radian, + wantStr: "6.283185307179586 rad", + wantDegs: 360.0 * geo.Degree, + wantTurns: 1.0 * geo.Turn, + }, + { + name: "negative-zero", + rads: geo.MustParseRadians("-0"), + wantStr: "-0 rad", + wantDegs: 0 * geo.Degree * -1, + wantTurns: 0 * geo.Turn * -1, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if got := tt.rads.String(); got != tt.wantStr { + t.Errorf("String got %q, want %q", got, tt.wantStr) + } + + r, err := geo.ParseRadians(tt.wantStr) + if err != nil { + t.Fatalf("ParseDegrees err %q, want nil", err.Error()) + } + if r != tt.rads { + t.Errorf("ParseDegrees got %q, want %q", r, tt.rads) + } + + d := tt.rads.Degrees() + if d != tt.wantDegs { + t.Errorf("Degrees got %v, want %v", d, tt.wantDegs) + } + if r := d.Radians(); r != tt.rads { // Roundtrip + t.Errorf("Radians got %v, want %v", r, tt.rads) + } + + o := tt.rads.Turns() + if o != tt.wantTurns { + t.Errorf("Turns got %v, want %v", o, tt.wantTurns) + } + }) + } +} + +func TestTurns(t *testing.T) { + for _, tt := range []struct { + name string + turns geo.Turns + wantStr string + wantText string + wantDegs geo.Degrees + wantRads geo.Radians + }{ + { + name: "zero", + turns: 0.0, + wantStr: "0", + wantDegs: 0.0 * geo.Degree, + wantRads: 0 * geo.Radian, + }, + { + name: "quarter-turn", + turns: 0.25, + wantStr: "0.25", + wantDegs: 90.0 * geo.Degree, + wantRads: 0.5 * math.Pi * geo.Radian, + }, + { + name: "half-turn", + turns: 0.5, + wantStr: "0.5", + wantDegs: 180.0 * geo.Degree, + wantRads: 1.0 * math.Pi * geo.Radian, + }, + { + name: "full-turn", + turns: 1.0, + wantStr: "1", + wantDegs: 360.0 * geo.Degree, + wantRads: 2.0 * math.Pi * geo.Radian, + }, + { + name: "negative-zero", + turns: geo.Turns(math.Copysign(0, -1)), + wantStr: "-0", + wantDegs: 0 * geo.Degree * -1, + wantRads: 0 * geo.Radian * -1, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if got := tt.turns.String(); got != tt.wantStr { + t.Errorf("String got %q, want %q", got, tt.wantStr) + } + + d := tt.turns.Degrees() + if d != tt.wantDegs { + t.Errorf("Degrees got %v, want %v", d, tt.wantDegs) + } + if o := d.Turns(); o != tt.turns { // Roundtrip + t.Errorf("Turns got %v, want %v", o, tt.turns) + } + + r := tt.turns.Radians() + if r != tt.wantRads { + t.Errorf("Turns got %v, want %v", r, tt.wantRads) + } + }) + } +} + +func TestDistance(t *testing.T) { + for _, tt := range []struct { + name string + dist geo.Distance + wantStr string + }{ + { + name: "zero", + dist: 0.0 * geo.Meter, + wantStr: "0m", + }, + { + name: "random", + dist: 4 * geo.Meter, + wantStr: "4m", + }, + { + name: "light-second", + dist: 299_792_458 * geo.Meter, + wantStr: "299792458m", + }, + { + name: "planck-length", + dist: 1.61625518e-35 * geo.Meter, + wantStr: "0.0000000000000000000000000000000000161625518m", + }, + { + name: "negative-zero", + dist: geo.Distance(math.Copysign(0, -1)), + wantStr: "-0m", + }, + } { + t.Run(tt.name, func(t *testing.T) { + if got := tt.dist.String(); got != tt.wantStr { + t.Errorf("String got %q, want %q", got, tt.wantStr) + } + + r, err := geo.ParseDistance(tt.wantStr) + if err != nil { + t.Fatalf("ParseDegrees err %q, want nil", err.Error()) + } + if r != tt.dist { + t.Errorf("ParseDegrees got %q, want %q", r, tt.dist) + } + }) + } +} + +func TestDistanceOnEarth(t *testing.T) { + for _, tt := range []struct { + name string + here geo.Point + there geo.Point + want geo.Distance + wantErr string + }{ + { + name: "no-points", + here: geo.Point{}, + there: geo.Point{}, + wantErr: "not a valid point", + }, + { + name: "not-here", + here: geo.Point{}, + there: geo.MakePoint(0, 0), + wantErr: "not a valid point", + }, + { + name: "not-there", + here: geo.MakePoint(0, 0), + there: geo.Point{}, + wantErr: "not a valid point", + }, + { + name: "null-island", + here: geo.MakePoint(0, 0), + there: geo.MakePoint(0, 0), + want: 0 * geo.Meter, + }, + { + name: "equator-to-south-pole", + here: geo.MakePoint(0, 0), + there: geo.MakePoint(-90, 0), + want: geo.EarthMeanCircumference / 4, + }, + { + name: "north-pole-to-south-pole", + here: geo.MakePoint(+90, 0), + there: geo.MakePoint(-90, 0), + want: geo.EarthMeanCircumference / 2, + }, + { + name: "meridian-to-antimeridian", + here: geo.MakePoint(0, 0), + there: geo.MakePoint(0, -180), + want: geo.EarthMeanCircumference / 2, + }, + { + name: "positive-to-negative-antimeridian", + here: geo.MakePoint(0, 180), + there: geo.MakePoint(0, -180), + want: 0 * geo.Meter, + }, + { + name: "toronto-to-montreal", + here: geo.MakePoint(+43.70011, -79.41630), + there: geo.MakePoint(+45.50884, -73.58781), + want: 503_200 * geo.Meter, + }, + { + name: "montreal-to-san-francisco", + here: geo.MakePoint(+45.50884, -73.58781), + there: geo.MakePoint(+37.77493, -122.41942), + want: 4_082_600 * geo.Meter, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.here.DistanceTo(tt.there) + if tt.wantErr == "" && err != nil { + t.Fatalf("err %q, want nil", err) + } + if tt.wantErr != "" && !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err %q, want %q", err, tt.wantErr) + } + + approx := func(x, y geo.Distance) bool { + return math.Abs(float64(x)-float64(y)) <= 10 + } + if !approx(got, tt.want) { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} From d334d9ba07fa8ae8abb5d39fa5a3e7a277f2dc32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus=20Lensb=C3=B8l?= Date: Fri, 18 Jul 2025 10:55:17 -0400 Subject: [PATCH 224/263] client/local,cmd/tailscale/cli,ipn/localapi: expose eventbus graph (#16597) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make it possible to dump the eventbus graph as JSON or DOT to both debug and document what is communicated via the bus. Updates #15160 Signed-off-by: Claus Lensbøl --- client/local/local.go | 6 ++++ cmd/tailscale/cli/debug.go | 56 ++++++++++++++++++++++++++++++++++++++ ipn/localapi/localapi.go | 50 ++++++++++++++++++++++++++++++++++ util/eventbus/debug.go | 13 +++++++++ 4 files changed, 125 insertions(+) diff --git a/client/local/local.go b/client/local/local.go index 74c4f0b6f8a2c..55d14f95eee5a 100644 --- a/client/local/local.go +++ b/client/local/local.go @@ -432,6 +432,12 @@ func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) { return res.Body, nil } +// EventBusGraph returns a graph of active publishers and subscribers in the eventbus +// as a [eventbus.DebugTopics] +func (lc *Client) EventBusGraph(ctx context.Context) ([]byte, error) { + return lc.get200(ctx, "/localapi/v0/debug-bus-graph") +} + // StreamBusEvents returns an iterator of Tailscale bus events as they arrive. // Each pair is a valid event and a nil error, or a zero event a non-nil error. // In case of error, the iterator ends after the pair reporting the error. diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index 8473c4a1707fa..fb062fd17c7aa 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -6,6 +6,7 @@ package cli import ( "bufio" "bytes" + "cmp" "context" "encoding/binary" "encoding/json" @@ -108,6 +109,17 @@ func debugCmd() *ffcli.Command { Exec: runDaemonBusEvents, ShortHelp: "Watch events on the tailscaled bus", }, + { + Name: "daemon-bus-graph", + ShortUsage: "tailscale debug daemon-bus-graph", + Exec: runDaemonBusGraph, + ShortHelp: "Print graph for the tailscaled bus", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("debug-bus-graph") + fs.StringVar(&daemonBusGraphArgs.format, "format", "json", "output format [json/dot]") + return fs + })(), + }, { Name: "metrics", ShortUsage: "tailscale debug metrics", @@ -807,6 +819,50 @@ func runDaemonBusEvents(ctx context.Context, args []string) error { return nil } +var daemonBusGraphArgs struct { + format string +} + +func runDaemonBusGraph(ctx context.Context, args []string) error { + graph, err := localClient.EventBusGraph(ctx) + if err != nil { + return err + } + if format := daemonBusGraphArgs.format; format != "json" && format != "dot" { + return fmt.Errorf("unrecognized output format %q", format) + } + if daemonBusGraphArgs.format == "dot" { + var topics eventbus.DebugTopics + if err := json.Unmarshal(graph, &topics); err != nil { + return fmt.Errorf("unable to parse json: %w", err) + } + fmt.Print(generateDOTGraph(topics.Topics)) + } else { + fmt.Print(string(graph)) + } + return nil +} + +// generateDOTGraph generates the DOT graph format based on the events +func generateDOTGraph(topics []eventbus.DebugTopic) string { + var sb strings.Builder + sb.WriteString("digraph event_bus {\n") + + for _, topic := range topics { + // If no subscribers, still ensure the topic is drawn + if len(topic.Subscribers) == 0 { + topic.Subscribers = append(topic.Subscribers, "no-subscribers") + } + for _, subscriber := range topic.Subscribers { + fmt.Fprintf(&sb, "\t%q -> %q [label=%q];\n", + topic.Publisher, subscriber, cmp.Or(topic.Name, "???")) + } + } + + sb.WriteString("}\n") + return sb.String() +} + var metricsArgs struct { watch bool } diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index d7c64b917ead4..2409aa1ae3a36 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -93,6 +93,7 @@ var handler = map[string]LocalAPIHandler{ "component-debug-logging": (*Handler).serveComponentDebugLogging, "debug": (*Handler).serveDebug, "debug-bus-events": (*Handler).serveDebugBusEvents, + "debug-bus-graph": (*Handler).serveEventBusGraph, "debug-derp-region": (*Handler).serveDebugDERPRegion, "debug-dial-types": (*Handler).serveDebugDialTypes, "debug-log": (*Handler).serveDebugLog, @@ -1004,6 +1005,55 @@ func (h *Handler) serveDebugBusEvents(w http.ResponseWriter, r *http.Request) { } } +// serveEventBusGraph taps into the event bus and dumps out the active graph of +// publishers and subscribers. It does not represent anything about the messages +// exchanged. +func (h *Handler) serveEventBusGraph(w http.ResponseWriter, r *http.Request) { + if r.Method != httpm.GET { + http.Error(w, "GET required", http.StatusMethodNotAllowed) + return + } + + bus, ok := h.LocalBackend().Sys().Bus.GetOK() + if !ok { + http.Error(w, "event bus not running", http.StatusPreconditionFailed) + return + } + + debugger := bus.Debugger() + clients := debugger.Clients() + + graph := map[string]eventbus.DebugTopic{} + + for _, client := range clients { + for _, pub := range debugger.PublishTypes(client) { + topic, ok := graph[pub.Name()] + if !ok { + topic = eventbus.DebugTopic{Name: pub.Name()} + } + topic.Publisher = client.Name() + graph[pub.Name()] = topic + } + for _, sub := range debugger.SubscribeTypes(client) { + topic, ok := graph[sub.Name()] + if !ok { + topic = eventbus.DebugTopic{Name: sub.Name()} + } + topic.Subscribers = append(topic.Subscribers, client.Name()) + graph[sub.Name()] = topic + } + } + + // The top level map is not really needed for the client, convert to a list. + topics := eventbus.DebugTopics{} + for _, v := range graph { + topics.Topics = append(topics.Topics, v) + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(topics) +} + func (h *Handler) serveComponentDebugLogging(w http.ResponseWriter, r *http.Request) { if !h.PermitWrite { http.Error(w, "debug access denied", http.StatusForbidden) diff --git a/util/eventbus/debug.go b/util/eventbus/debug.go index b6264f82fd0eb..a055f078fc4f2 100644 --- a/util/eventbus/debug.go +++ b/util/eventbus/debug.go @@ -195,3 +195,16 @@ type DebugEvent struct { To []string Event any } + +// DebugTopics provides the JSON encoding as a wrapper for a collection of [DebugTopic]. +type DebugTopics struct { + Topics []DebugTopic +} + +// DebugTopic provides the JSON encoding of publishers and subscribers for a +// given topic. +type DebugTopic struct { + Name string + Publisher string + Subscribers []string +} From 871f73d9924bc046a90d62fdbc0f74b783cc4630 Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Fri, 18 Jul 2025 10:55:43 -0400 Subject: [PATCH 225/263] Kevin/add drain sub command for serve services (#16502) * cmd/tailscale/cli: add drain subCommand for serve This commit adds the drain subcommand for serving services. After we merge advertise and serve service as one step, we now need a way to unadvertise service and this is it. Updates tailscale/corp#22954 Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * move runServeDrain and some update regarding pr comments Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * some code structure change Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --------- Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- cmd/tailscale/cli/serve_v2.go | 48 ++++++++++++++++++++++++++++++ cmd/tailscale/cli/serve_v2_test.go | 48 ++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/cmd/tailscale/cli/serve_v2.go b/cmd/tailscale/cli/serve_v2.go index 15de0609c72ad..6fa1a1b08c66c 100644 --- a/cmd/tailscale/cli/serve_v2.go +++ b/cmd/tailscale/cli/serve_v2.go @@ -203,6 +203,16 @@ func newServeV2Command(e *serveEnv, subcmd serveMode) *ffcli.Command { Exec: e.runServeReset, FlagSet: e.newFlags("serve-reset", nil), }, + { + Name: "drain", + ShortUsage: fmt.Sprintf("tailscale %s drain ", info.Name), + ShortHelp: "Drain a service from the current node", + LongHelp: "Make the current node no longer accept new connections for the specified service.\n" + + "Existing connections will continue to work until they are closed, but no new connections will be accepted.\n" + + "Use this command to gracefully remove a service from the current node without disrupting existing connections.\n" + + " should be a service name (e.g., svc:my-service).", + Exec: e.runServeDrain, + }, }, } } @@ -443,6 +453,44 @@ func (e *serveEnv) addServiceToPrefs(ctx context.Context, serviceName string) er return err } +func (e *serveEnv) removeServiceFromPrefs(ctx context.Context, serviceName tailcfg.ServiceName) error { + prefs, err := e.lc.GetPrefs(ctx) + if err != nil { + return fmt.Errorf("error getting prefs: %w", err) + } + if len(prefs.AdvertiseServices) == 0 { + return nil // nothing to remove + } + initialLen := len(prefs.AdvertiseServices) + prefs.AdvertiseServices = slices.DeleteFunc(prefs.AdvertiseServices, func(s string) bool { return s == serviceName.String() }) + if initialLen == len(prefs.AdvertiseServices) { + return nil // serviceName not advertised + } + _, err = e.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: prefs.AdvertiseServices, + }, + }) + return err +} + +func (e *serveEnv) runServeDrain(ctx context.Context, args []string) error { + if len(args) == 0 { + return errHelp + } + if len(args) != 1 { + fmt.Fprintf(Stderr, "error: invalid number of arguments\n\n") + return errHelp + } + svc := args[0] + svcName := tailcfg.ServiceName(svc) + if err := svcName.Validate(); err != nil { + return fmt.Errorf("invalid service name: %s", err) + } + return e.removeServiceFromPrefs(ctx, svcName) +} + const backgroundExistsMsg = "background configuration already exists, use `tailscale %s --%s=%d off` to remove the existing configuration" // validateConfig checks if the serve config is valid to serve the type wanted on the port. diff --git a/cmd/tailscale/cli/serve_v2_test.go b/cmd/tailscale/cli/serve_v2_test.go index b3e7ea773c698..2ba0b3f8434c8 100644 --- a/cmd/tailscale/cli/serve_v2_test.go +++ b/cmd/tailscale/cli/serve_v2_test.go @@ -1212,6 +1212,54 @@ func TestAddServiceToPrefs(t *testing.T) { } +func TestRemoveServiceFromPrefs(t *testing.T) { + tests := []struct { + name string + svcName tailcfg.ServiceName + startServices []string + expected []string + }{ + { + name: "remove service from empty prefs", + svcName: "svc:foo", + expected: []string{}, + }, + { + name: "remove existing service from prefs", + svcName: "svc:foo", + startServices: []string{"svc:foo"}, + expected: []string{}, + }, + { + name: "remove service not in prefs", + svcName: "svc:bar", + startServices: []string{"svc:foo"}, + expected: []string{"svc:foo"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lc := &fakeLocalServeClient{} + ctx := t.Context() + lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: tt.startServices, + }, + }) + e := &serveEnv{lc: lc, bg: bgBoolFlag{true, false}} + err := e.removeServiceFromPrefs(ctx, tt.svcName) + if err != nil { + t.Fatalf("removeServiceFromPrefs(%q) returned unexpected error: %v", tt.svcName, err) + } + if !slices.Equal(lc.prefs.AdvertiseServices, tt.expected) { + t.Errorf("removeServiceFromPrefs(%q) = %v, want %v", tt.svcName, lc.prefs.AdvertiseServices, tt.expected) + } + }) + } +} + func TestMessageForPort(t *testing.T) { svcIPMap := tailcfg.ServiceIPMappings{ "svc:foo": []netip.Addr{ From d1ceb62e2726ce0408a8376e22a27656dbb77d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Claus=20Lensb=C3=B8l?= Date: Thu, 17 Jul 2025 09:13:19 -0400 Subject: [PATCH 226/263] client/systray: look for ubuntu gnome MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ubuntu gnome has a different name on at least 25.04. Updates #1708 Signed-off-by: Claus Lensbøl --- client/systray/systray.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/systray/systray.go b/client/systray/systray.go index a87783c06ce5a..76c93ae18e781 100644 --- a/client/systray/systray.go +++ b/client/systray/systray.go @@ -128,7 +128,7 @@ func init() { desktop := strings.ToLower(os.Getenv("XDG_CURRENT_DESKTOP")) switch desktop { - case "gnome": + case "gnome", "ubuntu:gnome": // GNOME expands submenus downward in the main menu, rather than flyouts to the side. // Either as a result of that or another limitation, there seems to be a maximum depth of submenus. // Mullvad countries that have a city submenu are not being rendered, and so can't be selected. From 6c206fab58fc556b253e78547cc0073ef0c53975 Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Fri, 18 Jul 2025 10:17:40 -0700 Subject: [PATCH 227/263] feature/tpm: try opening /dev/tpmrm0 before /tmp/tpm0 on Linux (#16600) The tpmrm0 is a kernel-managed version of tpm0 that multiplexes multiple concurrent connections. The basic tpm0 can only be accessed by one application at a time, which can be pretty unreliable. Updates #15830 Signed-off-by: Andrew Lytvynov --- feature/tpm/tpm_linux.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/feature/tpm/tpm_linux.go b/feature/tpm/tpm_linux.go index f2d0f1402c16e..6c8131e8d8a28 100644 --- a/feature/tpm/tpm_linux.go +++ b/feature/tpm/tpm_linux.go @@ -9,5 +9,9 @@ import ( ) func open() (transport.TPMCloser, error) { + tpm, err := linuxtpm.Open("/dev/tpmrm0") + if err == nil { + return tpm, nil + } return linuxtpm.Open("/dev/tpm0") } From e01618a7c4eb5113f17f644b9b2ed8204c23a99b Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Fri, 18 Jul 2025 13:46:03 -0400 Subject: [PATCH 228/263] cmd/tailscale/cli: Add clear subcommand for serve services (#16509) * cmd/tailscale/cli: add clear subcommand for serve services This commit adds a clear subcommand for serve command, to remove all config for a passed service. This is a short cut for user to remove services after they drain a service. As an indipendent command it would avoid accidently remove a service on typo. Updates tailscale/corp#22954 Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * update regarding comments Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> * log when clearing a non-existing service but not error Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --------- Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- cmd/tailscale/cli/serve_v2.go | 36 ++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/cmd/tailscale/cli/serve_v2.go b/cmd/tailscale/cli/serve_v2.go index 6fa1a1b08c66c..8832a232d0f4e 100644 --- a/cmd/tailscale/cli/serve_v2.go +++ b/cmd/tailscale/cli/serve_v2.go @@ -213,6 +213,13 @@ func newServeV2Command(e *serveEnv, subcmd serveMode) *ffcli.Command { " should be a service name (e.g., svc:my-service).", Exec: e.runServeDrain, }, + { + Name: "clear", + ShortUsage: fmt.Sprintf("tailscale %s clear ", info.Name), + ShortHelp: "Remove all config for a service", + LongHelp: "Remove all handlers configured for the specified service.", + Exec: e.runServeClear, + }, }, } } @@ -486,11 +493,38 @@ func (e *serveEnv) runServeDrain(ctx context.Context, args []string) error { svc := args[0] svcName := tailcfg.ServiceName(svc) if err := svcName.Validate(); err != nil { - return fmt.Errorf("invalid service name: %s", err) + return fmt.Errorf("invalid service name: %w", err) } return e.removeServiceFromPrefs(ctx, svcName) } +func (e *serveEnv) runServeClear(ctx context.Context, args []string) error { + if len(args) == 0 { + return errHelp + } + if len(args) != 1 { + fmt.Fprintf(Stderr, "error: invalid number of arguments\n\n") + return errHelp + } + svc := tailcfg.ServiceName(args[0]) + if err := svc.Validate(); err != nil { + return fmt.Errorf("invalid service name: %w", err) + } + sc, err := e.lc.GetServeConfig(ctx) + if err != nil { + return fmt.Errorf("error getting serve config: %w", err) + } + if _, ok := sc.Services[svc]; !ok { + log.Printf("service %s not found in serve config, nothing to clear", svc) + return nil + } + delete(sc.Services, svc) + if err := e.removeServiceFromPrefs(ctx, svc); err != nil { + return fmt.Errorf("error removing service %s from prefs: %w", svc, err) + } + return e.lc.SetServeConfig(ctx, sc) +} + const backgroundExistsMsg = "background configuration already exists, use `tailscale %s --%s=%d off` to remove the existing configuration" // validateConfig checks if the serve config is valid to serve the type wanted on the port. From 5adde9e3f3f87cd9ce47832244aad49bcfb96bd8 Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Fri, 18 Jul 2025 15:06:09 -0400 Subject: [PATCH 229/263] cmd/tailscale/cli: remove advertise command (#16592) This commit removes the advertise command for service. The advertising is now embedded into serve command and unadvertising is moved to drain subcommand Fixes tailscale/corp#22954 Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- cmd/tailscale/cli/advertise.go | 76 ---------------------------------- cmd/tailscale/cli/cli.go | 1 - cmd/tailscale/cli/cli_test.go | 2 +- 3 files changed, 1 insertion(+), 78 deletions(-) delete mode 100644 cmd/tailscale/cli/advertise.go diff --git a/cmd/tailscale/cli/advertise.go b/cmd/tailscale/cli/advertise.go deleted file mode 100644 index 83d1a35aa8a14..0000000000000 --- a/cmd/tailscale/cli/advertise.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package cli - -import ( - "context" - "flag" - "fmt" - "strings" - - "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/envknob" - "tailscale.com/ipn" - "tailscale.com/tailcfg" -) - -var advertiseArgs struct { - services string // comma-separated list of services to advertise -} - -// TODO(naman): This flag may move to set.go or serve_v2.go after the WIPCode -// envknob is not needed. -func advertiseCmd() *ffcli.Command { - if !envknob.UseWIPCode() { - return nil - } - return &ffcli.Command{ - Name: "advertise", - ShortUsage: "tailscale advertise --services=", - ShortHelp: "Advertise this node as a destination for a service", - Exec: runAdvertise, - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("advertise") - fs.StringVar(&advertiseArgs.services, "services", "", "comma-separated services to advertise; each must start with \"svc:\" (e.g. \"svc:idp,svc:nas,svc:database\")") - return fs - })(), - } -} - -func runAdvertise(ctx context.Context, args []string) error { - if len(args) > 0 { - return flag.ErrHelp - } - - services, err := parseServiceNames(advertiseArgs.services) - if err != nil { - return err - } - - _, err = localClient.EditPrefs(ctx, &ipn.MaskedPrefs{ - AdvertiseServicesSet: true, - Prefs: ipn.Prefs{ - AdvertiseServices: services, - }, - }) - return err -} - -// parseServiceNames takes a comma-separated list of service names -// (eg. "svc:hello,svc:webserver,svc:catphotos"), splits them into -// a list and validates each service name. If valid, it returns -// the service names in a slice of strings. -func parseServiceNames(servicesArg string) ([]string, error) { - var services []string - if servicesArg != "" { - services = strings.Split(servicesArg, ",") - for _, svc := range services { - err := tailcfg.ServiceName(svc).Validate() - if err != nil { - return nil, fmt.Errorf("service %q: %s", svc, err) - } - } - } - return services, nil -} diff --git a/cmd/tailscale/cli/cli.go b/cmd/tailscale/cli/cli.go index d7e8e5ca22dce..bdfc7af423bf4 100644 --- a/cmd/tailscale/cli/cli.go +++ b/cmd/tailscale/cli/cli.go @@ -260,7 +260,6 @@ change in the future. debugCmd(), driveCmd, idTokenCmd, - advertiseCmd(), configureHostCmd(), ), FlagSet: rootfs, diff --git a/cmd/tailscale/cli/cli_test.go b/cmd/tailscale/cli/cli_test.go index 5dd4fa2340360..2e1bec8c9bcb0 100644 --- a/cmd/tailscale/cli/cli_test.go +++ b/cmd/tailscale/cli/cli_test.go @@ -964,7 +964,7 @@ func TestPrefFlagMapping(t *testing.T) { // flag for this. continue case "AdvertiseServices": - // Handled by the tailscale advertise subcommand, we don't want a + // Handled by the tailscale serve subcommand, we don't want a // CLI flag for this. continue case "InternalExitNodePrior": From f421907c38df057e1b293613644532f31e77b24b Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Mon, 21 Jul 2025 11:03:21 +0100 Subject: [PATCH 230/263] all-kube: create Tailscale Service for HA kube-apiserver ProxyGroup (#16572) Adds a new reconciler for ProxyGroups of type kube-apiserver that will provision a Tailscale Service for each replica to advertise. Adds two new condition types to the ProxyGroup, TailscaleServiceValid and TailscaleServiceConfigured, to post updates on the state of that reconciler in a way that's consistent with the service-pg reconciler. The created Tailscale Service name is configurable via a new ProxyGroup field spec.kubeAPISserver.ServiceName, which expects a string of the form "svc:". Lots of supporting changes were needed to implement this in a way that's consistent with other operator workflows, including: * Pulled containerboot's ensureServicesUnadvertised and certManager into kube/ libraries to be shared with k8s-proxy. Use those in k8s-proxy to aid Service cert sharing between replicas and graceful Service shutdown. * For certManager, add an initial wait to the cert loop to wait until the domain appears in the devices's netmap to avoid a guaranteed error on the first issue attempt when it's quick to start. * Made several methods in ingress-for-pg.go and svc-for-pg.go into functions to share with the new reconciler * Added a Resource struct to the owner refs stored in Tailscale Service annotations to be able to distinguish between Ingress- and ProxyGroup- based Services that need cleaning up in the Tailscale API. * Added a ListVIPServices method to the internal tailscale client to aid cleaning up orphaned Services * Support for reading config from a kube Secret, and partial support for config reloading, to prevent us having to force Pod restarts when config changes. * Fixed up the zap logger so it's possible to set debug log level. Updates #13358 Change-Id: Ia9607441157dd91fb9b6ecbc318eecbef446e116 Signed-off-by: Tom Proctor --- cmd/containerboot/main.go | 3 +- cmd/containerboot/serve.go | 10 +- cmd/k8s-operator/api-server-proxy-pg.go | 479 ++++++++++++++++++ cmd/k8s-operator/api-server-proxy-pg_test.go | 384 ++++++++++++++ .../{proxy.go => api-server-proxy.go} | 0 .../crds/tailscale.com_proxygroups.yaml | 56 +- .../deploy/manifests/operator.yaml | 56 +- cmd/k8s-operator/egress-eps_test.go | 3 +- cmd/k8s-operator/ingress-for-pg.go | 77 +-- cmd/k8s-operator/ingress-for-pg_test.go | 32 +- cmd/k8s-operator/operator.go | 70 ++- cmd/k8s-operator/proxygroup.go | 131 +++-- cmd/k8s-operator/proxygroup_specs.go | 41 +- cmd/k8s-operator/proxygroup_test.go | 162 +++++- cmd/k8s-operator/svc-for-pg.go | 18 +- cmd/k8s-operator/svc-for-pg_test.go | 13 +- cmd/k8s-operator/testutils_test.go | 8 +- cmd/k8s-operator/tsclient.go | 2 + cmd/k8s-proxy/internal/config/config.go | 264 ++++++++++ cmd/k8s-proxy/internal/config/config_test.go | 245 +++++++++ cmd/k8s-proxy/k8s-proxy.go | 268 ++++++++-- internal/client/tailscale/vip_service.go | 28 + ipn/store/kubestore/store_kube.go | 6 +- ipn/store/kubestore/store_kube_test.go | 7 +- k8s-operator/api-proxy/proxy.go | 65 ++- k8s-operator/api.md | 25 +- k8s-operator/apis/v1alpha1/types_connector.go | 3 + .../apis/v1alpha1/types_proxygroup.go | 54 +- k8s-operator/conditions.go | 10 + {cmd/containerboot => kube/certs}/certs.go | 107 ++-- .../certs}/certs_test.go | 35 +- kube/k8s-proxy/conf/conf.go | 54 +- kube/k8s-proxy/conf/conf_test.go | 9 +- kube/kubetypes/types.go | 6 + kube/localclient/fake-client.go | 35 ++ kube/localclient/local-client.go | 49 ++ .../services}/services.go | 20 +- kube/state/state.go | 16 +- kube/state/state_test.go | 97 ++-- 39 files changed, 2551 insertions(+), 397 deletions(-) create mode 100644 cmd/k8s-operator/api-server-proxy-pg.go create mode 100644 cmd/k8s-operator/api-server-proxy-pg_test.go rename cmd/k8s-operator/{proxy.go => api-server-proxy.go} (100%) create mode 100644 cmd/k8s-proxy/internal/config/config.go create mode 100644 cmd/k8s-proxy/internal/config/config_test.go rename {cmd/containerboot => kube/certs}/certs.go (60%) rename {cmd/containerboot => kube/certs}/certs_test.go (89%) create mode 100644 kube/localclient/fake-client.go create mode 100644 kube/localclient/local-client.go rename {cmd/containerboot => kube/services}/services.go (74%) diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 52b30b8375a4c..49c8a473a596d 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -122,6 +122,7 @@ import ( "tailscale.com/ipn" kubeutils "tailscale.com/k8s-operator" "tailscale.com/kube/kubetypes" + "tailscale.com/kube/services" "tailscale.com/tailcfg" "tailscale.com/types/logger" "tailscale.com/types/ptr" @@ -210,7 +211,7 @@ func run() error { ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) defer cancel() - if err := ensureServicesNotAdvertised(ctx, client); err != nil { + if err := services.EnsureServicesNotAdvertised(ctx, client, log.Printf); err != nil { log.Printf("Error ensuring services are not advertised: %v", err) } diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index 37fd497779c75..5fa8e580d5828 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -19,7 +19,9 @@ import ( "github.com/fsnotify/fsnotify" "tailscale.com/client/local" "tailscale.com/ipn" + "tailscale.com/kube/certs" "tailscale.com/kube/kubetypes" + klc "tailscale.com/kube/localclient" "tailscale.com/types/netmap" ) @@ -52,11 +54,9 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom var certDomain string var prevServeConfig *ipn.ServeConfig - var cm certManager + var cm *certs.CertManager if cfg.CertShareMode == "rw" { - cm = certManager{ - lc: lc, - } + cm = certs.NewCertManager(klc.New(lc), log.Printf) } for { select { @@ -93,7 +93,7 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom if cfg.CertShareMode != "rw" { continue } - if err := cm.ensureCertLoops(ctx, sc); err != nil { + if err := cm.EnsureCertLoops(ctx, sc); err != nil { log.Fatalf("serve proxy: error ensuring cert loops: %v", err) } } diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go new file mode 100644 index 0000000000000..252859eb37197 --- /dev/null +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -0,0 +1,479 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strings" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/internal/client/tailscale" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" + "tailscale.com/tstime" +) + +const ( + proxyPGFinalizerName = "tailscale.com/kube-apiserver-finalizer" + + // Reasons for KubeAPIServerProxyValid condition. + reasonKubeAPIServerProxyInvalid = "KubeAPIServerProxyInvalid" + reasonKubeAPIServerProxyValid = "KubeAPIServerProxyValid" + + // Reasons for KubeAPIServerProxyConfigured condition. + reasonKubeAPIServerProxyConfigured = "KubeAPIServerProxyConfigured" + reasonKubeAPIServerProxyNoBackends = "KubeAPIServerProxyNoBackends" +) + +// KubeAPIServerTSServiceReconciler reconciles the Tailscale Services required for an +// HA deployment of the API Server Proxy. +type KubeAPIServerTSServiceReconciler struct { + client.Client + recorder record.EventRecorder + logger *zap.SugaredLogger + tsClient tsClient + tsNamespace string + lc localClient + defaultTags []string + operatorID string // stableID of the operator's Tailscale device + + clock tstime.Clock +} + +// Reconcile is the entry point for the controller. +func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { + logger := r.logger.With("ProxyGroup", req.Name) + logger.Debugf("starting reconcile") + defer logger.Debugf("reconcile finished") + + pg := new(tsapi.ProxyGroup) + err = r.Get(ctx, req.NamespacedName, pg) + if apierrors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + logger.Debugf("ProxyGroup not found, assuming it was deleted") + return res, nil + } else if err != nil { + return res, fmt.Errorf("failed to get ProxyGroup: %w", err) + } + + serviceName := serviceNameForAPIServerProxy(pg) + logger = logger.With("Tailscale Service", serviceName) + + if markedForDeletion(pg) { + logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up") + if err = r.maybeCleanup(ctx, serviceName, pg, logger); err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error, retrying: %s", err) + return res, nil + } + + return res, err + } + + err = r.maybeProvision(ctx, serviceName, pg, logger) + if err != nil { + if strings.Contains(err.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error, retrying: %s", err) + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +// maybeProvision ensures that a Tailscale Service for this ProxyGroup exists +// and is up to date. +// +// Returns true if the operation resulted in a Tailscale Service update. +func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (err error) { + var dnsName string + oldPGStatus := pg.Status.DeepCopy() + defer func() { + podsAdvertising, podsErr := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName) + if podsErr != nil { + err = errors.Join(err, fmt.Errorf("failed to get number of advertised Pods: %w", podsErr)) + // Continue, updating the status with the best available information. + } + + // Update the ProxyGroup status with the Tailscale Service information + // Update the condition based on how many pods are advertising the service + conditionStatus := metav1.ConditionFalse + conditionReason := reasonKubeAPIServerProxyNoBackends + conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", podsAdvertising, pgReplicas(pg)) + + pg.Status.URL = "" + if podsAdvertising > 0 { + // At least one pod is advertising the service, consider it configured + conditionStatus = metav1.ConditionTrue + conditionReason = reasonKubeAPIServerProxyConfigured + if dnsName != "" { + pg.Status.URL = "https://" + dnsName + } + } + + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, conditionStatus, conditionReason, conditionMessage, pg.Generation, r.clock, logger) + + if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) { + // An error encountered here should get returned by the Reconcile function. + err = errors.Join(err, r.Client.Status().Update(ctx, pg)) + } + }() + + if !tsoperator.ProxyGroupAvailable(pg) { + return nil + } + + if !slices.Contains(pg.Finalizers, proxyPGFinalizerName) { + // This log line is printed exactly once during initial provisioning, + // because once the finalizer is in place this block gets skipped. So, + // this is a nice place to tell the operator that the high level, + // multi-reconcile operation is underway. + logger.Info("provisioning Tailscale Service for ProxyGroup") + pg.Finalizers = append(pg.Finalizers, proxyPGFinalizerName) + if err := r.Update(ctx, pg); err != nil { + return fmt.Errorf("failed to add finalizer: %w", err) + } + } + + // 1. Check there isn't a Tailscale Service with the same hostname + // already created and not owned by this ProxyGroup. + existingTSSvc, err := r.tsClient.GetVIPService(ctx, serviceName) + if isErrorFeatureFlagNotEnabled(err) { + logger.Warn(msgFeatureFlagNotEnabled) + r.recorder.Event(pg, corev1.EventTypeWarning, warningTailscaleServiceFeatureFlagNotEnabled, msgFeatureFlagNotEnabled) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msgFeatureFlagNotEnabled, pg.Generation, r.clock, logger) + return nil + } + if err != nil && !isErrorTailscaleServiceNotFound(err) { + return fmt.Errorf("error getting Tailscale Service %q: %w", serviceName, err) + } + + updatedAnnotations, err := exclusiveOwnerAnnotations(pg, r.operatorID, existingTSSvc) + if err != nil { + const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different Service name in the ProxyGroup's spec.kubeAPIServer.serviceName field" + msg := fmt.Sprintf("error ensuring exclusive ownership of Tailscale Service %s: %v. %s", serviceName, err, instr) + logger.Warn(msg) + r.recorder.Event(pg, corev1.EventTypeWarning, "InvalidTailscaleService", msg) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msg, pg.Generation, r.clock, logger) + return nil + } + + // After getting this far, we know the Tailscale Service is valid. + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, reasonKubeAPIServerProxyValid, pg.Generation, r.clock, logger) + + // Service tags are limited to matching the ProxyGroup's tags until we have + // support for querying peer caps for a Service-bound request. + serviceTags := r.defaultTags + if len(pg.Spec.Tags) > 0 { + serviceTags = pg.Spec.Tags.Stringify() + } + + tsSvc := &tailscale.VIPService{ + Name: serviceName, + Tags: serviceTags, + Ports: []string{"tcp:443"}, + Comment: managedTSServiceComment, + Annotations: updatedAnnotations, + } + if existingTSSvc != nil { + tsSvc.Addrs = existingTSSvc.Addrs + } + + // 2. Ensure the Tailscale Service exists and is up to date. + if existingTSSvc == nil || + !slices.Equal(tsSvc.Tags, existingTSSvc.Tags) || + !ownersAreSetAndEqual(tsSvc, existingTSSvc) || + !slices.Equal(tsSvc.Ports, existingTSSvc.Ports) { + logger.Infof("Ensuring Tailscale Service exists and is up to date") + if err := r.tsClient.CreateOrUpdateVIPService(ctx, tsSvc); err != nil { + return fmt.Errorf("error creating Tailscale Service: %w", err) + } + } + + // 3. Ensure that TLS Secret and RBAC exists. + tcd, err := tailnetCertDomain(ctx, r.lc) + if err != nil { + return fmt.Errorf("error determining DNS name base: %w", err) + } + dnsName = serviceName.WithoutPrefix() + "." + tcd + if err = r.ensureCertResources(ctx, pg, dnsName); err != nil { + return fmt.Errorf("error ensuring cert resources: %w", err) + } + + // 4. Configure the Pods to advertise the Tailscale Service. + if err = r.maybeAdvertiseServices(ctx, pg, serviceName, logger); err != nil { + return fmt.Errorf("error updating advertised Tailscale Services: %w", err) + } + + // 5. Clean up any stale Tailscale Services from previous resource versions. + if err = r.maybeDeleteStaleServices(ctx, pg, logger); err != nil { + return fmt.Errorf("failed to delete stale Tailscale Services: %w", err) + } + + return nil +} + +// maybeCleanup ensures that any resources, such as a Tailscale Service created for this Service, are cleaned up when the +// Service is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup- the Tailscale Service is only +// deleted if it does not contain any other owner references. If it does, the cleanup only removes the owner reference +// corresponding to this Service. +func (r *KubeAPIServerTSServiceReconciler) maybeCleanup(ctx context.Context, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (err error) { + ix := slices.Index(pg.Finalizers, proxyPGFinalizerName) + if ix < 0 { + logger.Debugf("no finalizer, nothing to do") + return nil + } + logger.Infof("Ensuring that Service %q is cleaned up", serviceName) + + defer func() { + if err == nil { + err = r.deleteFinalizer(ctx, pg, logger) + } + }() + + if _, err = cleanupTailscaleService(ctx, r.tsClient, serviceName, r.operatorID, logger); err != nil { + return fmt.Errorf("error deleting Tailscale Service: %w", err) + } + + if err = cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg.Name, serviceName); err != nil { + return fmt.Errorf("failed to clean up cert resources: %w", err) + } + + return nil +} + +// maybeDeleteStaleServices deletes Services that have previously been created for +// this ProxyGroup but are no longer needed. +func (r *KubeAPIServerTSServiceReconciler) maybeDeleteStaleServices(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) error { + serviceName := serviceNameForAPIServerProxy(pg) + + svcs, err := r.tsClient.ListVIPServices(ctx) + if err != nil { + return fmt.Errorf("error listing Tailscale Services: %w", err) + } + + for _, svc := range svcs.VIPServices { + if svc.Name == serviceName { + continue + } + + owners, err := parseOwnerAnnotation(&svc) + if err != nil { + logger.Warnf("error parsing owner annotation for Tailscale Service %s: %v", svc.Name, err) + continue + } + if owners == nil || len(owners.OwnerRefs) != 1 || owners.OwnerRefs[0].OperatorID != r.operatorID { + continue + } + + owner := owners.OwnerRefs[0] + if owner.Resource == nil || owner.Resource.Kind != "ProxyGroup" || owner.Resource.UID != string(pg.UID) { + continue + } + + logger.Infof("Deleting Tailscale Service %s", svc.Name) + if err := r.tsClient.DeleteVIPService(ctx, svc.Name); err != nil && !isErrorTailscaleServiceNotFound(err) { + return fmt.Errorf("error deleting Tailscale Service %s: %w", svc.Name, err) + } + + if err = cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg.Name, svc.Name); err != nil { + return fmt.Errorf("failed to clean up cert resources: %w", err) + } + } + + return nil +} + +func (r *KubeAPIServerTSServiceReconciler) deleteFinalizer(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) error { + pg.Finalizers = slices.DeleteFunc(pg.Finalizers, func(f string) bool { + return f == proxyPGFinalizerName + }) + logger.Debugf("ensure %q finalizer is removed", proxyPGFinalizerName) + + if err := r.Update(ctx, pg); err != nil { + return fmt.Errorf("failed to remove finalizer %q: %w", proxyPGFinalizerName, err) + } + return nil +} + +func (r *KubeAPIServerTSServiceReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string) error { + secret := certSecret(pg.Name, r.tsNamespace, domain, pg) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) { + s.Labels = secret.Labels + }); err != nil { + return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err) + } + role := certSecretRole(pg.Name, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { + r.Labels = role.Labels + r.Rules = role.Rules + }); err != nil { + return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err) + } + rolebinding := certSecretRoleBinding(pg, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) { + rb.Labels = rolebinding.Labels + rb.Subjects = rolebinding.Subjects + rb.RoleRef = rolebinding.RoleRef + }); err != nil { + return fmt.Errorf("failed to create or update RoleBinding %s: %w", rolebinding.Name, err) + } + return nil +} + +func (r *KubeAPIServerTSServiceReconciler) maybeAdvertiseServices(ctx context.Context, pg *tsapi.ProxyGroup, serviceName tailcfg.ServiceName, logger *zap.SugaredLogger) error { + // Get all config Secrets for this ProxyGroup + cfgSecrets := &corev1.SecretList{} + if err := r.List(ctx, cfgSecrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil { + return fmt.Errorf("failed to list config Secrets: %w", err) + } + + // Only advertise a Tailscale Service once the TLS certs required for + // serving it are available. + shouldBeAdvertised, err := hasCerts(ctx, r.Client, r.lc, r.tsNamespace, serviceName) + if err != nil { + return fmt.Errorf("error checking TLS credentials provisioned for Tailscale Service %q: %w", serviceName, err) + } + var advertiseServices []string + if shouldBeAdvertised { + advertiseServices = []string{serviceName.String()} + } + + for _, s := range cfgSecrets.Items { + if len(s.Data[kubetypes.KubeAPIServerConfigFile]) == 0 { + continue + } + + // Parse the existing config. + cfg, err := conf.Load(s.Data[kubetypes.KubeAPIServerConfigFile]) + if err != nil { + return fmt.Errorf("error loading config from Secret %q: %w", s.Name, err) + } + + if cfg.Parsed.APIServerProxy == nil { + return fmt.Errorf("config Secret %q does not contain APIServerProxy config", s.Name) + } + + existingCfgSecret := s.DeepCopy() + + var updated bool + if cfg.Parsed.APIServerProxy.ServiceName == nil || *cfg.Parsed.APIServerProxy.ServiceName != serviceName { + cfg.Parsed.APIServerProxy.ServiceName = &serviceName + updated = true + } + + // Update the services to advertise if required. + if !slices.Equal(cfg.Parsed.AdvertiseServices, advertiseServices) { + cfg.Parsed.AdvertiseServices = advertiseServices + updated = true + } + + if !updated { + continue + } + + // Update the config Secret. + cfgB, err := json.Marshal(conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &cfg.Parsed, + }) + if err != nil { + return err + } + + s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB + if !apiequality.Semantic.DeepEqual(existingCfgSecret, s) { + logger.Debugf("Updating the Tailscale Services in ProxyGroup config Secret %s", s.Name) + if err := r.Update(ctx, &s); err != nil { + return err + } + } + } + + return nil +} + +func serviceNameForAPIServerProxy(pg *tsapi.ProxyGroup) tailcfg.ServiceName { + if pg.Spec.KubeAPIServer != nil && pg.Spec.KubeAPIServer.Hostname != "" { + return tailcfg.ServiceName("svc:" + pg.Spec.KubeAPIServer.Hostname) + } + + return tailcfg.ServiceName("svc:" + pg.Name) +} + +// exclusiveOwnerAnnotations returns the updated annotations required to ensure this +// instance of the operator is the exclusive owner. If the Tailscale Service is not +// nil, but does not contain an owner reference we return an error as this likely means +// that the Service was created by something other than a Tailscale Kubernetes operator. +// We also error if it is already owned by another operator instance, as we do not +// want to load balance a kube-apiserver ProxyGroup across multiple clusters. +func exclusiveOwnerAnnotations(pg *tsapi.ProxyGroup, operatorID string, svc *tailscale.VIPService) (map[string]string, error) { + ref := OwnerRef{ + OperatorID: operatorID, + Resource: &Resource{ + Kind: "ProxyGroup", + Name: pg.Name, + UID: string(pg.UID), + }, + } + if svc == nil { + c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} + json, err := json.Marshal(c) + if err != nil { + return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service's owner annotation contents: %w, please report this", err) + } + return map[string]string{ + ownerAnnotation: string(json), + }, nil + } + o, err := parseOwnerAnnotation(svc) + if err != nil { + return nil, err + } + if o == nil || len(o.OwnerRefs) == 0 { + return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name) + } + if len(o.OwnerRefs) > 1 || o.OwnerRefs[0].OperatorID != operatorID { + return nil, fmt.Errorf("Tailscale Service %s is already owned by other operator(s) and cannot be shared across multiple clusters; configure a difference Service name to continue", svc.Name) + } + if o.OwnerRefs[0].Resource == nil { + return nil, fmt.Errorf("Tailscale Service %s exists, but does not reference an owning resource; not proceeding as this is likely a Service already owned by an Ingress", svc.Name) + } + if o.OwnerRefs[0].Resource.Kind != "ProxyGroup" || o.OwnerRefs[0].Resource.UID != string(pg.UID) { + return nil, fmt.Errorf("Tailscale Service %s is already owned by another resource: %#v; configure a difference Service name to continue", svc.Name, o.OwnerRefs[0].Resource) + } + if o.OwnerRefs[0].Resource.Name != pg.Name { + // ProxyGroup name can be updated in place. + o.OwnerRefs[0].Resource.Name = pg.Name + } + + oBytes, err := json.Marshal(o) + if err != nil { + return nil, err + } + + newAnnots := make(map[string]string, len(svc.Annotations)+1) + maps.Copy(newAnnots, svc.Annotations) + newAnnots[ownerAnnotation] = string(oBytes) + + return newAnnots, nil +} diff --git a/cmd/k8s-operator/api-server-proxy-pg_test.go b/cmd/k8s-operator/api-server-proxy-pg_test.go new file mode 100644 index 0000000000000..dfef63f22ff04 --- /dev/null +++ b/cmd/k8s-operator/api-server-proxy-pg_test.go @@ -0,0 +1,384 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "tailscale.com/internal/client/tailscale" + "tailscale.com/ipn/ipnstate" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" + "tailscale.com/tstest" + "tailscale.com/types/opt" + "tailscale.com/types/ptr" +) + +func TestAPIServerProxyReconciler(t *testing.T) { + const ( + pgName = "test-pg" + pgUID = "test-pg-uid" + ns = "operator-ns" + defaultDomain = "test-pg.ts.net" + ) + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgName, + Generation: 1, + UID: pgUID, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + }, + Status: tsapi.ProxyGroupStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.ProxyGroupAvailable), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + }, + }, + }, + } + initialCfg := &conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("test-key"), + APIServerProxy: &conf.APIServerProxyConfig{ + Enabled: opt.NewBool(true), + }, + }, + } + expectedCfg := *initialCfg + initialCfgB, err := json.Marshal(initialCfg) + if err != nil { + t.Fatalf("marshaling initial config: %v", err) + } + pgCfgSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgConfigSecretName(pgName, 0), + Namespace: ns, + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig), + }, + Data: map[string][]byte{ + // Existing config should be preserved. + kubetypes.KubeAPIServerConfigFile: initialCfgB, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg, pgCfgSecret). + WithStatusSubresource(pg). + Build() + expectCfg := func(c *conf.VersionedConfig) { + t.Helper() + cBytes, err := json.Marshal(c) + if err != nil { + t.Fatalf("marshaling expected config: %v", err) + } + pgCfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cBytes + expectEqual(t, fc, pgCfgSecret) + } + + ft := &fakeTSClient{} + ingressTSSvc := &tailscale.VIPService{ + Name: "svc:some-ingress-hostname", + Comment: managedTSServiceComment, + Annotations: map[string]string{ + // No resource field. + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, + }, + Ports: []string{"tcp:443"}, + Tags: []string{"tag:k8s"}, + Addrs: []string{"5.6.7.8"}, + } + ft.CreateOrUpdateVIPService(t.Context(), ingressTSSvc) + + lc := &fakeLocalClient{ + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{ + MagicDNSSuffix: "ts.net", + }, + }, + } + + r := &KubeAPIServerTSServiceReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + tsNamespace: ns, + logger: zap.Must(zap.NewDevelopment()).Sugar(), + recorder: record.NewFakeRecorder(10), + lc: lc, + clock: tstest.NewClock(tstest.ClockOpts{}), + operatorID: "self-id", + } + + // Create a Tailscale Service that will conflict with the initial config. + if err := ft.CreateOrUpdateVIPService(t.Context(), &tailscale.VIPService{ + Name: "svc:" + pgName, + }); err != nil { + t.Fatalf("creating initial Tailscale Service: %v", err) + } + expectReconciled(t, r, "", pgName) + pg.ObjectMeta.Finalizers = []string{proxyPGFinalizerName} + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, "", 1, r.clock, r.logger) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + expectMissing[corev1.Secret](t, fc, ns, defaultDomain) + expectMissing[rbacv1.Role](t, fc, ns, defaultDomain) + expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain) + expectEqual(t, fc, pgCfgSecret) // Unchanged. + + // Delete Tailscale Service; should see Service created and valid condition updated to true. + if err := ft.DeleteVIPService(t.Context(), "svc:"+pgName); err != nil { + t.Fatalf("deleting initial Tailscale Service: %v", err) + } + expectReconciled(t, r, "", pgName) + + tsSvc, err := ft.GetVIPService(t.Context(), "svc:"+pgName) + if err != nil { + t.Fatalf("getting Tailscale Service: %v", err) + } + if tsSvc == nil { + t.Fatalf("expected Tailscale Service to be created, but got nil") + } + expectedTSSvc := &tailscale.VIPService{ + Name: "svc:" + pgName, + Comment: managedTSServiceComment, + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"test-pg-uid"}}]}`, + }, + Ports: []string{"tcp:443"}, + Tags: []string{"tag:k8s"}, + Addrs: []string{"5.6.7.8"}, + } + if !reflect.DeepEqual(tsSvc, expectedTSSvc) { + t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc) + } + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.logger) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + expectedCfg.APIServerProxy.ServiceName = ptr.To(tailcfg.ServiceName("svc:" + pgName)) + expectCfg(&expectedCfg) + + expectEqual(t, fc, certSecret(pgName, ns, defaultDomain, pg)) + expectEqual(t, fc, certSecretRole(pgName, ns, defaultDomain)) + expectEqual(t, fc, certSecretRoleBinding(pg, ns, defaultDomain)) + + // Simulate certs being issued; should observe AdvertiseServices config change. + if err := populateTLSSecret(t.Context(), fc, pgName, defaultDomain); err != nil { + t.Fatalf("populating TLS Secret: %v", err) + } + expectReconciled(t, r, "", pgName) + + expectedCfg.AdvertiseServices = []string{"svc:" + pgName} + expectCfg(&expectedCfg) + + expectEqual(t, fc, pg, omitPGStatusConditionMessages) // Unchanged status. + + // Simulate Pod prefs updated with advertised services; should see Configured condition updated to true. + mustCreate(t, fc, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg-0", + Namespace: ns, + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeState), + }, + Data: map[string][]byte{ + "_current-profile": []byte("profile-foo"), + "profile-foo": []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`), + }, + }) + expectReconciled(t, r, "", pgName) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger) + pg.Status.URL = "https://" + defaultDomain + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Rename the Tailscale Service - old one + cert resources should be cleaned up. + updatedServiceName := tailcfg.ServiceName("svc:test-pg-renamed") + updatedDomain := "test-pg-renamed.ts.net" + pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{ + Hostname: updatedServiceName.WithoutPrefix(), + } + mustUpdate(t, fc, "", pgName, func(p *tsapi.ProxyGroup) { + p.Spec.KubeAPIServer = pg.Spec.KubeAPIServer + }) + expectReconciled(t, r, "", pgName) + _, err = ft.GetVIPService(t.Context(), "svc:"+pgName) + if !isErrorTailscaleServiceNotFound(err) { + t.Fatalf("Expected 404, got: %v", err) + } + tsSvc, err = ft.GetVIPService(t.Context(), updatedServiceName) + if err != nil { + t.Fatalf("Expected renamed svc, got error: %v", err) + } + expectedTSSvc.Name = updatedServiceName + if !reflect.DeepEqual(tsSvc, expectedTSSvc) { + t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc) + } + // Check cfg and status reset until TLS certs are available again. + expectedCfg.APIServerProxy.ServiceName = ptr.To(updatedServiceName) + expectedCfg.AdvertiseServices = nil + expectCfg(&expectedCfg) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + pg.Status.URL = "" + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + expectEqual(t, fc, certSecret(pgName, ns, updatedDomain, pg)) + expectEqual(t, fc, certSecretRole(pgName, ns, updatedDomain)) + expectEqual(t, fc, certSecretRoleBinding(pg, ns, updatedDomain)) + expectMissing[corev1.Secret](t, fc, ns, defaultDomain) + expectMissing[rbacv1.Role](t, fc, ns, defaultDomain) + expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain) + + // Check we get the new hostname in the status once ready. + if err := populateTLSSecret(t.Context(), fc, pgName, updatedDomain); err != nil { + t.Fatalf("populating TLS Secret: %v", err) + } + mustUpdate(t, fc, "operator-ns", "test-pg-0", func(s *corev1.Secret) { + s.Data["profile-foo"] = []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`) + }) + expectReconciled(t, r, "", pgName) + expectedCfg.AdvertiseServices = []string{updatedServiceName.String()} + expectCfg(&expectedCfg) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger) + pg.Status.URL = "https://" + updatedDomain + + // Delete the ProxyGroup and verify Tailscale Service and cert resources are cleaned up. + if err := fc.Delete(t.Context(), pg); err != nil { + t.Fatalf("deleting ProxyGroup: %v", err) + } + expectReconciled(t, r, "", pgName) + expectMissing[corev1.Secret](t, fc, ns, updatedDomain) + expectMissing[rbacv1.Role](t, fc, ns, updatedDomain) + expectMissing[rbacv1.RoleBinding](t, fc, ns, updatedDomain) + _, err = ft.GetVIPService(t.Context(), updatedServiceName) + if !isErrorTailscaleServiceNotFound(err) { + t.Fatalf("Expected 404, got: %v", err) + } + + // Ingress Tailscale Service should not be affected. + svc, err := ft.GetVIPService(t.Context(), ingressTSSvc.Name) + if err != nil { + t.Fatalf("getting ingress Tailscale Service: %v", err) + } + if !reflect.DeepEqual(svc, ingressTSSvc) { + t.Fatalf("expected ingress Tailscale Service to be unmodified %+v, got %+v", ingressTSSvc, svc) + } +} + +func TestExclusiveOwnerAnnotations(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pg1", + UID: "pg1-uid", + }, + } + const ( + selfOperatorID = "self-id" + pg1Owner = `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg1","uid":"pg1-uid"}}]}` + ) + + for name, tc := range map[string]struct { + svc *tailscale.VIPService + wantErr string + }{ + "no_svc": { + svc: nil, + }, + "empty_svc": { + svc: &tailscale.VIPService{}, + wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator", + }, + "already_owner": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: pg1Owner, + }, + }, + }, + "already_owner_name_updated": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"old-pg1-name","uid":"pg1-uid"}}]}`, + }, + }, + }, + "preserves_existing_annotations": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + "existing": "annotation", + ownerAnnotation: pg1Owner, + }, + }, + }, + "owned_by_another_operator": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`, + }, + }, + wantErr: "already owned by other operator(s)", + }, + "owned_by_an_ingress": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, // Ingress doesn't set Resource field (yet). + }, + }, + wantErr: "does not reference an owning resource", + }, + "owned_by_another_pg": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg2","uid":"pg2-uid"}}]}`, + }, + }, + wantErr: "already owned by another resource", + }, + } { + t.Run(name, func(t *testing.T) { + got, err := exclusiveOwnerAnnotations(pg, "self-id", tc.svc) + if tc.wantErr != "" { + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("exclusiveOwnerAnnotations() error = %v, wantErr %v", err, tc.wantErr) + } + } else if diff := cmp.Diff(pg1Owner, got[ownerAnnotation]); diff != "" { + t.Errorf("exclusiveOwnerAnnotations() mismatch (-want +got):\n%s", diff) + } + if tc.svc == nil { + return // Don't check annotations being preserved. + } + for k, v := range tc.svc.Annotations { + if k == ownerAnnotation { + continue + } + if got[k] != v { + t.Errorf("exclusiveOwnerAnnotations() did not preserve annotation %q: got %q, want %q", k, got[k], v) + } + } + }) + } +} + +func omitPGStatusConditionMessages(p *tsapi.ProxyGroup) { + for i := range p.Status.Conditions { + // Don't bother validating the message. + p.Status.Conditions[i].Message = "" + } +} diff --git a/cmd/k8s-operator/proxy.go b/cmd/k8s-operator/api-server-proxy.go similarity index 100% rename from cmd/k8s-operator/proxy.go rename to cmd/k8s-operator/api-server-proxy.go diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index 06c8479252873..98ca1c378ab8d 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -20,6 +20,10 @@ spec: jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason name: Status type: string + - description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver. + jsonPath: .status.url + name: URL + type: string - description: ProxyGroup type. jsonPath: .spec.type name: Type @@ -32,15 +36,22 @@ spec: openAPIV3Schema: description: |- ProxyGroup defines a set of Tailscale devices that will act as proxies. - Currently only egress ProxyGroups are supported. + Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver + proxies. In addition to running a highly available set of proxies, ingress + and egress ProxyGroups also allow for serving many annotated Services from a + single set of proxies to minimise resource consumption. + + For ingress and egress, use the tailscale.com/proxy-group annotation on a + Service to specify that the proxy should be implemented by a ProxyGroup + instead of a single dedicated proxy. - Use the tailscale.com/proxy-group annotation on a Service to specify that - the egress proxy should be implemented by a ProxyGroup instead of a single - dedicated proxy. In addition to running a highly available set of proxies, - ProxyGroup also allows for serving many annotated Services from a single - set of proxies to minimise resource consumption. + More info: + * https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + * https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress - More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + For kube-apiserver, the ProxyGroup is a standalone resource. Use the + spec.kubeAPIServer field to configure options specific to the kube-apiserver + ProxyGroup type. type: object required: - spec @@ -83,6 +94,14 @@ spec: ProxyGroup type. This field is only used when Type is set to "kube-apiserver". type: object properties: + hostname: + description: |- + Hostname is the hostname with which to expose the Kubernetes API server + proxies. Must be a valid DNS label no longer than 63 characters. If not + specified, the name of the ProxyGroup is used as the hostname. Must be + unique across the whole tailnet. + type: string + pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ mode: description: |- Mode to run the API server proxy in. Supported modes are auth and noauth. @@ -141,10 +160,20 @@ spec: conditions: description: |- List of status conditions to indicate the status of the ProxyGroup - resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`. - `ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled - and ready. `ProxyGroupAvailable` indicates that at least one proxy is - ready to serve traffic. + resources. Known condition types include `ProxyGroupReady` and + `ProxyGroupAvailable`. + + * `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and + all expected conditions are true. + * `ProxyGroupAvailable` indicates that at least one proxy is ready to + serve traffic. + + For ProxyGroups of type kube-apiserver, there are two additional conditions: + + * `KubeAPIServerProxyConfigured` indicates that at least one API server + proxy is configured and ready to serve traffic. + * `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is + valid. type: array items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -231,6 +260,11 @@ spec: x-kubernetes-list-map-keys: - hostname x-kubernetes-list-type: map + url: + description: |- + URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if + any. Only applies to ProxyGroups of type kube-apiserver. + type: string served: true storage: true subresources: diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index ff3705cb343ff..ac8143e98c22b 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -2873,6 +2873,10 @@ spec: jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason name: Status type: string + - description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver. + jsonPath: .status.url + name: URL + type: string - description: ProxyGroup type. jsonPath: .spec.type name: Type @@ -2885,15 +2889,22 @@ spec: openAPIV3Schema: description: |- ProxyGroup defines a set of Tailscale devices that will act as proxies. - Currently only egress ProxyGroups are supported. + Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver + proxies. In addition to running a highly available set of proxies, ingress + and egress ProxyGroups also allow for serving many annotated Services from a + single set of proxies to minimise resource consumption. + + For ingress and egress, use the tailscale.com/proxy-group annotation on a + Service to specify that the proxy should be implemented by a ProxyGroup + instead of a single dedicated proxy. - Use the tailscale.com/proxy-group annotation on a Service to specify that - the egress proxy should be implemented by a ProxyGroup instead of a single - dedicated proxy. In addition to running a highly available set of proxies, - ProxyGroup also allows for serving many annotated Services from a single - set of proxies to minimise resource consumption. + More info: + * https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + * https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress - More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + For kube-apiserver, the ProxyGroup is a standalone resource. Use the + spec.kubeAPIServer field to configure options specific to the kube-apiserver + ProxyGroup type. properties: apiVersion: description: |- @@ -2929,6 +2940,14 @@ spec: KubeAPIServer contains configuration specific to the kube-apiserver ProxyGroup type. This field is only used when Type is set to "kube-apiserver". properties: + hostname: + description: |- + Hostname is the hostname with which to expose the Kubernetes API server + proxies. Must be a valid DNS label no longer than 63 characters. If not + specified, the name of the ProxyGroup is used as the hostname. Must be + unique across the whole tailnet. + pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ + type: string mode: description: |- Mode to run the API server proxy in. Supported modes are auth and noauth. @@ -2990,10 +3009,20 @@ spec: conditions: description: |- List of status conditions to indicate the status of the ProxyGroup - resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`. - `ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled - and ready. `ProxyGroupAvailable` indicates that at least one proxy is - ready to serve traffic. + resources. Known condition types include `ProxyGroupReady` and + `ProxyGroupAvailable`. + + * `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and + all expected conditions are true. + * `ProxyGroupAvailable` indicates that at least one proxy is ready to + serve traffic. + + For ProxyGroups of type kube-apiserver, there are two additional conditions: + + * `KubeAPIServerProxyConfigured` indicates that at least one API server + proxy is configured and ready to serve traffic. + * `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is + valid. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: @@ -3080,6 +3109,11 @@ spec: x-kubernetes-list-map-keys: - hostname x-kubernetes-list-type: map + url: + description: |- + URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if + any. Only applies to ProxyGroups of type kube-apiserver. + type: string type: object required: - spec diff --git a/cmd/k8s-operator/egress-eps_test.go b/cmd/k8s-operator/egress-eps_test.go index bd81071cb5e4f..bd80112aeb8a2 100644 --- a/cmd/k8s-operator/egress-eps_test.go +++ b/cmd/k8s-operator/egress-eps_test.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" + "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/util/mak" ) @@ -200,7 +201,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-0", pg), Namespace: "operator-ns", - Labels: pgSecretLabels(pg, "state"), + Labels: pgSecretLabels(pg, kubetypes.LabelSecretTypeState), }, } return p, s diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index aaf22d471353f..3afeb528f7f8f 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -248,7 +248,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin return false, nil } // 3. Ensure that TLS Secret and RBAC exists - tcd, err := r.tailnetCertDomain(ctx) + tcd, err := tailnetCertDomain(ctx, r.lc) if err != nil { return false, fmt.Errorf("error determining DNS name base: %w", err) } @@ -358,7 +358,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } // 6. Update Ingress status if ProxyGroup Pods are ready. - count, err := r.numberPodsAdvertising(ctx, pg.Name, serviceName) + count, err := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName) if err != nil { return false, fmt.Errorf("failed to check if any Pods are configured: %w", err) } @@ -370,7 +370,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin ing.Status.LoadBalancer.Ingress = nil default: var ports []networkingv1.IngressPortStatus - hasCerts, err := r.hasCerts(ctx, serviceName) + hasCerts, err := hasCerts(ctx, r.Client, r.lc, r.tsNamespace, serviceName) if err != nil { return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err) } @@ -481,7 +481,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG delete(cfg.Services, tsSvcName) serveConfigChanged = true } - if err := r.cleanupCertResources(ctx, proxyGroupName, tsSvcName); err != nil { + if err := cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, proxyGroupName, tsSvcName); err != nil { return false, fmt.Errorf("failed to clean up cert resources: %w", err) } } @@ -557,7 +557,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, } // 3. Clean up any cluster resources - if err := r.cleanupCertResources(ctx, pg, serviceName); err != nil { + if err := cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg, serviceName); err != nil { return false, fmt.Errorf("failed to clean up cert resources: %w", err) } @@ -634,8 +634,8 @@ type localClient interface { } // tailnetCertDomain returns the base domain (TCD) of the current tailnet. -func (r *HAIngressReconciler) tailnetCertDomain(ctx context.Context) (string, error) { - st, err := r.lc.StatusWithoutPeers(ctx) +func tailnetCertDomain(ctx context.Context, lc localClient) (string, error) { + st, err := lc.StatusWithoutPeers(ctx) if err != nil { return "", fmt.Errorf("error getting tailscale status: %w", err) } @@ -761,7 +761,7 @@ const ( func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, logger *zap.SugaredLogger) (err error) { // Get all config Secrets for this ProxyGroup. secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil { + if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig))); err != nil { return fmt.Errorf("failed to list config Secrets: %w", err) } @@ -773,7 +773,7 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // The only exception is Ingresses with an HTTP endpoint enabled - if an // Ingress has an HTTP endpoint enabled, it will be advertised even if the // TLS cert is not yet provisioned. - hasCert, err := a.hasCerts(ctx, serviceName) + hasCert, err := hasCerts(ctx, a.Client, a.lc, a.tsNamespace, serviceName) if err != nil { return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err) } @@ -822,10 +822,10 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con return nil } -func (a *HAIngressReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) { +func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, pgName string, serviceName tailcfg.ServiceName) (int, error) { // Get all state Secrets for this ProxyGroup. secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil { + if err := cl.List(ctx, secrets, client.InNamespace(tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeState))); err != nil { return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err) } @@ -859,7 +859,14 @@ type ownerAnnotationValue struct { // Kubernetes operator instance. type OwnerRef struct { // OperatorID is the stable ID of the operator's Tailscale device. - OperatorID string `json:"operatorID,omitempty"` + OperatorID string `json:"operatorID,omitempty"` + Resource *Resource `json:"resource,omitempty"` // optional, used to identify the ProxyGroup that owns this Tailscale Service. +} + +type Resource struct { + Kind string `json:"kind,omitempty"` // "ProxyGroup" + Name string `json:"name,omitempty"` // Name of the ProxyGroup that owns this Tailscale Service. Informational only. + UID string `json:"uid,omitempty"` // UID of the ProxyGroup that owns this Tailscale Service. } // ownerAnnotations returns the updated annotations required to ensure this @@ -891,6 +898,9 @@ func ownerAnnotations(operatorID string, svc *tailscale.VIPService) (map[string] if slices.Contains(o.OwnerRefs, ref) { // up to date return svc.Annotations, nil } + if o.OwnerRefs[0].Resource != nil { + return nil, fmt.Errorf("Tailscale Service %s is owned by another resource: %#v; cannot be reused for an Ingress", svc.Name, o.OwnerRefs[0].Resource) + } o.OwnerRefs = append(o.OwnerRefs, ref) json, err := json.Marshal(o) if err != nil { @@ -949,7 +959,7 @@ func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi }); err != nil { return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err) } - rolebinding := certSecretRoleBinding(pg.Name, r.tsNamespace, domain) + rolebinding := certSecretRoleBinding(pg, r.tsNamespace, domain) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) { // Labels and subjects might have changed if the Ingress has been updated to use a // different ProxyGroup. @@ -963,19 +973,19 @@ func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi // cleanupCertResources ensures that the TLS Secret and associated RBAC // resources that allow proxies to read/write to the Secret are deleted. -func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, name tailcfg.ServiceName) error { - domainName, err := r.dnsNameForService(ctx, tailcfg.ServiceName(name)) +func cleanupCertResources(ctx context.Context, cl client.Client, lc localClient, tsNamespace, pgName string, serviceName tailcfg.ServiceName) error { + domainName, err := dnsNameForService(ctx, lc, serviceName) if err != nil { - return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", name, err) + return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", serviceName, err) } labels := certResourceLabels(pgName, domainName) - if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + if err := cl.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil { return fmt.Errorf("error deleting RoleBinding for domain name %s: %w", domainName, err) } - if err := r.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + if err := cl.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil { return fmt.Errorf("error deleting Role for domain name %s: %w", domainName, err) } - if err := r.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + if err := cl.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil { return fmt.Errorf("error deleting Secret for domain name %s: %w", domainName, err) } return nil @@ -1018,17 +1028,17 @@ func certSecretRole(pgName, namespace, domain string) *rbacv1.Role { // certSecretRoleBinding creates a RoleBinding for Role that will allow proxies // to manage the TLS Secret for the given domain. Domain must be a valid // Kubernetes resource name. -func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding { +func certSecretRoleBinding(pg *tsapi.ProxyGroup, namespace, domain string) *rbacv1.RoleBinding { return &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: domain, Namespace: namespace, - Labels: certResourceLabels(pgName, domain), + Labels: certResourceLabels(pg.Name, domain), }, Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: pgName, + Name: pgServiceAccountName(pg), Namespace: namespace, }, }, @@ -1041,14 +1051,17 @@ func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding // certSecret creates a Secret that will store the TLS certificate and private // key for the given domain. Domain must be a valid Kubernetes resource name. -func certSecret(pgName, namespace, domain string, ing *networkingv1.Ingress) *corev1.Secret { +func certSecret(pgName, namespace, domain string, parent client.Object) *corev1.Secret { labels := certResourceLabels(pgName, domain) - labels[kubetypes.LabelSecretType] = "certs" + labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeCerts // Labels that let us identify the Ingress resource lets us reconcile // the Ingress when the TLS Secret is updated (for example, when TLS // certs have been provisioned). - labels[LabelParentName] = ing.Name - labels[LabelParentNamespace] = ing.Namespace + labels[LabelParentType] = strings.ToLower(parent.GetObjectKind().GroupVersionKind().Kind) + labels[LabelParentName] = parent.GetName() + if ns := parent.GetNamespace(); ns != "" { + labels[LabelParentNamespace] = ns + } return &corev1.Secret{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", @@ -1076,9 +1089,9 @@ func certResourceLabels(pgName, domain string) map[string]string { } // dnsNameForService returns the DNS name for the given Tailscale Service's name. -func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) { +func dnsNameForService(ctx context.Context, lc localClient, svc tailcfg.ServiceName) (string, error) { s := svc.WithoutPrefix() - tcd, err := r.tailnetCertDomain(ctx) + tcd, err := tailnetCertDomain(ctx, lc) if err != nil { return "", fmt.Errorf("error determining DNS name base: %w", err) } @@ -1086,14 +1099,14 @@ func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg } // hasCerts checks if the TLS Secret for the given service has non-zero cert and key data. -func (r *HAIngressReconciler) hasCerts(ctx context.Context, svc tailcfg.ServiceName) (bool, error) { - domain, err := r.dnsNameForService(ctx, svc) +func hasCerts(ctx context.Context, cl client.Client, lc localClient, ns string, svc tailcfg.ServiceName) (bool, error) { + domain, err := dnsNameForService(ctx, lc, svc) if err != nil { return false, fmt.Errorf("failed to get DNS name for service: %w", err) } secret := &corev1.Secret{} - err = r.Get(ctx, client.ObjectKey{ - Namespace: r.tsNamespace, + err = cl.Get(ctx, client.ObjectKey{ + Namespace: ns, Name: domain, }, secret) if err != nil { diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index 5de86cdad573a..77e5ecb37a677 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -75,8 +75,13 @@ func TestIngressPGReconciler(t *testing.T) { // Verify that Role and RoleBinding have been created for the first Ingress. // Do not verify the cert Secret as that was already verified implicitly above. + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg", + }, + } expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net")) - expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net")) + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-svc.ts.net")) mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) { ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test" @@ -137,7 +142,7 @@ func TestIngressPGReconciler(t *testing.T) { // Verify that Role and RoleBinding have been created for the second Ingress. // Do not verify the cert Secret as that was already verified implicitly above. expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net")) - expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net")) + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-other-svc.ts.net")) // Verify first Ingress is still working verifyServeConfig(t, fc, "svc:my-svc", false) @@ -186,7 +191,12 @@ func TestIngressPGReconciler(t *testing.T) { }) expectReconciled(t, ingPGR, "default", "test-ingress") expectEqual(t, fc, certSecretRole("test-pg-second", "operator-ns", "my-svc.ts.net")) - expectEqual(t, fc, certSecretRoleBinding("test-pg-second", "operator-ns", "my-svc.ts.net")) + pg = &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg-second", + }, + } + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-svc.ts.net")) // Delete the first Ingress and verify cleanup if err := fc.Delete(context.Background(), ing); err != nil { @@ -515,7 +525,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test-pg-0", Namespace: "operator-ns", - Labels: pgSecretLabels("test-pg", "state"), + Labels: pgSecretLabels("test-pg", kubetypes.LabelSecretTypeState), }, Data: map[string][]byte{ "_current-profile": []byte("profile-foo"), @@ -686,6 +696,14 @@ func TestOwnerAnnotations(t *testing.T) { ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"},{"operatorID":"self-id"}]}`, }, }, + "owned_by_proxygroup": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"1234-UID"}}]}`, + }, + }, + wantErr: "owned by another resource", + }, } { t.Run(name, func(t *testing.T) { got, err := ownerAnnotations("self-id", tc.svc) @@ -708,7 +726,7 @@ func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain stri kubetypes.LabelManaged: "true", labelProxyGroup: pgName, labelDomain: domain, - kubetypes.LabelSecretType: "certs", + kubetypes.LabelSecretType: kubetypes.LabelSecretTypeCerts, }, }, Type: corev1.SecretTypeTLS, @@ -806,7 +824,7 @@ func verifyTailscaledConfig(t *testing.T, fc client.Client, pgName string, expec ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName(pgName, 0), Namespace: "operator-ns", - Labels: pgSecretLabels(pgName, "config"), + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig), }, Data: map[string][]byte{ tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)), @@ -845,7 +863,7 @@ func createPGResources(t *testing.T, fc client.Client, pgName string) { ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName(pgName, 0), Namespace: "operator-ns", - Labels: pgSecretLabels(pgName, "config"), + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig), }, Data: map[string][]byte{ tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte("{}"), diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 870a6f8b7f37e..94a0a6a781c4d 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -123,7 +123,7 @@ func main() { defer s.Close() restConfig := config.GetConfigOrDie() if mode != apiServerProxyModeDisabled { - ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, mode == apiServerProxyModeEnabled) + ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, mode == apiServerProxyModeEnabled, true) if err != nil { zlog.Fatalf("error creating API server proxy: %v", err) } @@ -633,6 +633,32 @@ func runReconcilers(opts reconcilerOpts) { startlog.Fatalf("could not create Recorder reconciler: %v", err) } + // kube-apiserver's Tailscale Service reconciler. + err = builder. + ControllerManagedBy(mgr). + For(&tsapi.ProxyGroup{}, builder.WithPredicates( + predicate.NewPredicateFuncs(func(obj client.Object) bool { + pg, ok := obj.(*tsapi.ProxyGroup) + return ok && pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer + }), + )). + Named("kube-apiserver-ts-service-reconciler"). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(kubeAPIServerPGsFromSecret(mgr.GetClient(), startlog))). + Complete(&KubeAPIServerTSServiceReconciler{ + Client: mgr.GetClient(), + recorder: eventRecorder, + logger: opts.log.Named("kube-apiserver-ts-service-reconciler"), + tsClient: opts.tsClient, + tsNamespace: opts.tailscaleNamespace, + lc: lc, + defaultTags: strings.Split(opts.proxyTags, ","), + operatorID: id, + clock: tstime.DefaultClock{}, + }) + if err != nil { + startlog.Fatalf("could not create Kubernetes API server Tailscale Service reconciler: %v", err) + } + // ProxyGroup reconciler. ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{}) proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog)) @@ -1214,7 +1240,7 @@ func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc { if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" { return nil } - if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" { + if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != kubetypes.LabelSecretTypeState { return nil } pg, ok := o.GetLabels()[LabelParentName] @@ -1304,7 +1330,7 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile. func isTLSSecret(secret *corev1.Secret) bool { return secret.Type == corev1.SecretTypeTLS && secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" && - secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == "certs" && + secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeCerts && secret.ObjectMeta.Labels[labelDomain] != "" && secret.ObjectMeta.Labels[labelProxyGroup] != "" } @@ -1312,7 +1338,7 @@ func isTLSSecret(secret *corev1.Secret) bool { func isPGStateSecret(secret *corev1.Secret) bool { return secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" && secret.ObjectMeta.Labels[LabelParentType] == "proxygroup" && - secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == "state" + secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeState } // HAIngressesFromSecret returns a handler that returns reconcile requests for @@ -1394,6 +1420,42 @@ func HAServicesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.M } } +// kubeAPIServerPGsFromSecret finds ProxyGroups of type "kube-apiserver" that +// need to be reconciled after a ProxyGroup-owned Secret is updated. +func kubeAPIServerPGsFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + secret, ok := o.(*corev1.Secret) + if !ok { + logger.Infof("[unexpected] Secret handler triggered for an object that is not a Secret") + return nil + } + if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" || + secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" { + return nil + } + + var pg tsapi.ProxyGroup + if err := cl.Get(ctx, types.NamespacedName{Name: secret.ObjectMeta.Labels[LabelParentName]}, &pg); err != nil { + logger.Infof("error getting ProxyGroup %s: %v", secret.ObjectMeta.Labels[LabelParentName], err) + return nil + } + + if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer { + return nil + } + + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: secret.ObjectMeta.Labels[LabelParentNamespace], + Name: secret.ObjectMeta.Labels[LabelParentName], + }, + }, + } + + } +} + // egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all // user-created ExternalName Services that should be exposed on this ProxyGroup. func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index 1fdc076f94cad..d62cb0f117a1d 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -68,8 +68,7 @@ const ( // // tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was // first introduced. - pgMinCapabilityVersion = 106 - kubeAPIServerConfigFile = "config.hujson" + pgMinCapabilityVersion = 106 ) var ( @@ -127,6 +126,10 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } if done, err := r.maybeCleanup(ctx, pg); err != nil { + if strings.Contains(err.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error, retrying: %s", err) + return reconcile.Result{}, nil + } return reconcile.Result{}, err } else if !done { logger.Debugf("ProxyGroup resource cleanup not yet finished, will retry...") @@ -158,7 +161,7 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG logger.Infof("ensuring ProxyGroup is set up") pg.Finalizers = append(pg.Finalizers, FinalizerName) if err := r.Update(ctx, pg); err != nil { - return r.notReadyErrf(pg, "error adding finalizer: %w", err) + return r.notReadyErrf(pg, logger, "error adding finalizer: %w", err) } } @@ -174,31 +177,25 @@ func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyG if apierrors.IsNotFound(err) { msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q does not (yet) exist", proxyClassName) logger.Info(msg) - return r.notReady(reasonProxyGroupCreating, msg) + return notReady(reasonProxyGroupCreating, msg) } if err != nil { - return r.notReadyErrf(pg, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err) + return r.notReadyErrf(pg, logger, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err) } if !tsoperator.ProxyClassIsReady(proxyClass) { msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName) logger.Info(msg) - return r.notReady(reasonProxyGroupCreating, msg) + return notReady(reasonProxyGroupCreating, msg) } } if err := r.validate(ctx, pg, proxyClass, logger); err != nil { - return r.notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err)) + return notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err)) } staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass) if err != nil { - if strings.Contains(err.Error(), optimisticLockErrorMsg) { - msg := fmt.Sprintf("optimistic lock error, retrying: %s", nrr.message) - logger.Info(msg) - return r.notReady(reasonProxyGroupCreating, msg) - } else { - return nil, nrr, err - } + return nil, nrr, err } return staticEndpoints, nrr, nil @@ -299,9 +296,9 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro reason := reasonProxyGroupCreationFailed msg := fmt.Sprintf("error provisioning NodePort Services for static endpoints: %v", err) r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg) - return r.notReady(reason, msg) + return notReady(reason, msg) } - return r.notReadyErrf(pg, "error provisioning NodePort Services for static endpoints: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning NodePort Services for static endpoints: %w", err) } } @@ -312,9 +309,9 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro reason := reasonProxyGroupCreationFailed msg := fmt.Sprintf("error provisioning config Secrets: %v", err) r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg) - return r.notReady(reason, msg) + return notReady(reason, msg) } - return r.notReadyErrf(pg, "error provisioning config Secrets: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning config Secrets: %w", err) } // State secrets are precreated so we can use the ProxyGroup CR as their owner ref. @@ -325,7 +322,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences }); err != nil { - return r.notReadyErrf(pg, "error provisioning state Secrets: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning state Secrets: %w", err) } } @@ -339,7 +336,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences }); err != nil { - return r.notReadyErrf(pg, "error provisioning ServiceAccount: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning ServiceAccount: %w", err) } } @@ -350,7 +347,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences r.Rules = role.Rules }); err != nil { - return r.notReadyErrf(pg, "error provisioning Role: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning Role: %w", err) } roleBinding := pgRoleBinding(pg, r.tsNamespace) @@ -361,7 +358,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.RoleRef = roleBinding.RoleRef r.Subjects = roleBinding.Subjects }); err != nil { - return r.notReadyErrf(pg, "error provisioning RoleBinding: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning RoleBinding: %w", err) } if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { @@ -371,7 +368,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp) }); err != nil { - return r.notReadyErrf(pg, "error provisioning egress ConfigMap %q: %w", cm.Name, err) + return r.notReadyErrf(pg, logger, "error provisioning egress ConfigMap %q: %w", cm.Name, err) } } @@ -381,7 +378,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro existing.ObjectMeta.Labels = cm.ObjectMeta.Labels existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences }); err != nil { - return r.notReadyErrf(pg, "error provisioning ingress ConfigMap %q: %w", cm.Name, err) + return r.notReadyErrf(pg, logger, "error provisioning ingress ConfigMap %q: %w", cm.Name, err) } } @@ -391,7 +388,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro } ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass) if err != nil { - return r.notReadyErrf(pg, "error generating StatefulSet spec: %w", err) + return r.notReadyErrf(pg, logger, "error generating StatefulSet spec: %w", err) } cfg := &tailscaleSTSConfig{ proxyType: string(pg.Spec.Type), @@ -404,7 +401,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences }); err != nil { - return r.notReadyErrf(pg, "error provisioning StatefulSet: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning StatefulSet: %w", err) } mo := &metricsOpts{ @@ -414,11 +411,11 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro proxyType: "proxygroup", } if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil { - return r.notReadyErrf(pg, "error reconciling metrics resources: %w", err) + return r.notReadyErrf(pg, logger, "error reconciling metrics resources: %w", err) } if err := r.cleanupDanglingResources(ctx, pg, proxyClass); err != nil { - return r.notReadyErrf(pg, "error cleaning up dangling resources: %w", err) + return r.notReadyErrf(pg, logger, "error cleaning up dangling resources: %w", err) } logger.Info("ProxyGroup resources synced") @@ -430,6 +427,10 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za defer func() { if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) { if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { + if strings.Contains(updateErr.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error updating status, retrying: %s", updateErr) + updateErr = nil + } err = errors.Join(err, updateErr) } } @@ -457,6 +458,7 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger) // Set ProxyGroupReady condition. + tsSvcValid, tsSvcSet := tsoperator.KubeAPIServerProxyValid(pg) status = metav1.ConditionFalse reason = reasonProxyGroupCreating switch { @@ -464,9 +466,15 @@ func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *za // If we failed earlier, that reason takes precedence. reason = nrr.reason message = nrr.message + case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && tsSvcSet && !tsSvcValid: + reason = reasonProxyGroupInvalid + message = "waiting for config in spec.kubeAPIServer to be marked valid" case len(devices) < desiredReplicas: case len(devices) > desiredReplicas: message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(devices)-desiredReplicas) + case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && !tsoperator.KubeAPIServerProxyConfigured(pg): + reason = reasonProxyGroupCreating + message = "waiting for proxies to start advertising the kube-apiserver proxy's hostname" default: status = metav1.ConditionTrue reason = reasonProxyGroupReady @@ -714,7 +722,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName(pg.Name, i), Namespace: r.tsNamespace, - Labels: pgSecretLabels(pg.Name, "config"), + Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig), OwnerReferences: pgOwnerReference(pg), }, } @@ -775,13 +783,6 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p } } - // AdvertiseServices config is set by ingress-pg-reconciler, so make sure we - // don't overwrite it if already set. - existingAdvertiseServices, err := extractAdvertiseServicesConfig(existingCfgSecret) - if err != nil { - return nil, err - } - if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { hostname := pgHostname(pg, i) @@ -795,7 +796,7 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p } if !deviceAuthed { existingCfg := conf.ConfigV1Alpha1{} - if err := json.Unmarshal(existingCfgSecret.Data[kubeAPIServerConfigFile], &existingCfg); err != nil { + if err := json.Unmarshal(existingCfgSecret.Data[kubetypes.KubeAPIServerConfigFile], &existingCfg); err != nil { return nil, fmt.Errorf("error unmarshalling existing config: %w", err) } if existingCfg.AuthKey != nil { @@ -803,19 +804,42 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p } } } + cfg := conf.VersionedConfig{ Version: "v1alpha1", ConfigV1Alpha1: &conf.ConfigV1Alpha1{ - Hostname: &hostname, + AuthKey: authKey, State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))), App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer), - AuthKey: authKey, - KubeAPIServer: &conf.KubeAPIServer{ + LogLevel: ptr.To(logger.Level().String()), + + // Reloadable fields. + Hostname: &hostname, + APIServerProxy: &conf.APIServerProxyConfig{ + Enabled: opt.NewBool(true), AuthMode: opt.NewBool(isAuthAPIServerProxy(pg)), + // The first replica is elected as the cert issuer, same + // as containerboot does for ingress-pg-reconciler. + IssueCerts: opt.NewBool(i == 0), }, }, } + // Copy over config that the apiserver-proxy-service-reconciler sets. + if existingCfgSecret != nil { + if k8sProxyCfg, ok := cfgSecret.Data[kubetypes.KubeAPIServerConfigFile]; ok { + k8sCfg := &conf.ConfigV1Alpha1{} + if err := json.Unmarshal(k8sProxyCfg, k8sCfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal kube-apiserver config: %w", err) + } + + cfg.AdvertiseServices = k8sCfg.AdvertiseServices + if k8sCfg.APIServerProxy != nil { + cfg.APIServerProxy.ServiceName = k8sCfg.APIServerProxy.ServiceName + } + } + } + if r.loginServer != "" { cfg.ServerURL = &r.loginServer } @@ -832,8 +856,15 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p if err != nil { return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err) } - mak.Set(&cfgSecret.Data, kubeAPIServerConfigFile, cfgB) + mak.Set(&cfgSecret.Data, kubetypes.KubeAPIServerConfigFile, cfgB) } else { + // AdvertiseServices config is set by ingress-pg-reconciler, so make sure we + // don't overwrite it if already set. + existingAdvertiseServices, err := extractAdvertiseServicesConfig(existingCfgSecret) + if err != nil { + return nil, err + } + configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer) if err != nil { return nil, fmt.Errorf("error creating tailscaled config: %w", err) @@ -1024,16 +1055,16 @@ func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error) return nil, nil } - conf, err := latestConfigFromSecret(cfgSecret) + cfg, err := latestConfigFromSecret(cfgSecret) if err != nil { return nil, err } - if conf == nil { + if cfg == nil { return nil, nil } - return conf.AdvertiseServices, nil + return cfg.AdvertiseServices, nil } // getNodeMetadata gets metadata for all the pods owned by this ProxyGroup by @@ -1045,7 +1076,7 @@ func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error) func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup) (metadata []nodeMetadata, _ error) { // List all state Secrets owned by this ProxyGroup. secrets := &corev1.SecretList{} - if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, "state"))); err != nil { + if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState))); err != nil { return nil, fmt.Errorf("failed to list state Secrets: %w", err) } for _, secret := range secrets.Items { @@ -1140,15 +1171,21 @@ type nodeMetadata struct { dnsName string } -func (r *ProxyGroupReconciler) notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason, error) { +func notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason, error) { return nil, ¬ReadyReason{ reason: reason, message: msg, }, nil } -func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) { +func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) { err := fmt.Errorf(format, a...) + if strings.Contains(err.Error(), optimisticLockErrorMsg) { + msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error()) + logger.Info(msg) + return notReady(reasonProxyGroupCreating, msg) + } + r.recorder.Event(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error()) return nil, ¬ReadyReason{ reason: reasonProxyGroupCreationFailed, diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 71398d0d54c2f..e185499f0e19d 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -7,7 +7,6 @@ package main import ( "fmt" - "path/filepath" "slices" "strconv" "strings" @@ -16,6 +15,7 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/yaml" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" @@ -341,8 +341,11 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por }, }, { - Name: "TS_K8S_PROXY_CONFIG", - Value: filepath.Join("/etc/tsconfig/$(POD_NAME)/", kubeAPIServerConfigFile), + Name: "TS_K8S_PROXY_CONFIG", + Value: "kube:" + types.NamespacedName{ + Namespace: namespace, + Name: "$(POD_NAME)-config", + }.String(), }, } @@ -355,20 +358,6 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por return envs }(), - VolumeMounts: func() []corev1.VolumeMount { - var mounts []corev1.VolumeMount - - // TODO(tomhjp): Read config directly from the Secret instead. - for i := range pgReplicas(pg) { - mounts = append(mounts, corev1.VolumeMount{ - Name: fmt.Sprintf("k8s-proxy-config-%d", i), - ReadOnly: true, - MountPath: fmt.Sprintf("/etc/tsconfig/%s-%d", pg.Name, i), - }) - } - - return mounts - }(), Ports: []corev1.ContainerPort{ { Name: "k8s-proxy", @@ -378,21 +367,6 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por }, }, }, - Volumes: func() []corev1.Volume { - var volumes []corev1.Volume - for i := range pgReplicas(pg) { - volumes = append(volumes, corev1.Volume{ - Name: fmt.Sprintf("k8s-proxy-config-%d", i), - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: pgConfigSecretName(pg.Name, i), - }, - }, - }) - } - - return volumes - }(), }, }, }, @@ -426,6 +400,7 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role { Resources: []string{"secrets"}, Verbs: []string{ "list", + "watch", // For k8s-proxy. }, }, { @@ -508,7 +483,7 @@ func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.S ObjectMeta: metav1.ObjectMeta{ Name: pgStateSecretName(pg.Name, i), Namespace: namespace, - Labels: pgSecretLabels(pg.Name, "state"), + Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState), OwnerReferences: pgOwnerReference(pg), }, }) diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 6f143c0566dff..ef6babc5679cc 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -31,8 +31,11 @@ import ( kube "tailscale.com/k8s-operator" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" "tailscale.com/tstest" + "tailscale.com/types/opt" "tailscale.com/types/ptr" ) @@ -1256,6 +1259,163 @@ func TestProxyGroupTypes(t *testing.T) { }) } +func TestKubeAPIServerStatusConditionFlow(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-k8s-apiserver", + UID: "test-k8s-apiserver-uid", + Generation: 1, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + Replicas: ptr.To[int32](1), + KubeAPIServer: &tsapi.KubeAPIServerConfig{ + Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), + }, + }, + } + stateSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgStateSecretName(pg.Name, 0), + Namespace: tsNamespace, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg, stateSecret). + WithStatusSubresource(pg). + Build() + r := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + l: zap.Must(zap.NewDevelopment()).Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), + } + + expectReconciled(t, r, "", pg.Name) + pg.ObjectMeta.Finalizers = append(pg.ObjectMeta.Finalizers, FinalizerName) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "", 0, r.clock, r.l) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Set kube-apiserver valid. + mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.l) + }) + expectReconciled(t, r, "", pg.Name) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.l) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Set available. + addNodeIDToStateSecrets(t, fc, pg) + expectReconciled(t, r, "", pg.Name) + pg.Status.Devices = []tsapi.TailnetDevice{ + { + Hostname: "hostname-nodeid-0", + TailnetIPs: []string{"1.2.3.4", "::1"}, + }, + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "", 0, r.clock, r.l) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Set kube-apiserver configured. + mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.l) + }) + expectReconciled(t, r, "", pg.Name) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.l) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, "", 1, r.clock, r.l) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) +} + +func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.ProxyGroup{}). + Build() + + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + l: zap.Must(zap.NewDevelopment()).Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), + } + + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-k8s-apiserver", + UID: "test-k8s-apiserver-uid", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + Replicas: ptr.To[int32](1), + KubeAPIServer: &tsapi.KubeAPIServerConfig{ + Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), // Avoid needing to pre-create the static ServiceAccount. + }, + }, + } + if err := fc.Create(t.Context(), pg); err != nil { + t.Fatal(err) + } + expectReconciled(t, reconciler, "", pg.Name) + + cfg := conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("secret-authkey"), + State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, 0))), + App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer), + LogLevel: ptr.To("debug"), + + Hostname: ptr.To("test-k8s-apiserver-0"), + APIServerProxy: &conf.APIServerProxyConfig{ + Enabled: opt.NewBool(true), + AuthMode: opt.NewBool(false), + IssueCerts: opt.NewBool(true), + }, + }, + } + cfgB, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("failed to marshal config: %v", err) + } + + cfgSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgConfigSecretName(pg.Name, 0), + Namespace: tsNamespace, + Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig), + OwnerReferences: pgOwnerReference(pg), + }, + Data: map[string][]byte{ + kubetypes.KubeAPIServerConfigFile: cfgB, + }, + } + expectEqual(t, fc, cfgSecret) + + // Now simulate the kube-apiserver services reconciler updating config, + // then check the proxygroup reconciler doesn't overwrite it. + cfg.APIServerProxy.ServiceName = ptr.To(tailcfg.ServiceName("svc:some-svc-name")) + cfg.AdvertiseServices = []string{"svc:should-not-be-overwritten"} + cfgB, err = json.Marshal(cfg) + if err != nil { + t.Fatalf("failed to marshal config: %v", err) + } + mustUpdate(t, fc, tsNamespace, cfgSecret.Name, func(s *corev1.Secret) { + s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB + }) + expectReconciled(t, reconciler, "", pg.Name) + + cfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cfgB + expectEqual(t, fc, cfgSecret) +} + func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -1660,7 +1820,7 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG if _, err := createOrUpdate(t.Context(), fc, "tailscale", pod, nil); err != nil { t.Fatalf("failed to create or update Pod %s: %v", pod.Name, err) } - mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) { + mustUpdate(t, fc, tsNamespace, pgStateSecretName(pg.Name, i), func(s *corev1.Secret) { s.Data = map[string][]byte{ currentProfileKey: []byte(key), key: bytes, diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index 4247eaaa0040f..62cc36bd4a82b 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -41,7 +41,7 @@ import ( ) const ( - finalizerName = "tailscale.com/service-pg-finalizer" + svcPGFinalizerName = "tailscale.com/service-pg-finalizer" reasonIngressSvcInvalid = "IngressSvcInvalid" reasonIngressSvcValid = "IngressSvcValid" @@ -174,13 +174,13 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin return false, nil } - if !slices.Contains(svc.Finalizers, finalizerName) { + if !slices.Contains(svc.Finalizers, svcPGFinalizerName) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, // this is a nice place to tell the operator that the high level, // multi-reconcile operation is underway. logger.Infof("exposing Service over tailscale") - svc.Finalizers = append(svc.Finalizers, finalizerName) + svc.Finalizers = append(svc.Finalizers, svcPGFinalizerName) if err := r.Update(ctx, svc); err != nil { return false, fmt.Errorf("failed to add finalizer: %w", err) } @@ -378,7 +378,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin // corresponding to this Service. func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, svc *corev1.Service, logger *zap.SugaredLogger) (svcChanged bool, err error) { logger.Debugf("Ensuring any resources for Service are cleaned up") - ix := slices.Index(svc.Finalizers, finalizerName) + ix := slices.Index(svc.Finalizers, svcPGFinalizerName) if ix < 0 { logger.Debugf("no finalizer, nothing to do") return false, nil @@ -485,12 +485,12 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG func (r *HAServiceReconciler) deleteFinalizer(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error { svc.Finalizers = slices.DeleteFunc(svc.Finalizers, func(f string) bool { - return f == finalizerName + return f == svcPGFinalizerName }) - logger.Debugf("ensure %q finalizer is removed", finalizerName) + logger.Debugf("ensure %q finalizer is removed", svcPGFinalizerName) if err := r.Update(ctx, svc); err != nil { - return fmt.Errorf("failed to remove finalizer %q: %w", finalizerName, err) + return fmt.Errorf("failed to remove finalizer %q: %w", svcPGFinalizerName, err) } r.mu.Lock() defer r.mu.Unlock() @@ -653,7 +653,7 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // Get all config Secrets for this ProxyGroup. // Get all Pods secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil { + if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig))); err != nil { return fmt.Errorf("failed to list config Secrets: %w", err) } @@ -720,7 +720,7 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) { // Get all state Secrets for this ProxyGroup. secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil { + if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeState))); err != nil { return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err) } diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go index 054c3ed49f5cb..baaa07727df06 100644 --- a/cmd/k8s-operator/svc-for-pg_test.go +++ b/cmd/k8s-operator/svc-for-pg_test.go @@ -26,6 +26,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/ingressservices" + "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/types/ptr" "tailscale.com/util/mak" @@ -139,7 +140,7 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName("test-pg", 0), Namespace: "operator-ns", - Labels: pgSecretLabels("test-pg", "config"), + Labels: pgSecretLabels("test-pg", kubetypes.LabelSecretTypeConfig), }, Data: map[string][]byte{ tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(`{"Version":""}`), @@ -298,12 +299,12 @@ func TestServicePGReconciler_MultiCluster(t *testing.T) { t.Fatalf("getting Tailscale Service: %v", err) } - if len(tsSvcs) != 1 { - t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs)) + if len(tsSvcs.VIPServices) != 1 { + t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs.VIPServices)) } - for name := range tsSvcs { - t.Logf("found Tailscale Service with name %q", name.String()) + for _, svc := range tsSvcs.VIPServices { + t.Logf("found Tailscale Service with name %q", svc.Name) } } } @@ -336,7 +337,7 @@ func TestIgnoreRegularService(t *testing.T) { tsSvcs, err := ft.ListVIPServices(context.Background()) if err == nil { - if len(tsSvcs) > 0 { + if len(tsSvcs.VIPServices) > 0 { t.Fatal("unexpected Tailscale Services found") } } diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 56542700d951c..6ae32d6fbac13 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -891,13 +891,17 @@ func (c *fakeTSClient) GetVIPService(ctx context.Context, name tailcfg.ServiceNa return svc, nil } -func (c *fakeTSClient) ListVIPServices(ctx context.Context) (map[tailcfg.ServiceName]*tailscale.VIPService, error) { +func (c *fakeTSClient) ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error) { c.Lock() defer c.Unlock() if c.vipServices == nil { return nil, &tailscale.ErrResponse{Status: http.StatusNotFound} } - return c.vipServices, nil + result := &tailscale.VIPServiceList{} + for _, svc := range c.vipServices { + result.VIPServices = append(result.VIPServices, *svc) + } + return result, nil } func (c *fakeTSClient) CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error { diff --git a/cmd/k8s-operator/tsclient.go b/cmd/k8s-operator/tsclient.go index a94d55afed604..50620c26ddf27 100644 --- a/cmd/k8s-operator/tsclient.go +++ b/cmd/k8s-operator/tsclient.go @@ -56,6 +56,8 @@ type tsClient interface { DeleteDevice(ctx context.Context, nodeStableID string) error // GetVIPService is a method for getting a Tailscale Service. VIPService is the original name for Tailscale Service. GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*tailscale.VIPService, error) + // ListVIPServices is a method for listing all Tailscale Services. VIPService is the original name for Tailscale Service. + ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error) // CreateOrUpdateVIPService is a method for creating or updating a Tailscale Service. CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error // DeleteVIPService is a method for deleting a Tailscale Service. diff --git a/cmd/k8s-proxy/internal/config/config.go b/cmd/k8s-proxy/internal/config/config.go new file mode 100644 index 0000000000000..4013047e76f0c --- /dev/null +++ b/cmd/k8s-proxy/internal/config/config.go @@ -0,0 +1,264 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package config provides watchers for the various supported ways to load a +// config file for k8s-proxy; currently file or Kubernetes Secret. +package config + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/types/ptr" + "tailscale.com/util/testenv" +) + +type configLoader struct { + logger *zap.SugaredLogger + client clientcorev1.CoreV1Interface + + cfgChan chan<- *conf.Config + previous []byte + + once sync.Once // For use in tests. To close cfgIgnored. + cfgIgnored chan struct{} // For use in tests. +} + +func NewConfigLoader(logger *zap.SugaredLogger, client clientcorev1.CoreV1Interface, cfgChan chan<- *conf.Config) *configLoader { + return &configLoader{ + logger: logger, + client: client, + cfgChan: cfgChan, + } +} + +func (l *configLoader) WatchConfig(ctx context.Context, path string) error { + secretNamespacedName, isKubeSecret := strings.CutPrefix(path, "kube:") + if isKubeSecret { + secretNamespace, secretName, ok := strings.Cut(secretNamespacedName, string(types.Separator)) + if !ok { + return fmt.Errorf("invalid Kubernetes Secret reference %q, expected format /", path) + } + if err := l.watchConfigSecretChanges(ctx, secretNamespace, secretName); err != nil && !errors.Is(err, context.Canceled) { + return fmt.Errorf("error watching config Secret %q: %w", secretNamespacedName, err) + } + + return nil + } + + if err := l.watchConfigFileChanges(ctx, path); err != nil && !errors.Is(err, context.Canceled) { + return fmt.Errorf("error watching config file %q: %w", path, err) + } + + return nil +} + +func (l *configLoader) reloadConfig(ctx context.Context, raw []byte) error { + if bytes.Equal(raw, l.previous) { + if l.cfgIgnored != nil && testenv.InTest() { + l.once.Do(func() { + close(l.cfgIgnored) + }) + } + return nil + } + + cfg, err := conf.Load(raw) + if err != nil { + return fmt.Errorf("error loading config: %w", err) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case l.cfgChan <- &cfg: + } + + l.previous = raw + return nil +} + +func (l *configLoader) watchConfigFileChanges(ctx context.Context, path string) error { + var ( + tickChan <-chan time.Time + eventChan <-chan fsnotify.Event + errChan <-chan error + ) + + if w, err := fsnotify.NewWatcher(); err != nil { + // Creating a new fsnotify watcher would fail for example if inotify was not able to create a new file descriptor. + // See https://github.com/tailscale/tailscale/issues/15081 + l.logger.Infof("Failed to create fsnotify watcher on config file %q; watching for changes on 5s timer: %v", path, err) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + tickChan = ticker.C + } else { + dir := filepath.Dir(path) + file := filepath.Base(path) + l.logger.Infof("Watching directory %q for changes to config file %q", dir, file) + defer w.Close() + if err := w.Add(dir); err != nil { + return fmt.Errorf("failed to add fsnotify watch: %w", err) + } + eventChan = w.Events + errChan = w.Errors + } + + // Read the initial config file, but after the watcher is already set up to + // avoid an unlucky race condition if the config file is edited in between. + b, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading config file %q: %w", path, err) + } + if err := l.reloadConfig(ctx, b); err != nil { + return fmt.Errorf("error loading initial config file %q: %w", path, err) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err, ok := <-errChan: + if !ok { + // Watcher was closed. + return nil + } + return fmt.Errorf("watcher error: %w", err) + case <-tickChan: + case ev, ok := <-eventChan: + if !ok { + // Watcher was closed. + return nil + } + if ev.Name != path || ev.Op&fsnotify.Write == 0 { + // Ignore irrelevant events. + continue + } + } + b, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading config file: %w", err) + } + // Writers such as os.WriteFile may truncate the file before writing + // new contents, so it's possible to read an empty file if we read before + // the write has completed. + if len(b) == 0 { + continue + } + if err := l.reloadConfig(ctx, b); err != nil { + return fmt.Errorf("error reloading config file %q: %v", path, err) + } + } +} + +func (l *configLoader) watchConfigSecretChanges(ctx context.Context, secretNamespace, secretName string) error { + secrets := l.client.Secrets(secretNamespace) + w, err := secrets.Watch(ctx, metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + // Re-watch regularly to avoid relying on long-lived connections. + // See https://github.com/kubernetes-client/javascript/issues/596#issuecomment-786419380 + TimeoutSeconds: ptr.To(int64(600)), + FieldSelector: fmt.Sprintf("metadata.name=%s", secretName), + Watch: true, + }) + if err != nil { + return fmt.Errorf("failed to watch config Secret %q: %w", secretName, err) + } + defer func() { + // May not be the original watcher by the time we exit. + if w != nil { + w.Stop() + } + }() + + // Get the initial config Secret now we've got the watcher set up. + secret, err := secrets.Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get config Secret %q: %w", secretName, err) + } + + if err := l.configFromSecret(ctx, secret); err != nil { + return fmt.Errorf("error loading initial config: %w", err) + } + + l.logger.Infof("Watching config Secret %q for changes", secretName) + for { + var secret *corev1.Secret + select { + case <-ctx.Done(): + return ctx.Err() + case ev, ok := <-w.ResultChan(): + if !ok { + w.Stop() + w, err = secrets.Watch(ctx, metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + TimeoutSeconds: ptr.To(int64(600)), + FieldSelector: fmt.Sprintf("metadata.name=%s", secretName), + Watch: true, + }) + if err != nil { + return fmt.Errorf("failed to re-watch config Secret %q: %w", secretName, err) + } + continue + } + + switch ev.Type { + case watch.Added, watch.Modified: + // New config available to load. + var ok bool + secret, ok = ev.Object.(*corev1.Secret) + if !ok { + return fmt.Errorf("unexpected object type %T in watch event for config Secret %q", ev.Object, secretName) + } + if secret == nil || secret.Data == nil { + continue + } + if err := l.configFromSecret(ctx, secret); err != nil { + return fmt.Errorf("error reloading config Secret %q: %v", secret.Name, err) + } + case watch.Error: + return fmt.Errorf("error watching config Secret %q: %v", secretName, ev.Object) + default: + // Ignore, no action required. + continue + } + } + } +} + +func (l *configLoader) configFromSecret(ctx context.Context, s *corev1.Secret) error { + b := s.Data[kubetypes.KubeAPIServerConfigFile] + if len(b) == 0 { + return fmt.Errorf("config Secret %q does not contain expected config in key %q", s.Name, kubetypes.KubeAPIServerConfigFile) + } + + if err := l.reloadConfig(ctx, b); err != nil { + return err + } + + return nil +} diff --git a/cmd/k8s-proxy/internal/config/config_test.go b/cmd/k8s-proxy/internal/config/config_test.go new file mode 100644 index 0000000000000..1603dbe1f398f --- /dev/null +++ b/cmd/k8s-proxy/internal/config/config_test.go @@ -0,0 +1,245 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package config + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/types/ptr" +) + +func TestWatchConfig(t *testing.T) { + type phase struct { + config string + cancel bool + expectedConf *conf.ConfigV1Alpha1 + expectedErr string + } + + // Same set of behaviour tests for each config source. + for _, env := range []string{"file", "kube"} { + t.Run(env, func(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + initialConfig string + phases []phase + }{ + { + name: "no_config", + phases: []phase{{ + expectedErr: "error loading initial config", + }}, + }, + { + name: "valid_config", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{{ + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("abc123"), + }, + }}, + }, + { + name: "can_cancel", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{ + { + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("abc123"), + }, + }, + { + cancel: true, + }, + }, + }, + { + name: "can_reload", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{ + { + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("abc123"), + }, + }, + { + config: `{"version": "v1alpha1", "authKey": "def456"}`, + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("def456"), + }, + }, + }, + }, + { + name: "ignores_events_with_no_changes", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{ + { + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("abc123"), + }, + }, + { + config: `{"version": "v1alpha1", "authKey": "abc123"}`, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + cl := fake.NewClientset() + + var cfgPath string + var writeFile func(*testing.T, string) + if env == "file" { + cfgPath = filepath.Join(root, kubetypes.KubeAPIServerConfigFile) + writeFile = func(t *testing.T, content string) { + if err := os.WriteFile(cfgPath, []byte(content), 0o644); err != nil { + t.Fatalf("error writing config file %q: %v", cfgPath, err) + } + } + } else { + cfgPath = "kube:default/config-secret" + writeFile = func(t *testing.T, content string) { + s := secretFrom(content) + mustCreateOrUpdate(t, cl, s) + } + } + configChan := make(chan *conf.Config) + l := NewConfigLoader(zap.Must(zap.NewDevelopment()).Sugar(), cl.CoreV1(), configChan) + l.cfgIgnored = make(chan struct{}) + errs := make(chan error) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + writeFile(t, tc.initialConfig) + go func() { + errs <- l.WatchConfig(ctx, cfgPath) + }() + + for i, p := range tc.phases { + if p.config != "" { + writeFile(t, p.config) + } + if p.cancel { + cancel() + } + + select { + case cfg := <-configChan: + if diff := cmp.Diff(*p.expectedConf, cfg.Parsed); diff != "" { + t.Errorf("unexpected config (-want +got):\n%s", diff) + } + case err := <-errs: + if p.cancel { + if err != nil { + t.Fatalf("unexpected error after cancel: %v", err) + } + } else if p.expectedErr == "" { + t.Fatalf("unexpected error: %v", err) + } else if !strings.Contains(err.Error(), p.expectedErr) { + t.Fatalf("expected error to contain %q, got %q", p.expectedErr, err.Error()) + } + case <-l.cfgIgnored: + if p.expectedConf != nil { + t.Fatalf("expected config to be reloaded, but got ignored signal") + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for expected event in phase: %d", i) + } + } + }) + } + }) + } +} + +func TestWatchConfigSecret_Rewatches(t *testing.T) { + cl := fake.NewClientset() + var watchCount int + var watcher *watch.RaceFreeFakeWatcher + expected := []string{ + `{"version": "v1alpha1", "authKey": "abc123"}`, + `{"version": "v1alpha1", "authKey": "def456"}`, + `{"version": "v1alpha1", "authKey": "ghi789"}`, + } + cl.PrependWatchReactor("secrets", func(action ktesting.Action) (handled bool, ret watch.Interface, err error) { + watcher = watch.NewRaceFreeFake() + watcher.Add(secretFrom(expected[watchCount])) + if action.GetVerb() == "watch" && action.GetResource().Resource == "secrets" { + watchCount++ + } + return true, watcher, nil + }) + + configChan := make(chan *conf.Config) + l := NewConfigLoader(zap.Must(zap.NewDevelopment()).Sugar(), cl.CoreV1(), configChan) + + mustCreateOrUpdate(t, cl, secretFrom(expected[0])) + + errs := make(chan error) + go func() { + errs <- l.watchConfigSecretChanges(t.Context(), "default", "config-secret") + }() + + for i := range 2 { + select { + case cfg := <-configChan: + if exp := expected[i]; cfg.Parsed.AuthKey == nil || !strings.Contains(exp, *cfg.Parsed.AuthKey) { + t.Fatalf("expected config to have authKey %q, got: %v", exp, cfg.Parsed.AuthKey) + } + if i == 0 { + watcher.Stop() + } + case err := <-errs: + t.Fatalf("unexpected error: %v", err) + case <-l.cfgIgnored: + t.Fatalf("expected config to be reloaded, but got ignored signal") + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for expected event") + } + } + + if watchCount != 2 { + t.Fatalf("expected 2 watch API calls, got %d", watchCount) + } +} + +func secretFrom(content string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "config-secret", + }, + Data: map[string][]byte{ + kubetypes.KubeAPIServerConfigFile: []byte(content), + }, + } +} + +func mustCreateOrUpdate(t *testing.T, cl *fake.Clientset, s *corev1.Secret) { + t.Helper() + if _, err := cl.CoreV1().Secrets("default").Create(t.Context(), s, metav1.CreateOptions{}); err != nil { + if _, updateErr := cl.CoreV1().Secrets("default").Update(t.Context(), s, metav1.UpdateOptions{}); updateErr != nil { + t.Fatalf("error writing config Secret %q: %v", s.Name, updateErr) + } + } +} diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go index b7f3d9535a071..eea1f15f7fdd8 100644 --- a/cmd/k8s-proxy/k8s-proxy.go +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -14,6 +14,7 @@ import ( "fmt" "os" "os/signal" + "reflect" "strings" "syscall" "time" @@ -21,20 +22,37 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "k8s.io/utils/strings/slices" + "tailscale.com/client/local" + "tailscale.com/cmd/k8s-proxy/internal/config" "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/store" apiproxy "tailscale.com/k8s-operator/api-proxy" + "tailscale.com/kube/certs" "tailscale.com/kube/k8s-proxy/conf" + klc "tailscale.com/kube/localclient" + "tailscale.com/kube/services" "tailscale.com/kube/state" + "tailscale.com/tailcfg" "tailscale.com/tsnet" ) func main() { - logger := zap.Must(zap.NewProduction()).Sugar() + encoderCfg := zap.NewProductionEncoderConfig() + encoderCfg.EncodeTime = zapcore.RFC3339TimeEncoder + logger := zap.Must(zap.Config{ + Level: zap.NewAtomicLevelAt(zap.DebugLevel), + Encoding: "json", + OutputPaths: []string{"stderr"}, + ErrorOutputPaths: []string{"stderr"}, + EncoderConfig: encoderCfg, + }.Build()).Sugar() defer logger.Sync() + if err := run(logger); err != nil { logger.Fatal(err.Error()) } @@ -42,18 +60,58 @@ func main() { func run(logger *zap.SugaredLogger) error { var ( - configFile = os.Getenv("TS_K8S_PROXY_CONFIG") + configPath = os.Getenv("TS_K8S_PROXY_CONFIG") podUID = os.Getenv("POD_UID") ) - if configFile == "" { + if configPath == "" { return errors.New("TS_K8S_PROXY_CONFIG unset") } - // TODO(tomhjp): Support reloading config. - // TODO(tomhjp): Support reading config from a Secret. - cfg, err := conf.Load(configFile) + // serveCtx to live for the lifetime of the process, only gets cancelled + // once the Tailscale Service has been drained + serveCtx, serveCancel := context.WithCancel(context.Background()) + defer serveCancel() + + // ctx to cancel to start the shutdown process. + ctx, cancel := context.WithCancel(serveCtx) + defer cancel() + + sigsChan := make(chan os.Signal, 1) + signal.Notify(sigsChan, syscall.SIGINT, syscall.SIGTERM) + go func() { + select { + case <-ctx.Done(): + case s := <-sigsChan: + logger.Infof("Received shutdown signal %s, exiting", s) + cancel() + } + }() + + var group *errgroup.Group + group, ctx = errgroup.WithContext(ctx) + + restConfig, err := getRestConfig(logger) + if err != nil { + return fmt.Errorf("error getting rest config: %w", err) + } + clientset, err := kubernetes.NewForConfig(restConfig) if err != nil { - return fmt.Errorf("error loading config file %q: %w", configFile, err) + return fmt.Errorf("error creating Kubernetes clientset: %w", err) + } + + // Load and watch config. + cfgChan := make(chan *conf.Config) + cfgLoader := config.NewConfigLoader(logger, clientset.CoreV1(), cfgChan) + group.Go(func() error { + return cfgLoader.WatchConfig(ctx, configPath) + }) + + // Get initial config. + var cfg *conf.Config + select { + case <-ctx.Done(): + return group.Wait() + case cfg = <-cfgChan: } if cfg.Parsed.LogLevel != nil { @@ -82,6 +140,14 @@ func run(logger *zap.SugaredLogger) error { hostinfo.SetApp(*cfg.Parsed.App) } + // TODO(tomhjp): Pass this setting directly into the store instead of using + // environment variables. + if cfg.Parsed.APIServerProxy != nil && cfg.Parsed.APIServerProxy.IssueCerts.EqualBool(true) { + os.Setenv("TS_CERT_SHARE_MODE", "rw") + } else { + os.Setenv("TS_CERT_SHARE_MODE", "ro") + } + st, err := getStateStore(cfg.Parsed.State, logger) if err != nil { return err @@ -115,10 +181,6 @@ func run(logger *zap.SugaredLogger) error { ts.Hostname = *cfg.Parsed.Hostname } - // ctx to live for the lifetime of the process. - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer cancel() - // Make sure we crash loop if Up doesn't complete in reasonable time. upCtx, upCancel := context.WithTimeout(ctx, time.Minute) defer upCancel() @@ -126,9 +188,6 @@ func run(logger *zap.SugaredLogger) error { return fmt.Errorf("error starting tailscale server: %w", err) } defer ts.Close() - - group, groupCtx := errgroup.WithContext(ctx) - lc, err := ts.LocalClient() if err != nil { return fmt.Errorf("error getting local client: %w", err) @@ -136,23 +195,13 @@ func run(logger *zap.SugaredLogger) error { // Setup for updating state keys. if podUID != "" { - w, err := lc.WatchIPNBus(groupCtx, ipn.NotifyInitialNetMap) - if err != nil { - return fmt.Errorf("error watching IPN bus: %w", err) - } - defer w.Close() - group.Go(func() error { - if err := state.KeepKeysUpdated(st, w.Next); err != nil && err != groupCtx.Err() { - return fmt.Errorf("error keeping state keys updated: %w", err) - } - - return nil + return state.KeepKeysUpdated(ctx, st, klc.New(lc)) }) } if cfg.Parsed.AcceptRoutes != nil { - _, err = lc.EditPrefs(groupCtx, &ipn.MaskedPrefs{ + _, err = lc.EditPrefs(ctx, &ipn.MaskedPrefs{ RouteAllSet: true, Prefs: ipn.Prefs{RouteAll: *cfg.Parsed.AcceptRoutes}, }) @@ -161,34 +210,97 @@ func run(logger *zap.SugaredLogger) error { } } - // Setup for the API server proxy. - restConfig, err := getRestConfig(logger) - if err != nil { - return fmt.Errorf("error getting rest config: %w", err) + // TODO(tomhjp): There seems to be a bug that on restart the device does + // not get reassigned it's already working Service IPs unless we clear and + // reset the serve config. + if err := lc.SetServeConfig(ctx, &ipn.ServeConfig{}); err != nil { + return fmt.Errorf("error clearing existing ServeConfig: %w", err) } - authMode := true - if cfg.Parsed.KubeAPIServer != nil { - v, ok := cfg.Parsed.KubeAPIServer.AuthMode.Get() - if ok { - authMode = v + + var cm *certs.CertManager + if shouldIssueCerts(cfg) { + logger.Infof("Will issue TLS certs for Tailscale Service") + cm = certs.NewCertManager(klc.New(lc), logger.Infof) + } + if err := setServeConfig(ctx, lc, cm, apiServerProxyService(cfg)); err != nil { + return err + } + + if cfg.Parsed.AdvertiseServices != nil { + if _, err := lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: cfg.Parsed.AdvertiseServices, + }, + }); err != nil { + return fmt.Errorf("error setting prefs AdvertiseServices: %w", err) } } - ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, authMode) + + // Setup for the API server proxy. + authMode := true + if cfg.Parsed.APIServerProxy != nil && cfg.Parsed.APIServerProxy.AuthMode.EqualBool(false) { + authMode = false + } + ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, authMode, false) if err != nil { return fmt.Errorf("error creating api server proxy: %w", err) } - // TODO(tomhjp): Work out whether we should use TS_CERT_SHARE_MODE or not, - // and possibly issue certs upfront here before serving. group.Go(func() error { - if err := ap.Run(groupCtx); err != nil { + if err := ap.Run(serveCtx); err != nil { return fmt.Errorf("error running API server proxy: %w", err) } return nil }) - return group.Wait() + for { + select { + case <-ctx.Done(): + // Context cancelled, exit. + logger.Info("Context cancelled, exiting") + shutdownCtx, shutdownCancel := context.WithTimeout(serveCtx, 20*time.Second) + unadvertiseErr := services.EnsureServicesNotAdvertised(shutdownCtx, lc, logger.Infof) + shutdownCancel() + serveCancel() + return errors.Join(unadvertiseErr, group.Wait()) + case cfg = <-cfgChan: + // Handle config reload. + // TODO(tomhjp): Make auth mode reloadable. + var prefs ipn.MaskedPrefs + cfgLogger := logger + currentPrefs, err := lc.GetPrefs(ctx) + if err != nil { + return fmt.Errorf("error getting current prefs: %w", err) + } + if !slices.Equal(currentPrefs.AdvertiseServices, cfg.Parsed.AdvertiseServices) { + cfgLogger = cfgLogger.With("AdvertiseServices", fmt.Sprintf("%v -> %v", currentPrefs.AdvertiseServices, cfg.Parsed.AdvertiseServices)) + prefs.AdvertiseServicesSet = true + prefs.Prefs.AdvertiseServices = cfg.Parsed.AdvertiseServices + } + if cfg.Parsed.Hostname != nil && *cfg.Parsed.Hostname != currentPrefs.Hostname { + cfgLogger = cfgLogger.With("Hostname", fmt.Sprintf("%s -> %s", currentPrefs.Hostname, *cfg.Parsed.Hostname)) + prefs.HostnameSet = true + prefs.Hostname = *cfg.Parsed.Hostname + } + if cfg.Parsed.AcceptRoutes != nil && *cfg.Parsed.AcceptRoutes != currentPrefs.RouteAll { + cfgLogger = cfgLogger.With("AcceptRoutes", fmt.Sprintf("%v -> %v", currentPrefs.RouteAll, *cfg.Parsed.AcceptRoutes)) + prefs.RouteAllSet = true + prefs.Prefs.RouteAll = *cfg.Parsed.AcceptRoutes + } + if !prefs.IsEmpty() { + if _, err := lc.EditPrefs(ctx, &prefs); err != nil { + return fmt.Errorf("error editing prefs: %w", err) + } + } + if err := setServeConfig(ctx, lc, cm, apiServerProxyService(cfg)); err != nil { + return fmt.Errorf("error setting serve config: %w", err) + } + + cfgLogger.Infof("Config reloaded") + } + } } func getStateStore(path *string, logger *zap.SugaredLogger) (ipn.StateStore, error) { @@ -226,3 +338,79 @@ func getRestConfig(logger *zap.SugaredLogger) (*rest.Config, error) { return restConfig, nil } + +func apiServerProxyService(cfg *conf.Config) tailcfg.ServiceName { + if cfg.Parsed.APIServerProxy != nil && + cfg.Parsed.APIServerProxy.Enabled.EqualBool(true) && + cfg.Parsed.APIServerProxy.ServiceName != nil && + *cfg.Parsed.APIServerProxy.ServiceName != "" { + return tailcfg.ServiceName(*cfg.Parsed.APIServerProxy.ServiceName) + } + + return "" +} + +func shouldIssueCerts(cfg *conf.Config) bool { + return cfg.Parsed.APIServerProxy != nil && + cfg.Parsed.APIServerProxy.IssueCerts.EqualBool(true) +} + +// setServeConfig sets up serve config such that it's serving for the passed in +// Tailscale Service, and does nothing if it's already up to date. +func setServeConfig(ctx context.Context, lc *local.Client, cm *certs.CertManager, name tailcfg.ServiceName) error { + existingServeConfig, err := lc.GetServeConfig(ctx) + if err != nil { + return fmt.Errorf("error getting existing serve config: %w", err) + } + + // Ensure serve config is cleared if no Tailscale Service. + if name == "" { + if reflect.DeepEqual(*existingServeConfig, ipn.ServeConfig{}) { + // Already up to date. + return nil + } + + if cm != nil { + cm.EnsureCertLoops(ctx, &ipn.ServeConfig{}) + } + return lc.SetServeConfig(ctx, &ipn.ServeConfig{}) + } + + status, err := lc.StatusWithoutPeers(ctx) + if err != nil { + return fmt.Errorf("error getting local client status: %w", err) + } + serviceHostPort := ipn.HostPort(fmt.Sprintf("%s.%s:443", name.WithoutPrefix(), status.CurrentTailnet.MagicDNSSuffix)) + + serveConfig := ipn.ServeConfig{ + // Configure for the Service hostname. + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + name: { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: { + HTTPS: true, + }, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + serviceHostPort: { + Handlers: map[string]*ipn.HTTPHandler{ + "/": { + Proxy: fmt.Sprintf("http://%s:80", strings.TrimSuffix(status.Self.DNSName, ".")), + }, + }, + }, + }, + }, + }, + } + + if reflect.DeepEqual(*existingServeConfig, serveConfig) { + // Already up to date. + return nil + } + + if cm != nil { + cm.EnsureCertLoops(ctx, &serveConfig) + } + return lc.SetServeConfig(ctx, &serveConfig) +} diff --git a/internal/client/tailscale/vip_service.go b/internal/client/tailscale/vip_service.go index 64fcfdf5e86d6..48c59ce4569da 100644 --- a/internal/client/tailscale/vip_service.go +++ b/internal/client/tailscale/vip_service.go @@ -36,6 +36,11 @@ type VIPService struct { Tags []string `json:"tags,omitempty"` } +// VIPServiceList represents the JSON response to the list VIP Services API. +type VIPServiceList struct { + VIPServices []VIPService `json:"vipServices"` +} + // GetVIPService retrieves a VIPService by its name. It returns 404 if the VIPService is not found. func (client *Client) GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*VIPService, error) { path := client.BuildTailnetURL("vip-services", name.String()) @@ -59,6 +64,29 @@ func (client *Client) GetVIPService(ctx context.Context, name tailcfg.ServiceNam return svc, nil } +// ListVIPServices retrieves all existing Services and returns them as a list. +func (client *Client) ListVIPServices(ctx context.Context) (*VIPServiceList, error) { + path := client.BuildTailnetURL("vip-services") + req, err := http.NewRequestWithContext(ctx, httpm.GET, path, nil) + if err != nil { + return nil, fmt.Errorf("error creating new HTTP request: %w", err) + } + b, resp, err := SendRequest(client, req) + if err != nil { + return nil, fmt.Errorf("error making Tailsale API request: %w", err) + } + // If status code was not successful, return the error. + // TODO: Change the check for the StatusCode to include other 2XX success codes. + if resp.StatusCode != http.StatusOK { + return nil, HandleErrorResponse(b, resp) + } + result := &VIPServiceList{} + if err := json.Unmarshal(b, result); err != nil { + return nil, err + } + return result, nil +} + // CreateOrUpdateVIPService creates or updates a VIPService by its name. Caller must ensure that, if the // VIPService already exists, the VIPService is fetched first to ensure that any auto-allocated IP addresses are not // lost during the update. If the VIPService was created without any IP addresses explicitly set (so that they were diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go index 14025bbb4150a..a9ad514e755b2 100644 --- a/ipn/store/kubestore/store_kube.go +++ b/ipn/store/kubestore/store_kube.go @@ -394,8 +394,8 @@ func (s *Store) canPatchSecret(secret string) bool { // certSecretSelector returns a label selector that can be used to list all // Secrets that aren't Tailscale state Secrets and contain TLS certificates for // HTTPS endpoints that this node serves. -// Currently (3/2025) this only applies to the Kubernetes Operator's ingress -// ProxyGroup. +// Currently (7/2025) this only applies to the Kubernetes Operator's ProxyGroup +// when spec.Type is "ingress" or "kube-apiserver". func (s *Store) certSecretSelector() map[string]string { if s.podName == "" { return map[string]string{} @@ -406,7 +406,7 @@ func (s *Store) certSecretSelector() map[string]string { } pgName := s.podName[:p] return map[string]string{ - kubetypes.LabelSecretType: "certs", + kubetypes.LabelSecretType: kubetypes.LabelSecretTypeCerts, kubetypes.LabelManaged: "true", "tailscale.com/proxy-group": pgName, } diff --git a/ipn/store/kubestore/store_kube_test.go b/ipn/store/kubestore/store_kube_test.go index 0d709264e5c08..9a49f30288840 100644 --- a/ipn/store/kubestore/store_kube_test.go +++ b/ipn/store/kubestore/store_kube_test.go @@ -17,6 +17,7 @@ import ( "tailscale.com/ipn/store/mem" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" ) func TestWriteState(t *testing.T) { @@ -516,7 +517,7 @@ func TestNewWithClient(t *testing.T) { ) certSecretsLabels := map[string]string{ - "tailscale.com/secret-type": "certs", + "tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts, "tailscale.com/managed": "true", "tailscale.com/proxy-group": "ingress-proxies", } @@ -582,7 +583,7 @@ func TestNewWithClient(t *testing.T) { makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"), makeSecret("some-other-secret", nil, "3"), makeSecret("app3.other-proxies.ts.net", map[string]string{ - "tailscale.com/secret-type": "certs", + "tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts, "tailscale.com/managed": "true", "tailscale.com/proxy-group": "some-other-proxygroup", }, "4"), @@ -606,7 +607,7 @@ func TestNewWithClient(t *testing.T) { makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"), makeSecret("some-other-secret", nil, "3"), makeSecret("app3.other-proxies.ts.net", map[string]string{ - "tailscale.com/secret-type": "certs", + "tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts, "tailscale.com/managed": "true", "tailscale.com/proxy-group": "some-other-proxygroup", }, "4"), diff --git a/k8s-operator/api-proxy/proxy.go b/k8s-operator/api-proxy/proxy.go index d33c088de78db..e079e984ff5a1 100644 --- a/k8s-operator/api-proxy/proxy.go +++ b/k8s-operator/api-proxy/proxy.go @@ -10,6 +10,7 @@ import ( "crypto/tls" "errors" "fmt" + "net" "net/http" "net/http/httputil" "net/netip" @@ -46,7 +47,7 @@ var ( // caller's Tailscale identity and the rules defined in the tailnet ACLs. // - false: the proxy is started and requests are passed through to the // Kubernetes API without any auth modifications. -func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsnet.Server, authMode bool) (*APIServerProxy, error) { +func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsnet.Server, authMode bool, https bool) (*APIServerProxy, error) { if !authMode { restConfig = rest.AnonymousClientConfig(restConfig) } @@ -85,6 +86,7 @@ func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsn log: zlog, lc: lc, authMode: authMode, + https: https, upstreamURL: u, ts: ts, } @@ -104,11 +106,6 @@ func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsn // // It return when ctx is cancelled or ServeTLS fails. func (ap *APIServerProxy) Run(ctx context.Context) error { - ln, err := ap.ts.Listen("tcp", ":443") - if err != nil { - return fmt.Errorf("could not listen on :443: %v", err) - } - mux := http.NewServeMux() mux.HandleFunc("/", ap.serveDefault) mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY) @@ -117,32 +114,61 @@ func (ap *APIServerProxy) Run(ctx context.Context) error { mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/attach", ap.serveAttachWS) ap.hs = &http.Server{ + Handler: mux, + ErrorLog: zap.NewStdLog(ap.log.Desugar()), + } + + mode := "noauth" + if ap.authMode { + mode = "auth" + } + var tsLn net.Listener + var serve func(ln net.Listener) error + if ap.https { + var err error + tsLn, err = ap.ts.Listen("tcp", ":443") + if err != nil { + return fmt.Errorf("could not listen on :443: %w", err) + } + serve = func(ln net.Listener) error { + return ap.hs.ServeTLS(ln, "", "") + } + // Kubernetes uses SPDY for exec and port-forward, however SPDY is // incompatible with HTTP/2; so disable HTTP/2 in the proxy. - TLSConfig: &tls.Config{ + ap.hs.TLSConfig = &tls.Config{ GetCertificate: ap.lc.GetCertificate, NextProtos: []string{"http/1.1"}, - }, - TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), - Handler: mux, + } + ap.hs.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) + } else { + var err error + tsLn, err = ap.ts.Listen("tcp", ":80") + if err != nil { + return fmt.Errorf("could not listen on :80: %w", err) + } + serve = ap.hs.Serve } errs := make(chan error) go func() { - ap.log.Infof("API server proxy is listening on %s with auth mode: %v", ln.Addr(), ap.authMode) - if err := ap.hs.ServeTLS(ln, "", ""); err != nil && err != http.ErrServerClosed { - errs <- fmt.Errorf("failed to serve: %w", err) + ap.log.Infof("API server proxy in %s mode is listening on tailnet addresses %s", mode, tsLn.Addr()) + if err := serve(tsLn); err != nil && err != http.ErrServerClosed { + errs <- fmt.Errorf("error serving: %w", err) } }() select { case <-ctx.Done(): - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - return ap.hs.Shutdown(shutdownCtx) case err := <-errs: + ap.hs.Close() return err } + + // Graceful shutdown with a timeout of 10s. + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return ap.hs.Shutdown(shutdownCtx) } // APIServerProxy is an [net/http.Handler] that authenticates requests using the Tailscale @@ -152,7 +178,8 @@ type APIServerProxy struct { lc *local.Client rp *httputil.ReverseProxy - authMode bool + authMode bool // Whether to run with impersonation using caller's tailnet identity. + https bool // Whether to serve on https for the device hostname; true for k8s-operator, false for k8s-proxy. ts *tsnet.Server hs *http.Server upstreamURL *url.URL @@ -181,13 +208,13 @@ func (ap *APIServerProxy) serveExecWS(w http.ResponseWriter, r *http.Request) { ap.sessionForProto(w, r, ksr.ExecSessionType, ksr.WSProtocol) } -// serveExecSPDY serves '/attach' requests for sessions streamed over SPDY, +// serveAttachSPDY serves '/attach' requests for sessions streamed over SPDY, // optionally configuring the kubectl exec sessions to be recorded. func (ap *APIServerProxy) serveAttachSPDY(w http.ResponseWriter, r *http.Request) { ap.sessionForProto(w, r, ksr.AttachSessionType, ksr.SPDYProtocol) } -// serveExecWS serves '/attach' requests for sessions streamed over WebSocket, +// serveAttachWS serves '/attach' requests for sessions streamed over WebSocket, // optionally configuring the kubectl exec sessions to be recorded. func (ap *APIServerProxy) serveAttachWS(w http.ResponseWriter, r *http.Request) { ap.sessionForProto(w, r, ksr.AttachSessionType, ksr.WSProtocol) diff --git a/k8s-operator/api.md b/k8s-operator/api.md index c09152da6f6c1..cd36798d69f8b 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -342,6 +342,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `mode` _[APIServerProxyMode](#apiserverproxymode)_ | Mode to run the API server proxy in. Supported modes are auth and noauth.
          In auth mode, requests from the tailnet proxied over to the Kubernetes
          API server are additionally impersonated using the sender's tailnet identity.
          If not specified, defaults to auth mode. | | Enum: [auth noauth]
          Type: string
          | +| `hostname` _string_ | Hostname is the hostname with which to expose the Kubernetes API server
          proxies. Must be a valid DNS label no longer than 63 characters. If not
          specified, the name of the ProxyGroup is used as the hostname. Must be
          unique across the whole tailnet. | | Pattern: `^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`
          Type: string
          | #### LabelValue @@ -610,15 +611,22 @@ _Appears in:_ ProxyGroup defines a set of Tailscale devices that will act as proxies. -Currently only egress ProxyGroups are supported. +Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver +proxies. In addition to running a highly available set of proxies, ingress +and egress ProxyGroups also allow for serving many annotated Services from a +single set of proxies to minimise resource consumption. -Use the tailscale.com/proxy-group annotation on a Service to specify that -the egress proxy should be implemented by a ProxyGroup instead of a single -dedicated proxy. In addition to running a highly available set of proxies, -ProxyGroup also allows for serving many annotated Services from a single -set of proxies to minimise resource consumption. +For ingress and egress, use the tailscale.com/proxy-group annotation on a +Service to specify that the proxy should be implemented by a ProxyGroup +instead of a single dedicated proxy. -More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +More info: +* https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +* https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress + +For kube-apiserver, the ProxyGroup is a standalone resource. Use the +spec.kubeAPIServer field to configure options specific to the kube-apiserver +ProxyGroup type. @@ -690,8 +698,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the ProxyGroup
          resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`.
          `ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled
          and ready. `ProxyGroupAvailable` indicates that at least one proxy is
          ready to serve traffic. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the ProxyGroup
          resources. Known condition types include `ProxyGroupReady` and
          `ProxyGroupAvailable`.
          * `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and
          all expected conditions are true.
          * `ProxyGroupAvailable` indicates that at least one proxy is ready to
          serve traffic.
          For ProxyGroups of type kube-apiserver, there are two additional conditions:
          * `KubeAPIServerProxyConfigured` indicates that at least one API server
          proxy is configured and ready to serve traffic.
          * `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is
          valid. | | | | `devices` _[TailnetDevice](#tailnetdevice) array_ | List of tailnet devices associated with the ProxyGroup StatefulSet. | | | +| `url` _string_ | URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if
          any. Only applies to ProxyGroups of type kube-apiserver. | | | #### ProxyGroupType diff --git a/k8s-operator/apis/v1alpha1/types_connector.go b/k8s-operator/apis/v1alpha1/types_connector.go index 88fd07346cd5b..ce6a1411b9ea8 100644 --- a/k8s-operator/apis/v1alpha1/types_connector.go +++ b/k8s-operator/apis/v1alpha1/types_connector.go @@ -226,4 +226,7 @@ const ( IngressSvcValid ConditionType = `TailscaleIngressSvcValid` IngressSvcConfigured ConditionType = `TailscaleIngressSvcConfigured` + + KubeAPIServerProxyValid ConditionType = `KubeAPIServerProxyValid` // The kubeAPIServer config for the ProxyGroup is valid. + KubeAPIServerProxyConfigured ConditionType = `KubeAPIServerProxyConfigured` // At least one of the ProxyGroup's Pods is advertising the kube-apiserver proxy's hostname. ) diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index ad5b113612bbf..28fd9e00973c5 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -13,19 +13,27 @@ import ( // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster,shortName=pg // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ProxyGroupReady")].reason`,description="Status of the deployed ProxyGroup resources." +// +kubebuilder:printcolumn:name="URL",type="string",JSONPath=`.status.url`,description="URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver." // +kubebuilder:printcolumn:name="Type",type="string",JSONPath=`.spec.type`,description="ProxyGroup type." // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // ProxyGroup defines a set of Tailscale devices that will act as proxies. -// Currently only egress ProxyGroups are supported. +// Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver +// proxies. In addition to running a highly available set of proxies, ingress +// and egress ProxyGroups also allow for serving many annotated Services from a +// single set of proxies to minimise resource consumption. // -// Use the tailscale.com/proxy-group annotation on a Service to specify that -// the egress proxy should be implemented by a ProxyGroup instead of a single -// dedicated proxy. In addition to running a highly available set of proxies, -// ProxyGroup also allows for serving many annotated Services from a single -// set of proxies to minimise resource consumption. +// For ingress and egress, use the tailscale.com/proxy-group annotation on a +// Service to specify that the proxy should be implemented by a ProxyGroup +// instead of a single dedicated proxy. // -// More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +// More info: +// * https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +// * https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress +// +// For kube-apiserver, the ProxyGroup is a standalone resource. Use the +// spec.kubeAPIServer field to configure options specific to the kube-apiserver +// ProxyGroup type. type ProxyGroup struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -93,10 +101,20 @@ type ProxyGroupSpec struct { type ProxyGroupStatus struct { // List of status conditions to indicate the status of the ProxyGroup - // resources. Known condition types are `ProxyGroupReady`, `ProxyGroupAvailable`. - // `ProxyGroupReady` indicates all ProxyGroup resources are fully reconciled - // and ready. `ProxyGroupAvailable` indicates that at least one proxy is - // ready to serve traffic. + // resources. Known condition types include `ProxyGroupReady` and + // `ProxyGroupAvailable`. + // + // * `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and + // all expected conditions are true. + // * `ProxyGroupAvailable` indicates that at least one proxy is ready to + // serve traffic. + // + // For ProxyGroups of type kube-apiserver, there are two additional conditions: + // + // * `KubeAPIServerProxyConfigured` indicates that at least one API server + // proxy is configured and ready to serve traffic. + // * `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is + // valid. // // +listType=map // +listMapKey=type @@ -108,6 +126,11 @@ type ProxyGroupStatus struct { // +listMapKey=hostname // +optional Devices []TailnetDevice `json:"devices,omitempty"` + + // URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if + // any. Only applies to ProxyGroups of type kube-apiserver. + // +optional + URL string `json:"url,omitempty"` } type TailnetDevice struct { @@ -157,4 +180,13 @@ type KubeAPIServerConfig struct { // If not specified, defaults to auth mode. // +optional Mode *APIServerProxyMode `json:"mode,omitempty"` + + // Hostname is the hostname with which to expose the Kubernetes API server + // proxies. Must be a valid DNS label no longer than 63 characters. If not + // specified, the name of the ProxyGroup is used as the hostname. Must be + // unique across the whole tailnet. + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Pattern=`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$` + // +optional + Hostname string `json:"hostname,omitempty"` } diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go index f6858c0059162..ae465a728f0ff 100644 --- a/k8s-operator/conditions.go +++ b/k8s-operator/conditions.go @@ -146,6 +146,16 @@ func ProxyGroupAvailable(pg *tsapi.ProxyGroup) bool { return cond != nil && cond.Status == metav1.ConditionTrue } +func KubeAPIServerProxyValid(pg *tsapi.ProxyGroup) (valid bool, set bool) { + cond := proxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation, cond != nil +} + +func KubeAPIServerProxyConfigured(pg *tsapi.ProxyGroup) bool { + cond := proxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation +} + func proxyGroupCondition(pg *tsapi.ProxyGroup, condType tsapi.ConditionType) *metav1.Condition { idx := xslices.IndexFunc(pg.Status.Conditions, func(cond metav1.Condition) bool { return cond.Type == string(condType) diff --git a/cmd/containerboot/certs.go b/kube/certs/certs.go similarity index 60% rename from cmd/containerboot/certs.go rename to kube/certs/certs.go index 504ef7988072b..8e2e5fb43a8ac 100644 --- a/cmd/containerboot/certs.go +++ b/kube/certs/certs.go @@ -1,29 +1,32 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux - -package main +// Package certs implements logic to help multiple Kubernetes replicas share TLS +// certs for a common Tailscale Service. +package certs import ( "context" "fmt" - "log" "net" + "slices" "sync" "time" "tailscale.com/ipn" + "tailscale.com/kube/localclient" + "tailscale.com/types/logger" "tailscale.com/util/goroutines" "tailscale.com/util/mak" ) -// certManager is responsible for issuing certificates for known domains and for +// CertManager is responsible for issuing certificates for known domains and for // maintaining a loop that re-attempts issuance daily. // Currently cert manager logic is only run on ingress ProxyGroup replicas that are responsible for managing certs for // HA Ingress HTTPS endpoints ('write' replicas). -type certManager struct { - lc localClient +type CertManager struct { + lc localclient.LocalClient + logf logger.Logf tracker goroutines.Tracker // tracks running goroutines mu sync.Mutex // guards the following // certLoops contains a map of DNS names, for which we currently need to @@ -32,11 +35,18 @@ type certManager struct { certLoops map[string]context.CancelFunc } -// ensureCertLoops ensures that, for all currently managed Service HTTPS +func NewCertManager(lc localclient.LocalClient, logf logger.Logf) *CertManager { + return &CertManager{ + lc: lc, + logf: logf, + } +} + +// EnsureCertLoops ensures that, for all currently managed Service HTTPS // endpoints, there is a cert loop responsible for issuing and ensuring the // renewal of the TLS certs. // ServeConfig must not be nil. -func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error { +func (cm *CertManager) EnsureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error { if sc == nil { return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig") } @@ -87,12 +97,18 @@ func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig) // renewed at that point. Renewal here is needed to prevent the shared certs from expiry in edge cases where the 'write' // replica does not get any HTTPS requests. // https://letsencrypt.org/docs/integration-guide/#retrying-failures -func (cm *certManager) runCertLoop(ctx context.Context, domain string) { +func (cm *CertManager) runCertLoop(ctx context.Context, domain string) { const ( normalInterval = 24 * time.Hour // regular renewal check initialRetry = 1 * time.Minute // initial backoff after a failure maxRetryInterval = 24 * time.Hour // max backoff period ) + + if err := cm.waitForCertDomain(ctx, domain); err != nil { + // Best-effort, log and continue with the issuing loop. + cm.logf("error waiting for cert domain %s: %v", domain, err) + } + timer := time.NewTimer(0) // fire off timer immediately defer timer.Stop() retryCount := 0 @@ -101,38 +117,31 @@ func (cm *certManager) runCertLoop(ctx context.Context, domain string) { case <-ctx.Done(): return case <-timer.C: - // We call the certificate endpoint, but don't do anything - // with the returned certs here. - // The call to the certificate endpoint will ensure that - // certs are issued/renewed as needed and stored in the - // relevant state store. For example, for HA Ingress - // 'write' replica, the cert and key will be stored in a - // Kubernetes Secret named after the domain for which we - // are issuing. - // Note that renewals triggered by the call to the - // certificates endpoint here and by renewal check - // triggered during a call to node's HTTPS endpoint - // share the same state/renewal lock mechanism, so we - // should not run into redundant issuances during - // concurrent renewal checks. - // TODO(irbekrm): maybe it is worth adding a new - // issuance endpoint that explicitly only triggers - // issuance and stores certs in the relevant store, but - // does not return certs to the caller? + // We call the certificate endpoint, but don't do anything with the + // returned certs here. The call to the certificate endpoint will + // ensure that certs are issued/renewed as needed and stored in the + // relevant state store. For example, for HA Ingress 'write' replica, + // the cert and key will be stored in a Kubernetes Secret named after + // the domain for which we are issuing. + // + // Note that renewals triggered by the call to the certificates + // endpoint here and by renewal check triggered during a call to + // node's HTTPS endpoint share the same state/renewal lock mechanism, + // so we should not run into redundant issuances during concurrent + // renewal checks. - // An issuance holds a shared lock, so we need to avoid - // a situation where other services cannot issue certs - // because a single one is holding the lock. + // An issuance holds a shared lock, so we need to avoid a situation + // where other services cannot issue certs because a single one is + // holding the lock. ctxT, cancel := context.WithTimeout(ctx, time.Second*300) - defer cancel() _, _, err := cm.lc.CertPair(ctxT, domain) + cancel() if err != nil { - log.Printf("error refreshing certificate for %s: %v", domain, err) + cm.logf("error refreshing certificate for %s: %v", domain, err) } var nextInterval time.Duration - // TODO(irbekrm): distinguish between LE rate limit - // errors and other error types like transient network - // errors. + // TODO(irbekrm): distinguish between LE rate limit errors and other + // error types like transient network errors. if err == nil { retryCount = 0 nextInterval = normalInterval @@ -147,10 +156,34 @@ func (cm *certManager) runCertLoop(ctx context.Context, domain string) { backoff = maxRetryInterval } nextInterval = backoff - log.Printf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n", + cm.logf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n", domain, retryCount, err, nextInterval) } timer.Reset(nextInterval) } } } + +// waitForCertDomain ensures the requested domain is in the list of allowed +// domains before issuing the cert for the first time. +func (cm *CertManager) waitForCertDomain(ctx context.Context, domain string) error { + w, err := cm.lc.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) + if err != nil { + return fmt.Errorf("error watching IPN bus: %w", err) + } + defer w.Close() + + for { + n, err := w.Next() + if err != nil { + return err + } + if n.NetMap == nil { + continue + } + + if slices.Contains(n.NetMap.DNS.CertDomains, domain) { + return nil + } + } +} diff --git a/cmd/containerboot/certs_test.go b/kube/certs/certs_test.go similarity index 89% rename from cmd/containerboot/certs_test.go rename to kube/certs/certs_test.go index 577311ea36a64..8434f21ae6976 100644 --- a/cmd/containerboot/certs_test.go +++ b/kube/certs/certs_test.go @@ -1,17 +1,18 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux - -package main +package certs import ( "context" + "log" "testing" "time" "tailscale.com/ipn" + "tailscale.com/kube/localclient" "tailscale.com/tailcfg" + "tailscale.com/types/netmap" ) // TestEnsureCertLoops tests that the certManager correctly starts and stops @@ -161,8 +162,28 @@ func TestEnsureCertLoops(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cm := &certManager{ - lc: &fakeLocalClient{}, + notifyChan := make(chan ipn.Notify) + go func() { + for { + notifyChan <- ipn.Notify{ + NetMap: &netmap.NetworkMap{ + DNS: tailcfg.DNSConfig{ + CertDomains: []string{ + "my-app.tailnetxyz.ts.net", + "my-other-app.tailnetxyz.ts.net", + }, + }, + }, + } + } + }() + cm := &CertManager{ + lc: &localclient.FakeLocalClient{ + FakeIPNBusWatcher: localclient.FakeIPNBusWatcher{ + NotifyChan: notifyChan, + }, + }, + logf: log.Printf, certLoops: make(map[string]context.CancelFunc), } @@ -179,7 +200,7 @@ func TestEnsureCertLoops(t *testing.T) { } })() - err := cm.ensureCertLoops(ctx, tt.initialConfig) + err := cm.EnsureCertLoops(ctx, tt.initialConfig) if (err != nil) != tt.wantErr { t.Fatalf("ensureCertLoops() error = %v", err) } @@ -189,7 +210,7 @@ func TestEnsureCertLoops(t *testing.T) { } if tt.updatedConfig != nil { - if err := cm.ensureCertLoops(ctx, tt.updatedConfig); err != nil { + if err := cm.EnsureCertLoops(ctx, tt.updatedConfig); err != nil { t.Fatalf("ensureCertLoops() error on update = %v", err) } diff --git a/kube/k8s-proxy/conf/conf.go b/kube/k8s-proxy/conf/conf.go index 8882360c5ea21..a32e0c03ef2bc 100644 --- a/kube/k8s-proxy/conf/conf.go +++ b/kube/k8s-proxy/conf/conf.go @@ -9,11 +9,12 @@ package conf import ( "encoding/json" + "errors" "fmt" "net/netip" - "os" "github.com/tailscale/hujson" + "tailscale.com/tailcfg" "tailscale.com/types/opt" ) @@ -21,12 +22,11 @@ const v1Alpha1 = "v1alpha1" // Config describes a config file. type Config struct { - Path string // disk path of HuJSON - Raw []byte // raw bytes from disk, in HuJSON form + Raw []byte // raw bytes, in HuJSON form Std []byte // standardized JSON form Version string // "v1alpha1" - // Parsed is the parsed config, converted from its on-disk version to the + // Parsed is the parsed config, converted from its raw bytes version to the // latest known format. Parsed ConfigV1Alpha1 } @@ -48,47 +48,49 @@ type VersionedConfig struct { } type ConfigV1Alpha1 struct { - AuthKey *string `json:",omitempty"` // Tailscale auth key to use. - Hostname *string `json:",omitempty"` // Tailscale device hostname. - State *string `json:",omitempty"` // Path to the Tailscale state. - LogLevel *string `json:",omitempty"` // "debug", "info". Defaults to "info". - App *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer - KubeAPIServer *KubeAPIServer `json:",omitempty"` // Config specific to the API Server proxy. - ServerURL *string `json:",omitempty"` // URL of the Tailscale coordination server. - AcceptRoutes *bool `json:",omitempty"` // Accepts routes advertised by other Tailscale nodes. + AuthKey *string `json:",omitempty"` // Tailscale auth key to use. + State *string `json:",omitempty"` // Path to the Tailscale state. + LogLevel *string `json:",omitempty"` // "debug", "info". Defaults to "info". + App *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer + ServerURL *string `json:",omitempty"` // URL of the Tailscale coordination server. // StaticEndpoints are additional, user-defined endpoints that this node // should advertise amongst its wireguard endpoints. StaticEndpoints []netip.AddrPort `json:",omitempty"` + + // TODO(tomhjp): The remaining fields should all be reloadable during + // runtime, but currently missing most of the APIServerProxy fields. + Hostname *string `json:",omitempty"` // Tailscale device hostname. + AcceptRoutes *bool `json:",omitempty"` // Accepts routes advertised by other Tailscale nodes. + AdvertiseServices []string `json:",omitempty"` // Tailscale Services to advertise. + APIServerProxy *APIServerProxyConfig `json:",omitempty"` // Config specific to the API Server proxy. } -type KubeAPIServer struct { - AuthMode opt.Bool `json:",omitempty"` +type APIServerProxyConfig struct { + Enabled opt.Bool `json:",omitempty"` // Whether to enable the API Server proxy. + AuthMode opt.Bool `json:",omitempty"` // Run in auth or noauth mode. + ServiceName *tailcfg.ServiceName `json:",omitempty"` // Name of the Tailscale Service to advertise. + IssueCerts opt.Bool `json:",omitempty"` // Whether this replica should issue TLS certs for the Tailscale Service. } // Load reads and parses the config file at the provided path on disk. -func Load(path string) (c Config, err error) { - c.Path = path - - c.Raw, err = os.ReadFile(path) - if err != nil { - return c, fmt.Errorf("error reading config file %q: %w", path, err) - } +func Load(raw []byte) (c Config, err error) { + c.Raw = raw c.Std, err = hujson.Standardize(c.Raw) if err != nil { - return c, fmt.Errorf("error parsing config file %q HuJSON/JSON: %w", path, err) + return c, fmt.Errorf("error parsing config as HuJSON/JSON: %w", err) } var ver VersionedConfig if err := json.Unmarshal(c.Std, &ver); err != nil { - return c, fmt.Errorf("error parsing config file %q: %w", path, err) + return c, fmt.Errorf("error parsing config: %w", err) } rootV1Alpha1 := (ver.Version == v1Alpha1) backCompatV1Alpha1 := (ver.V1Alpha1 != nil) switch { case ver.Version == "": - return c, fmt.Errorf("error parsing config file %q: no \"version\" field provided", path) + return c, errors.New("error parsing config: no \"version\" field provided") case rootV1Alpha1 && backCompatV1Alpha1: // Exactly one of these should be set. - return c, fmt.Errorf("error parsing config file %q: both root and v1alpha1 config provided", path) + return c, errors.New("error parsing config: both root and v1alpha1 config provided") case rootV1Alpha1 != backCompatV1Alpha1: c.Version = v1Alpha1 switch { @@ -100,7 +102,7 @@ func Load(path string) (c Config, err error) { c.Parsed = ConfigV1Alpha1{} } default: - return c, fmt.Errorf("error parsing config file %q: unsupported \"version\" value %q; want \"%s\"", path, ver.Version, v1Alpha1) + return c, fmt.Errorf("error parsing config: unsupported \"version\" value %q; want \"%s\"", ver.Version, v1Alpha1) } return c, nil diff --git a/kube/k8s-proxy/conf/conf_test.go b/kube/k8s-proxy/conf/conf_test.go index a47391dc90ade..3082be1ba9dcd 100644 --- a/kube/k8s-proxy/conf/conf_test.go +++ b/kube/k8s-proxy/conf/conf_test.go @@ -6,8 +6,6 @@ package conf import ( - "os" - "path/filepath" "strings" "testing" @@ -57,12 +55,7 @@ func TestVersionedConfig(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "config.json") - if err := os.WriteFile(path, []byte(tc.inputConfig), 0644); err != nil { - t.Fatalf("failed to write config file: %v", err) - } - cfg, err := Load(path) + cfg, err := Load([]byte(tc.inputConfig)) switch { case tc.expectedError == "" && err != nil: t.Fatalf("unexpected error: %v", err) diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index 20b0050143c93..5e7d4cd1f1fd1 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -54,4 +54,10 @@ const ( LabelManaged = "tailscale.com/managed" LabelSecretType = "tailscale.com/secret-type" // "config", "state" "certs" + + LabelSecretTypeConfig = "config" + LabelSecretTypeState = "state" + LabelSecretTypeCerts = "certs" + + KubeAPIServerConfigFile = "config.hujson" ) diff --git a/kube/localclient/fake-client.go b/kube/localclient/fake-client.go new file mode 100644 index 0000000000000..7f0a08316634e --- /dev/null +++ b/kube/localclient/fake-client.go @@ -0,0 +1,35 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package localclient + +import ( + "context" + "fmt" + + "tailscale.com/ipn" +) + +type FakeLocalClient struct { + FakeIPNBusWatcher +} + +func (f *FakeLocalClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (IPNBusWatcher, error) { + return &f.FakeIPNBusWatcher, nil +} + +func (f *FakeLocalClient) CertPair(ctx context.Context, domain string) ([]byte, []byte, error) { + return nil, nil, fmt.Errorf("CertPair not implemented") +} + +type FakeIPNBusWatcher struct { + NotifyChan chan ipn.Notify +} + +func (f *FakeIPNBusWatcher) Close() error { + return nil +} + +func (f *FakeIPNBusWatcher) Next() (ipn.Notify, error) { + return <-f.NotifyChan, nil +} diff --git a/kube/localclient/local-client.go b/kube/localclient/local-client.go new file mode 100644 index 0000000000000..5d541e3655ddb --- /dev/null +++ b/kube/localclient/local-client.go @@ -0,0 +1,49 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package localclient provides an interface for all the local.Client methods +// kube needs to use, so that we can easily mock it in tests. +package localclient + +import ( + "context" + "io" + + "tailscale.com/client/local" + "tailscale.com/ipn" +) + +// LocalClient is roughly a subset of the local.Client struct's methods, used +// for easier testing. +type LocalClient interface { + WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (IPNBusWatcher, error) + CertIssuer +} + +// IPNBusWatcher is local.IPNBusWatcher's methods restated in an interface to +// allow for easier mocking in tests. +type IPNBusWatcher interface { + io.Closer + Next() (ipn.Notify, error) +} + +type CertIssuer interface { + CertPair(context.Context, string) ([]byte, []byte, error) +} + +// New returns a LocalClient that wraps the provided local.Client. +func New(lc *local.Client) LocalClient { + return &localClient{lc: lc} +} + +type localClient struct { + lc *local.Client +} + +func (l *localClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (IPNBusWatcher, error) { + return l.lc.WatchIPNBus(ctx, mask) +} + +func (l *localClient) CertPair(ctx context.Context, domain string) ([]byte, []byte, error) { + return l.lc.CertPair(ctx, domain) +} diff --git a/cmd/containerboot/services.go b/kube/services/services.go similarity index 74% rename from cmd/containerboot/services.go rename to kube/services/services.go index 6079128c02b19..a9e50975ca9f1 100644 --- a/cmd/containerboot/services.go +++ b/kube/services/services.go @@ -1,25 +1,25 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux - -package main +// Package services manages graceful shutdown of Tailscale Services advertised +// by Kubernetes clients. +package services import ( "context" "fmt" - "log" "time" "tailscale.com/client/local" "tailscale.com/ipn" + "tailscale.com/types/logger" ) -// ensureServicesNotAdvertised is a function that gets called on containerboot -// termination and ensures that any currently advertised VIPServices get -// unadvertised to give clients time to switch to another node before this one -// is shut down. -func ensureServicesNotAdvertised(ctx context.Context, lc *local.Client) error { +// EnsureServicesNotAdvertised is a function that gets called on containerboot +// or k8s-proxy termination and ensures that any currently advertised Services +// get unadvertised to give clients time to switch to another node before this +// one is shut down. +func EnsureServicesNotAdvertised(ctx context.Context, lc *local.Client, logf logger.Logf) error { prefs, err := lc.GetPrefs(ctx) if err != nil { return fmt.Errorf("error getting prefs: %w", err) @@ -28,7 +28,7 @@ func ensureServicesNotAdvertised(ctx context.Context, lc *local.Client) error { return nil } - log.Printf("unadvertising services: %v", prefs.AdvertiseServices) + logf("unadvertising services: %v", prefs.AdvertiseServices) if _, err := lc.EditPrefs(ctx, &ipn.MaskedPrefs{ AdvertiseServicesSet: true, Prefs: ipn.Prefs{ diff --git a/kube/state/state.go b/kube/state/state.go index 4831a5f5b367a..2605f0952f708 100644 --- a/kube/state/state.go +++ b/kube/state/state.go @@ -11,11 +11,13 @@ package state import ( + "context" "encoding/json" "fmt" "tailscale.com/ipn" "tailscale.com/kube/kubetypes" + klc "tailscale.com/kube/localclient" "tailscale.com/tailcfg" "tailscale.com/util/deephash" ) @@ -56,12 +58,20 @@ func SetInitialKeys(store ipn.StateStore, podUID string) error { // cancelled or it hits an error. The passed in next function is expected to be // from a local.IPNBusWatcher that is at least subscribed to // ipn.NotifyInitialNetMap. -func KeepKeysUpdated(store ipn.StateStore, next func() (ipn.Notify, error)) error { - var currentDeviceID, currentDeviceIPs, currentDeviceFQDN deephash.Sum +func KeepKeysUpdated(ctx context.Context, store ipn.StateStore, lc klc.LocalClient) error { + w, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) + if err != nil { + return fmt.Errorf("error watching IPN bus: %w", err) + } + defer w.Close() + var currentDeviceID, currentDeviceIPs, currentDeviceFQDN deephash.Sum for { - n, err := next() // Blocks on a streaming LocalAPI HTTP call. + n, err := w.Next() // Blocks on a streaming LocalAPI HTTP call. if err != nil { + if err == ctx.Err() { + return nil + } return err } if n.NetMap == nil { diff --git a/kube/state/state_test.go b/kube/state/state_test.go index 0375b1c01d91a..8701aa1b7fa65 100644 --- a/kube/state/state_test.go +++ b/kube/state/state_test.go @@ -15,6 +15,7 @@ import ( "github.com/google/go-cmp/cmp" "tailscale.com/ipn" "tailscale.com/ipn/store" + klc "tailscale.com/kube/localclient" "tailscale.com/tailcfg" "tailscale.com/types/logger" "tailscale.com/types/netmap" @@ -100,24 +101,20 @@ func TestSetInitialStateKeys(t *testing.T) { } func TestKeepStateKeysUpdated(t *testing.T) { - store, err := store.New(logger.Discard, "mem:") - if err != nil { - t.Fatalf("error creating in-memory store: %v", err) + store := fakeStore{ + writeChan: make(chan string), } - nextWaiting := make(chan struct{}) - go func() { - <-nextWaiting // Acknowledge the initial signal. - }() - notifyCh := make(chan ipn.Notify) - next := func() (ipn.Notify, error) { - nextWaiting <- struct{}{} // Send signal to test that state is consistent. - return <-notifyCh, nil // Wait for test input. + errs := make(chan error) + notifyChan := make(chan ipn.Notify) + lc := &klc.FakeLocalClient{ + FakeIPNBusWatcher: klc.FakeIPNBusWatcher{ + NotifyChan: notifyChan, + }, } - errs := make(chan error, 1) go func() { - err := KeepKeysUpdated(store, next) + err := KeepKeysUpdated(t.Context(), store, lc) if err != nil { errs <- fmt.Errorf("keepStateKeysUpdated returned with error: %w", err) } @@ -126,16 +123,12 @@ func TestKeepStateKeysUpdated(t *testing.T) { for _, tc := range []struct { name string notify ipn.Notify - expected map[ipn.StateKey][]byte + expected []string }{ { - name: "initial_not_authed", - notify: ipn.Notify{}, - expected: map[ipn.StateKey][]byte{ - keyDeviceID: nil, - keyDeviceFQDN: nil, - keyDeviceIPs: nil, - }, + name: "initial_not_authed", + notify: ipn.Notify{}, + expected: nil, }, { name: "authed", @@ -148,10 +141,10 @@ func TestKeepStateKeysUpdated(t *testing.T) { }).View(), }, }, - expected: map[ipn.StateKey][]byte{ - keyDeviceID: []byte("TESTCTRL00000001"), - keyDeviceFQDN: []byte("test-node.test.ts.net"), - keyDeviceIPs: []byte(`["100.64.0.1","fd7a:115c:a1e0:ab12:4843:cd96:0:1"]`), + expected: []string{ + fmt.Sprintf("%s=%s", keyDeviceID, "TESTCTRL00000001"), + fmt.Sprintf("%s=%s", keyDeviceFQDN, "test-node.test.ts.net"), + fmt.Sprintf("%s=%s", keyDeviceIPs, `["100.64.0.1","fd7a:115c:a1e0:ab12:4843:cd96:0:1"]`), }, }, { @@ -165,39 +158,39 @@ func TestKeepStateKeysUpdated(t *testing.T) { }).View(), }, }, - expected: map[ipn.StateKey][]byte{ - keyDeviceID: []byte("TESTCTRL00000001"), - keyDeviceFQDN: []byte("updated.test.ts.net"), - keyDeviceIPs: []byte(`["100.64.0.250"]`), + expected: []string{ + fmt.Sprintf("%s=%s", keyDeviceFQDN, "updated.test.ts.net"), + fmt.Sprintf("%s=%s", keyDeviceIPs, `["100.64.0.250"]`), }, }, } { t.Run(tc.name, func(t *testing.T) { - // Send test input. - select { - case notifyCh <- tc.notify: - case <-errs: - t.Fatal("keepStateKeysUpdated returned before test input") - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for next() to be called again") - } - - // Wait for next() to be called again so we know the goroutine has - // processed the event. - select { - case <-nextWaiting: - case <-errs: - t.Fatal("keepStateKeysUpdated returned before test input") - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for next() to be called again") - } - - for key, value := range tc.expected { - got, _ := store.ReadState(key) - if !bytes.Equal(got, value) { - t.Errorf("state key %q mismatch: expected %q, got %q", key, value, got) + notifyChan <- tc.notify + for _, expected := range tc.expected { + select { + case got := <-store.writeChan: + if got != expected { + t.Errorf("expected %q, got %q", expected, got) + } + case err := <-errs: + t.Fatalf("unexpected error: %v", err) + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for expected write %q", expected) } } }) } } + +type fakeStore struct { + writeChan chan string +} + +func (f fakeStore) ReadState(key ipn.StateKey) ([]byte, error) { + return nil, fmt.Errorf("ReadState not implemented") +} + +func (f fakeStore) WriteState(key ipn.StateKey, value []byte) error { + f.writeChan <- fmt.Sprintf("%s=%s", key, value) + return nil +} From d6d29abbb6878fc777a9a21dd631ec3a8455e4ec Mon Sep 17 00:00:00 2001 From: Raj Singh Date: Mon, 14 Jul 2025 15:23:45 -0500 Subject: [PATCH 231/263] tstest/integration/testcontrol: include peer CapMaps in MapResponses Fixes #16560 Signed-off-by: Raj Singh --- tstest/integration/capmap_test.go | 147 ++++++++++++++++++ tstest/integration/testcontrol/testcontrol.go | 4 + 2 files changed, 151 insertions(+) create mode 100644 tstest/integration/capmap_test.go diff --git a/tstest/integration/capmap_test.go b/tstest/integration/capmap_test.go new file mode 100644 index 0000000000000..0ee05be2f57d7 --- /dev/null +++ b/tstest/integration/capmap_test.go @@ -0,0 +1,147 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package integration + +import ( + "errors" + "testing" + "time" + + "tailscale.com/tailcfg" + "tailscale.com/tstest" +) + +// TestPeerCapMap tests that the node capability map (CapMap) is included in peer information. +func TestPeerCapMap(t *testing.T) { + tstest.Shard(t) + tstest.Parallel(t) + env := NewTestEnv(t) + + // Spin up two nodes. + n1 := NewTestNode(t, env) + d1 := n1.StartDaemon() + n1.AwaitListening() + n1.MustUp() + n1.AwaitRunning() + + n2 := NewTestNode(t, env) + d2 := n2.StartDaemon() + n2.AwaitListening() + n2.MustUp() + n2.AwaitRunning() + + n1.AwaitIP4() + n2.AwaitIP4() + + // Get the nodes from the control server. + nodes := env.Control.AllNodes() + if len(nodes) != 2 { + t.Fatalf("expected 2 nodes, got %d nodes", len(nodes)) + } + + // Figure out which node is which by comparing keys. + st1 := n1.MustStatus() + var tn1, tn2 *tailcfg.Node + for _, n := range nodes { + if n.Key == st1.Self.PublicKey { + tn1 = n + } else { + tn2 = n + } + } + + // Set CapMap on both nodes. + caps := make(tailcfg.NodeCapMap) + caps["example:custom"] = []tailcfg.RawMessage{`"value"`} + caps["example:enabled"] = []tailcfg.RawMessage{`true`} + + env.Control.SetNodeCapMap(tn1.Key, caps) + env.Control.SetNodeCapMap(tn2.Key, caps) + + // Check that nodes see each other's CapMap. + if err := tstest.WaitFor(10*time.Second, func() error { + st1 := n1.MustStatus() + st2 := n2.MustStatus() + + if len(st1.Peer) == 0 || len(st2.Peer) == 0 { + return errors.New("no peers") + } + + // Check n1 sees n2's CapMap. + p1 := st1.Peer[st1.Peers()[0]] + if p1.CapMap == nil { + return errors.New("peer CapMap is nil") + } + if p1.CapMap["example:custom"] == nil || p1.CapMap["example:enabled"] == nil { + return errors.New("peer CapMap missing entries") + } + + // Check n2 sees n1's CapMap. + p2 := st2.Peer[st2.Peers()[0]] + if p2.CapMap == nil { + return errors.New("peer CapMap is nil") + } + if p2.CapMap["example:custom"] == nil || p2.CapMap["example:enabled"] == nil { + return errors.New("peer CapMap missing entries") + } + + return nil + }); err != nil { + t.Fatal(err) + } + + d1.MustCleanShutdown(t) + d2.MustCleanShutdown(t) +} + +// TestSetNodeCapMap tests that SetNodeCapMap updates are propagated to peers. +func TestSetNodeCapMap(t *testing.T) { + tstest.Shard(t) + tstest.Parallel(t) + env := NewTestEnv(t) + + n1 := NewTestNode(t, env) + d1 := n1.StartDaemon() + n1.AwaitListening() + n1.MustUp() + n1.AwaitRunning() + + nodes := env.Control.AllNodes() + if len(nodes) != 1 { + t.Fatalf("expected 1 node, got %d nodes", len(nodes)) + } + node1 := nodes[0] + + // Set initial CapMap. + caps := make(tailcfg.NodeCapMap) + caps["test:state"] = []tailcfg.RawMessage{`"initial"`} + env.Control.SetNodeCapMap(node1.Key, caps) + + // Start second node and verify it sees the first node's CapMap. + n2 := NewTestNode(t, env) + d2 := n2.StartDaemon() + n2.AwaitListening() + n2.MustUp() + n2.AwaitRunning() + + if err := tstest.WaitFor(5*time.Second, func() error { + st := n2.MustStatus() + if len(st.Peer) == 0 { + return errors.New("no peers") + } + p := st.Peer[st.Peers()[0]] + if p.CapMap == nil || p.CapMap["test:state"] == nil { + return errors.New("peer CapMap not set") + } + if string(p.CapMap["test:state"][0]) != `"initial"` { + return errors.New("wrong CapMap value") + } + return nil + }); err != nil { + t.Fatal(err) + } + + d1.MustCleanShutdown(t) + d2.MustCleanShutdown(t) +} diff --git a/tstest/integration/testcontrol/testcontrol.go b/tstest/integration/testcontrol/testcontrol.go index 71205f897aad8..739795bb3d245 100644 --- a/tstest/integration/testcontrol/testcontrol.go +++ b/tstest/integration/testcontrol/testcontrol.go @@ -1000,7 +1000,11 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse, s.mu.Lock() peerAddress := s.masquerades[p.Key][node.Key] routes := s.nodeSubnetRoutes[p.Key] + peerCapMap := maps.Clone(s.nodeCapMaps[p.Key]) s.mu.Unlock() + if peerCapMap != nil { + p.CapMap = peerCapMap + } if peerAddress.IsValid() { if peerAddress.Is6() { p.Addresses[1] = netip.PrefixFrom(peerAddress, peerAddress.BitLen()) From 5d4e67fd937bef4f3ad5ec8e93174a5b6bd7dceb Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 21 Jul 2025 08:36:43 -0700 Subject: [PATCH 232/263] net/dns/recursive: set EDNS on queries Updates tailscale/corp#30631 Change-Id: Ib88ea1bb51dd917c04f8d41bcaa6d59b9abd4f73 Signed-off-by: Brad Fitzpatrick --- net/dns/recursive/recursive.go | 1 + 1 file changed, 1 insertion(+) diff --git a/net/dns/recursive/recursive.go b/net/dns/recursive/recursive.go index eb23004d88190..fd865e37ab737 100644 --- a/net/dns/recursive/recursive.go +++ b/net/dns/recursive/recursive.go @@ -547,6 +547,7 @@ func (r *Resolver) queryNameserverProto( // Prepare a message asking for an appropriately-typed record // for the name we're querying. m := new(dns.Msg) + m.SetEdns0(1232, false /* no DNSSEC */) m.SetQuestion(name.WithTrailingDot(), uint16(qtype)) // Allow mocking out the network components with our exchange hook. From 1677fb190519710d66354600f659b50af77d7759 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 21 Jul 2025 10:02:37 -0700 Subject: [PATCH 233/263] wgengine/magicsock,all: allocate peer relay over disco instead of PeerAPI (#16603) Updates tailscale/corp#30583 Updates tailscale/corp#30534 Updates tailscale/corp#30557 Signed-off-by: Dylan Bargatze Signed-off-by: Jordan Whited Co-authored-by: Dylan Bargatze --- disco/disco.go | 261 +++++++++---- disco/disco_test.go | 41 +- feature/relayserver/relayserver.go | 151 +++---- feature/relayserver/relayserver_test.go | 10 + ipn/ipnlocal/local.go | 2 +- ipn/localapi/localapi.go | 2 +- net/udprelay/endpoint/endpoint.go | 9 + net/udprelay/server.go | 45 +-- tailcfg/tailcfg.go | 3 +- types/key/disco.go | 38 ++ types/key/disco_test.go | 18 + wgengine/magicsock/endpoint.go | 17 +- wgengine/magicsock/magicsock.go | 355 +++++++++++++---- wgengine/magicsock/magicsock_test.go | 335 ++++++++-------- wgengine/magicsock/relaymanager.go | 498 ++++++++++++++---------- wgengine/magicsock/relaymanager_test.go | 42 +- 16 files changed, 1187 insertions(+), 640 deletions(-) diff --git a/disco/disco.go b/disco/disco.go index d4623c119dbcb..1689d2a93da77 100644 --- a/disco/disco.go +++ b/disco/disco.go @@ -42,13 +42,15 @@ const NonceLen = 24 type MessageType byte const ( - TypePing = MessageType(0x01) - TypePong = MessageType(0x02) - TypeCallMeMaybe = MessageType(0x03) - TypeBindUDPRelayEndpoint = MessageType(0x04) - TypeBindUDPRelayEndpointChallenge = MessageType(0x05) - TypeBindUDPRelayEndpointAnswer = MessageType(0x06) - TypeCallMeMaybeVia = MessageType(0x07) + TypePing = MessageType(0x01) + TypePong = MessageType(0x02) + TypeCallMeMaybe = MessageType(0x03) + TypeBindUDPRelayEndpoint = MessageType(0x04) + TypeBindUDPRelayEndpointChallenge = MessageType(0x05) + TypeBindUDPRelayEndpointAnswer = MessageType(0x06) + TypeCallMeMaybeVia = MessageType(0x07) + TypeAllocateUDPRelayEndpointRequest = MessageType(0x08) + TypeAllocateUDPRelayEndpointResponse = MessageType(0x09) ) const v0 = byte(0) @@ -97,6 +99,10 @@ func Parse(p []byte) (Message, error) { return parseBindUDPRelayEndpointAnswer(ver, p) case TypeCallMeMaybeVia: return parseCallMeMaybeVia(ver, p) + case TypeAllocateUDPRelayEndpointRequest: + return parseAllocateUDPRelayEndpointRequest(ver, p) + case TypeAllocateUDPRelayEndpointResponse: + return parseAllocateUDPRelayEndpointResponse(ver, p) default: return nil, fmt.Errorf("unknown message type 0x%02x", byte(t)) } @@ -381,9 +387,7 @@ func (m *BindUDPRelayEndpointCommon) decode(b []byte) error { } // BindUDPRelayEndpoint is the first messaged transmitted from UDP relay client -// towards UDP relay server as part of the 3-way bind handshake. This message -// type is currently considered experimental and is not yet tied to a -// tailcfg.CapabilityVersion. +// towards UDP relay server as part of the 3-way bind handshake. type BindUDPRelayEndpoint struct { BindUDPRelayEndpointCommon } @@ -405,8 +409,7 @@ func parseBindUDPRelayEndpoint(ver uint8, p []byte) (m *BindUDPRelayEndpoint, er // BindUDPRelayEndpointChallenge is transmitted from UDP relay server towards // UDP relay client in response to a BindUDPRelayEndpoint message as part of the -// 3-way bind handshake. This message type is currently considered experimental -// and is not yet tied to a tailcfg.CapabilityVersion. +// 3-way bind handshake. type BindUDPRelayEndpointChallenge struct { BindUDPRelayEndpointCommon } @@ -427,9 +430,7 @@ func parseBindUDPRelayEndpointChallenge(ver uint8, p []byte) (m *BindUDPRelayEnd } // BindUDPRelayEndpointAnswer is transmitted from UDP relay client to UDP relay -// server in response to a BindUDPRelayEndpointChallenge message. This message -// type is currently considered experimental and is not yet tied to a -// tailcfg.CapabilityVersion. +// server in response to a BindUDPRelayEndpointChallenge message. type BindUDPRelayEndpointAnswer struct { BindUDPRelayEndpointCommon } @@ -449,6 +450,168 @@ func parseBindUDPRelayEndpointAnswer(ver uint8, p []byte) (m *BindUDPRelayEndpoi return m, nil } +// AllocateUDPRelayEndpointRequest is a message sent only over DERP to request +// allocation of a relay endpoint on a [tailscale.com/net/udprelay.Server] +type AllocateUDPRelayEndpointRequest struct { + // ClientDisco are the Disco public keys of the clients that should be + // permitted to handshake with the endpoint. + ClientDisco [2]key.DiscoPublic + // Generation represents the allocation request generation. The server must + // echo it back in the [AllocateUDPRelayEndpointResponse] to enable request + // and response alignment client-side. + Generation uint32 +} + +// allocateUDPRelayEndpointRequestLen is the length of a marshaled +// [AllocateUDPRelayEndpointRequest] message without the message header. +const allocateUDPRelayEndpointRequestLen = key.DiscoPublicRawLen*2 + // ClientDisco + 4 // Generation + +func (m *AllocateUDPRelayEndpointRequest) AppendMarshal(b []byte) []byte { + ret, p := appendMsgHeader(b, TypeAllocateUDPRelayEndpointRequest, v0, allocateUDPRelayEndpointRequestLen) + for i := 0; i < len(m.ClientDisco); i++ { + disco := m.ClientDisco[i].AppendTo(nil) + copy(p, disco) + p = p[key.DiscoPublicRawLen:] + } + binary.BigEndian.PutUint32(p, m.Generation) + return ret +} + +func parseAllocateUDPRelayEndpointRequest(ver uint8, p []byte) (m *AllocateUDPRelayEndpointRequest, err error) { + m = new(AllocateUDPRelayEndpointRequest) + if ver != 0 { + return + } + if len(p) < allocateUDPRelayEndpointRequestLen { + return m, errShort + } + for i := 0; i < len(m.ClientDisco); i++ { + m.ClientDisco[i] = key.DiscoPublicFromRaw32(mem.B(p[:key.DiscoPublicRawLen])) + p = p[key.DiscoPublicRawLen:] + } + m.Generation = binary.BigEndian.Uint32(p) + return m, nil +} + +// AllocateUDPRelayEndpointResponse is a message sent only over DERP in response +// to a [AllocateUDPRelayEndpointRequest]. +type AllocateUDPRelayEndpointResponse struct { + // Generation represents the allocation request generation. The server must + // echo back the [AllocateUDPRelayEndpointRequest.Generation] here to enable + // request and response alignment client-side. + Generation uint32 + UDPRelayEndpoint +} + +func (m *AllocateUDPRelayEndpointResponse) AppendMarshal(b []byte) []byte { + endpointsLen := epLength * len(m.AddrPorts) + generationLen := 4 + ret, d := appendMsgHeader(b, TypeAllocateUDPRelayEndpointResponse, v0, generationLen+udpRelayEndpointLenMinusAddrPorts+endpointsLen) + binary.BigEndian.PutUint32(d, m.Generation) + m.encode(d[4:]) + return ret +} + +func parseAllocateUDPRelayEndpointResponse(ver uint8, p []byte) (m *AllocateUDPRelayEndpointResponse, err error) { + m = new(AllocateUDPRelayEndpointResponse) + if ver != 0 { + return m, nil + } + if len(p) < 4 { + return m, errShort + } + m.Generation = binary.BigEndian.Uint32(p) + err = m.decode(p[4:]) + return m, err +} + +const udpRelayEndpointLenMinusAddrPorts = key.DiscoPublicRawLen + // ServerDisco + (key.DiscoPublicRawLen * 2) + // ClientDisco + 8 + // LamportID + 4 + // VNI + 8 + // BindLifetime + 8 // SteadyStateLifetime + +// UDPRelayEndpoint is a mirror of [tailscale.com/net/udprelay/endpoint.ServerEndpoint], +// refer to it for field documentation. [UDPRelayEndpoint] is carried in both +// [CallMeMaybeVia] and [AllocateUDPRelayEndpointResponse] messages. +type UDPRelayEndpoint struct { + // ServerDisco is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.ServerDisco] + ServerDisco key.DiscoPublic + // ClientDisco is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.ClientDisco] + ClientDisco [2]key.DiscoPublic + // LamportID is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.LamportID] + LamportID uint64 + // VNI is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.VNI] + VNI uint32 + // BindLifetime is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.BindLifetime] + BindLifetime time.Duration + // SteadyStateLifetime is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.SteadyStateLifetime] + SteadyStateLifetime time.Duration + // AddrPorts is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.AddrPorts] + AddrPorts []netip.AddrPort +} + +// encode encodes m in b. b must be at least [udpRelayEndpointLenMinusAddrPorts] +// + [epLength] * len(m.AddrPorts) bytes long. +func (m *UDPRelayEndpoint) encode(b []byte) { + disco := m.ServerDisco.AppendTo(nil) + copy(b, disco) + b = b[key.DiscoPublicRawLen:] + for i := 0; i < len(m.ClientDisco); i++ { + disco = m.ClientDisco[i].AppendTo(nil) + copy(b, disco) + b = b[key.DiscoPublicRawLen:] + } + binary.BigEndian.PutUint64(b[:8], m.LamportID) + b = b[8:] + binary.BigEndian.PutUint32(b[:4], m.VNI) + b = b[4:] + binary.BigEndian.PutUint64(b[:8], uint64(m.BindLifetime)) + b = b[8:] + binary.BigEndian.PutUint64(b[:8], uint64(m.SteadyStateLifetime)) + b = b[8:] + for _, ipp := range m.AddrPorts { + a := ipp.Addr().As16() + copy(b, a[:]) + binary.BigEndian.PutUint16(b[16:18], ipp.Port()) + b = b[epLength:] + } +} + +// decode decodes m from b. +func (m *UDPRelayEndpoint) decode(b []byte) error { + if len(b) < udpRelayEndpointLenMinusAddrPorts+epLength || + (len(b)-udpRelayEndpointLenMinusAddrPorts)%epLength != 0 { + return errShort + } + m.ServerDisco = key.DiscoPublicFromRaw32(mem.B(b[:key.DiscoPublicRawLen])) + b = b[key.DiscoPublicRawLen:] + for i := 0; i < len(m.ClientDisco); i++ { + m.ClientDisco[i] = key.DiscoPublicFromRaw32(mem.B(b[:key.DiscoPublicRawLen])) + b = b[key.DiscoPublicRawLen:] + } + m.LamportID = binary.BigEndian.Uint64(b[:8]) + b = b[8:] + m.VNI = binary.BigEndian.Uint32(b[:4]) + b = b[4:] + m.BindLifetime = time.Duration(binary.BigEndian.Uint64(b[:8])) + b = b[8:] + m.SteadyStateLifetime = time.Duration(binary.BigEndian.Uint64(b[:8])) + b = b[8:] + m.AddrPorts = make([]netip.AddrPort, 0, len(b)-udpRelayEndpointLenMinusAddrPorts/epLength) + for len(b) > 0 { + var a [16]byte + copy(a[:], b) + m.AddrPorts = append(m.AddrPorts, netip.AddrPortFrom( + netip.AddrFrom16(a).Unmap(), + binary.BigEndian.Uint16(b[16:18]))) + b = b[epLength:] + } + return nil +} + // CallMeMaybeVia is a message sent only over DERP to request that the recipient // try to open up a magicsock path back to the sender. The 'Via' in // CallMeMaybeVia highlights that candidate paths are served through an @@ -464,78 +627,22 @@ func parseBindUDPRelayEndpointAnswer(ver uint8, p []byte) (m *BindUDPRelayEndpoi // The recipient may choose to not open a path back if it's already happy with // its path. Direct connections, e.g. [CallMeMaybe]-signaled, take priority over // CallMeMaybeVia paths. -// -// This message type is currently considered experimental and is not yet tied to -// a [tailscale.com/tailcfg.CapabilityVersion]. type CallMeMaybeVia struct { - // ServerDisco is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.ServerDisco] - ServerDisco key.DiscoPublic - // LamportID is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.LamportID] - LamportID uint64 - // VNI is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.VNI] - VNI uint32 - // BindLifetime is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.BindLifetime] - BindLifetime time.Duration - // SteadyStateLifetime is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.SteadyStateLifetime] - SteadyStateLifetime time.Duration - // AddrPorts is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.AddrPorts] - AddrPorts []netip.AddrPort + UDPRelayEndpoint } -const cmmvDataLenMinusEndpoints = key.DiscoPublicRawLen + // ServerDisco - 8 + // LamportID - 4 + // VNI - 8 + // BindLifetime - 8 // SteadyStateLifetime - func (m *CallMeMaybeVia) AppendMarshal(b []byte) []byte { endpointsLen := epLength * len(m.AddrPorts) - ret, p := appendMsgHeader(b, TypeCallMeMaybeVia, v0, cmmvDataLenMinusEndpoints+endpointsLen) - disco := m.ServerDisco.AppendTo(nil) - copy(p, disco) - p = p[key.DiscoPublicRawLen:] - binary.BigEndian.PutUint64(p[:8], m.LamportID) - p = p[8:] - binary.BigEndian.PutUint32(p[:4], m.VNI) - p = p[4:] - binary.BigEndian.PutUint64(p[:8], uint64(m.BindLifetime)) - p = p[8:] - binary.BigEndian.PutUint64(p[:8], uint64(m.SteadyStateLifetime)) - p = p[8:] - for _, ipp := range m.AddrPorts { - a := ipp.Addr().As16() - copy(p, a[:]) - binary.BigEndian.PutUint16(p[16:18], ipp.Port()) - p = p[epLength:] - } + ret, p := appendMsgHeader(b, TypeCallMeMaybeVia, v0, udpRelayEndpointLenMinusAddrPorts+endpointsLen) + m.encode(p) return ret } func parseCallMeMaybeVia(ver uint8, p []byte) (m *CallMeMaybeVia, err error) { m = new(CallMeMaybeVia) - if len(p) < cmmvDataLenMinusEndpoints+epLength || - (len(p)-cmmvDataLenMinusEndpoints)%epLength != 0 || - ver != 0 { + if ver != 0 { return m, nil } - m.ServerDisco = key.DiscoPublicFromRaw32(mem.B(p[:key.DiscoPublicRawLen])) - p = p[key.DiscoPublicRawLen:] - m.LamportID = binary.BigEndian.Uint64(p[:8]) - p = p[8:] - m.VNI = binary.BigEndian.Uint32(p[:4]) - p = p[4:] - m.BindLifetime = time.Duration(binary.BigEndian.Uint64(p[:8])) - p = p[8:] - m.SteadyStateLifetime = time.Duration(binary.BigEndian.Uint64(p[:8])) - p = p[8:] - m.AddrPorts = make([]netip.AddrPort, 0, len(p)-cmmvDataLenMinusEndpoints/epLength) - for len(p) > 0 { - var a [16]byte - copy(a[:], p) - m.AddrPorts = append(m.AddrPorts, netip.AddrPortFrom( - netip.AddrFrom16(a).Unmap(), - binary.BigEndian.Uint16(p[16:18]))) - p = p[epLength:] - } - return m, nil + err = m.decode(p) + return m, err } diff --git a/disco/disco_test.go b/disco/disco_test.go index 9fb71ff83b73b..71b68338a8c90 100644 --- a/disco/disco_test.go +++ b/disco/disco_test.go @@ -25,6 +25,19 @@ func TestMarshalAndParse(t *testing.T) { }, } + udpRelayEndpoint := UDPRelayEndpoint{ + ServerDisco: key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})), + ClientDisco: [2]key.DiscoPublic{key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 3: 3, 30: 30, 31: 31})), key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 4: 4, 30: 30, 31: 31}))}, + LamportID: 123, + VNI: 456, + BindLifetime: time.Second, + SteadyStateLifetime: time.Minute, + AddrPorts: []netip.AddrPort{ + netip.MustParseAddrPort("1.2.3.4:567"), + netip.MustParseAddrPort("[2001::3456]:789"), + }, + } + tests := []struct { name string want string @@ -117,17 +130,25 @@ func TestMarshalAndParse(t *testing.T) { { name: "call_me_maybe_via", m: &CallMeMaybeVia{ - ServerDisco: key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})), - LamportID: 123, - VNI: 456, - BindLifetime: time.Second, - SteadyStateLifetime: time.Minute, - AddrPorts: []netip.AddrPort{ - netip.MustParseAddrPort("1.2.3.4:567"), - netip.MustParseAddrPort("[2001::3456]:789"), - }, + UDPRelayEndpoint: udpRelayEndpoint, + }, + want: "07 00 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 00 00 00 00 00 00 7b 00 00 01 c8 00 00 00 00 3b 9a ca 00 00 00 00 0d f8 47 58 00 00 00 00 00 00 00 00 00 00 00 ff ff 01 02 03 04 02 37 20 01 00 00 00 00 00 00 00 00 00 00 00 00 34 56 03 15", + }, + { + name: "allocate_udp_relay_endpoint_request", + m: &AllocateUDPRelayEndpointRequest{ + ClientDisco: [2]key.DiscoPublic{key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 3: 3, 30: 30, 31: 31})), key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 4: 4, 30: 30, 31: 31}))}, + Generation: 1, + }, + want: "08 00 00 01 02 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 00 00 01", + }, + { + name: "allocate_udp_relay_endpoint_response", + m: &AllocateUDPRelayEndpointResponse{ + Generation: 1, + UDPRelayEndpoint: udpRelayEndpoint, }, - want: "07 00 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 00 00 00 00 00 00 7b 00 00 01 c8 00 00 00 00 3b 9a ca 00 00 00 00 0d f8 47 58 00 00 00 00 00 00 00 00 00 00 00 ff ff 01 02 03 04 02 37 20 01 00 00 00 00 00 00 00 00 00 00 00 00 34 56 03 15", + want: "09 00 00 00 00 01 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 00 00 00 00 00 00 7b 00 00 01 c8 00 00 00 00 3b 9a ca 00 00 00 00 0d f8 47 58 00 00 00 00 00 00 00 00 00 00 00 ff ff 01 02 03 04 02 37 20 01 00 00 00 00 00 00 00 00 00 00 00 00 34 56 03 15", }, } for _, tt := range tests { diff --git a/feature/relayserver/relayserver.go b/feature/relayserver/relayserver.go index d0ad27624f09f..f4077b5f9da0b 100644 --- a/feature/relayserver/relayserver.go +++ b/feature/relayserver/relayserver.go @@ -6,25 +6,21 @@ package relayserver import ( - "encoding/json" "errors" - "fmt" - "io" - "net/http" "sync" - "time" + "tailscale.com/disco" "tailscale.com/feature" "tailscale.com/ipn" "tailscale.com/ipn/ipnext" - "tailscale.com/ipn/ipnlocal" "tailscale.com/net/udprelay" "tailscale.com/net/udprelay/endpoint" "tailscale.com/tailcfg" "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/types/ptr" - "tailscale.com/util/httpm" + "tailscale.com/util/eventbus" + "tailscale.com/wgengine/magicsock" ) // featureName is the name of the feature implemented by this package. @@ -34,26 +30,34 @@ const featureName = "relayserver" func init() { feature.Register(featureName) ipnext.RegisterExtension(featureName, newExtension) - ipnlocal.RegisterPeerAPIHandler("/v0/relay/endpoint", handlePeerAPIRelayAllocateEndpoint) } // newExtension is an [ipnext.NewExtensionFn] that creates a new relay server // extension. It is registered with [ipnext.RegisterExtension] if the package is // imported. -func newExtension(logf logger.Logf, _ ipnext.SafeBackend) (ipnext.Extension, error) { - return &extension{logf: logger.WithPrefix(logf, featureName+": ")}, nil +func newExtension(logf logger.Logf, sb ipnext.SafeBackend) (ipnext.Extension, error) { + return &extension{ + logf: logger.WithPrefix(logf, featureName+": "), + bus: sb.Sys().Bus.Get(), + }, nil } // extension is an [ipnext.Extension] managing the relay server on platforms // that import this package. type extension struct { logf logger.Logf + bus *eventbus.Bus - mu sync.Mutex // guards the following fields + mu sync.Mutex // guards the following fields + eventClient *eventbus.Client // closed to stop consumeEventbusTopics + reqSub *eventbus.Subscriber[magicsock.UDPRelayAllocReq] // receives endpoint alloc requests from magicsock + respPub *eventbus.Publisher[magicsock.UDPRelayAllocResp] // publishes endpoint alloc responses to magicsock shutdown bool - port *int // ipn.Prefs.RelayServerPort, nil if disabled - hasNodeAttrDisableRelayServer bool // tailcfg.NodeAttrDisableRelayServer - server relayServer // lazily initialized + port *int // ipn.Prefs.RelayServerPort, nil if disabled + busDoneCh chan struct{} // non-nil if port is non-nil, closed when consumeEventbusTopics returns + hasNodeAttrDisableRelayServer bool // tailcfg.NodeAttrDisableRelayServer + server relayServer // lazily initialized + } // relayServer is the interface of [udprelay.Server]. @@ -77,6 +81,18 @@ func (e *extension) Init(host ipnext.Host) error { return nil } +// initBusConnection initializes the [*eventbus.Client], [*eventbus.Subscriber], +// [*eventbus.Publisher], and [chan struct{}] used to publish/receive endpoint +// allocation messages to/from the [*eventbus.Bus]. It also starts +// consumeEventbusTopics in a separate goroutine. +func (e *extension) initBusConnection() { + e.eventClient = e.bus.Client("relayserver.extension") + e.reqSub = eventbus.Subscribe[magicsock.UDPRelayAllocReq](e.eventClient) + e.respPub = eventbus.Publish[magicsock.UDPRelayAllocResp](e.eventClient) + e.busDoneCh = make(chan struct{}) + go e.consumeEventbusTopics() +} + func (e *extension) selfNodeViewChanged(nodeView tailcfg.NodeView) { e.mu.Lock() defer e.mu.Unlock() @@ -98,11 +114,57 @@ func (e *extension) profileStateChanged(_ ipn.LoginProfileView, prefs ipn.PrefsV e.server.Close() e.server = nil } + if e.port != nil { + e.eventClient.Close() + <-e.busDoneCh + } e.port = nil if ok { e.port = ptr.To(newPort) + e.initBusConnection() + } + } +} + +func (e *extension) consumeEventbusTopics() { + defer close(e.busDoneCh) + + for { + select { + case <-e.reqSub.Done(): + // If reqSub is done, the eventClient has been closed, which is a + // signal to return. + return + case req := <-e.reqSub.Events(): + rs, err := e.relayServerOrInit() + if err != nil { + e.logf("error initializing server: %v", err) + continue + } + se, err := rs.AllocateEndpoint(req.Message.ClientDisco[0], req.Message.ClientDisco[1]) + if err != nil { + e.logf("error allocating endpoint: %v", err) + continue + } + e.respPub.Publish(magicsock.UDPRelayAllocResp{ + ReqRxFromNodeKey: req.RxFromNodeKey, + ReqRxFromDiscoKey: req.RxFromDiscoKey, + Message: &disco.AllocateUDPRelayEndpointResponse{ + Generation: req.Message.Generation, + UDPRelayEndpoint: disco.UDPRelayEndpoint{ + ServerDisco: se.ServerDisco, + ClientDisco: se.ClientDisco, + LamportID: se.LamportID, + VNI: se.VNI, + BindLifetime: se.BindLifetime.Duration, + SteadyStateLifetime: se.SteadyStateLifetime.Duration, + AddrPorts: se.AddrPorts, + }, + }, + }) } } + } // Shutdown implements [ipnlocal.Extension]. @@ -114,6 +176,10 @@ func (e *extension) Shutdown() error { e.server.Close() e.server = nil } + if e.port != nil { + e.eventClient.Close() + <-e.busDoneCh + } return nil } @@ -139,60 +205,3 @@ func (e *extension) relayServerOrInit() (relayServer, error) { } return e.server, nil } - -func handlePeerAPIRelayAllocateEndpoint(h ipnlocal.PeerAPIHandler, w http.ResponseWriter, r *http.Request) { - e, ok := ipnlocal.GetExt[*extension](h.LocalBackend()) - if !ok { - http.Error(w, "relay failed to initialize", http.StatusServiceUnavailable) - return - } - - httpErrAndLog := func(message string, code int) { - http.Error(w, message, code) - h.Logf("relayserver: request from %v returned code %d: %s", h.RemoteAddr(), code, message) - } - - if !h.PeerCaps().HasCapability(tailcfg.PeerCapabilityRelay) { - httpErrAndLog("relay not permitted", http.StatusForbidden) - return - } - - if r.Method != httpm.POST { - httpErrAndLog("only POST method is allowed", http.StatusMethodNotAllowed) - return - } - - var allocateEndpointReq struct { - DiscoKeys []key.DiscoPublic - } - err := json.NewDecoder(io.LimitReader(r.Body, 512)).Decode(&allocateEndpointReq) - if err != nil { - httpErrAndLog(err.Error(), http.StatusBadRequest) - return - } - if len(allocateEndpointReq.DiscoKeys) != 2 { - httpErrAndLog("2 disco public keys must be supplied", http.StatusBadRequest) - return - } - - rs, err := e.relayServerOrInit() - if err != nil { - httpErrAndLog(err.Error(), http.StatusServiceUnavailable) - return - } - ep, err := rs.AllocateEndpoint(allocateEndpointReq.DiscoKeys[0], allocateEndpointReq.DiscoKeys[1]) - if err != nil { - var notReady udprelay.ErrServerNotReady - if errors.As(err, ¬Ready) { - w.Header().Set("Retry-After", fmt.Sprintf("%d", notReady.RetryAfter.Round(time.Second)/time.Second)) - httpErrAndLog(err.Error(), http.StatusServiceUnavailable) - return - } - httpErrAndLog(err.Error(), http.StatusInternalServerError) - return - } - err = json.NewEncoder(w).Encode(&ep) - if err != nil { - httpErrAndLog(err.Error(), http.StatusInternalServerError) - } -} diff --git a/feature/relayserver/relayserver_test.go b/feature/relayserver/relayserver_test.go index cc7f05f67fbdd..84158188e90fb 100644 --- a/feature/relayserver/relayserver_test.go +++ b/feature/relayserver/relayserver_test.go @@ -9,6 +9,7 @@ import ( "tailscale.com/ipn" "tailscale.com/net/udprelay/endpoint" + "tailscale.com/tsd" "tailscale.com/types/key" "tailscale.com/types/ptr" ) @@ -108,9 +109,18 @@ func Test_extension_profileStateChanged(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + sys := tsd.NewSystem() + bus := sys.Bus.Get() e := &extension{ port: tt.fields.port, server: tt.fields.server, + bus: bus, + } + if e.port != nil { + // Entering profileStateChanged with a non-nil port requires + // bus init, which is called in profileStateChanged when + // transitioning port from nil to non-nil. + e.initBusConnection() } e.profileStateChanged(ipn.LoginProfileView{}, tt.args.prefs, tt.args.sameNode) if tt.wantNilServer != (e.server == nil) { diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index d3754e5409c1a..8665a88c4f867 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -6957,7 +6957,7 @@ func (b *LocalBackend) DebugReSTUN() error { return nil } -func (b *LocalBackend) DebugPeerRelayServers() set.Set[netip.AddrPort] { +func (b *LocalBackend) DebugPeerRelayServers() set.Set[netip.Addr] { return b.MagicConn().PeerRelays() } diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index 2409aa1ae3a36..0acc5a65fca8a 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -699,7 +699,7 @@ func (h *Handler) serveDebug(w http.ResponseWriter, r *http.Request) { h.b.DebugForcePreferDERP(n) case "peer-relay-servers": servers := h.b.DebugPeerRelayServers().Slice() - slices.SortFunc(servers, func(a, b netip.AddrPort) int { + slices.SortFunc(servers, func(a, b netip.Addr) int { return a.Compare(b) }) err = json.NewEncoder(w).Encode(servers) diff --git a/net/udprelay/endpoint/endpoint.go b/net/udprelay/endpoint/endpoint.go index 2672a856b797b..0d2a14e965a4a 100644 --- a/net/udprelay/endpoint/endpoint.go +++ b/net/udprelay/endpoint/endpoint.go @@ -7,11 +7,16 @@ package endpoint import ( "net/netip" + "time" "tailscale.com/tstime" "tailscale.com/types/key" ) +// ServerRetryAfter is the default +// [tailscale.com/net/udprelay.ErrServerNotReady.RetryAfter] value. +const ServerRetryAfter = time.Second * 3 + // ServerEndpoint contains details for an endpoint served by a // [tailscale.com/net/udprelay.Server]. type ServerEndpoint struct { @@ -21,6 +26,10 @@ type ServerEndpoint struct { // unique ServerEndpoint allocation. ServerDisco key.DiscoPublic + // ClientDisco are the Disco public keys of the relay participants permitted + // to handshake with this endpoint. + ClientDisco [2]key.DiscoPublic + // LamportID is unique and monotonically non-decreasing across // ServerEndpoint allocations for the lifetime of Server. It enables clients // to dedup and resolve allocation event order. Clients may race to allocate diff --git a/net/udprelay/server.go b/net/udprelay/server.go index 7651bf295a233..c34a4b5f6835c 100644 --- a/net/udprelay/server.go +++ b/net/udprelay/server.go @@ -73,23 +73,7 @@ type Server struct { lamportID uint64 vniPool []uint32 // the pool of available VNIs byVNI map[uint32]*serverEndpoint - byDisco map[pairOfDiscoPubKeys]*serverEndpoint -} - -// pairOfDiscoPubKeys is a pair of key.DiscoPublic. It must be constructed via -// newPairOfDiscoPubKeys to ensure lexicographical ordering. -type pairOfDiscoPubKeys [2]key.DiscoPublic - -func (p pairOfDiscoPubKeys) String() string { - return fmt.Sprintf("%s <=> %s", p[0].ShortString(), p[1].ShortString()) -} - -func newPairOfDiscoPubKeys(discoA, discoB key.DiscoPublic) pairOfDiscoPubKeys { - pair := pairOfDiscoPubKeys([2]key.DiscoPublic{discoA, discoB}) - slices.SortFunc(pair[:], func(a, b key.DiscoPublic) int { - return a.Compare(b) - }) - return pair + byDisco map[key.SortedPairOfDiscoPublic]*serverEndpoint } // serverEndpoint contains Server-internal [endpoint.ServerEndpoint] state. @@ -99,7 +83,7 @@ type serverEndpoint struct { // indexing of this array aligns with the following fields, e.g. // discoSharedSecrets[0] is the shared secret to use when sealing // Disco protocol messages for transmission towards discoPubKeys[0]. - discoPubKeys pairOfDiscoPubKeys + discoPubKeys key.SortedPairOfDiscoPublic discoSharedSecrets [2]key.DiscoShared handshakeGeneration [2]uint32 // or zero if a handshake has never started for that relay leg handshakeAddrPorts [2]netip.AddrPort // or zero value if a handshake has never started for that relay leg @@ -126,7 +110,7 @@ func (e *serverEndpoint) handleDiscoControlMsg(from netip.AddrPort, senderIndex if common.VNI != e.vni { return errors.New("mismatching VNI") } - if common.RemoteKey.Compare(e.discoPubKeys[otherSender]) != 0 { + if common.RemoteKey.Compare(e.discoPubKeys.Get()[otherSender]) != 0 { return errors.New("mismatching RemoteKey") } return nil @@ -152,7 +136,7 @@ func (e *serverEndpoint) handleDiscoControlMsg(from netip.AddrPort, senderIndex m := new(disco.BindUDPRelayEndpointChallenge) m.VNI = e.vni m.Generation = discoMsg.Generation - m.RemoteKey = e.discoPubKeys[otherSender] + m.RemoteKey = e.discoPubKeys.Get()[otherSender] rand.Read(e.challenge[senderIndex][:]) copy(m.Challenge[:], e.challenge[senderIndex][:]) reply := make([]byte, packet.GeneveFixedHeaderLength, 512) @@ -200,9 +184,9 @@ func (e *serverEndpoint) handleSealedDiscoControlMsg(from netip.AddrPort, b []by sender := key.DiscoPublicFromRaw32(mem.B(senderRaw)) senderIndex := -1 switch { - case sender.Compare(e.discoPubKeys[0]) == 0: + case sender.Compare(e.discoPubKeys.Get()[0]) == 0: senderIndex = 0 - case sender.Compare(e.discoPubKeys[1]) == 0: + case sender.Compare(e.discoPubKeys.Get()[1]) == 0: senderIndex = 1 default: // unknown Disco public key @@ -291,12 +275,12 @@ func (e *serverEndpoint) isBound() bool { // which is useful to override in tests. func NewServer(logf logger.Logf, port int, overrideAddrs []netip.Addr) (s *Server, err error) { s = &Server{ - logf: logger.WithPrefix(logf, "relayserver"), + logf: logf, disco: key.NewDisco(), bindLifetime: defaultBindLifetime, steadyStateLifetime: defaultSteadyStateLifetime, closeCh: make(chan struct{}), - byDisco: make(map[pairOfDiscoPubKeys]*serverEndpoint), + byDisco: make(map[key.SortedPairOfDiscoPublic]*serverEndpoint), byVNI: make(map[uint32]*serverEndpoint), } s.discoPublic = s.disco.Public() @@ -315,7 +299,7 @@ func NewServer(logf logger.Logf, port int, overrideAddrs []netip.Addr) (s *Serve } s.netChecker = &netcheck.Client{ NetMon: netMon, - Logf: logger.WithPrefix(logf, "relayserver: netcheck:"), + Logf: logger.WithPrefix(logf, "netcheck: "), SendPacket: func(b []byte, addrPort netip.AddrPort) (int, error) { if addrPort.Addr().Is4() { return s.uc4.WriteToUDPAddrPort(b, addrPort) @@ -615,7 +599,7 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv if len(s.addrPorts) == 0 { if !s.addrDiscoveryOnce { - return endpoint.ServerEndpoint{}, ErrServerNotReady{RetryAfter: 3 * time.Second} + return endpoint.ServerEndpoint{}, ErrServerNotReady{RetryAfter: endpoint.ServerRetryAfter} } return endpoint.ServerEndpoint{}, errors.New("server addrPorts are not yet known") } @@ -624,7 +608,7 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv return endpoint.ServerEndpoint{}, fmt.Errorf("client disco equals server disco: %s", s.discoPublic.ShortString()) } - pair := newPairOfDiscoPubKeys(discoA, discoB) + pair := key.NewSortedPairOfDiscoPublic(discoA, discoB) e, ok := s.byDisco[pair] if ok { // Return the existing allocation. Clients can resolve duplicate @@ -639,6 +623,7 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv // behaviors and endpoint state (bound or not). We might want to // consider storing them (maybe interning) in the [*serverEndpoint] // at allocation time. + ClientDisco: pair.Get(), AddrPorts: slices.Clone(s.addrPorts), VNI: e.vni, LamportID: e.lamportID, @@ -657,15 +642,17 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv lamportID: s.lamportID, allocatedAt: time.Now(), } - e.discoSharedSecrets[0] = s.disco.Shared(e.discoPubKeys[0]) - e.discoSharedSecrets[1] = s.disco.Shared(e.discoPubKeys[1]) + e.discoSharedSecrets[0] = s.disco.Shared(e.discoPubKeys.Get()[0]) + e.discoSharedSecrets[1] = s.disco.Shared(e.discoPubKeys.Get()[1]) e.vni, s.vniPool = s.vniPool[0], s.vniPool[1:] s.byDisco[pair] = e s.byVNI[e.vni] = e + s.logf("allocated endpoint vni=%d lamportID=%d disco[0]=%v disco[1]=%v", e.vni, e.lamportID, pair.Get()[0].ShortString(), pair.Get()[1].ShortString()) return endpoint.ServerEndpoint{ ServerDisco: s.discoPublic, + ClientDisco: pair.Get(), AddrPorts: slices.Clone(s.addrPorts), VNI: e.vni, LamportID: e.lamportID, diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 398a2c8a2b93a..550914b96e31a 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -165,7 +165,8 @@ type CapabilityVersion int // - 118: 2025-07-01: Client sends Hostinfo.StateEncrypted to report whether the state file is encrypted at rest (#15830) // - 119: 2025-07-10: Client uses Hostinfo.Location.Priority to prioritize one route over another. // - 120: 2025-07-15: Client understands peer relay disco messages, and implements peer client and relay server functions -const CurrentCapabilityVersion CapabilityVersion = 120 +// - 121: 2025-07-19: Client understands peer relay endpoint alloc with [disco.AllocateUDPRelayEndpointRequest] & [disco.AllocateUDPRelayEndpointResponse] +const CurrentCapabilityVersion CapabilityVersion = 121 // ID is an integer ID for a user, node, or login allocated by the // control plane. diff --git a/types/key/disco.go b/types/key/disco.go index 1013ce5bf89af..ce5f9b36fd9a1 100644 --- a/types/key/disco.go +++ b/types/key/disco.go @@ -73,6 +73,44 @@ func (k DiscoPrivate) Shared(p DiscoPublic) DiscoShared { return ret } +// SortedPairOfDiscoPublic is a lexicographically sorted container of two +// [DiscoPublic] keys. +type SortedPairOfDiscoPublic struct { + k [2]DiscoPublic +} + +// Get returns the underlying keys. +func (s SortedPairOfDiscoPublic) Get() [2]DiscoPublic { + return s.k +} + +// NewSortedPairOfDiscoPublic returns a SortedPairOfDiscoPublic from a and b. +func NewSortedPairOfDiscoPublic(a, b DiscoPublic) SortedPairOfDiscoPublic { + s := SortedPairOfDiscoPublic{} + if a.Compare(b) < 0 { + s.k[0] = a + s.k[1] = b + } else { + s.k[0] = b + s.k[1] = a + } + return s +} + +func (s SortedPairOfDiscoPublic) String() string { + return fmt.Sprintf("%s <=> %s", s.k[0].ShortString(), s.k[1].ShortString()) +} + +// Equal returns true if s and b are equal, otherwise it returns false. +func (s SortedPairOfDiscoPublic) Equal(b SortedPairOfDiscoPublic) bool { + for i := range s.k { + if s.k[i].Compare(b.k[i]) != 0 { + return false + } + } + return true +} + // DiscoPublic is the public portion of a DiscoPrivate. type DiscoPublic struct { k [32]byte diff --git a/types/key/disco_test.go b/types/key/disco_test.go index c62c13cbf8970..131fe350f508a 100644 --- a/types/key/disco_test.go +++ b/types/key/disco_test.go @@ -81,3 +81,21 @@ func TestDiscoShared(t *testing.T) { t.Error("k1.Shared(k2) != k2.Shared(k1)") } } + +func TestSortedPairOfDiscoPublic(t *testing.T) { + pubA := DiscoPublic{} + pubA.k[0] = 0x01 + pubB := DiscoPublic{} + pubB.k[0] = 0x02 + sortedInput := NewSortedPairOfDiscoPublic(pubA, pubB) + unsortedInput := NewSortedPairOfDiscoPublic(pubB, pubA) + if sortedInput.Get() != unsortedInput.Get() { + t.Fatal("sortedInput.Get() != unsortedInput.Get()") + } + if unsortedInput.Get()[0] != pubA { + t.Fatal("unsortedInput.Get()[0] != pubA") + } + if unsortedInput.Get()[1] != pubB { + t.Fatal("unsortedInput.Get()[1] != pubB") + } +} diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 48d5ef5a11338..6381b021088b6 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -879,8 +879,14 @@ func (de *endpoint) setHeartbeatDisabled(v bool) { // discoverUDPRelayPathsLocked starts UDP relay path discovery. func (de *endpoint) discoverUDPRelayPathsLocked(now mono.Time) { - // TODO(jwhited): return early if there are no relay servers set, otherwise - // we spin up and down relayManager.runLoop unnecessarily. + if !de.c.hasPeerRelayServers.Load() { + // Changes in this value between its access and the logic following + // are fine, we will eventually do the "right" thing during future path + // discovery. The worst case is we suppress path discovery for the + // current cycle, or we unnecessarily call into [relayManager] and do + // some wasted work. + return + } de.lastUDPRelayPathDiscovery = now lastBest := de.bestAddr lastBestIsTrusted := mono.Now().Before(de.trustBestAddrUntil) @@ -2035,8 +2041,15 @@ func (de *endpoint) numStopAndReset() int64 { return atomic.LoadInt64(&de.numStopAndResetAtomic) } +// setDERPHome sets the provided regionID as home for de. Calls to setDERPHome +// must never run concurrent to [Conn.updateRelayServersSet], otherwise +// [candidatePeerRelay] DERP home changes may be missed from the perspective of +// [relayManager]. func (de *endpoint) setDERPHome(regionID uint16) { de.mu.Lock() defer de.mu.Unlock() de.derpAddr = netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, uint16(regionID)) + if de.c.hasPeerRelayServers.Load() { + de.c.relayManager.handleDERPHomeChange(de.publicKey, regionID) + } } diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index ad07003f72fbb..ee0ee40ca1d13 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -175,13 +175,15 @@ type Conn struct { // These [eventbus.Subscriber] fields are solely accessed by // consumeEventbusTopics once initialized. - pmSub *eventbus.Subscriber[portmapper.Mapping] - filterSub *eventbus.Subscriber[FilterUpdate] - nodeViewsSub *eventbus.Subscriber[NodeViewsUpdate] - nodeMutsSub *eventbus.Subscriber[NodeMutationsUpdate] - syncSub *eventbus.Subscriber[syncPoint] - syncPub *eventbus.Publisher[syncPoint] - subsDoneCh chan struct{} // closed when consumeEventbusTopics returns + pmSub *eventbus.Subscriber[portmapper.Mapping] + filterSub *eventbus.Subscriber[FilterUpdate] + nodeViewsSub *eventbus.Subscriber[NodeViewsUpdate] + nodeMutsSub *eventbus.Subscriber[NodeMutationsUpdate] + syncSub *eventbus.Subscriber[syncPoint] + syncPub *eventbus.Publisher[syncPoint] + allocRelayEndpointPub *eventbus.Publisher[UDPRelayAllocReq] + allocRelayEndpointSub *eventbus.Subscriber[UDPRelayAllocResp] + subsDoneCh chan struct{} // closed when consumeEventbusTopics returns // pconn4 and pconn6 are the underlying UDP sockets used to // send/receive packets for wireguard and other magicsock @@ -271,6 +273,14 @@ type Conn struct { // captureHook, if non-nil, is the pcap logging callback when capturing. captureHook syncs.AtomicValue[packet.CaptureCallback] + // hasPeerRelayServers is whether [relayManager] is configured with at least + // one peer relay server via [relayManager.handleRelayServersSet]. It is + // only accessed by [Conn.updateRelayServersSet], [endpoint.setDERPHome], + // and [endpoint.discoverUDPRelayPathsLocked]. It exists to suppress + // calls into [relayManager] leading to wasted work involving channel + // operations and goroutine creation. + hasPeerRelayServers atomic.Bool + // discoPrivate is the private naclbox key used for active // discovery traffic. It is always present, and immutable. discoPrivate key.DiscoPrivate @@ -567,6 +577,36 @@ func (s syncPoint) Signal() { close(s) } +// UDPRelayAllocReq represents a [*disco.AllocateUDPRelayEndpointRequest] +// reception event. This is signaled over an [eventbus.Bus] from +// [magicsock.Conn] towards [relayserver.extension]. +type UDPRelayAllocReq struct { + // RxFromNodeKey is the unauthenticated (DERP server claimed src) node key + // of the transmitting party, noted at disco message reception time over + // DERP. This node key is unambiguously-aligned with RxFromDiscoKey being + // that the disco message is received over DERP. + RxFromNodeKey key.NodePublic + // RxFromDiscoKey is the disco key of the transmitting party, noted and + // authenticated at reception time. + RxFromDiscoKey key.DiscoPublic + // Message is the disco message. + Message *disco.AllocateUDPRelayEndpointRequest +} + +// UDPRelayAllocResp represents a [*disco.AllocateUDPRelayEndpointResponse] +// that is yet to be transmitted over DERP (or delivered locally if +// ReqRxFromNodeKey is self). This is signaled over an [eventbus.Bus] from +// [relayserver.extension] towards [magicsock.Conn]. +type UDPRelayAllocResp struct { + // ReqRxFromNodeKey is copied from [UDPRelayAllocReq.RxFromNodeKey]. It + // enables peer lookup leading up to transmission over DERP. + ReqRxFromNodeKey key.NodePublic + // ReqRxFromDiscoKey is copied from [UDPRelayAllocReq.RxFromDiscoKey]. + ReqRxFromDiscoKey key.DiscoPublic + // Message is the disco message. + Message *disco.AllocateUDPRelayEndpointResponse +} + // newConn is the error-free, network-listening-side-effect-free based // of NewConn. Mostly for tests. func newConn(logf logger.Logf) *Conn { @@ -625,10 +665,40 @@ func (c *Conn) consumeEventbusTopics() { case syncPoint := <-c.syncSub.Events(): c.dlogf("magicsock: received sync point after reconfig") syncPoint.Signal() + case allocResp := <-c.allocRelayEndpointSub.Events(): + c.onUDPRelayAllocResp(allocResp) } } } +func (c *Conn) onUDPRelayAllocResp(allocResp UDPRelayAllocResp) { + c.mu.Lock() + defer c.mu.Unlock() + ep, ok := c.peerMap.endpointForNodeKey(allocResp.ReqRxFromNodeKey) + if !ok { + // If it's not a peer, it might be for self (we can peer relay through + // ourselves), in which case we want to hand it down to [relayManager] + // now versus taking a network round-trip through DERP. + selfNodeKey := c.publicKeyAtomic.Load() + if selfNodeKey.Compare(allocResp.ReqRxFromNodeKey) == 0 && + allocResp.ReqRxFromDiscoKey.Compare(c.discoPublic) == 0 { + c.relayManager.handleRxDiscoMsg(c, allocResp.Message, selfNodeKey, allocResp.ReqRxFromDiscoKey, epAddr{}) + } + return + } + disco := ep.disco.Load() + if disco == nil { + return + } + if disco.key.Compare(allocResp.ReqRxFromDiscoKey) != 0 { + return + } + ep.mu.Lock() + defer ep.mu.Unlock() + derpAddr := ep.derpAddr + go c.sendDiscoMessage(epAddr{ap: derpAddr}, ep.publicKey, disco.key, allocResp.Message, discoVerboseLog) +} + // Synchronize waits for all [eventbus] events published // prior to this call to be processed by the receiver. func (c *Conn) Synchronize() { @@ -670,6 +740,8 @@ func NewConn(opts Options) (*Conn, error) { c.nodeMutsSub = eventbus.Subscribe[NodeMutationsUpdate](c.eventClient) c.syncSub = eventbus.Subscribe[syncPoint](c.eventClient) c.syncPub = eventbus.Publish[syncPoint](c.eventClient) + c.allocRelayEndpointPub = eventbus.Publish[UDPRelayAllocReq](c.eventClient) + c.allocRelayEndpointSub = eventbus.Subscribe[UDPRelayAllocResp](c.eventClient) c.subsDoneCh = make(chan struct{}) go c.consumeEventbusTopics() } @@ -1847,6 +1919,24 @@ func (v *virtualNetworkID) get() uint32 { return v._vni & vniGetMask } +// sendDiscoAllocateUDPRelayEndpointRequest is primarily an alias for +// sendDiscoMessage, but it will alternatively send m over the eventbus if dst +// is a DERP IP:port, and dstKey is self. This saves a round-trip through DERP +// when we are attempting to allocate on a self (in-process) peer relay server. +func (c *Conn) sendDiscoAllocateUDPRelayEndpointRequest(dst epAddr, dstKey key.NodePublic, dstDisco key.DiscoPublic, allocReq *disco.AllocateUDPRelayEndpointRequest, logLevel discoLogLevel) (sent bool, err error) { + isDERP := dst.ap.Addr() == tailcfg.DerpMagicIPAddr + selfNodeKey := c.publicKeyAtomic.Load() + if isDERP && dstKey.Compare(selfNodeKey) == 0 { + c.allocRelayEndpointPub.Publish(UDPRelayAllocReq{ + RxFromNodeKey: selfNodeKey, + RxFromDiscoKey: c.discoPublic, + Message: allocReq, + }) + return true, nil + } + return c.sendDiscoMessage(dst, dstKey, dstDisco, allocReq, logLevel) +} + // sendDiscoMessage sends discovery message m to dstDisco at dst. // // If dst.ap is a DERP IP:port, then dstKey must be non-zero. @@ -2176,7 +2266,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake c.logf("[unexpected] %T packets should not come from a relay server with Geneve control bit set", dm) return } - c.relayManager.handleGeneveEncapDiscoMsg(c, challenge, di, src) + c.relayManager.handleRxDiscoMsg(c, challenge, key.NodePublic{}, di.discoKey, src) return } @@ -2201,7 +2291,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake // If it's an unknown TxID, and it's Geneve-encapsulated, then // make [relayManager] aware. It might be in the middle of probing // src. - c.relayManager.handleGeneveEncapDiscoMsg(c, dm, di, src) + c.relayManager.handleRxDiscoMsg(c, dm, key.NodePublic{}, di.discoKey, src) } case *disco.CallMeMaybe, *disco.CallMeMaybeVia: var via *disco.CallMeMaybeVia @@ -2276,7 +2366,95 @@ func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshake len(cmm.MyNumber)) go ep.handleCallMeMaybe(cmm) } + case *disco.AllocateUDPRelayEndpointRequest, *disco.AllocateUDPRelayEndpointResponse: + var resp *disco.AllocateUDPRelayEndpointResponse + isResp := false + msgType := "AllocateUDPRelayEndpointRequest" + req, ok := dm.(*disco.AllocateUDPRelayEndpointRequest) + if ok { + metricRecvDiscoAllocUDPRelayEndpointRequest.Add(1) + } else { + metricRecvDiscoAllocUDPRelayEndpointResponse.Add(1) + resp = dm.(*disco.AllocateUDPRelayEndpointResponse) + msgType = "AllocateUDPRelayEndpointResponse" + isResp = true + } + + if !isDERP { + // These messages should only come via DERP. + c.logf("[unexpected] %s packets should only come via DERP", msgType) + return + } + nodeKey := derpNodeSrc + ep, ok := c.peerMap.endpointForNodeKey(nodeKey) + if !ok { + c.logf("magicsock: disco: ignoring %s from %v; %v is unknown", msgType, sender.ShortString(), derpNodeSrc.ShortString()) + return + } + epDisco := ep.disco.Load() + if epDisco == nil { + return + } + if epDisco.key != di.discoKey { + if isResp { + metricRecvDiscoAllocUDPRelayEndpointResponseBadDisco.Add(1) + } else { + metricRecvDiscoAllocUDPRelayEndpointRequestBadDisco.Add(1) + } + c.logf("[unexpected] %s from peer via DERP whose netmap discokey != disco source", msgType) + return + } + if isResp { + c.dlogf("[v1] magicsock: disco: %v<-%v (%v, %v) got %s, %d endpoints", + c.discoShort, epDisco.short, + ep.publicKey.ShortString(), derpStr(src.String()), + msgType, + len(resp.AddrPorts)) + c.relayManager.handleRxDiscoMsg(c, resp, nodeKey, di.discoKey, src) + return + } else if sender.Compare(req.ClientDisco[0]) != 0 && sender.Compare(req.ClientDisco[1]) != 0 { + // An allocation request must contain the sender's disco key in + // ClientDisco. One of the relay participants must be the sender. + c.logf("magicsock: disco: %s from %v; %v does not contain sender's disco key", + msgType, sender.ShortString(), derpNodeSrc.ShortString()) + return + } else { + c.dlogf("[v1] magicsock: disco: %v<-%v (%v, %v) got %s, for %d<->%d", + c.discoShort, epDisco.short, + ep.publicKey.ShortString(), derpStr(src.String()), + msgType, + req.ClientDisco[0], req.ClientDisco[1]) + } + + if c.filt == nil { + return + } + // Binary search of peers is O(log n) while c.mu is held. + // TODO: We might be able to use ep.nodeAddr instead of all addresses, + // or we might be able to release c.mu before doing this work. Keep it + // simple and slow for now. c.peers.AsSlice is a copy. We may need to + // write our own binary search for a [views.Slice]. + peerI, ok := slices.BinarySearchFunc(c.peers.AsSlice(), ep.nodeID, func(peer tailcfg.NodeView, target tailcfg.NodeID) int { + if peer.ID() < target { + return -1 + } else if peer.ID() > target { + return 1 + } + return 0 + }) + if !ok { + // unexpected + return + } + if !nodeHasCap(c.filt, c.peers.At(peerI), c.self, tailcfg.PeerCapabilityRelay) { + return + } + c.allocRelayEndpointPub.Publish(UDPRelayAllocReq{ + RxFromDiscoKey: sender, + RxFromNodeKey: nodeKey, + Message: req, + }) } return } @@ -2337,7 +2515,7 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src epAddr, di *discoInfo, derpN // Geneve-encapsulated [disco.Ping] messages in the interest of // simplicity. It might be in the middle of probing src, so it must be // made aware. - c.relayManager.handleGeneveEncapDiscoMsg(c, dm, di, src) + c.relayManager.handleRxDiscoMsg(c, dm, key.NodePublic{}, di.discoKey, src) return } @@ -2687,7 +2865,7 @@ func (c *Conn) SetProbeUDPLifetime(v bool) { // capVerIsRelayCapable returns true if version is relay client and server // capable, otherwise it returns false. func capVerIsRelayCapable(version tailcfg.CapabilityVersion) bool { - return version >= 120 + return version >= 121 } // onFilterUpdate is called when a [FilterUpdate] is received over the @@ -2715,6 +2893,11 @@ func (c *Conn) onFilterUpdate(f FilterUpdate) { // peers are passed as args (vs c.mu-guarded fields) to enable callers to // release c.mu before calling as this is O(m * n) (we iterate all cap rules 'm' // in filt for every peer 'n'). +// +// Calls to updateRelayServersSet must never run concurrent to +// [endpoint.setDERPHome], otherwise [candidatePeerRelay] DERP home changes may +// be missed from the perspective of [relayManager]. +// // TODO: Optimize this so that it's not O(m * n). This might involve: // 1. Changes to [filter.Filter], e.g. adding a CapsWithValues() to check for // a given capability instead of building and returning a map of all of @@ -2722,69 +2905,75 @@ func (c *Conn) onFilterUpdate(f FilterUpdate) { // 2. Moving this work upstream into [nodeBackend] or similar, and publishing // the computed result over the eventbus instead. func (c *Conn) updateRelayServersSet(filt *filter.Filter, self tailcfg.NodeView, peers views.Slice[tailcfg.NodeView]) { - relayServers := make(set.Set[netip.AddrPort]) + relayServers := make(set.Set[candidatePeerRelay]) nodes := append(peers.AsSlice(), self) for _, maybeCandidate := range nodes { - peerAPI := peerAPIIfCandidateRelayServer(filt, self, maybeCandidate) - if peerAPI.IsValid() { - relayServers.Add(peerAPI) + if maybeCandidate.ID() != self.ID() && !capVerIsRelayCapable(maybeCandidate.Cap()) { + // If maybeCandidate's [tailcfg.CapabilityVersion] is not relay-capable, + // we skip it. If maybeCandidate happens to be self, then this check is + // unnecessary as self is always capable from this point (the statically + // compiled [tailcfg.CurrentCapabilityVersion]) forward. + continue + } + if !nodeHasCap(filt, maybeCandidate, self, tailcfg.PeerCapabilityRelayTarget) { + continue } + relayServers.Add(candidatePeerRelay{ + nodeKey: maybeCandidate.Key(), + discoKey: maybeCandidate.DiscoKey(), + derpHomeRegionID: uint16(maybeCandidate.HomeDERP()), + }) } c.relayManager.handleRelayServersSet(relayServers) + if len(relayServers) > 0 { + c.hasPeerRelayServers.Store(true) + } else { + c.hasPeerRelayServers.Store(false) + } } -// peerAPIIfCandidateRelayServer returns the peer API address of maybeCandidate -// if it is considered to be a candidate relay server upon evaluation against -// filt and self, otherwise it returns a zero value. self and maybeCandidate -// may be equal. -func peerAPIIfCandidateRelayServer(filt *filter.Filter, self, maybeCandidate tailcfg.NodeView) netip.AddrPort { +// nodeHasCap returns true if src has cap on dst, otherwise it returns false. +func nodeHasCap(filt *filter.Filter, src, dst tailcfg.NodeView, cap tailcfg.PeerCapability) bool { if filt == nil || - !self.Valid() || - !maybeCandidate.Valid() || - !maybeCandidate.Hostinfo().Valid() { - return netip.AddrPort{} - } - if maybeCandidate.ID() != self.ID() && !capVerIsRelayCapable(maybeCandidate.Cap()) { - // If maybeCandidate's [tailcfg.CapabilityVersion] is not relay-capable, - // we skip it. If maybeCandidate happens to be self, then this check is - // unnecessary as self is always capable from this point (the statically - // compiled [tailcfg.CurrentCapabilityVersion]) forward. - return netip.AddrPort{} - } - for _, maybeCandidatePrefix := range maybeCandidate.Addresses().All() { - if !maybeCandidatePrefix.IsSingleIP() { + !src.Valid() || + !dst.Valid() { + return false + } + for _, srcPrefix := range src.Addresses().All() { + if !srcPrefix.IsSingleIP() { continue } - maybeCandidateAddr := maybeCandidatePrefix.Addr() - for _, selfPrefix := range self.Addresses().All() { - if !selfPrefix.IsSingleIP() { + srcAddr := srcPrefix.Addr() + for _, dstPrefix := range dst.Addresses().All() { + if !dstPrefix.IsSingleIP() { continue } - selfAddr := selfPrefix.Addr() - if selfAddr.BitLen() == maybeCandidateAddr.BitLen() { // same address family - if filt.CapsWithValues(maybeCandidateAddr, selfAddr).HasCapability(tailcfg.PeerCapabilityRelayTarget) { - for _, s := range maybeCandidate.Hostinfo().Services().All() { - if maybeCandidateAddr.Is4() && s.Proto == tailcfg.PeerAPI4 || - maybeCandidateAddr.Is6() && s.Proto == tailcfg.PeerAPI6 { - return netip.AddrPortFrom(maybeCandidateAddr, s.Port) - } - } - return netip.AddrPort{} // no peerAPI - } else { - // [nodeBackend.peerCapsLocked] only returns/considers the - // [tailcfg.PeerCapMap] between the passed src and the - // _first_ host (/32 or /128) address for self. We are - // consistent with that behavior here. If self and - // maybeCandidate host addresses are of the same address - // family they either have the capability or not. We do not - // check against additional host addresses of the same - // address family. - return netip.AddrPort{} - } + dstAddr := dstPrefix.Addr() + if dstAddr.BitLen() == srcAddr.BitLen() { // same address family + // [nodeBackend.peerCapsLocked] only returns/considers the + // [tailcfg.PeerCapMap] between the passed src and the _first_ + // host (/32 or /128) address for self. We are consistent with + // that behavior here. If src and dst host addresses are of the + // same address family they either have the capability or not. + // We do not check against additional host addresses of the same + // address family. + return filt.CapsWithValues(srcAddr, dstAddr).HasCapability(cap) } } } - return netip.AddrPort{} + return false +} + +// candidatePeerRelay represents the identifiers and DERP home region ID for a +// peer relay server. +type candidatePeerRelay struct { + nodeKey key.NodePublic + discoKey key.DiscoPublic + derpHomeRegionID uint16 +} + +func (c *candidatePeerRelay) isValid() bool { + return !c.nodeKey.IsZero() && !c.discoKey.IsZero() } // onNodeViewsUpdate is called when a [NodeViewsUpdate] is received over the @@ -3792,18 +3981,22 @@ var ( metricRecvDiscoBadKey = clientmetric.NewCounter("magicsock_disco_recv_bad_key") metricRecvDiscoBadParse = clientmetric.NewCounter("magicsock_disco_recv_bad_parse") - metricRecvDiscoUDP = clientmetric.NewCounter("magicsock_disco_recv_udp") - metricRecvDiscoDERP = clientmetric.NewCounter("magicsock_disco_recv_derp") - metricRecvDiscoPing = clientmetric.NewCounter("magicsock_disco_recv_ping") - metricRecvDiscoPong = clientmetric.NewCounter("magicsock_disco_recv_pong") - metricRecvDiscoCallMeMaybe = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe") - metricRecvDiscoCallMeMaybeVia = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia") - metricRecvDiscoCallMeMaybeBadNode = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_node") - metricRecvDiscoCallMeMaybeViaBadNode = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia_bad_node") - metricRecvDiscoCallMeMaybeBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_disco") - metricRecvDiscoCallMeMaybeViaBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia_bad_disco") - metricRecvDiscoDERPPeerNotHere = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_not_here") - metricRecvDiscoDERPPeerGoneUnknown = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_gone_unknown") + metricRecvDiscoUDP = clientmetric.NewCounter("magicsock_disco_recv_udp") + metricRecvDiscoDERP = clientmetric.NewCounter("magicsock_disco_recv_derp") + metricRecvDiscoPing = clientmetric.NewCounter("magicsock_disco_recv_ping") + metricRecvDiscoPong = clientmetric.NewCounter("magicsock_disco_recv_pong") + metricRecvDiscoCallMeMaybe = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe") + metricRecvDiscoCallMeMaybeVia = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia") + metricRecvDiscoCallMeMaybeBadNode = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_node") + metricRecvDiscoCallMeMaybeViaBadNode = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia_bad_node") + metricRecvDiscoCallMeMaybeBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_disco") + metricRecvDiscoCallMeMaybeViaBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia_bad_disco") + metricRecvDiscoAllocUDPRelayEndpointRequest = clientmetric.NewCounter("magicsock_disco_recv_alloc_udp_relay_endpoint_request") + metricRecvDiscoAllocUDPRelayEndpointRequestBadDisco = clientmetric.NewCounter("magicsock_disco_recv_alloc_udp_relay_endpoint_request_bad_disco") + metricRecvDiscoAllocUDPRelayEndpointResponseBadDisco = clientmetric.NewCounter("magicsock_disco_recv_alloc_udp_relay_endpoint_response_bad_disco") + metricRecvDiscoAllocUDPRelayEndpointResponse = clientmetric.NewCounter("magicsock_disco_recv_alloc_udp_relay_endpoint_response") + metricRecvDiscoDERPPeerNotHere = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_not_here") + metricRecvDiscoDERPPeerGoneUnknown = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_gone_unknown") // metricDERPHomeChange is how many times our DERP home region DI has // changed from non-zero to a different non-zero. metricDERPHomeChange = clientmetric.NewCounter("derp_home_change") @@ -3985,6 +4178,22 @@ func (le *lazyEndpoint) FromPeer(peerPublicKey [32]byte) { } // PeerRelays returns the current set of candidate peer relays. -func (c *Conn) PeerRelays() set.Set[netip.AddrPort] { - return c.relayManager.getServers() +func (c *Conn) PeerRelays() set.Set[netip.Addr] { + candidatePeerRelays := c.relayManager.getServers() + servers := make(set.Set[netip.Addr], len(candidatePeerRelays)) + c.mu.Lock() + defer c.mu.Unlock() + for relay := range candidatePeerRelays { + pi, ok := c.peerMap.byNodeKey[relay.nodeKey] + if !ok { + if c.self.Key().Compare(relay.nodeKey) == 0 { + if c.self.Addresses().Len() > 0 { + servers.Add(c.self.Addresses().At(0).Addr()) + } + } + continue + } + servers.Add(pi.ep.nodeAddr) + } + return servers } diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index 1d76e6c595a47..8a09df27d2ce7 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -19,7 +19,6 @@ import ( "net/http/httptest" "net/netip" "os" - "reflect" "runtime" "strconv" "strings" @@ -64,6 +63,7 @@ import ( "tailscale.com/types/netmap" "tailscale.com/types/nettype" "tailscale.com/types/ptr" + "tailscale.com/types/views" "tailscale.com/util/cibuild" "tailscale.com/util/eventbus" "tailscale.com/util/must" @@ -3384,61 +3384,72 @@ func Test_virtualNetworkID(t *testing.T) { } } -func Test_peerAPIIfCandidateRelayServer(t *testing.T) { - hostInfo := &tailcfg.Hostinfo{ - Services: []tailcfg.Service{ - { - Proto: tailcfg.PeerAPI4, - Port: 4, - }, - { - Proto: tailcfg.PeerAPI6, - Port: 6, - }, +func Test_looksLikeInitiationMsg(t *testing.T) { + initMsg := make([]byte, device.MessageInitiationSize) + binary.BigEndian.PutUint32(initMsg, device.MessageInitiationType) + initMsgSizeTransportType := make([]byte, device.MessageInitiationSize) + binary.BigEndian.PutUint32(initMsgSizeTransportType, device.MessageTransportType) + tests := []struct { + name string + b []byte + want bool + }{ + { + name: "valid initiation", + b: initMsg, + want: true, + }, + { + name: "invalid message type field", + b: initMsgSizeTransportType, + want: false, + }, + { + name: "too small", + b: initMsg[:device.MessageInitiationSize-1], + want: false, + }, + { + name: "too big", + b: append(initMsg, 0), + want: false, }, } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := looksLikeInitiationMsg(tt.b); got != tt.want { + t.Errorf("looksLikeInitiationMsg() = %v, want %v", got, tt.want) + } + }) + } +} - selfOnlyIPv4 := &tailcfg.Node{ +func Test_nodeHasCap(t *testing.T) { + nodeAOnlyIPv4 := &tailcfg.Node{ ID: 1, - // Intentionally set a value < 120 to verify the statically compiled - // [tailcfg.CurrentCapabilityVersion] is used when self is - // maybeCandidate. - Cap: 119, Addresses: []netip.Prefix{ netip.MustParsePrefix("1.1.1.1/32"), }, - Hostinfo: hostInfo.View(), } - selfOnlyIPv6 := selfOnlyIPv4.Clone() - selfOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::1/128") + nodeBOnlyIPv6 := nodeAOnlyIPv4.Clone() + nodeBOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::1/128") - peerOnlyIPv4 := &tailcfg.Node{ - ID: 2, - Cap: 120, + nodeCOnlyIPv4 := &tailcfg.Node{ + ID: 2, Addresses: []netip.Prefix{ netip.MustParsePrefix("2.2.2.2/32"), }, - Hostinfo: hostInfo.View(), } - - peerOnlyIPv4NotCapable := peerOnlyIPv4.Clone() - peerOnlyIPv4NotCapable.Cap = 119 - - peerOnlyIPv6 := peerOnlyIPv4.Clone() - peerOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::2/128") - - peerOnlyIPv4ZeroCapVer := peerOnlyIPv4.Clone() - peerOnlyIPv4ZeroCapVer.Cap = 0 - - peerOnlyIPv4NilHostinfo := peerOnlyIPv4.Clone() - peerOnlyIPv4NilHostinfo.Hostinfo = tailcfg.HostinfoView{} + nodeDOnlyIPv6 := nodeCOnlyIPv4.Clone() + nodeDOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::2/128") tests := []struct { - name string - filt *filter.Filter - self tailcfg.NodeView - maybeCandidate tailcfg.NodeView - want netip.AddrPort + name string + filt *filter.Filter + src tailcfg.NodeView + dst tailcfg.NodeView + cap tailcfg.PeerCapability + want bool }{ { name: "match v4", @@ -3453,26 +3464,10 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv4.View(), - maybeCandidate: peerOnlyIPv4.View(), - want: netip.MustParseAddrPort("2.2.2.2:4"), - }, - { - name: "match v4 self", - filt: filter.New([]filtertype.Match{ - { - Srcs: []netip.Prefix{selfOnlyIPv4.Addresses[0]}, - Caps: []filtertype.CapMatch{ - { - Dst: selfOnlyIPv4.Addresses[0], - Cap: tailcfg.PeerCapabilityRelayTarget, - }, - }, - }, - }, nil, nil, nil, nil, nil), - self: selfOnlyIPv4.View(), - maybeCandidate: selfOnlyIPv4.View(), - want: netip.AddrPortFrom(selfOnlyIPv4.Addresses[0].Addr(), 4), + src: nodeCOnlyIPv4.View(), + dst: nodeAOnlyIPv4.View(), + cap: tailcfg.PeerCapabilityRelayTarget, + want: true, }, { name: "match v6", @@ -3487,77 +3482,67 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv6.View(), - maybeCandidate: peerOnlyIPv6.View(), - want: netip.MustParseAddrPort("[::2]:6"), - }, - { - name: "match v6 self", - filt: filter.New([]filtertype.Match{ - { - Srcs: []netip.Prefix{selfOnlyIPv6.Addresses[0]}, - Caps: []filtertype.CapMatch{ - { - Dst: selfOnlyIPv6.Addresses[0], - Cap: tailcfg.PeerCapabilityRelayTarget, - }, - }, - }, - }, nil, nil, nil, nil, nil), - self: selfOnlyIPv6.View(), - maybeCandidate: selfOnlyIPv6.View(), - want: netip.AddrPortFrom(selfOnlyIPv6.Addresses[0].Addr(), 6), + src: nodeDOnlyIPv6.View(), + dst: nodeBOnlyIPv6.View(), + cap: tailcfg.PeerCapabilityRelayTarget, + want: true, }, { - name: "peer incapable", + name: "no match CapMatch Dst", filt: filter.New([]filtertype.Match{ { - Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Srcs: []netip.Prefix{netip.MustParsePrefix("::2/128")}, Caps: []filtertype.CapMatch{ { - Dst: netip.MustParsePrefix("1.1.1.1/32"), + Dst: netip.MustParsePrefix("::3/128"), Cap: tailcfg.PeerCapabilityRelayTarget, }, }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv4.View(), - maybeCandidate: peerOnlyIPv4NotCapable.View(), + src: nodeDOnlyIPv6.View(), + dst: nodeBOnlyIPv6.View(), + cap: tailcfg.PeerCapabilityRelayTarget, + want: false, }, { - name: "no match dst", + name: "no match peer cap", filt: filter.New([]filtertype.Match{ { Srcs: []netip.Prefix{netip.MustParsePrefix("::2/128")}, Caps: []filtertype.CapMatch{ { - Dst: netip.MustParsePrefix("::3/128"), - Cap: tailcfg.PeerCapabilityRelayTarget, + Dst: netip.MustParsePrefix("::1/128"), + Cap: tailcfg.PeerCapabilityIngress, }, }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv6.View(), - maybeCandidate: peerOnlyIPv6.View(), + src: nodeDOnlyIPv6.View(), + dst: nodeBOnlyIPv6.View(), + cap: tailcfg.PeerCapabilityRelayTarget, + want: false, }, { - name: "no match peer cap", + name: "nil src", filt: filter.New([]filtertype.Match{ { - Srcs: []netip.Prefix{netip.MustParsePrefix("::2/128")}, + Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, Caps: []filtertype.CapMatch{ { - Dst: netip.MustParsePrefix("::1/128"), - Cap: tailcfg.PeerCapabilityIngress, + Dst: netip.MustParsePrefix("1.1.1.1/32"), + Cap: tailcfg.PeerCapabilityRelayTarget, }, }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv6.View(), - maybeCandidate: peerOnlyIPv6.View(), + src: tailcfg.NodeView{}, + dst: nodeAOnlyIPv4.View(), + cap: tailcfg.PeerCapabilityRelayTarget, + want: false, }, { - name: "cap ver not relay capable", + name: "nil dst", filt: filter.New([]filtertype.Match{ { Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, @@ -3569,108 +3554,136 @@ func Test_peerAPIIfCandidateRelayServer(t *testing.T) { }, }, }, nil, nil, nil, nil, nil), - self: peerOnlyIPv4.View(), - maybeCandidate: peerOnlyIPv4ZeroCapVer.View(), + src: nodeCOnlyIPv4.View(), + dst: tailcfg.NodeView{}, + cap: tailcfg.PeerCapabilityRelayTarget, + want: false, }, - { - name: "nil filt", - filt: nil, - self: selfOnlyIPv4.View(), - maybeCandidate: peerOnlyIPv4.View(), + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := nodeHasCap(tt.filt, tt.src, tt.dst, tt.cap); got != tt.want { + t.Errorf("nodeHasCap() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestConn_updateRelayServersSet(t *testing.T) { + peerNodeCandidateRelay := &tailcfg.Node{ + Cap: 121, + ID: 1, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("1.1.1.1/32"), }, + HomeDERP: 1, + Key: key.NewNode().Public(), + DiscoKey: key.NewDisco().Public(), + } + + peerNodeNotCandidateRelayCapVer := &tailcfg.Node{ + Cap: 120, // intentionally lower to fail capVer check + ID: 1, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("1.1.1.1/32"), + }, + HomeDERP: 1, + Key: key.NewNode().Public(), + DiscoKey: key.NewDisco().Public(), + } + + selfNode := &tailcfg.Node{ + Cap: 120, // intentionally lower than capVerIsRelayCapable to verify self check + ID: 2, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("2.2.2.2/32"), + }, + HomeDERP: 2, + Key: key.NewNode().Public(), + DiscoKey: key.NewDisco().Public(), + } + + tests := []struct { + name string + filt *filter.Filter + self tailcfg.NodeView + peers views.Slice[tailcfg.NodeView] + wantRelayServers set.Set[candidatePeerRelay] + }{ { - name: "nil self", + name: "candidate relay server", filt: filter.New([]filtertype.Match{ { - Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Srcs: peerNodeCandidateRelay.Addresses, Caps: []filtertype.CapMatch{ { - Dst: netip.MustParsePrefix("1.1.1.1/32"), + Dst: selfNode.Addresses[0], Cap: tailcfg.PeerCapabilityRelayTarget, }, }, }, }, nil, nil, nil, nil, nil), - self: tailcfg.NodeView{}, - maybeCandidate: peerOnlyIPv4.View(), + self: selfNode.View(), + peers: views.SliceOf([]tailcfg.NodeView{peerNodeCandidateRelay.View()}), + wantRelayServers: set.SetOf([]candidatePeerRelay{ + { + nodeKey: peerNodeCandidateRelay.Key, + discoKey: peerNodeCandidateRelay.DiscoKey, + derpHomeRegionID: 1, + }, + }), }, { - name: "nil peer", + name: "self candidate relay server", filt: filter.New([]filtertype.Match{ { - Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Srcs: selfNode.Addresses, Caps: []filtertype.CapMatch{ { - Dst: netip.MustParsePrefix("1.1.1.1/32"), + Dst: selfNode.Addresses[0], Cap: tailcfg.PeerCapabilityRelayTarget, }, }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv4.View(), - maybeCandidate: tailcfg.NodeView{}, + self: selfNode.View(), + peers: views.SliceOf([]tailcfg.NodeView{selfNode.View()}), + wantRelayServers: set.SetOf([]candidatePeerRelay{ + { + nodeKey: selfNode.Key, + discoKey: selfNode.DiscoKey, + derpHomeRegionID: 2, + }, + }), }, { - name: "nil peer hostinfo", + name: "no candidate relay server", filt: filter.New([]filtertype.Match{ { - Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Srcs: peerNodeNotCandidateRelayCapVer.Addresses, Caps: []filtertype.CapMatch{ { - Dst: netip.MustParsePrefix("1.1.1.1/32"), + Dst: selfNode.Addresses[0], Cap: tailcfg.PeerCapabilityRelayTarget, }, }, }, }, nil, nil, nil, nil, nil), - self: selfOnlyIPv4.View(), - maybeCandidate: peerOnlyIPv4NilHostinfo.View(), + self: selfNode.View(), + peers: views.SliceOf([]tailcfg.NodeView{peerNodeNotCandidateRelayCapVer.View()}), + wantRelayServers: make(set.Set[candidatePeerRelay]), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := peerAPIIfCandidateRelayServer(tt.filt, tt.self, tt.maybeCandidate); !reflect.DeepEqual(got, tt.want) { - t.Errorf("peerAPIIfCandidateRelayServer() = %v, want %v", got, tt.want) + c := &Conn{} + c.updateRelayServersSet(tt.filt, tt.self, tt.peers) + got := c.relayManager.getServers() + if !got.Equal(tt.wantRelayServers) { + t.Fatalf("got: %v != want: %v", got, tt.wantRelayServers) } - }) - } -} - -func Test_looksLikeInitiationMsg(t *testing.T) { - initMsg := make([]byte, device.MessageInitiationSize) - binary.BigEndian.PutUint32(initMsg, device.MessageInitiationType) - initMsgSizeTransportType := make([]byte, device.MessageInitiationSize) - binary.BigEndian.PutUint32(initMsgSizeTransportType, device.MessageTransportType) - tests := []struct { - name string - b []byte - want bool - }{ - { - name: "valid initiation", - b: initMsg, - want: true, - }, - { - name: "invalid message type field", - b: initMsgSizeTransportType, - want: false, - }, - { - name: "too small", - b: initMsg[:device.MessageInitiationSize-1], - want: false, - }, - { - name: "too big", - b: append(initMsg, 0), - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := looksLikeInitiationMsg(tt.b); got != tt.want { - t.Errorf("looksLikeInitiationMsg() = %v, want %v", got, tt.want) + if len(tt.wantRelayServers) > 0 != c.hasPeerRelayServers.Load() { + t.Fatalf("c.hasPeerRelayServers: %v != wantRelayServers: %v", c.hasPeerRelayServers.Load(), tt.wantRelayServers) } }) } diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index d7acf80b51a58..ad8c5fc763adb 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -4,23 +4,18 @@ package magicsock import ( - "bytes" "context" - "encoding/json" "errors" - "fmt" - "io" - "net/http" "net/netip" - "strconv" "sync" "time" "tailscale.com/disco" "tailscale.com/net/stun" udprelay "tailscale.com/net/udprelay/endpoint" + "tailscale.com/tailcfg" + "tailscale.com/tstime" "tailscale.com/types/key" - "tailscale.com/util/httpm" "tailscale.com/util/set" ) @@ -38,26 +33,28 @@ type relayManager struct { // =================================================================== // The following fields are owned by a single goroutine, runLoop(). - serversByAddrPort map[netip.AddrPort]key.DiscoPublic - serversByDisco map[key.DiscoPublic]netip.AddrPort - allocWorkByEndpoint map[*endpoint]*relayEndpointAllocWork - handshakeWorkByEndpointByServerDisco map[*endpoint]map[key.DiscoPublic]*relayHandshakeWork - handshakeWorkByServerDiscoVNI map[serverDiscoVNI]*relayHandshakeWork - handshakeWorkAwaitingPong map[*relayHandshakeWork]addrPortVNI - addrPortVNIToHandshakeWork map[addrPortVNI]*relayHandshakeWork - handshakeGeneration uint32 + serversByNodeKey map[key.NodePublic]candidatePeerRelay + allocWorkByCandidatePeerRelayByEndpoint map[*endpoint]map[candidatePeerRelay]*relayEndpointAllocWork + allocWorkByDiscoKeysByServerNodeKey map[key.NodePublic]map[key.SortedPairOfDiscoPublic]*relayEndpointAllocWork + handshakeWorkByServerDiscoByEndpoint map[*endpoint]map[key.DiscoPublic]*relayHandshakeWork + handshakeWorkByServerDiscoVNI map[serverDiscoVNI]*relayHandshakeWork + handshakeWorkAwaitingPong map[*relayHandshakeWork]addrPortVNI + addrPortVNIToHandshakeWork map[addrPortVNI]*relayHandshakeWork + handshakeGeneration uint32 + allocGeneration uint32 // =================================================================== // The following chan fields serve event inputs to a single goroutine, // runLoop(). - startDiscoveryCh chan endpointWithLastBest - allocateWorkDoneCh chan relayEndpointAllocWorkDoneEvent - handshakeWorkDoneCh chan relayEndpointHandshakeWorkDoneEvent - cancelWorkCh chan *endpoint - newServerEndpointCh chan newRelayServerEndpointEvent - rxHandshakeDiscoMsgCh chan relayHandshakeDiscoMsgEvent - serversCh chan set.Set[netip.AddrPort] - getServersCh chan chan set.Set[netip.AddrPort] + startDiscoveryCh chan endpointWithLastBest + allocateWorkDoneCh chan relayEndpointAllocWorkDoneEvent + handshakeWorkDoneCh chan relayEndpointHandshakeWorkDoneEvent + cancelWorkCh chan *endpoint + newServerEndpointCh chan newRelayServerEndpointEvent + rxDiscoMsgCh chan relayDiscoMsgEvent + serversCh chan set.Set[candidatePeerRelay] + getServersCh chan chan set.Set[candidatePeerRelay] + derpHomeChangeCh chan derpHomeChangeEvent discoInfoMu sync.Mutex // guards the following field discoInfoByServerDisco map[key.DiscoPublic]*relayHandshakeDiscoInfo @@ -86,7 +83,7 @@ type relayHandshakeWork struct { // relayManager.handshakeWorkDoneCh if runLoop() can receive it. runLoop() // must select{} read on doneCh to prevent deadlock when attempting to write // to rxDiscoMsgCh. - rxDiscoMsgCh chan relayHandshakeDiscoMsgEvent + rxDiscoMsgCh chan relayDiscoMsgEvent doneCh chan relayEndpointHandshakeWorkDoneEvent ctx context.Context @@ -100,14 +97,15 @@ type relayHandshakeWork struct { type newRelayServerEndpointEvent struct { wlb endpointWithLastBest se udprelay.ServerEndpoint - server netip.AddrPort // zero value if learned via [disco.CallMeMaybeVia] + server candidatePeerRelay // zero value if learned via [disco.CallMeMaybeVia] } // relayEndpointAllocWorkDoneEvent indicates relay server endpoint allocation // work for an [*endpoint] has completed. This structure is immutable once // initialized. type relayEndpointAllocWorkDoneEvent struct { - work *relayEndpointAllocWork + work *relayEndpointAllocWork + allocated udprelay.ServerEndpoint // !allocated.ServerDisco.IsZero() if allocation succeeded } // relayEndpointHandshakeWorkDoneEvent indicates relay server endpoint handshake @@ -122,18 +120,42 @@ type relayEndpointHandshakeWorkDoneEvent struct { // hasActiveWorkRunLoop returns true if there is outstanding allocation or // handshaking work for any endpoint, otherwise it returns false. func (r *relayManager) hasActiveWorkRunLoop() bool { - return len(r.allocWorkByEndpoint) > 0 || len(r.handshakeWorkByEndpointByServerDisco) > 0 + return len(r.allocWorkByCandidatePeerRelayByEndpoint) > 0 || len(r.handshakeWorkByServerDiscoByEndpoint) > 0 } // hasActiveWorkForEndpointRunLoop returns true if there is outstanding // allocation or handshaking work for the provided endpoint, otherwise it // returns false. func (r *relayManager) hasActiveWorkForEndpointRunLoop(ep *endpoint) bool { - _, handshakeWork := r.handshakeWorkByEndpointByServerDisco[ep] - _, allocWork := r.allocWorkByEndpoint[ep] + _, handshakeWork := r.handshakeWorkByServerDiscoByEndpoint[ep] + _, allocWork := r.allocWorkByCandidatePeerRelayByEndpoint[ep] return handshakeWork || allocWork } +// derpHomeChangeEvent represents a change in the DERP home region for the +// node identified by nodeKey. This structure is immutable once initialized. +type derpHomeChangeEvent struct { + nodeKey key.NodePublic + regionID uint16 +} + +// handleDERPHomeChange handles a DERP home change event for nodeKey and +// regionID. +func (r *relayManager) handleDERPHomeChange(nodeKey key.NodePublic, regionID uint16) { + relayManagerInputEvent(r, nil, &r.derpHomeChangeCh, derpHomeChangeEvent{ + nodeKey: nodeKey, + regionID: regionID, + }) +} + +func (r *relayManager) handleDERPHomeChangeRunLoop(event derpHomeChangeEvent) { + c, ok := r.serversByNodeKey[event.nodeKey] + if ok { + c.derpHomeRegionID = event.regionID + r.serversByNodeKey[event.nodeKey] = c + } +} + // runLoop is a form of event loop. It ensures exclusive access to most of // [relayManager] state. func (r *relayManager) runLoop() { @@ -151,13 +173,7 @@ func (r *relayManager) runLoop() { return } case done := <-r.allocateWorkDoneCh: - work, ok := r.allocWorkByEndpoint[done.work.ep] - if ok && work == done.work { - // Verify the work in the map is the same as the one that we're - // cleaning up. New events on r.startDiscoveryCh can - // overwrite pre-existing keys. - delete(r.allocWorkByEndpoint, done.work.ep) - } + r.handleAllocWorkDoneRunLoop(done) if !r.hasActiveWorkRunLoop() { return } @@ -176,8 +192,8 @@ func (r *relayManager) runLoop() { if !r.hasActiveWorkRunLoop() { return } - case discoMsgEvent := <-r.rxHandshakeDiscoMsgCh: - r.handleRxHandshakeDiscoMsgRunLoop(discoMsgEvent) + case discoMsgEvent := <-r.rxDiscoMsgCh: + r.handleRxDiscoMsgRunLoop(discoMsgEvent) if !r.hasActiveWorkRunLoop() { return } @@ -191,69 +207,77 @@ func (r *relayManager) runLoop() { if !r.hasActiveWorkRunLoop() { return } + case derpHomeChange := <-r.derpHomeChangeCh: + r.handleDERPHomeChangeRunLoop(derpHomeChange) + if !r.hasActiveWorkRunLoop() { + return + } } } } -func (r *relayManager) handleGetServersRunLoop(getServersCh chan set.Set[netip.AddrPort]) { - servers := make(set.Set[netip.AddrPort], len(r.serversByAddrPort)) - for server := range r.serversByAddrPort { - servers.Add(server) +func (r *relayManager) handleGetServersRunLoop(getServersCh chan set.Set[candidatePeerRelay]) { + servers := make(set.Set[candidatePeerRelay], len(r.serversByNodeKey)) + for _, v := range r.serversByNodeKey { + servers.Add(v) } getServersCh <- servers } -func (r *relayManager) getServers() set.Set[netip.AddrPort] { - ch := make(chan set.Set[netip.AddrPort]) +func (r *relayManager) getServers() set.Set[candidatePeerRelay] { + ch := make(chan set.Set[candidatePeerRelay]) relayManagerInputEvent(r, nil, &r.getServersCh, ch) return <-ch } -func (r *relayManager) handleServersUpdateRunLoop(update set.Set[netip.AddrPort]) { - for k, v := range r.serversByAddrPort { - if !update.Contains(k) { - delete(r.serversByAddrPort, k) - delete(r.serversByDisco, v) +func (r *relayManager) handleServersUpdateRunLoop(update set.Set[candidatePeerRelay]) { + for _, v := range r.serversByNodeKey { + if !update.Contains(v) { + delete(r.serversByNodeKey, v.nodeKey) } } for _, v := range update.Slice() { - _, ok := r.serversByAddrPort[v] - if ok { - // don't zero known disco keys - continue - } - r.serversByAddrPort[v] = key.DiscoPublic{} + r.serversByNodeKey[v.nodeKey] = v } } -type relayHandshakeDiscoMsgEvent struct { - conn *Conn // for access to [Conn] if there is no associated [relayHandshakeWork] - msg disco.Message - disco key.DiscoPublic - from netip.AddrPort - vni uint32 - at time.Time +type relayDiscoMsgEvent struct { + conn *Conn // for access to [Conn] if there is no associated [relayHandshakeWork] + msg disco.Message + relayServerNodeKey key.NodePublic // nonzero if msg is a [*disco.AllocateUDPRelayEndpointResponse] + disco key.DiscoPublic + from netip.AddrPort + vni uint32 + at time.Time } // relayEndpointAllocWork serves to track in-progress relay endpoint allocation // for an [*endpoint]. This structure is immutable once initialized. type relayEndpointAllocWork struct { - // ep is the [*endpoint] associated with the work - ep *endpoint - // cancel() will signal all associated goroutines to return + wlb endpointWithLastBest + discoKeys key.SortedPairOfDiscoPublic + candidatePeerRelay candidatePeerRelay + + // allocateServerEndpoint() always writes to doneCh (len 1) when it + // returns. It may end up writing the same event afterward to + // [relayManager.allocateWorkDoneCh] if runLoop() can receive it. runLoop() + // must select{} read on doneCh to prevent deadlock when attempting to write + // to rxDiscoMsgCh. + rxDiscoMsgCh chan *disco.AllocateUDPRelayEndpointResponse + doneCh chan relayEndpointAllocWorkDoneEvent + + ctx context.Context cancel context.CancelFunc - // wg.Wait() will return once all associated goroutines have returned - wg *sync.WaitGroup } // init initializes [relayManager] if it is not already initialized. func (r *relayManager) init() { r.initOnce.Do(func() { r.discoInfoByServerDisco = make(map[key.DiscoPublic]*relayHandshakeDiscoInfo) - r.serversByDisco = make(map[key.DiscoPublic]netip.AddrPort) - r.serversByAddrPort = make(map[netip.AddrPort]key.DiscoPublic) - r.allocWorkByEndpoint = make(map[*endpoint]*relayEndpointAllocWork) - r.handshakeWorkByEndpointByServerDisco = make(map[*endpoint]map[key.DiscoPublic]*relayHandshakeWork) + r.serversByNodeKey = make(map[key.NodePublic]candidatePeerRelay) + r.allocWorkByCandidatePeerRelayByEndpoint = make(map[*endpoint]map[candidatePeerRelay]*relayEndpointAllocWork) + r.allocWorkByDiscoKeysByServerNodeKey = make(map[key.NodePublic]map[key.SortedPairOfDiscoPublic]*relayEndpointAllocWork) + r.handshakeWorkByServerDiscoByEndpoint = make(map[*endpoint]map[key.DiscoPublic]*relayHandshakeWork) r.handshakeWorkByServerDiscoVNI = make(map[serverDiscoVNI]*relayHandshakeWork) r.handshakeWorkAwaitingPong = make(map[*relayHandshakeWork]addrPortVNI) r.addrPortVNIToHandshakeWork = make(map[addrPortVNI]*relayHandshakeWork) @@ -262,9 +286,10 @@ func (r *relayManager) init() { r.handshakeWorkDoneCh = make(chan relayEndpointHandshakeWorkDoneEvent) r.cancelWorkCh = make(chan *endpoint) r.newServerEndpointCh = make(chan newRelayServerEndpointEvent) - r.rxHandshakeDiscoMsgCh = make(chan relayHandshakeDiscoMsgEvent) - r.serversCh = make(chan set.Set[netip.AddrPort]) - r.getServersCh = make(chan chan set.Set[netip.AddrPort]) + r.rxDiscoMsgCh = make(chan relayDiscoMsgEvent) + r.serversCh = make(chan set.Set[candidatePeerRelay]) + r.getServersCh = make(chan chan set.Set[candidatePeerRelay]) + r.derpHomeChangeCh = make(chan derpHomeChangeEvent) r.runLoopStoppedCh = make(chan struct{}, 1) r.runLoopStoppedCh <- struct{}{} }) @@ -330,6 +355,7 @@ func (r *relayManager) discoInfo(serverDisco key.DiscoPublic) (_ *discoInfo, ok func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, lastBest addrQuality, lastBestIsTrusted bool, dm *disco.CallMeMaybeVia) { se := udprelay.ServerEndpoint{ ServerDisco: dm.ServerDisco, + ClientDisco: dm.ClientDisco, LamportID: dm.LamportID, AddrPorts: dm.AddrPorts, VNI: dm.VNI, @@ -346,14 +372,25 @@ func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, lastBest addrQuality, }) } -// handleGeneveEncapDiscoMsg handles reception of Geneve-encapsulated disco -// messages. -func (r *relayManager) handleGeneveEncapDiscoMsg(conn *Conn, dm disco.Message, di *discoInfo, src epAddr) { - relayManagerInputEvent(r, nil, &r.rxHandshakeDiscoMsgCh, relayHandshakeDiscoMsgEvent{conn: conn, msg: dm, disco: di.discoKey, from: src.ap, vni: src.vni.get(), at: time.Now()}) +// handleRxDiscoMsg handles reception of disco messages that [relayManager] +// may be interested in. This includes all Geneve-encapsulated disco messages +// and [*disco.AllocateUDPRelayEndpointResponse]. If dm is a +// [*disco.AllocateUDPRelayEndpointResponse] then relayServerNodeKey must be +// nonzero. +func (r *relayManager) handleRxDiscoMsg(conn *Conn, dm disco.Message, relayServerNodeKey key.NodePublic, discoKey key.DiscoPublic, src epAddr) { + relayManagerInputEvent(r, nil, &r.rxDiscoMsgCh, relayDiscoMsgEvent{ + conn: conn, + msg: dm, + relayServerNodeKey: relayServerNodeKey, + disco: discoKey, + from: src.ap, + vni: src.vni.get(), + at: time.Now(), + }) } // handleRelayServersSet handles an update of the complete relay server set. -func (r *relayManager) handleRelayServersSet(servers set.Set[netip.AddrPort]) { +func (r *relayManager) handleRelayServersSet(servers set.Set[candidatePeerRelay]) { relayManagerInputEvent(r, nil, &r.serversCh, servers) } @@ -396,7 +433,11 @@ type endpointWithLastBest struct { // startUDPRelayPathDiscoveryFor starts UDP relay path discovery for ep on all // known relay servers if ep has no in-progress work. func (r *relayManager) startUDPRelayPathDiscoveryFor(ep *endpoint, lastBest addrQuality, lastBestIsTrusted bool) { - relayManagerInputEvent(r, nil, &r.startDiscoveryCh, endpointWithLastBest{ep, lastBest, lastBestIsTrusted}) + relayManagerInputEvent(r, nil, &r.startDiscoveryCh, endpointWithLastBest{ + ep: ep, + lastBest: lastBest, + lastBestIsTrusted: lastBestIsTrusted, + }) } // stopWork stops all outstanding allocation & handshaking work for 'ep'. @@ -407,13 +448,15 @@ func (r *relayManager) stopWork(ep *endpoint) { // stopWorkRunLoop cancels & clears outstanding allocation and handshaking // work for 'ep'. func (r *relayManager) stopWorkRunLoop(ep *endpoint) { - allocWork, ok := r.allocWorkByEndpoint[ep] + byDiscoKeys, ok := r.allocWorkByCandidatePeerRelayByEndpoint[ep] if ok { - allocWork.cancel() - allocWork.wg.Wait() - delete(r.allocWorkByEndpoint, ep) + for _, work := range byDiscoKeys { + work.cancel() + done := <-work.doneCh + r.handleAllocWorkDoneRunLoop(done) + } } - byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[ep] + byServerDisco, ok := r.handshakeWorkByServerDiscoByEndpoint[ep] if ok { for _, handshakeWork := range byServerDisco { handshakeWork.cancel() @@ -430,13 +473,33 @@ type addrPortVNI struct { vni uint32 } -func (r *relayManager) handleRxHandshakeDiscoMsgRunLoop(event relayHandshakeDiscoMsgEvent) { +func (r *relayManager) handleRxDiscoMsgRunLoop(event relayDiscoMsgEvent) { var ( work *relayHandshakeWork ok bool ) apv := addrPortVNI{event.from, event.vni} switch msg := event.msg.(type) { + case *disco.AllocateUDPRelayEndpointResponse: + sorted := key.NewSortedPairOfDiscoPublic(msg.ClientDisco[0], msg.ClientDisco[1]) + byDiscoKeys, ok := r.allocWorkByDiscoKeysByServerNodeKey[event.relayServerNodeKey] + if !ok { + // No outstanding work tied to this relay sever, discard. + return + } + allocWork, ok := byDiscoKeys[sorted] + if !ok { + // No outstanding work tied to these disco keys, discard. + return + } + select { + case done := <-allocWork.doneCh: + // allocateServerEndpoint returned, clean up its state + r.handleAllocWorkDoneRunLoop(done) + return + case allocWork.rxDiscoMsgCh <- msg: + return + } case *disco.BindUDPRelayEndpointChallenge: work, ok = r.handshakeWorkByServerDiscoVNI[serverDiscoVNI{event.disco, event.vni}] if !ok { @@ -504,8 +567,39 @@ func (r *relayManager) handleRxHandshakeDiscoMsgRunLoop(event relayHandshakeDisc } } +func (r *relayManager) handleAllocWorkDoneRunLoop(done relayEndpointAllocWorkDoneEvent) { + byCandidatePeerRelay, ok := r.allocWorkByCandidatePeerRelayByEndpoint[done.work.wlb.ep] + if !ok { + return + } + work, ok := byCandidatePeerRelay[done.work.candidatePeerRelay] + if !ok || work != done.work { + return + } + delete(byCandidatePeerRelay, done.work.candidatePeerRelay) + if len(byCandidatePeerRelay) == 0 { + delete(r.allocWorkByCandidatePeerRelayByEndpoint, done.work.wlb.ep) + } + byDiscoKeys, ok := r.allocWorkByDiscoKeysByServerNodeKey[done.work.candidatePeerRelay.nodeKey] + if !ok { + // unexpected + return + } + delete(byDiscoKeys, done.work.discoKeys) + if len(byDiscoKeys) == 0 { + delete(r.allocWorkByDiscoKeysByServerNodeKey, done.work.candidatePeerRelay.nodeKey) + } + if !done.allocated.ServerDisco.IsZero() { + r.handleNewServerEndpointRunLoop(newRelayServerEndpointEvent{ + wlb: done.work.wlb, + se: done.allocated, + server: done.work.candidatePeerRelay, + }) + } +} + func (r *relayManager) handleHandshakeWorkDoneRunLoop(done relayEndpointHandshakeWorkDoneEvent) { - byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[done.work.wlb.ep] + byServerDisco, ok := r.handshakeWorkByServerDiscoByEndpoint[done.work.wlb.ep] if !ok { return } @@ -515,7 +609,7 @@ func (r *relayManager) handleHandshakeWorkDoneRunLoop(done relayEndpointHandshak } delete(byServerDisco, done.work.se.ServerDisco) if len(byServerDisco) == 0 { - delete(r.handshakeWorkByEndpointByServerDisco, done.work.wlb.ep) + delete(r.handshakeWorkByServerDiscoByEndpoint, done.work.wlb.ep) } delete(r.handshakeWorkByServerDiscoVNI, serverDiscoVNI{done.work.se.ServerDisco, done.work.se.VNI}) apv, ok := r.handshakeWorkAwaitingPong[work] @@ -562,7 +656,7 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay } // Check for duplicate work by [*endpoint] + server disco. - byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[newServerEndpoint.wlb.ep] + byServerDisco, ok := r.handshakeWorkByServerDiscoByEndpoint[newServerEndpoint.wlb.ep] if ok { existingWork, ok := byServerDisco[newServerEndpoint.se.ServerDisco] if ok { @@ -580,33 +674,9 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay // We're now reasonably sure we're dealing with the latest // [udprelay.ServerEndpoint] from a server event order perspective - // (LamportID). Update server disco key tracking if appropriate. - if newServerEndpoint.server.IsValid() { - serverDisco, ok := r.serversByAddrPort[newServerEndpoint.server] - if !ok { - // Allocation raced with an update to our known servers set. This - // server is no longer known. Return early. - return - } - if serverDisco.Compare(newServerEndpoint.se.ServerDisco) != 0 { - // The server's disco key has either changed, or simply become - // known for the first time. In the former case we end up detaching - // any in-progress handshake work from a "known" relay server. - // Practically speaking we expect the detached work to fail - // if the server key did in fact change (server restart) while we - // were attempting to handshake with it. It is possible, though - // unlikely, for a server addr:port to effectively move between - // nodes. Either way, there is no harm in detaching existing work, - // and we explicitly let that happen for the rare case the detached - // handshake would complete and remain functional. - delete(r.serversByDisco, serverDisco) - delete(r.serversByAddrPort, newServerEndpoint.server) - r.serversByDisco[serverDisco] = newServerEndpoint.server - r.serversByAddrPort[newServerEndpoint.server] = serverDisco - } - } + // (LamportID). - if newServerEndpoint.server.IsValid() { + if newServerEndpoint.server.isValid() { // Send a [disco.CallMeMaybeVia] to the remote peer if we allocated this // endpoint, regardless of if we start a handshake below. go r.sendCallMeMaybeVia(newServerEndpoint.wlb.ep, newServerEndpoint.se) @@ -641,14 +711,14 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay work := &relayHandshakeWork{ wlb: newServerEndpoint.wlb, se: newServerEndpoint.se, - rxDiscoMsgCh: make(chan relayHandshakeDiscoMsgEvent), + rxDiscoMsgCh: make(chan relayDiscoMsgEvent), doneCh: make(chan relayEndpointHandshakeWorkDoneEvent, 1), ctx: ctx, cancel: cancel, } if byServerDisco == nil { byServerDisco = make(map[key.DiscoPublic]*relayHandshakeWork) - r.handshakeWorkByEndpointByServerDisco[newServerEndpoint.wlb.ep] = byServerDisco + r.handshakeWorkByServerDiscoByEndpoint[newServerEndpoint.wlb.ep] = byServerDisco } byServerDisco[newServerEndpoint.se.ServerDisco] = work r.handshakeWorkByServerDiscoVNI[sdv] = work @@ -674,12 +744,15 @@ func (r *relayManager) sendCallMeMaybeVia(ep *endpoint, se udprelay.ServerEndpoi return } callMeMaybeVia := &disco.CallMeMaybeVia{ - ServerDisco: se.ServerDisco, - LamportID: se.LamportID, - VNI: se.VNI, - BindLifetime: se.BindLifetime.Duration, - SteadyStateLifetime: se.SteadyStateLifetime.Duration, - AddrPorts: se.AddrPorts, + UDPRelayEndpoint: disco.UDPRelayEndpoint{ + ServerDisco: se.ServerDisco, + ClientDisco: se.ClientDisco, + LamportID: se.LamportID, + VNI: se.VNI, + BindLifetime: se.BindLifetime.Duration, + SteadyStateLifetime: se.SteadyStateLifetime.Duration, + AddrPorts: se.AddrPorts, + }, } ep.c.sendDiscoMessage(epAddr{ap: derpAddr}, ep.publicKey, epDisco.key, callMeMaybeVia, discoVerboseLog) } @@ -800,7 +873,7 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork, generat // one. // // We don't need to TX a pong, that was already handled for us - // in handleRxHandshakeDiscoMsgRunLoop(). + // in handleRxDiscoMsgRunLoop(). txPing(msgEvent.from, nil) case *disco.Pong: at, ok := sentPingAt[msg.TxID] @@ -823,104 +896,113 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork, generat } } -func (r *relayManager) allocateAllServersRunLoop(wlb endpointWithLastBest) { - if len(r.serversByAddrPort) == 0 { - return - } - ctx, cancel := context.WithCancel(context.Background()) - started := &relayEndpointAllocWork{ep: wlb.ep, cancel: cancel, wg: &sync.WaitGroup{}} - for k := range r.serversByAddrPort { - started.wg.Add(1) - go r.allocateSingleServer(ctx, started.wg, k, wlb) - } - r.allocWorkByEndpoint[wlb.ep] = started - go func() { - started.wg.Wait() - relayManagerInputEvent(r, ctx, &r.allocateWorkDoneCh, relayEndpointAllocWorkDoneEvent{work: started}) - // cleanup context cancellation must come after the - // relayManagerInputEvent call, otherwise it returns early without - // writing the event to runLoop(). - started.cancel() - }() -} +const allocateUDPRelayEndpointRequestTimeout = time.Second * 10 -type errNotReady struct{ retryAfter time.Duration } +func (r *relayManager) allocateServerEndpoint(work *relayEndpointAllocWork, generation uint32) { + done := relayEndpointAllocWorkDoneEvent{work: work} -func (e errNotReady) Error() string { - return fmt.Sprintf("server not ready, retry after %v", e.retryAfter) -} + defer func() { + work.doneCh <- done + relayManagerInputEvent(r, work.ctx, &r.allocateWorkDoneCh, done) + work.cancel() + }() -const reqTimeout = time.Second * 10 + dm := &disco.AllocateUDPRelayEndpointRequest{ + ClientDisco: work.discoKeys.Get(), + Generation: generation, + } + + sendAllocReq := func() { + work.wlb.ep.c.sendDiscoAllocateUDPRelayEndpointRequest( + epAddr{ + ap: netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, work.candidatePeerRelay.derpHomeRegionID), + }, + work.candidatePeerRelay.nodeKey, + work.candidatePeerRelay.discoKey, + dm, + discoVerboseLog, + ) + } + go sendAllocReq() + + returnAfterTimer := time.NewTimer(allocateUDPRelayEndpointRequestTimeout) + defer returnAfterTimer.Stop() + // While connections to DERP are over TCP, they can be lossy on the DERP + // server when data moves between the two independent streams. Also, the + // peer relay server may not be "ready" (see [tailscale.com/net/udprelay.ErrServerNotReady]). + // So, start a timer to retry once if needed. + retryAfterTimer := time.NewTimer(udprelay.ServerRetryAfter) + defer retryAfterTimer.Stop() -func doAllocate(ctx context.Context, server netip.AddrPort, discoKeys [2]key.DiscoPublic) (udprelay.ServerEndpoint, error) { - var reqBody bytes.Buffer - type allocateRelayEndpointReq struct { - DiscoKeys []key.DiscoPublic - } - a := &allocateRelayEndpointReq{ - DiscoKeys: []key.DiscoPublic{discoKeys[0], discoKeys[1]}, - } - err := json.NewEncoder(&reqBody).Encode(a) - if err != nil { - return udprelay.ServerEndpoint{}, err - } - reqCtx, cancel := context.WithTimeout(ctx, reqTimeout) - defer cancel() - req, err := http.NewRequestWithContext(reqCtx, httpm.POST, "http://"+server.String()+"/v0/relay/endpoint", &reqBody) - if err != nil { - return udprelay.ServerEndpoint{}, err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return udprelay.ServerEndpoint{}, err - } - defer resp.Body.Close() - switch resp.StatusCode { - case http.StatusOK: - var se udprelay.ServerEndpoint - err = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&se) - return se, err - case http.StatusServiceUnavailable: - raHeader := resp.Header.Get("Retry-After") - raSeconds, err := strconv.ParseUint(raHeader, 10, 32) - if err == nil { - return udprelay.ServerEndpoint{}, errNotReady{retryAfter: time.Second * time.Duration(raSeconds)} + for { + select { + case <-work.ctx.Done(): + return + case <-returnAfterTimer.C: + return + case <-retryAfterTimer.C: + go sendAllocReq() + case resp := <-work.rxDiscoMsgCh: + if resp.Generation != generation || + !work.discoKeys.Equal(key.NewSortedPairOfDiscoPublic(resp.ClientDisco[0], resp.ClientDisco[1])) { + continue + } + done.allocated = udprelay.ServerEndpoint{ + ServerDisco: resp.ServerDisco, + ClientDisco: resp.ClientDisco, + LamportID: resp.LamportID, + AddrPorts: resp.AddrPorts, + VNI: resp.VNI, + BindLifetime: tstime.GoDuration{Duration: resp.BindLifetime}, + SteadyStateLifetime: tstime.GoDuration{Duration: resp.SteadyStateLifetime}, + } + return } - fallthrough - default: - return udprelay.ServerEndpoint{}, fmt.Errorf("non-200 status: %d", resp.StatusCode) } } -func (r *relayManager) allocateSingleServer(ctx context.Context, wg *sync.WaitGroup, server netip.AddrPort, wlb endpointWithLastBest) { - // TODO(jwhited): introduce client metrics counters for notable failures - defer wg.Done() +func (r *relayManager) allocateAllServersRunLoop(wlb endpointWithLastBest) { + if len(r.serversByNodeKey) == 0 { + return + } remoteDisco := wlb.ep.disco.Load() if remoteDisco == nil { return } - firstTry := true - for { - se, err := doAllocate(ctx, server, [2]key.DiscoPublic{wlb.ep.c.discoPublic, remoteDisco.key}) - if err == nil { - relayManagerInputEvent(r, ctx, &r.newServerEndpointCh, newRelayServerEndpointEvent{ - wlb: wlb, - se: se, - server: server, // we allocated this endpoint (vs CallMeMaybeVia reception), mark it as such - }) - return - } - wlb.ep.c.logf("[v1] magicsock: relayManager: error allocating endpoint on %v for %v: %v", server, wlb.ep.discoShort(), err) - var notReady errNotReady - if firstTry && errors.As(err, ¬Ready) { - select { - case <-ctx.Done(): - return - case <-time.After(min(notReady.retryAfter, reqTimeout)): - firstTry = false + discoKeys := key.NewSortedPairOfDiscoPublic(wlb.ep.c.discoPublic, remoteDisco.key) + for _, v := range r.serversByNodeKey { + byDiscoKeys, ok := r.allocWorkByDiscoKeysByServerNodeKey[v.nodeKey] + if !ok { + byDiscoKeys = make(map[key.SortedPairOfDiscoPublic]*relayEndpointAllocWork) + r.allocWorkByDiscoKeysByServerNodeKey[v.nodeKey] = byDiscoKeys + } else { + _, ok = byDiscoKeys[discoKeys] + if ok { + // If there is an existing key, a disco key collision may have + // occurred across peers ([*endpoint]). Do not overwrite the + // existing work, let it finish. + wlb.ep.c.logf("[unexpected] magicsock: relayManager: suspected disco key collision on server %v for keys: %v", v.nodeKey.ShortString(), discoKeys) continue } } - return + ctx, cancel := context.WithCancel(context.Background()) + started := &relayEndpointAllocWork{ + wlb: wlb, + discoKeys: discoKeys, + candidatePeerRelay: v, + rxDiscoMsgCh: make(chan *disco.AllocateUDPRelayEndpointResponse), + doneCh: make(chan relayEndpointAllocWorkDoneEvent, 1), + ctx: ctx, + cancel: cancel, + } + byDiscoKeys[discoKeys] = started + byCandidatePeerRelay, ok := r.allocWorkByCandidatePeerRelayByEndpoint[wlb.ep] + if !ok { + byCandidatePeerRelay = make(map[candidatePeerRelay]*relayEndpointAllocWork) + r.allocWorkByCandidatePeerRelayByEndpoint[wlb.ep] = byCandidatePeerRelay + } + byCandidatePeerRelay[v] = started + r.allocGeneration++ + go r.allocateServerEndpoint(started, r.allocGeneration) } } diff --git a/wgengine/magicsock/relaymanager_test.go b/wgengine/magicsock/relaymanager_test.go index 01f9258ad7521..e4891f5678a24 100644 --- a/wgengine/magicsock/relaymanager_test.go +++ b/wgengine/magicsock/relaymanager_test.go @@ -4,7 +4,6 @@ package magicsock import ( - "net/netip" "testing" "tailscale.com/disco" @@ -22,26 +21,57 @@ func TestRelayManagerInitAndIdle(t *testing.T) { <-rm.runLoopStoppedCh rm = relayManager{} - rm.handleCallMeMaybeVia(&endpoint{c: &Conn{discoPrivate: key.NewDisco()}}, addrQuality{}, false, &disco.CallMeMaybeVia{ServerDisco: key.NewDisco().Public()}) + rm.handleCallMeMaybeVia(&endpoint{c: &Conn{discoPrivate: key.NewDisco()}}, addrQuality{}, false, &disco.CallMeMaybeVia{UDPRelayEndpoint: disco.UDPRelayEndpoint{ServerDisco: key.NewDisco().Public()}}) <-rm.runLoopStoppedCh rm = relayManager{} - rm.handleGeneveEncapDiscoMsg(&Conn{discoPrivate: key.NewDisco()}, &disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, epAddr{}) + rm.handleRxDiscoMsg(&Conn{discoPrivate: key.NewDisco()}, &disco.BindUDPRelayEndpointChallenge{}, key.NodePublic{}, key.DiscoPublic{}, epAddr{}) <-rm.runLoopStoppedCh rm = relayManager{} - rm.handleRelayServersSet(make(set.Set[netip.AddrPort])) + rm.handleRelayServersSet(make(set.Set[candidatePeerRelay])) <-rm.runLoopStoppedCh rm = relayManager{} rm.getServers() <-rm.runLoopStoppedCh + + rm = relayManager{} + rm.handleDERPHomeChange(key.NodePublic{}, 1) + <-rm.runLoopStoppedCh +} + +func TestRelayManagerHandleDERPHomeChange(t *testing.T) { + rm := relayManager{} + servers := make(set.Set[candidatePeerRelay], 1) + c := candidatePeerRelay{ + nodeKey: key.NewNode().Public(), + discoKey: key.NewDisco().Public(), + derpHomeRegionID: 1, + } + servers.Add(c) + rm.handleRelayServersSet(servers) + want := c + want.derpHomeRegionID = 2 + rm.handleDERPHomeChange(c.nodeKey, 2) + got := rm.getServers() + if len(got) != 1 { + t.Fatalf("got %d servers, want 1", len(got)) + } + _, ok := got[want] + if !ok { + t.Fatal("DERP home change failed to propagate") + } } func TestRelayManagerGetServers(t *testing.T) { rm := relayManager{} - servers := make(set.Set[netip.AddrPort], 1) - servers.Add(netip.MustParseAddrPort("192.0.2.1:7")) + servers := make(set.Set[candidatePeerRelay], 1) + c := candidatePeerRelay{ + nodeKey: key.NewNode().Public(), + discoKey: key.NewDisco().Public(), + } + servers.Add(c) rm.handleRelayServersSet(servers) got := rm.getServers() if !servers.Equal(got) { From 0d03a3746a0229fe749b94b1d60491de64b135cd Mon Sep 17 00:00:00 2001 From: Andrew Lytvynov Date: Mon, 21 Jul 2025 10:35:53 -0700 Subject: [PATCH 234/263] feature/tpm: log errors on the initial info fetch (#16574) This function is behind a sync.Once so we should only see errors at startup. In particular the error from `open` is useful to diagnose why TPM might not be accessible. Updates #15830 Signed-off-by: Andrew Lytvynov --- feature/tpm/tpm.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/feature/tpm/tpm.go b/feature/tpm/tpm.go index 9499ed02a8b2f..0260cca586e13 100644 --- a/feature/tpm/tpm.go +++ b/feature/tpm/tpm.go @@ -44,8 +44,10 @@ func init() { func info() *tailcfg.TPMInfo { tpm, err := open() if err != nil { + log.Printf("TPM: error opening: %v", err) return nil } + log.Printf("TPM: successfully opened") defer tpm.Close() info := new(tailcfg.TPMInfo) @@ -74,10 +76,12 @@ func info() *tailcfg.TPMInfo { PropertyCount: 1, }.Execute(tpm) if err != nil { + log.Printf("TPM: GetCapability %v: %v", cap.prop, err) continue } props, err := resp.CapabilityData.Data.TPMProperties() if err != nil { + log.Printf("TPM: GetCapability %v: %v", cap.prop, err) continue } if len(props.TPMProperty) == 0 { From c989824aac0df05b00275ae8911b7bbf26797d9d Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 21 Jul 2025 19:06:36 +0100 Subject: [PATCH 235/263] cmd/k8s-operator: Allow specifying cluster ips for nameservers (#16477) This commit modifies the kubernetes operator's `DNSConfig` resource with the addition of a new field at `nameserver.service.clusterIP`. This field allows users to specify a static in-cluster IP address of the nameserver when deployed. Fixes #14305 Signed-off-by: David Bond --- .../deploy/crds/tailscale.com_dnsconfigs.yaml | 9 +- .../deploy/manifests/operator.yaml | 9 +- cmd/k8s-operator/nameserver.go | 9 +- cmd/k8s-operator/nameserver_test.go | 177 +++++++++++------- k8s-operator/api.md | 19 +- .../apis/v1alpha1/types_tsdnsconfig.go | 11 +- .../apis/v1alpha1/zz_generated.deepcopy.go | 20 ++ 7 files changed, 179 insertions(+), 75 deletions(-) diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml index 268d978c15f37..bffad47f97191 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml @@ -101,6 +101,13 @@ spec: tag: description: Tag defaults to unstable. type: string + service: + description: Service configuration. + type: object + properties: + clusterIP: + description: ClusterIP sets the static IP of the service used by the nameserver. + type: string status: description: |- Status describes the status of the DNSConfig. This is set @@ -172,7 +179,7 @@ spec: ip: description: |- IP is the ClusterIP of the Service fronting the deployed ts.net nameserver. - Currently you must manually update your cluster DNS config to add + Currently, you must manually update your cluster DNS config to add this address as a stub nameserver for ts.net for cluster workloads to be able to resolve MagicDNS names associated with egress or Ingress proxies. diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index ac8143e98c22b..175f2a7fbe9ba 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -389,6 +389,13 @@ spec: description: Tag defaults to unstable. type: string type: object + service: + description: Service configuration. + properties: + clusterIP: + description: ClusterIP sets the static IP of the service used by the nameserver. + type: string + type: object type: object required: - nameserver @@ -462,7 +469,7 @@ spec: ip: description: |- IP is the ClusterIP of the Service fronting the deployed ts.net nameserver. - Currently you must manually update your cluster DNS config to add + Currently, you must manually update your cluster DNS config to add this address as a stub nameserver for ts.net for cluster workloads to be able to resolve MagicDNS names associated with egress or Ingress proxies. diff --git a/cmd/k8s-operator/nameserver.go b/cmd/k8s-operator/nameserver.go index 20d66f7d0766a..983a28c918276 100644 --- a/cmd/k8s-operator/nameserver.go +++ b/cmd/k8s-operator/nameserver.go @@ -7,14 +7,13 @@ package main import ( "context" + _ "embed" "errors" "fmt" "slices" "strings" "sync" - _ "embed" - "go.uber.org/zap" xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" @@ -183,6 +182,10 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa if tsDNSCfg.Spec.Nameserver.Image != nil && tsDNSCfg.Spec.Nameserver.Image.Tag != "" { dCfg.imageTag = tsDNSCfg.Spec.Nameserver.Image.Tag } + if tsDNSCfg.Spec.Nameserver.Service != nil { + dCfg.clusterIP = tsDNSCfg.Spec.Nameserver.Service.ClusterIP + } + for _, deployable := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} { if err := deployable.updateObj(ctx, dCfg, a.Client); err != nil { return fmt.Errorf("error reconciling %s: %w", deployable.kind, err) @@ -213,6 +216,7 @@ type deployConfig struct { labels map[string]string ownerRefs []metav1.OwnerReference namespace string + clusterIP string } var ( @@ -267,6 +271,7 @@ var ( svc.ObjectMeta.Labels = cfg.labels svc.ObjectMeta.OwnerReferences = cfg.ownerRefs svc.ObjectMeta.Namespace = cfg.namespace + svc.Spec.ClusterIP = cfg.clusterIP _, err := createOrUpdate[corev1.Service](ctx, kubeClient, cfg.namespace, svc, func(*corev1.Service) {}) return err }, diff --git a/cmd/k8s-operator/nameserver_test.go b/cmd/k8s-operator/nameserver_test.go index cec95b84ee719..55a998ac31979 100644 --- a/cmd/k8s-operator/nameserver_test.go +++ b/cmd/k8s-operator/nameserver_test.go @@ -26,7 +26,7 @@ import ( ) func TestNameserverReconciler(t *testing.T) { - dnsCfg := &tsapi.DNSConfig{ + dnsConfig := &tsapi.DNSConfig{ TypeMeta: metav1.TypeMeta{Kind: "DNSConfig", APIVersion: "tailscale.com/v1alpha1"}, ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -37,91 +37,130 @@ func TestNameserverReconciler(t *testing.T) { Repo: "test", Tag: "v0.0.1", }, + Service: &tsapi.NameserverService{ + ClusterIP: "5.4.3.2", + }, }, }, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). - WithObjects(dnsCfg). - WithStatusSubresource(dnsCfg). + WithObjects(dnsConfig). + WithStatusSubresource(dnsConfig). Build() - zl, err := zap.NewDevelopment() + + logger, err := zap.NewDevelopment() if err != nil { t.Fatal(err) } - cl := tstest.NewClock(tstest.ClockOpts{}) - nr := &NameserverReconciler{ + + clock := tstest.NewClock(tstest.ClockOpts{}) + reconciler := &NameserverReconciler{ Client: fc, - clock: cl, - logger: zl.Sugar(), - tsNamespace: "tailscale", + clock: clock, + logger: logger.Sugar(), + tsNamespace: tsNamespace, } - expectReconciled(t, nr, "", "test") - // Verify that nameserver Deployment has been created and has the expected fields. - wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: "tailscale"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}} - if err := yaml.Unmarshal(deployYaml, wantsDeploy); err != nil { - t.Fatalf("unmarshalling yaml: %v", err) - } - dnsCfgOwnerRef := metav1.NewControllerRef(dnsCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig")) - wantsDeploy.OwnerReferences = []metav1.OwnerReference{*dnsCfgOwnerRef} - wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1" - wantsDeploy.Namespace = "tailscale" - labels := nameserverResourceLabels("test", "tailscale") - wantsDeploy.ObjectMeta.Labels = labels - expectEqual(t, fc, wantsDeploy) - - // Verify that DNSConfig advertizes the nameserver's Service IP address, - // has the ready status condition and tailscale finalizer. - mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { - svc.Spec.ClusterIP = "1.2.3.4" + expectReconciled(t, reconciler, "", "test") + + ownerReference := metav1.NewControllerRef(dnsConfig, tsapi.SchemeGroupVersion.WithKind("DNSConfig")) + nameserverLabels := nameserverResourceLabels(dnsConfig.Name, tsNamespace) + + wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}} + t.Run("deployment has expected fields", func(t *testing.T) { + if err = yaml.Unmarshal(deployYaml, wantsDeploy); err != nil { + t.Fatalf("unmarshalling yaml: %v", err) + } + wantsDeploy.OwnerReferences = []metav1.OwnerReference{*ownerReference} + wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1" + wantsDeploy.Namespace = tsNamespace + wantsDeploy.ObjectMeta.Labels = nameserverLabels + expectEqual(t, fc, wantsDeploy) }) - expectReconciled(t, nr, "", "test") - dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{ - IP: "1.2.3.4", - } - dnsCfg.Finalizers = []string{FinalizerName} - dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, metav1.Condition{ - Type: string(tsapi.NameserverReady), - Status: metav1.ConditionTrue, - Reason: reasonNameserverCreated, - Message: reasonNameserverCreated, - LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + + wantsSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: corev1.SchemeGroupVersion.Identifier()}} + t.Run("service has expected fields", func(t *testing.T) { + if err = yaml.Unmarshal(svcYaml, wantsSvc); err != nil { + t.Fatalf("unmarshalling yaml: %v", err) + } + wantsSvc.Spec.ClusterIP = dnsConfig.Spec.Nameserver.Service.ClusterIP + wantsSvc.OwnerReferences = []metav1.OwnerReference{*ownerReference} + wantsSvc.Namespace = tsNamespace + wantsSvc.ObjectMeta.Labels = nameserverLabels + expectEqual(t, fc, wantsSvc) }) - expectEqual(t, fc, dnsCfg) - // // Verify that nameserver image gets updated to match DNSConfig spec. - mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { - dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2" + t.Run("dns config status is set", func(t *testing.T) { + // Verify that DNSConfig advertizes the nameserver's Service IP address, + // has the ready status condition and tailscale finalizer. + mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { + svc.Spec.ClusterIP = "1.2.3.4" + }) + expectReconciled(t, reconciler, "", "test") + + dnsConfig.Finalizers = []string{FinalizerName} + dnsConfig.Status.Nameserver = &tsapi.NameserverStatus{ + IP: "1.2.3.4", + } + dnsConfig.Status.Conditions = append(dnsConfig.Status.Conditions, metav1.Condition{ + Type: string(tsapi.NameserverReady), + Status: metav1.ConditionTrue, + Reason: reasonNameserverCreated, + Message: reasonNameserverCreated, + LastTransitionTime: metav1.Time{Time: clock.Now().Truncate(time.Second)}, + }) + + expectEqual(t, fc, dnsConfig) }) - expectReconciled(t, nr, "", "test") - wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2" - expectEqual(t, fc, wantsDeploy) - - // Verify that when another actor sets ConfigMap data, it does not get - // overwritten by nameserver reconciler. - dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}} - bs, err := json.Marshal(dnsRecords) - if err != nil { - t.Fatalf("error marshalling ConfigMap contents: %v", err) - } - mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) { - mak.Set(&cm.Data, "records.json", string(bs)) + + t.Run("nameserver image can be updated", func(t *testing.T) { + // Verify that nameserver image gets updated to match DNSConfig spec. + mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { + dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2" + }) + expectReconciled(t, reconciler, "", "test") + wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2" + expectEqual(t, fc, wantsDeploy) + }) + + t.Run("reconciler does not overwrite custom configuration", func(t *testing.T) { + // Verify that when another actor sets ConfigMap data, it does not get + // overwritten by nameserver reconciler. + dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}} + bs, err := json.Marshal(dnsRecords) + if err != nil { + t.Fatalf("error marshalling ConfigMap contents: %v", err) + } + + mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) { + mak.Set(&cm.Data, "records.json", string(bs)) + }) + + expectReconciled(t, reconciler, "", "test") + + wantCm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dnsrecords", + Namespace: "tailscale", + Labels: nameserverLabels, + OwnerReferences: []metav1.OwnerReference{*ownerReference}, + }, + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + Data: map[string]string{"records.json": string(bs)}, + } + + expectEqual(t, fc, wantCm) }) - expectReconciled(t, nr, "", "test") - wantCm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords", - Namespace: "tailscale", Labels: labels, OwnerReferences: []metav1.OwnerReference{*dnsCfgOwnerRef}}, - TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, - Data: map[string]string{"records.json": string(bs)}, - } - expectEqual(t, fc, wantCm) - // Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset, - // the nameserver image defaults to tailscale/k8s-nameserver:unstable. - mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { - dnsCfg.Spec.Nameserver.Image = nil + t.Run("uses default nameserver image", func(t *testing.T) { + // Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset, + // the nameserver image defaults to tailscale/k8s-nameserver:unstable. + mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { + dnsCfg.Spec.Nameserver.Image = nil + }) + expectReconciled(t, reconciler, "", "test") + wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable" + expectEqual(t, fc, wantsDeploy) }) - expectReconciled(t, nr, "", "test") - wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable" - expectEqual(t, fc, wantsDeploy) } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index cd36798d69f8b..564c87f503a22 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -422,6 +422,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `image` _[NameserverImage](#nameserverimage)_ | Nameserver image. Defaults to tailscale/k8s-nameserver:unstable. | | | +| `service` _[NameserverService](#nameserverservice)_ | Service configuration. | | | #### NameserverImage @@ -441,6 +442,22 @@ _Appears in:_ | `tag` _string_ | Tag defaults to unstable. | | | +#### NameserverService + + + + + + + +_Appears in:_ +- [Nameserver](#nameserver) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `clusterIP` _string_ | ClusterIP sets the static IP of the service used by the nameserver. | | | + + #### NameserverStatus @@ -454,7 +471,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
          Currently you must manually update your cluster DNS config to add
          this address as a stub nameserver for ts.net for cluster workloads to be
          able to resolve MagicDNS names associated with egress or Ingress
          proxies.
          The IP address will change if you delete and recreate the DNSConfig. | | | +| `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
          Currently, you must manually update your cluster DNS config to add
          this address as a stub nameserver for ts.net for cluster workloads to be
          able to resolve MagicDNS names associated with egress or Ingress
          proxies.
          The IP address will change if you delete and recreate the DNSConfig. | | | #### NodePortConfig diff --git a/k8s-operator/apis/v1alpha1/types_tsdnsconfig.go b/k8s-operator/apis/v1alpha1/types_tsdnsconfig.go index 0178d60eab606..0e26ee6476d7a 100644 --- a/k8s-operator/apis/v1alpha1/types_tsdnsconfig.go +++ b/k8s-operator/apis/v1alpha1/types_tsdnsconfig.go @@ -82,6 +82,9 @@ type Nameserver struct { // Nameserver image. Defaults to tailscale/k8s-nameserver:unstable. // +optional Image *NameserverImage `json:"image,omitempty"` + // Service configuration. + // +optional + Service *NameserverService `json:"service,omitempty"` } type NameserverImage struct { @@ -93,6 +96,12 @@ type NameserverImage struct { Tag string `json:"tag,omitempty"` } +type NameserverService struct { + // ClusterIP sets the static IP of the service used by the nameserver. + // +optional + ClusterIP string `json:"clusterIP,omitempty"` +} + type DNSConfigStatus struct { // +listType=map // +listMapKey=type @@ -105,7 +114,7 @@ type DNSConfigStatus struct { type NameserverStatus struct { // IP is the ClusterIP of the Service fronting the deployed ts.net nameserver. - // Currently you must manually update your cluster DNS config to add + // Currently, you must manually update your cluster DNS config to add // this address as a stub nameserver for ts.net for cluster workloads to be // able to resolve MagicDNS names associated with egress or Ingress // proxies. diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index 32adbd6804ed0..6586c13546f4f 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -385,6 +385,11 @@ func (in *Nameserver) DeepCopyInto(out *Nameserver) { *out = new(NameserverImage) **out = **in } + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = new(NameserverService) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Nameserver. @@ -412,6 +417,21 @@ func (in *NameserverImage) DeepCopy() *NameserverImage { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NameserverService) DeepCopyInto(out *NameserverService) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NameserverService. +func (in *NameserverService) DeepCopy() *NameserverService { + if in == nil { + return nil + } + out := new(NameserverService) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NameserverStatus) DeepCopyInto(out *NameserverStatus) { *out = *in From 8453170aa120227dfec3c3141f081d9495a0a7c1 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 21 Jul 2025 12:36:16 -0700 Subject: [PATCH 236/263] feature/relayserver: fix consumeEventbusTopics deadlock (#16618) consumeEventbusTopics now owns server and related eventbus machinery. Updates tailscale/corp#30651 Signed-off-by: Jordan Whited --- feature/relayserver/relayserver.go | 121 +++++++++------------ feature/relayserver/relayserver_test.go | 134 +++++++++++++++--------- 2 files changed, 136 insertions(+), 119 deletions(-) diff --git a/feature/relayserver/relayserver.go b/feature/relayserver/relayserver.go index f4077b5f9da0b..b90a6234508f2 100644 --- a/feature/relayserver/relayserver.go +++ b/feature/relayserver/relayserver.go @@ -6,7 +6,6 @@ package relayserver import ( - "errors" "sync" "tailscale.com/disco" @@ -48,16 +47,12 @@ type extension struct { logf logger.Logf bus *eventbus.Bus - mu sync.Mutex // guards the following fields - eventClient *eventbus.Client // closed to stop consumeEventbusTopics - reqSub *eventbus.Subscriber[magicsock.UDPRelayAllocReq] // receives endpoint alloc requests from magicsock - respPub *eventbus.Publisher[magicsock.UDPRelayAllocResp] // publishes endpoint alloc responses to magicsock + mu sync.Mutex // guards the following fields shutdown bool port *int // ipn.Prefs.RelayServerPort, nil if disabled - busDoneCh chan struct{} // non-nil if port is non-nil, closed when consumeEventbusTopics returns + disconnectFromBusCh chan struct{} // non-nil if consumeEventbusTopics is running, closed to signal it to return + busDoneCh chan struct{} // non-nil if consumeEventbusTopics is running, closed when it returns hasNodeAttrDisableRelayServer bool // tailcfg.NodeAttrDisableRelayServer - server relayServer // lazily initialized - } // relayServer is the interface of [udprelay.Server]. @@ -81,26 +76,27 @@ func (e *extension) Init(host ipnext.Host) error { return nil } -// initBusConnection initializes the [*eventbus.Client], [*eventbus.Subscriber], -// [*eventbus.Publisher], and [chan struct{}] used to publish/receive endpoint -// allocation messages to/from the [*eventbus.Bus]. It also starts -// consumeEventbusTopics in a separate goroutine. -func (e *extension) initBusConnection() { - e.eventClient = e.bus.Client("relayserver.extension") - e.reqSub = eventbus.Subscribe[magicsock.UDPRelayAllocReq](e.eventClient) - e.respPub = eventbus.Publish[magicsock.UDPRelayAllocResp](e.eventClient) +// handleBusLifetimeLocked handles the lifetime of consumeEventbusTopics. +func (e *extension) handleBusLifetimeLocked() { + busShouldBeRunning := !e.shutdown && e.port != nil && !e.hasNodeAttrDisableRelayServer + if !busShouldBeRunning { + e.disconnectFromBusLocked() + return + } + if e.busDoneCh != nil { + return // already running + } + port := *e.port + e.disconnectFromBusCh = make(chan struct{}) e.busDoneCh = make(chan struct{}) - go e.consumeEventbusTopics() + go e.consumeEventbusTopics(port) } func (e *extension) selfNodeViewChanged(nodeView tailcfg.NodeView) { e.mu.Lock() defer e.mu.Unlock() e.hasNodeAttrDisableRelayServer = nodeView.HasCap(tailcfg.NodeAttrDisableRelayServer) - if e.hasNodeAttrDisableRelayServer && e.server != nil { - e.server.Close() - e.server = nil - } + e.handleBusLifetimeLocked() } func (e *extension) profileStateChanged(_ ipn.LoginProfileView, prefs ipn.PrefsView, sameNode bool) { @@ -110,43 +106,52 @@ func (e *extension) profileStateChanged(_ ipn.LoginProfileView, prefs ipn.PrefsV enableOrDisableServer := ok != (e.port != nil) portChanged := ok && e.port != nil && newPort != *e.port if enableOrDisableServer || portChanged || !sameNode { - if e.server != nil { - e.server.Close() - e.server = nil - } - if e.port != nil { - e.eventClient.Close() - <-e.busDoneCh - } + e.disconnectFromBusLocked() e.port = nil if ok { e.port = ptr.To(newPort) - e.initBusConnection() } } + e.handleBusLifetimeLocked() } -func (e *extension) consumeEventbusTopics() { +func (e *extension) consumeEventbusTopics(port int) { defer close(e.busDoneCh) + eventClient := e.bus.Client("relayserver.extension") + reqSub := eventbus.Subscribe[magicsock.UDPRelayAllocReq](eventClient) + respPub := eventbus.Publish[magicsock.UDPRelayAllocResp](eventClient) + defer eventClient.Close() + + var rs relayServer // lazily initialized + defer func() { + if rs != nil { + rs.Close() + } + }() for { select { - case <-e.reqSub.Done(): + case <-e.disconnectFromBusCh: + return + case <-reqSub.Done(): // If reqSub is done, the eventClient has been closed, which is a // signal to return. return - case req := <-e.reqSub.Events(): - rs, err := e.relayServerOrInit() - if err != nil { - e.logf("error initializing server: %v", err) - continue + case req := <-reqSub.Events(): + if rs == nil { + var err error + rs, err = udprelay.NewServer(e.logf, port, nil) + if err != nil { + e.logf("error initializing server: %v", err) + continue + } } se, err := rs.AllocateEndpoint(req.Message.ClientDisco[0], req.Message.ClientDisco[1]) if err != nil { e.logf("error allocating endpoint: %v", err) continue } - e.respPub.Publish(magicsock.UDPRelayAllocResp{ + respPub.Publish(magicsock.UDPRelayAllocResp{ ReqRxFromNodeKey: req.RxFromNodeKey, ReqRxFromDiscoKey: req.RxFromDiscoKey, Message: &disco.AllocateUDPRelayEndpointResponse{ @@ -164,44 +169,22 @@ func (e *extension) consumeEventbusTopics() { }) } } +} +func (e *extension) disconnectFromBusLocked() { + if e.busDoneCh != nil { + close(e.disconnectFromBusCh) + <-e.busDoneCh + e.busDoneCh = nil + e.disconnectFromBusCh = nil + } } // Shutdown implements [ipnlocal.Extension]. func (e *extension) Shutdown() error { e.mu.Lock() defer e.mu.Unlock() + e.disconnectFromBusLocked() e.shutdown = true - if e.server != nil { - e.server.Close() - e.server = nil - } - if e.port != nil { - e.eventClient.Close() - <-e.busDoneCh - } return nil } - -func (e *extension) relayServerOrInit() (relayServer, error) { - e.mu.Lock() - defer e.mu.Unlock() - if e.shutdown { - return nil, errors.New("relay server is shutdown") - } - if e.server != nil { - return e.server, nil - } - if e.port == nil { - return nil, errors.New("relay server is not configured") - } - if e.hasNodeAttrDisableRelayServer { - return nil, errors.New("disable-relay-server node attribute is present") - } - var err error - e.server, err = udprelay.NewServer(e.logf, *e.port, nil) - if err != nil { - return nil, err - } - return e.server, nil -} diff --git a/feature/relayserver/relayserver_test.go b/feature/relayserver/relayserver_test.go index 84158188e90fb..d3fc36a83674a 100644 --- a/feature/relayserver/relayserver_test.go +++ b/feature/relayserver/relayserver_test.go @@ -4,107 +4,91 @@ package relayserver import ( - "errors" "testing" "tailscale.com/ipn" - "tailscale.com/net/udprelay/endpoint" "tailscale.com/tsd" - "tailscale.com/types/key" "tailscale.com/types/ptr" + "tailscale.com/util/eventbus" ) -type fakeRelayServer struct{} - -func (f *fakeRelayServer) Close() error { return nil } - -func (f *fakeRelayServer) AllocateEndpoint(_, _ key.DiscoPublic) (endpoint.ServerEndpoint, error) { - return endpoint.ServerEndpoint{}, errors.New("fake relay server") -} - func Test_extension_profileStateChanged(t *testing.T) { prefsWithPortOne := ipn.Prefs{RelayServerPort: ptr.To(1)} prefsWithNilPort := ipn.Prefs{RelayServerPort: nil} type fields struct { - server relayServer - port *int + port *int } type args struct { prefs ipn.PrefsView sameNode bool } tests := []struct { - name string - fields fields - args args - wantPort *int - wantNilServer bool + name string + fields fields + args args + wantPort *int + wantBusRunning bool }{ { - name: "no changes non-nil server", + name: "no changes non-nil port", fields: fields{ - server: &fakeRelayServer{}, - port: ptr.To(1), + port: ptr.To(1), }, args: args{ prefs: prefsWithPortOne.View(), sameNode: true, }, - wantPort: ptr.To(1), - wantNilServer: false, + wantPort: ptr.To(1), + wantBusRunning: true, }, { name: "prefs port nil", fields: fields{ - server: &fakeRelayServer{}, - port: ptr.To(1), + port: ptr.To(1), }, args: args{ prefs: prefsWithNilPort.View(), sameNode: true, }, - wantPort: nil, - wantNilServer: true, + wantPort: nil, + wantBusRunning: false, }, { name: "prefs port changed", fields: fields{ - server: &fakeRelayServer{}, - port: ptr.To(2), + port: ptr.To(2), }, args: args{ prefs: prefsWithPortOne.View(), sameNode: true, }, - wantPort: ptr.To(1), - wantNilServer: true, + wantPort: ptr.To(1), + wantBusRunning: true, }, { name: "sameNode false", fields: fields{ - server: &fakeRelayServer{}, - port: ptr.To(1), + port: ptr.To(1), }, args: args{ prefs: prefsWithPortOne.View(), sameNode: false, }, - wantPort: ptr.To(1), - wantNilServer: true, + wantPort: ptr.To(1), + wantBusRunning: true, }, { name: "prefs port non-nil extension port nil", fields: fields{ - server: nil, - port: nil, + port: nil, }, args: args{ prefs: prefsWithPortOne.View(), sameNode: false, }, - wantPort: ptr.To(1), - wantNilServer: true, + wantPort: ptr.To(1), + wantBusRunning: true, }, } for _, tt := range tests { @@ -112,19 +96,13 @@ func Test_extension_profileStateChanged(t *testing.T) { sys := tsd.NewSystem() bus := sys.Bus.Get() e := &extension{ - port: tt.fields.port, - server: tt.fields.server, - bus: bus, - } - if e.port != nil { - // Entering profileStateChanged with a non-nil port requires - // bus init, which is called in profileStateChanged when - // transitioning port from nil to non-nil. - e.initBusConnection() + port: tt.fields.port, + bus: bus, } + defer e.disconnectFromBusLocked() e.profileStateChanged(ipn.LoginProfileView{}, tt.args.prefs, tt.args.sameNode) - if tt.wantNilServer != (e.server == nil) { - t.Errorf("wantNilServer: %v != (e.server == nil): %v", tt.wantNilServer, e.server == nil) + if tt.wantBusRunning != (e.busDoneCh != nil) { + t.Errorf("wantBusRunning: %v != (e.busDoneCh != nil): %v", tt.wantBusRunning, e.busDoneCh != nil) } if (tt.wantPort == nil) != (e.port == nil) { t.Errorf("(tt.wantPort == nil): %v != (e.port == nil): %v", tt.wantPort == nil, e.port == nil) @@ -134,3 +112,59 @@ func Test_extension_profileStateChanged(t *testing.T) { }) } } + +func Test_extension_handleBusLifetimeLocked(t *testing.T) { + tests := []struct { + name string + shutdown bool + port *int + busDoneCh chan struct{} + hasNodeAttrDisableRelayServer bool + wantBusRunning bool + }{ + { + name: "want running", + shutdown: false, + port: ptr.To(1), + hasNodeAttrDisableRelayServer: false, + wantBusRunning: true, + }, + { + name: "shutdown true", + shutdown: true, + port: ptr.To(1), + hasNodeAttrDisableRelayServer: false, + wantBusRunning: false, + }, + { + name: "port nil", + shutdown: false, + port: nil, + hasNodeAttrDisableRelayServer: false, + wantBusRunning: false, + }, + { + name: "hasNodeAttrDisableRelayServer true", + shutdown: false, + port: nil, + hasNodeAttrDisableRelayServer: true, + wantBusRunning: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &extension{ + bus: eventbus.New(), + shutdown: tt.shutdown, + port: tt.port, + busDoneCh: tt.busDoneCh, + hasNodeAttrDisableRelayServer: tt.hasNodeAttrDisableRelayServer, + } + e.handleBusLifetimeLocked() + defer e.disconnectFromBusLocked() + if tt.wantBusRunning != (e.busDoneCh != nil) { + t.Errorf("wantBusRunning: %v != (e.busDoneCh != nil): %v", tt.wantBusRunning, e.busDoneCh != nil) + } + }) + } +} From 6f7e78b10ffac8f1dcd79aebe12b38ee96e76ce7 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Tue, 22 Jul 2025 10:07:09 +0100 Subject: [PATCH 237/263] cmd/tailscale/cli: make configure kubeconfig accept Tailscale Services (#16601) The Kubernetes API server proxy is getting the ability to serve on a Tailscale Service instead of individual node names. Update the configure kubeconfig sub-command to accept arguments that look like a Tailscale Service. Note, we can't know for sure whether a peer is advertising a Tailscale Service, we can only guess based on the ExtraRecords in the netmap and that IP showing up in a peer's AllowedIPs. Also adds an --http flag to allow targeting individual proxies that can be adverting on http for their node name, and makes the command a bit more forgiving on the range of inputs it accepts and how eager it is to print the help text when the input is obviously wrong. Updates #13358 Change-Id: Ica0509c6b2c707252a43d7c18b530ec1acf7508f Signed-off-by: Tom Proctor --- cmd/tailscale/cli/configure-kube.go | 151 +++++++++++++++++++++-- cmd/tailscale/cli/configure-kube_test.go | 56 ++++++++- 2 files changed, 194 insertions(+), 13 deletions(-) diff --git a/cmd/tailscale/cli/configure-kube.go b/cmd/tailscale/cli/configure-kube.go index 6bc4e202efd4e..e74e8877996fe 100644 --- a/cmd/tailscale/cli/configure-kube.go +++ b/cmd/tailscale/cli/configure-kube.go @@ -9,17 +9,29 @@ import ( "errors" "flag" "fmt" + "net/netip" + "net/url" "os" "path/filepath" "slices" "strings" + "time" "github.com/peterbourgon/ff/v3/ffcli" "k8s.io/client-go/util/homedir" "sigs.k8s.io/yaml" + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/types/netmap" + "tailscale.com/util/dnsname" "tailscale.com/version" ) +var configureKubeconfigArgs struct { + http bool // Use HTTP instead of HTTPS (default) for the auth proxy. +} + func configureKubeconfigCmd() *ffcli.Command { return &ffcli.Command{ Name: "kubeconfig", @@ -34,6 +46,7 @@ See: https://tailscale.com/s/k8s-auth-proxy `), FlagSet: (func() *flag.FlagSet { fs := newFlagSet("kubeconfig") + fs.BoolVar(&configureKubeconfigArgs.http, "http", false, "Use HTTP instead of HTTPS to connect to the auth proxy. Ignored if you include a scheme in the hostname argument.") return fs })(), Exec: runConfigureKubeconfig, @@ -70,10 +83,13 @@ func kubeconfigPath() (string, error) { } func runConfigureKubeconfig(ctx context.Context, args []string) error { - if len(args) != 1 { - return errors.New("unknown arguments") + if len(args) != 1 || args[0] == "" { + return flag.ErrHelp + } + hostOrFQDNOrIP, http, err := getInputs(args[0], configureKubeconfigArgs.http) + if err != nil { + return fmt.Errorf("error parsing inputs: %w", err) } - hostOrFQDN := args[0] st, err := localClient.Status(ctx) if err != nil { @@ -82,22 +98,45 @@ func runConfigureKubeconfig(ctx context.Context, args []string) error { if st.BackendState != "Running" { return errors.New("Tailscale is not running") } - targetFQDN, ok := nodeDNSNameFromArg(st, hostOrFQDN) - if !ok { - return fmt.Errorf("no peer found with hostname %q", hostOrFQDN) + nm, err := getNetMap(ctx) + if err != nil { + return err + } + + targetFQDN, err := nodeOrServiceDNSNameFromArg(st, nm, hostOrFQDNOrIP) + if err != nil { + return err } targetFQDN = strings.TrimSuffix(targetFQDN, ".") var kubeconfig string if kubeconfig, err = kubeconfigPath(); err != nil { return err } - if err = setKubeconfigForPeer(targetFQDN, kubeconfig); err != nil { + scheme := "https://" + if http { + scheme = "http://" + } + if err = setKubeconfigForPeer(scheme, targetFQDN, kubeconfig); err != nil { return err } - printf("kubeconfig configured for %q\n", hostOrFQDN) + printf("kubeconfig configured for %q at URL %q\n", targetFQDN, scheme+targetFQDN) return nil } +func getInputs(arg string, httpArg bool) (string, bool, error) { + u, err := url.Parse(arg) + if err != nil { + return "", false, err + } + + switch u.Scheme { + case "http", "https": + return u.Host, u.Scheme == "http", nil + default: + return arg, httpArg, nil + } +} + // appendOrSetNamed finds a map with a "name" key matching name in dst, and // replaces it with val. If no such map is found, val is appended to dst. func appendOrSetNamed(dst []any, name string, val map[string]any) []any { @@ -116,7 +155,7 @@ func appendOrSetNamed(dst []any, name string, val map[string]any) []any { var errInvalidKubeconfig = errors.New("invalid kubeconfig") -func updateKubeconfig(cfgYaml []byte, fqdn string) ([]byte, error) { +func updateKubeconfig(cfgYaml []byte, scheme, fqdn string) ([]byte, error) { var cfg map[string]any if len(cfgYaml) > 0 { if err := yaml.Unmarshal(cfgYaml, &cfg); err != nil { @@ -139,7 +178,7 @@ func updateKubeconfig(cfgYaml []byte, fqdn string) ([]byte, error) { cfg["clusters"] = appendOrSetNamed(clusters, fqdn, map[string]any{ "name": fqdn, "cluster": map[string]string{ - "server": "https://" + fqdn, + "server": scheme + fqdn, }, }) @@ -172,7 +211,7 @@ func updateKubeconfig(cfgYaml []byte, fqdn string) ([]byte, error) { return yaml.Marshal(cfg) } -func setKubeconfigForPeer(fqdn, filePath string) error { +func setKubeconfigForPeer(scheme, fqdn, filePath string) error { dir := filepath.Dir(filePath) if _, err := os.Stat(dir); err != nil { if !os.IsNotExist(err) { @@ -191,9 +230,97 @@ func setKubeconfigForPeer(fqdn, filePath string) error { if err != nil && !os.IsNotExist(err) { return fmt.Errorf("reading kubeconfig: %w", err) } - b, err = updateKubeconfig(b, fqdn) + b, err = updateKubeconfig(b, scheme, fqdn) if err != nil { return err } return os.WriteFile(filePath, b, 0600) } + +// nodeOrServiceDNSNameFromArg returns the PeerStatus.DNSName value from a peer +// in st that matches the input arg which can be a base name, full DNS name, or +// an IP. If none is found, it looks for a Tailscale Service +func nodeOrServiceDNSNameFromArg(st *ipnstate.Status, nm *netmap.NetworkMap, arg string) (string, error) { + // First check for a node DNS name. + if dnsName, ok := nodeDNSNameFromArg(st, arg); ok { + return dnsName, nil + } + + // If not found, check for a Tailscale Service DNS name. + rec, ok := serviceDNSRecordFromNetMap(nm, st.CurrentTailnet.MagicDNSSuffix, arg) + if !ok { + return "", fmt.Errorf("no peer found for %q", arg) + } + + // Validate we can see a peer advertising the Tailscale Service. + ip, err := netip.ParseAddr(rec.Value) + if err != nil { + return "", fmt.Errorf("error parsing ExtraRecord IP address %q: %w", rec.Value, err) + } + ipPrefix := netip.PrefixFrom(ip, ip.BitLen()) + for _, ps := range st.Peer { + for _, allowedIP := range ps.AllowedIPs.All() { + if allowedIP == ipPrefix { + return rec.Name, nil + } + } + } + + return "", fmt.Errorf("%q is in MagicDNS, but is not currently reachable on any known peer", arg) +} + +func getNetMap(ctx context.Context) (*netmap.NetworkMap, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) + if err != nil { + return nil, err + } + defer watcher.Close() + + n, err := watcher.Next() + if err != nil { + return nil, err + } + + return n.NetMap, nil +} + +func serviceDNSRecordFromNetMap(nm *netmap.NetworkMap, tcd, arg string) (rec tailcfg.DNSRecord, ok bool) { + argIP, _ := netip.ParseAddr(arg) + argFQDN, err := dnsname.ToFQDN(arg) + argFQDNValid := err == nil + if !argIP.IsValid() && !argFQDNValid { + return rec, false + } + + for _, rec := range nm.DNS.ExtraRecords { + if argIP.IsValid() { + recIP, _ := netip.ParseAddr(rec.Value) + if recIP == argIP { + return rec, true + } + continue + } + + if !argFQDNValid { + continue + } + + recFirstLabel := dnsname.FirstLabel(rec.Name) + if strings.EqualFold(arg, recFirstLabel) { + return rec, true + } + + recFQDN, err := dnsname.ToFQDN(rec.Name) + if err != nil { + continue + } + if strings.EqualFold(argFQDN.WithTrailingDot(), recFQDN.WithTrailingDot()) { + return rec, true + } + } + + return tailcfg.DNSRecord{}, false +} diff --git a/cmd/tailscale/cli/configure-kube_test.go b/cmd/tailscale/cli/configure-kube_test.go index d71a9b627e7f0..0c8b6b2b6cc0e 100644 --- a/cmd/tailscale/cli/configure-kube_test.go +++ b/cmd/tailscale/cli/configure-kube_test.go @@ -6,6 +6,7 @@ package cli import ( "bytes" + "fmt" "strings" "testing" @@ -16,6 +17,7 @@ func TestKubeconfig(t *testing.T) { const fqdn = "foo.tail-scale.ts.net" tests := []struct { name string + http bool in string want string wantErr error @@ -48,6 +50,27 @@ contexts: current-context: foo.tail-scale.ts.net kind: Config users: +- name: tailscale-auth + user: + token: unused`, + }, + { + name: "empty_http", + http: true, + in: "", + want: `apiVersion: v1 +clusters: +- cluster: + server: http://foo.tail-scale.ts.net + name: foo.tail-scale.ts.net +contexts: +- context: + cluster: foo.tail-scale.ts.net + user: tailscale-auth + name: foo.tail-scale.ts.net +current-context: foo.tail-scale.ts.net +kind: Config +users: - name: tailscale-auth user: token: unused`, @@ -202,7 +225,11 @@ users: } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := updateKubeconfig([]byte(tt.in), fqdn) + scheme := "https://" + if tt.http { + scheme = "http://" + } + got, err := updateKubeconfig([]byte(tt.in), scheme, fqdn) if err != nil { if err != tt.wantErr { t.Fatalf("updateKubeconfig() error = %v, wantErr %v", err, tt.wantErr) @@ -219,3 +246,30 @@ users: }) } } + +func TestGetInputs(t *testing.T) { + for _, arg := range []string{ + "foo.tail-scale.ts.net", + "foo", + "127.0.0.1", + } { + for _, prefix := range []string{"", "https://", "http://"} { + for _, httpFlag := range []bool{false, true} { + expectedHost := arg + expectedHTTP := (httpFlag && !strings.HasPrefix(prefix, "https://")) || strings.HasPrefix(prefix, "http://") + t.Run(fmt.Sprintf("%s%s_http=%v", prefix, arg, httpFlag), func(t *testing.T) { + host, http, err := getInputs(prefix+arg, httpFlag) + if err != nil { + t.Fatal(err) + } + if host != expectedHost { + t.Errorf("host = %v, want %v", host, expectedHost) + } + if http != expectedHTTP { + t.Errorf("http = %v, want %v", http, expectedHTTP) + } + }) + } + } + } +} From 22a8e0ac50ee2211e013fae2f2dbd8a9622657d8 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Tue, 22 Jul 2025 14:46:38 +0100 Subject: [PATCH 238/263] cmd/{k8s-operator,k8s-proxy},kube: use consistent type for auth mode config (#16626) Updates k8s-proxy's config so its auth mode config matches that we set in kube-apiserver ProxyGroups for consistency. Updates #13358 Change-Id: I95e29cec6ded2dc7c6d2d03f968a25c822bc0e01 Signed-off-by: Tom Proctor --- cmd/k8s-operator/api-server-proxy.go | 36 +++++-------------- cmd/k8s-operator/operator.go | 6 ++-- cmd/k8s-operator/proxygroup.go | 8 +++-- cmd/k8s-operator/proxygroup_test.go | 2 +- cmd/k8s-proxy/k8s-proxy.go | 9 ++--- k8s-operator/api-proxy/proxy.go | 8 ++--- k8s-operator/sessionrecording/hijacker.go | 2 +- kube/k8s-proxy/conf/conf.go | 9 ++--- kube/kubetypes/types.go | 23 ++++++++++++- kube/kubetypes/types_test.go | 42 +++++++++++++++++++++++ 10 files changed, 98 insertions(+), 47 deletions(-) create mode 100644 kube/kubetypes/types_test.go diff --git a/cmd/k8s-operator/api-server-proxy.go b/cmd/k8s-operator/api-server-proxy.go index 09a7b8c6232d2..70333d2c48d41 100644 --- a/cmd/k8s-operator/api-server-proxy.go +++ b/cmd/k8s-operator/api-server-proxy.go @@ -9,30 +9,12 @@ import ( "fmt" "log" "os" -) - -type apiServerProxyMode int - -func (a apiServerProxyMode) String() string { - switch a { - case apiServerProxyModeDisabled: - return "disabled" - case apiServerProxyModeEnabled: - return "auth" - case apiServerProxyModeNoAuth: - return "noauth" - default: - return "unknown" - } -} -const ( - apiServerProxyModeDisabled apiServerProxyMode = iota - apiServerProxyModeEnabled - apiServerProxyModeNoAuth + "tailscale.com/kube/kubetypes" + "tailscale.com/types/ptr" ) -func parseAPIProxyMode() apiServerProxyMode { +func parseAPIProxyMode() *kubetypes.APIServerProxyMode { haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != "" haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != "" switch { @@ -41,21 +23,21 @@ func parseAPIProxyMode() apiServerProxyMode { case haveAuthProxyEnv: var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated if authProxyEnv { - return apiServerProxyModeEnabled + return ptr.To(kubetypes.APIServerProxyModeAuth) } - return apiServerProxyModeDisabled + return nil case haveAPIProxyEnv: var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth" switch apiProxyEnv { case "true": - return apiServerProxyModeEnabled + return ptr.To(kubetypes.APIServerProxyModeAuth) case "false", "": - return apiServerProxyModeDisabled + return nil case "noauth": - return apiServerProxyModeNoAuth + return ptr.To(kubetypes.APIServerProxyModeNoAuth) default: panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv)) } } - return apiServerProxyModeDisabled + return nil } diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index 94a0a6a781c4d..76d2df51d47d2 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -113,7 +113,7 @@ func main() { // additionally act as api-server proxy // https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy. mode := parseAPIProxyMode() - if mode == apiServerProxyModeDisabled { + if mode == nil { hostinfo.SetApp(kubetypes.AppOperator) } else { hostinfo.SetApp(kubetypes.AppInProcessAPIServerProxy) @@ -122,8 +122,8 @@ func main() { s, tsc := initTSNet(zlog, loginServer) defer s.Close() restConfig := config.GetConfigOrDie() - if mode != apiServerProxyModeDisabled { - ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, mode == apiServerProxyModeEnabled, true) + if mode != nil { + ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, *mode, true) if err != nil { zlog.Fatalf("error creating API server proxy: %v", err) } diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index d62cb0f117a1d..f9c12797d523d 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -805,6 +805,10 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p } } + mode := kubetypes.APIServerProxyModeAuth + if !isAuthAPIServerProxy(pg) { + mode = kubetypes.APIServerProxyModeNoAuth + } cfg := conf.VersionedConfig{ Version: "v1alpha1", ConfigV1Alpha1: &conf.ConfigV1Alpha1{ @@ -816,8 +820,8 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p // Reloadable fields. Hostname: &hostname, APIServerProxy: &conf.APIServerProxyConfig{ - Enabled: opt.NewBool(true), - AuthMode: opt.NewBool(isAuthAPIServerProxy(pg)), + Enabled: opt.NewBool(true), + Mode: &mode, // The first replica is elected as the cert issuer, same // as containerboot does for ingress-pg-reconciler. IssueCerts: opt.NewBool(i == 0), diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index ef6babc5679cc..0dc791b0412de 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -1376,7 +1376,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { Hostname: ptr.To("test-k8s-apiserver-0"), APIServerProxy: &conf.APIServerProxyConfig{ Enabled: opt.NewBool(true), - AuthMode: opt.NewBool(false), + Mode: ptr.To(kubetypes.APIServerProxyModeNoAuth), IssueCerts: opt.NewBool(true), }, }, diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go index eea1f15f7fdd8..b56ceaab0d5ca 100644 --- a/cmd/k8s-proxy/k8s-proxy.go +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -34,6 +34,7 @@ import ( apiproxy "tailscale.com/k8s-operator/api-proxy" "tailscale.com/kube/certs" "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" klc "tailscale.com/kube/localclient" "tailscale.com/kube/services" "tailscale.com/kube/state" @@ -238,11 +239,11 @@ func run(logger *zap.SugaredLogger) error { } // Setup for the API server proxy. - authMode := true - if cfg.Parsed.APIServerProxy != nil && cfg.Parsed.APIServerProxy.AuthMode.EqualBool(false) { - authMode = false + mode := kubetypes.APIServerProxyModeAuth + if cfg.Parsed.APIServerProxy != nil && cfg.Parsed.APIServerProxy.Mode != nil { + mode = *cfg.Parsed.APIServerProxy.Mode } - ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, authMode, false) + ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, mode, false) if err != nil { return fmt.Errorf("error creating api server proxy: %w", err) } diff --git a/k8s-operator/api-proxy/proxy.go b/k8s-operator/api-proxy/proxy.go index e079e984ff5a1..c648e1622537d 100644 --- a/k8s-operator/api-proxy/proxy.go +++ b/k8s-operator/api-proxy/proxy.go @@ -47,8 +47,8 @@ var ( // caller's Tailscale identity and the rules defined in the tailnet ACLs. // - false: the proxy is started and requests are passed through to the // Kubernetes API without any auth modifications. -func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsnet.Server, authMode bool, https bool) (*APIServerProxy, error) { - if !authMode { +func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsnet.Server, mode kubetypes.APIServerProxyMode, https bool) (*APIServerProxy, error) { + if mode == kubetypes.APIServerProxyModeNoAuth { restConfig = rest.AnonymousClientConfig(restConfig) } @@ -85,7 +85,7 @@ func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsn ap := &APIServerProxy{ log: zlog, lc: lc, - authMode: authMode, + authMode: mode == kubetypes.APIServerProxyModeAuth, https: https, upstreamURL: u, ts: ts, @@ -278,7 +278,7 @@ func (ap *APIServerProxy) sessionForProto(w http.ResponseWriter, r *http.Request Namespace: r.PathValue(namespaceNameKey), Log: ap.log, } - h := ksr.New(opts) + h := ksr.NewHijacker(opts) ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who))) } diff --git a/k8s-operator/sessionrecording/hijacker.go b/k8s-operator/sessionrecording/hijacker.go index e8c534afc9319..675a9b1ddacc6 100644 --- a/k8s-operator/sessionrecording/hijacker.go +++ b/k8s-operator/sessionrecording/hijacker.go @@ -57,7 +57,7 @@ var ( counterSessionRecordingsUploaded = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_uploaded") ) -func New(opts HijackerOpts) *Hijacker { +func NewHijacker(opts HijackerOpts) *Hijacker { return &Hijacker{ ts: opts.TS, req: opts.Req, diff --git a/kube/k8s-proxy/conf/conf.go b/kube/k8s-proxy/conf/conf.go index a32e0c03ef2bc..fdb6301ac5a1d 100644 --- a/kube/k8s-proxy/conf/conf.go +++ b/kube/k8s-proxy/conf/conf.go @@ -14,6 +14,7 @@ import ( "net/netip" "github.com/tailscale/hujson" + "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/types/opt" ) @@ -66,10 +67,10 @@ type ConfigV1Alpha1 struct { } type APIServerProxyConfig struct { - Enabled opt.Bool `json:",omitempty"` // Whether to enable the API Server proxy. - AuthMode opt.Bool `json:",omitempty"` // Run in auth or noauth mode. - ServiceName *tailcfg.ServiceName `json:",omitempty"` // Name of the Tailscale Service to advertise. - IssueCerts opt.Bool `json:",omitempty"` // Whether this replica should issue TLS certs for the Tailscale Service. + Enabled opt.Bool `json:",omitempty"` // Whether to enable the API Server proxy. + Mode *kubetypes.APIServerProxyMode `json:",omitempty"` // "auth" or "noauth" mode. + ServiceName *tailcfg.ServiceName `json:",omitempty"` // Name of the Tailscale Service to advertise. + IssueCerts opt.Bool `json:",omitempty"` // Whether this replica should issue TLS certs for the Tailscale Service. } // Load reads and parses the config file at the provided path on disk. diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index 5e7d4cd1f1fd1..44b01fe1ad1f5 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -3,6 +3,8 @@ package kubetypes +import "fmt" + const ( // Hostinfo App values for the Tailscale Kubernetes Operator components. AppOperator = "k8s-operator" @@ -59,5 +61,24 @@ const ( LabelSecretTypeState = "state" LabelSecretTypeCerts = "certs" - KubeAPIServerConfigFile = "config.hujson" + KubeAPIServerConfigFile = "config.hujson" + APIServerProxyModeAuth APIServerProxyMode = "auth" + APIServerProxyModeNoAuth APIServerProxyMode = "noauth" ) + +// APIServerProxyMode specifies whether the API server proxy will add +// impersonation headers to requests based on the caller's Tailscale identity. +// May be "auth" or "noauth". +type APIServerProxyMode string + +func (a *APIServerProxyMode) UnmarshalJSON(data []byte) error { + switch string(data) { + case `"auth"`: + *a = APIServerProxyModeAuth + case `"noauth"`: + *a = APIServerProxyModeNoAuth + default: + return fmt.Errorf("unknown APIServerProxyMode %q", data) + } + return nil +} diff --git a/kube/kubetypes/types_test.go b/kube/kubetypes/types_test.go new file mode 100644 index 0000000000000..ea1846b3253e8 --- /dev/null +++ b/kube/kubetypes/types_test.go @@ -0,0 +1,42 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package kubetypes + +import ( + "encoding/json" + "testing" +) + +func TestUnmarshalAPIServerProxyMode(t *testing.T) { + tests := []struct { + data string + expected APIServerProxyMode + }{ + {data: `{"mode":"auth"}`, expected: APIServerProxyModeAuth}, + {data: `{"mode":"noauth"}`, expected: APIServerProxyModeNoAuth}, + {data: `{"mode":""}`, expected: ""}, + {data: `{"mode":"Auth"}`, expected: ""}, + {data: `{"mode":"unknown"}`, expected: ""}, + } + + for _, tc := range tests { + var s struct { + Mode *APIServerProxyMode `json:",omitempty"` + } + err := json.Unmarshal([]byte(tc.data), &s) + if tc.expected == "" { + if err == nil { + t.Errorf("expected error for %q, got none", tc.data) + } + continue + } + if err != nil { + t.Errorf("unexpected error for %q: %v", tc.data, err) + continue + } + if *s.Mode != tc.expected { + t.Errorf("for %q expected %q, got %q", tc.data, tc.expected, *s.Mode) + } + } +} From 44947054967e3eda476c92206e0a14fd1ffc4ec0 Mon Sep 17 00:00:00 2001 From: David Bond Date: Tue, 22 Jul 2025 17:07:51 +0100 Subject: [PATCH 239/263] cmd/{k8s-proxy,containerboot,k8s-operator},kube: add health check and metrics endpoints for k8s-proxy (#16540) * Modifies the k8s-proxy to expose health check and metrics endpoints on the Pod's IP. * Moves cmd/containerboot/healthz.go and cmd/containerboot/metrics.go to /kube to be shared with /k8s-proxy. Updates #13358 Signed-off-by: David Bond --- cmd/containerboot/healthz.go | 57 ------------- cmd/containerboot/main.go | 16 ++-- cmd/k8s-operator/proxygroup.go | 8 +- cmd/k8s-operator/proxygroup_test.go | 2 + cmd/k8s-proxy/k8s-proxy.go | 67 +++++++++++++-- kube/health/healthz.go | 84 +++++++++++++++++++ kube/k8s-proxy/conf/conf.go | 36 ++++++-- .../containerboot => kube/metrics}/metrics.go | 8 +- 8 files changed, 196 insertions(+), 82 deletions(-) delete mode 100644 cmd/containerboot/healthz.go create mode 100644 kube/health/healthz.go rename {cmd/containerboot => kube/metrics}/metrics.go (90%) diff --git a/cmd/containerboot/healthz.go b/cmd/containerboot/healthz.go deleted file mode 100644 index d6a64a37c4ac5..0000000000000 --- a/cmd/containerboot/healthz.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -//go:build linux - -package main - -import ( - "fmt" - "log" - "net/http" - "sync" - - "tailscale.com/kube/kubetypes" -) - -// healthz is a simple health check server, if enabled it returns 200 OK if -// this tailscale node currently has at least one tailnet IP address else -// returns 503. -type healthz struct { - sync.Mutex - hasAddrs bool - podIPv4 string -} - -func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) { - h.Lock() - defer h.Unlock() - - if h.hasAddrs { - w.Header().Add(kubetypes.PodIPv4Header, h.podIPv4) - if _, err := w.Write([]byte("ok")); err != nil { - http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError) - } - } else { - http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable) - } -} - -func (h *healthz) update(healthy bool) { - h.Lock() - defer h.Unlock() - - if h.hasAddrs != healthy { - log.Println("Setting healthy", healthy) - } - h.hasAddrs = healthy -} - -// registerHealthHandlers registers a simple health handler at /healthz. -// A containerized tailscale instance is considered healthy if -// it has at least one tailnet IP address. -func registerHealthHandlers(mux *http.ServeMux, podIPv4 string) *healthz { - h := &healthz{podIPv4: podIPv4} - mux.Handle("GET /healthz", h) - return h -} diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 49c8a473a596d..f056d26f3c2c0 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -121,7 +121,9 @@ import ( "tailscale.com/client/tailscale" "tailscale.com/ipn" kubeutils "tailscale.com/k8s-operator" + healthz "tailscale.com/kube/health" "tailscale.com/kube/kubetypes" + "tailscale.com/kube/metrics" "tailscale.com/kube/services" "tailscale.com/tailcfg" "tailscale.com/types/logger" @@ -232,13 +234,13 @@ func run() error { } defer killTailscaled() - var healthCheck *healthz + var healthCheck *healthz.Healthz ep := &egressProxy{} if cfg.HealthCheckAddrPort != "" { mux := http.NewServeMux() log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort) - healthCheck = registerHealthHandlers(mux, cfg.PodIPv4) + healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, log.Printf) close := runHTTPServer(mux, cfg.HealthCheckAddrPort) defer close() @@ -249,12 +251,12 @@ func run() error { if cfg.localMetricsEnabled() { log.Printf("Running metrics endpoint at %s/metrics", cfg.LocalAddrPort) - registerMetricsHandlers(mux, client, cfg.DebugAddrPort) + metrics.RegisterMetricsHandlers(mux, client, cfg.DebugAddrPort) } if cfg.localHealthEnabled() { log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort) - healthCheck = registerHealthHandlers(mux, cfg.PodIPv4) + healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, log.Printf) } if cfg.egressSvcsTerminateEPEnabled() { @@ -438,8 +440,8 @@ authLoop: ) // egressSvcsErrorChan will get an error sent to it if this containerboot instance is configured to expose 1+ // egress services in HA mode and errored. - var egressSvcsErrorChan = make(chan error) - var ingressSvcsErrorChan = make(chan error) + egressSvcsErrorChan := make(chan error) + ingressSvcsErrorChan := make(chan error) defer t.Stop() // resetTimer resets timer for when to next attempt to resolve the DNS // name for the proxy configured with TS_EXPERIMENTAL_DEST_DNS_NAME. The @@ -644,7 +646,7 @@ runLoop: } if healthCheck != nil { - healthCheck.update(len(addrs) != 0) + healthCheck.Update(len(addrs) != 0) } if cfg.ServeConfigPath != "" { diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index f9c12797d523d..debeb5c6b3442 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -826,6 +826,8 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p // as containerboot does for ingress-pg-reconciler. IssueCerts: opt.NewBool(i == 0), }, + LocalPort: ptr.To(uint16(9002)), + HealthCheckEnabled: opt.NewBool(true), }, } @@ -849,7 +851,11 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p } if proxyClass != nil && proxyClass.Spec.TailscaleConfig != nil { - cfg.AcceptRoutes = &proxyClass.Spec.TailscaleConfig.AcceptRoutes + cfg.AcceptRoutes = opt.NewBool(proxyClass.Spec.TailscaleConfig.AcceptRoutes) + } + + if proxyClass != nil && proxyClass.Spec.Metrics != nil { + cfg.MetricsEnabled = opt.NewBool(proxyClass.Spec.Metrics.Enable) } if len(endpoints[nodePortSvcName]) > 0 { diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 0dc791b0412de..d763cf92276ec 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -1379,6 +1379,8 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { Mode: ptr.To(kubetypes.APIServerProxyModeNoAuth), IssueCerts: opt.NewBool(true), }, + LocalPort: ptr.To(uint16(9002)), + HealthCheckEnabled: opt.NewBool(true), }, } cfgB, err := json.Marshal(cfg) diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go index b56ceaab0d5ca..448bbe3971c0d 100644 --- a/cmd/k8s-proxy/k8s-proxy.go +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -12,9 +12,12 @@ import ( "context" "errors" "fmt" + "net" + "net/http" "os" "os/signal" "reflect" + "strconv" "strings" "syscall" "time" @@ -33,9 +36,11 @@ import ( "tailscale.com/ipn/store" apiproxy "tailscale.com/k8s-operator/api-proxy" "tailscale.com/kube/certs" + healthz "tailscale.com/kube/health" "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" klc "tailscale.com/kube/localclient" + "tailscale.com/kube/metrics" "tailscale.com/kube/services" "tailscale.com/kube/state" "tailscale.com/tailcfg" @@ -63,6 +68,7 @@ func run(logger *zap.SugaredLogger) error { var ( configPath = os.Getenv("TS_K8S_PROXY_CONFIG") podUID = os.Getenv("POD_UID") + podIP = os.Getenv("POD_IP") ) if configPath == "" { return errors.New("TS_K8S_PROXY_CONFIG unset") @@ -201,10 +207,57 @@ func run(logger *zap.SugaredLogger) error { }) } - if cfg.Parsed.AcceptRoutes != nil { + if cfg.Parsed.HealthCheckEnabled.EqualBool(true) || cfg.Parsed.MetricsEnabled.EqualBool(true) { + addr := podIP + if addr == "" { + addr = cfg.GetLocalAddr() + } + + addrPort := getLocalAddrPort(addr, cfg.GetLocalPort()) + mux := http.NewServeMux() + localSrv := &http.Server{Addr: addrPort, Handler: mux} + + if cfg.Parsed.MetricsEnabled.EqualBool(true) { + logger.Infof("Running metrics endpoint at %s/metrics", addrPort) + metrics.RegisterMetricsHandlers(mux, lc, "") + } + + if cfg.Parsed.HealthCheckEnabled.EqualBool(true) { + ipV4, _ := ts.TailscaleIPs() + hz := healthz.RegisterHealthHandlers(mux, ipV4.String(), logger.Infof) + group.Go(func() error { + err := hz.MonitorHealth(ctx, lc) + if err == nil || errors.Is(err, context.Canceled) { + return nil + } + return err + }) + } + + group.Go(func() error { + errChan := make(chan error) + go func() { + if err := localSrv.ListenAndServe(); err != nil { + errChan <- err + } + close(errChan) + }() + + select { + case <-ctx.Done(): + sCtx, scancel := context.WithTimeout(serveCtx, 10*time.Second) + defer scancel() + return localSrv.Shutdown(sCtx) + case err := <-errChan: + return err + } + }) + } + + if v, ok := cfg.Parsed.AcceptRoutes.Get(); ok { _, err = lc.EditPrefs(ctx, &ipn.MaskedPrefs{ RouteAllSet: true, - Prefs: ipn.Prefs{RouteAll: *cfg.Parsed.AcceptRoutes}, + Prefs: ipn.Prefs{RouteAll: v}, }) if err != nil { return fmt.Errorf("error editing prefs: %w", err) @@ -285,10 +338,10 @@ func run(logger *zap.SugaredLogger) error { prefs.HostnameSet = true prefs.Hostname = *cfg.Parsed.Hostname } - if cfg.Parsed.AcceptRoutes != nil && *cfg.Parsed.AcceptRoutes != currentPrefs.RouteAll { - cfgLogger = cfgLogger.With("AcceptRoutes", fmt.Sprintf("%v -> %v", currentPrefs.RouteAll, *cfg.Parsed.AcceptRoutes)) + if v, ok := cfg.Parsed.AcceptRoutes.Get(); ok && v != currentPrefs.RouteAll { + cfgLogger = cfgLogger.With("AcceptRoutes", fmt.Sprintf("%v -> %v", currentPrefs.RouteAll, v)) prefs.RouteAllSet = true - prefs.Prefs.RouteAll = *cfg.Parsed.AcceptRoutes + prefs.Prefs.RouteAll = v } if !prefs.IsEmpty() { if _, err := lc.EditPrefs(ctx, &prefs); err != nil { @@ -304,6 +357,10 @@ func run(logger *zap.SugaredLogger) error { } } +func getLocalAddrPort(addr string, port uint16) string { + return net.JoinHostPort(addr, strconv.FormatUint(uint64(port), 10)) +} + func getStateStore(path *string, logger *zap.SugaredLogger) (ipn.StateStore, error) { p := "mem:" if path != nil { diff --git a/kube/health/healthz.go b/kube/health/healthz.go new file mode 100644 index 0000000000000..c8cfcc7ec01b4 --- /dev/null +++ b/kube/health/healthz.go @@ -0,0 +1,84 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package health contains shared types and underlying methods for serving +// a `/healthz` endpoint for containerboot and k8s-proxy. +package health + +import ( + "context" + "fmt" + "net/http" + "sync" + + "tailscale.com/client/local" + "tailscale.com/ipn" + "tailscale.com/kube/kubetypes" + "tailscale.com/types/logger" +) + +// Healthz is a simple health check server, if enabled it returns 200 OK if +// this tailscale node currently has at least one tailnet IP address else +// returns 503. +type Healthz struct { + sync.Mutex + hasAddrs bool + podIPv4 string + logger logger.Logf +} + +func (h *Healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.Lock() + defer h.Unlock() + + if h.hasAddrs { + w.Header().Add(kubetypes.PodIPv4Header, h.podIPv4) + if _, err := w.Write([]byte("ok")); err != nil { + http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError) + } + } else { + http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable) + } +} + +func (h *Healthz) Update(healthy bool) { + h.Lock() + defer h.Unlock() + + if h.hasAddrs != healthy { + h.logger("Setting healthy %v", healthy) + } + h.hasAddrs = healthy +} + +func (h *Healthz) MonitorHealth(ctx context.Context, lc *local.Client) error { + w, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) + if err != nil { + return fmt.Errorf("failed to watch IPN bus: %w", err) + } + + for { + n, err := w.Next() + if err != nil { + return err + } + + if n.NetMap != nil { + h.Update(n.NetMap.SelfNode.Addresses().Len() != 0) + } + } +} + +// RegisterHealthHandlers registers a simple health handler at /healthz. +// A containerized tailscale instance is considered healthy if +// it has at least one tailnet IP address. +func RegisterHealthHandlers(mux *http.ServeMux, podIPv4 string, logger logger.Logf) *Healthz { + h := &Healthz{ + podIPv4: podIPv4, + logger: logger, + } + mux.Handle("GET /healthz", h) + return h +} diff --git a/kube/k8s-proxy/conf/conf.go b/kube/k8s-proxy/conf/conf.go index fdb6301ac5a1d..5294952438896 100644 --- a/kube/k8s-proxy/conf/conf.go +++ b/kube/k8s-proxy/conf/conf.go @@ -49,21 +49,23 @@ type VersionedConfig struct { } type ConfigV1Alpha1 struct { - AuthKey *string `json:",omitempty"` // Tailscale auth key to use. - State *string `json:",omitempty"` // Path to the Tailscale state. - LogLevel *string `json:",omitempty"` // "debug", "info". Defaults to "info". - App *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer - ServerURL *string `json:",omitempty"` // URL of the Tailscale coordination server. - // StaticEndpoints are additional, user-defined endpoints that this node - // should advertise amongst its wireguard endpoints. - StaticEndpoints []netip.AddrPort `json:",omitempty"` + AuthKey *string `json:",omitempty"` // Tailscale auth key to use. + State *string `json:",omitempty"` // Path to the Tailscale state. + LogLevel *string `json:",omitempty"` // "debug", "info". Defaults to "info". + App *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer + ServerURL *string `json:",omitempty"` // URL of the Tailscale coordination server. + LocalAddr *string `json:",omitempty"` // The address to use for serving HTTP health checks and metrics (defaults to all interfaces). + LocalPort *uint16 `json:",omitempty"` // The port to use for serving HTTP health checks and metrics (defaults to 9002). + MetricsEnabled opt.Bool `json:",omitempty"` // Serve metrics on :/metrics. + HealthCheckEnabled opt.Bool `json:",omitempty"` // Serve health check on :/metrics. // TODO(tomhjp): The remaining fields should all be reloadable during // runtime, but currently missing most of the APIServerProxy fields. Hostname *string `json:",omitempty"` // Tailscale device hostname. - AcceptRoutes *bool `json:",omitempty"` // Accepts routes advertised by other Tailscale nodes. + AcceptRoutes opt.Bool `json:",omitempty"` // Accepts routes advertised by other Tailscale nodes. AdvertiseServices []string `json:",omitempty"` // Tailscale Services to advertise. APIServerProxy *APIServerProxyConfig `json:",omitempty"` // Config specific to the API Server proxy. + StaticEndpoints []netip.AddrPort `json:",omitempty"` // StaticEndpoints are additional, user-defined endpoints that this node should advertise amongst its wireguard endpoints. } type APIServerProxyConfig struct { @@ -108,3 +110,19 @@ func Load(raw []byte) (c Config, err error) { return c, nil } + +func (c *Config) GetLocalAddr() string { + if c.Parsed.LocalAddr == nil { + return "[::]" + } + + return *c.Parsed.LocalAddr +} + +func (c *Config) GetLocalPort() uint16 { + if c.Parsed.LocalPort == nil { + return uint16(9002) + } + + return *c.Parsed.LocalPort +} diff --git a/cmd/containerboot/metrics.go b/kube/metrics/metrics.go similarity index 90% rename from cmd/containerboot/metrics.go rename to kube/metrics/metrics.go index bbd050de6df26..0db683008f91e 100644 --- a/cmd/containerboot/metrics.go +++ b/kube/metrics/metrics.go @@ -1,9 +1,11 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux +//go:build !plan9 -package main +// Package metrics contains shared types and underlying methods for serving +// localapi metrics. This is primarily consumed by containerboot and k8s-proxy. +package metrics import ( "fmt" @@ -68,7 +70,7 @@ func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) { // In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug // endpoint if configured to ease migration for a breaking change serving user // metrics instead of debug metrics on the "metrics" port. -func registerMetricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) { +func RegisterMetricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) { m := &metrics{ lc: lc, debugEndpoint: debugAddrPort, From 0de5e7b94f0bb89bcaed108f656d3ed50da85d02 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Tue, 22 Jul 2025 09:22:17 -1000 Subject: [PATCH 240/263] util/set: add IntSet (#16602) IntSet is a set optimized for integers. Updates tailscale/corp#29809 Signed-off-by: Joe Tsai --- util/set/intset.go | 172 +++++++++++++++++++++++++++++++++++++++ util/set/intset_test.go | 174 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 346 insertions(+) create mode 100644 util/set/intset.go create mode 100644 util/set/intset_test.go diff --git a/util/set/intset.go b/util/set/intset.go new file mode 100644 index 0000000000000..b747d3bffa9fd --- /dev/null +++ b/util/set/intset.go @@ -0,0 +1,172 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package set + +import ( + "iter" + "maps" + "math/bits" + "math/rand/v2" + + "golang.org/x/exp/constraints" + "tailscale.com/util/mak" +) + +// IntSet is a set optimized for integer values close to zero +// or set of integers that are close in value. +type IntSet[T constraints.Integer] struct { + // bits is a [bitSet] for numbers less than [bits.UintSize]. + bits bitSet + + // extra is a mapping of [bitSet] for numbers not in bits, + // where the key is a number modulo [bits.UintSize]. + extra map[uint64]bitSet + + // extraLen is the count of numbers in extra since len(extra) + // does not reflect that each bitSet may have multiple numbers. + extraLen int +} + +// Values returns an iterator over the elements of the set. +// The iterator will yield the elements in no particular order. +func (s IntSet[T]) Values() iter.Seq[T] { + return func(yield func(T) bool) { + if s.bits != 0 { + for i := range s.bits.values() { + if !yield(decodeZigZag[T](i)) { + return + } + } + } + if s.extra != nil { + for hi, bs := range s.extra { + for lo := range bs.values() { + if !yield(decodeZigZag[T](hi*bits.UintSize + lo)) { + return + } + } + } + } + } +} + +// Contains reports whether e is in the set. +func (s IntSet[T]) Contains(e T) bool { + if v := encodeZigZag(e); v < bits.UintSize { + return s.bits.contains(v) + } else { + hi, lo := v/uint64(bits.UintSize), v%uint64(bits.UintSize) + return s.extra[hi].contains(lo) + } +} + +// Add adds e to the set. +// +// When storing a IntSet in a map as a value type, +// it is important to re-assign the map entry after calling Add or Delete, +// as the IntSet's representation may change. +func (s *IntSet[T]) Add(e T) { + if v := encodeZigZag(e); v < bits.UintSize { + s.bits.add(v) + } else { + hi, lo := v/uint64(bits.UintSize), v%uint64(bits.UintSize) + if bs := s.extra[hi]; !bs.contains(lo) { + bs.add(lo) + mak.Set(&s.extra, hi, bs) + s.extra[hi] = bs + s.extraLen++ + } + } +} + +// AddSeq adds the values from seq to the set. +func (s *IntSet[T]) AddSeq(seq iter.Seq[T]) { + for e := range seq { + s.Add(e) + } +} + +// Len reports the number of elements in the set. +func (s IntSet[T]) Len() int { + return s.bits.len() + s.extraLen +} + +// Delete removes e from the set. +// +// When storing a IntSet in a map as a value type, +// it is important to re-assign the map entry after calling Add or Delete, +// as the IntSet's representation may change. +func (s *IntSet[T]) Delete(e T) { + if v := encodeZigZag(e); v < bits.UintSize { + s.bits.delete(v) + } else { + hi, lo := v/uint64(bits.UintSize), v%uint64(bits.UintSize) + if bs := s.extra[hi]; bs.contains(lo) { + bs.delete(lo) + mak.Set(&s.extra, hi, bs) + s.extra[hi] = bs + s.extraLen-- + } + } +} + +// Clone returns a copy of s that doesn't alias the original. +func (s IntSet[T]) Clone() IntSet[T] { + return IntSet[T]{ + bits: s.bits, + extra: maps.Clone(s.extra), + extraLen: s.extraLen, + } +} + +type bitSet uint + +func (s bitSet) values() iter.Seq[uint64] { + return func(yield func(uint64) bool) { + // Hyrum-proofing: randomly iterate in forwards or reverse. + if rand.Uint64()%2 == 0 { + for i := 0; i < bits.UintSize; i++ { + if s.contains(uint64(i)) && !yield(uint64(i)) { + return + } + } + } else { + for i := bits.UintSize; i >= 0; i-- { + if s.contains(uint64(i)) && !yield(uint64(i)) { + return + } + } + } + } +} +func (s bitSet) len() int { return bits.OnesCount(uint(s)) } +func (s bitSet) contains(i uint64) bool { return s&(1< 0 } +func (s *bitSet) add(i uint64) { *s |= 1 << i } +func (s *bitSet) delete(i uint64) { *s &= ^(1 << i) } + +// encodeZigZag encodes an integer as an unsigned integer ensuring that +// negative integers near zero still have a near zero positive value. +// For unsigned integers, it returns the value verbatim. +func encodeZigZag[T constraints.Integer](v T) uint64 { + var zero T + if ^zero >= 0 { // must be constraints.Unsigned + return uint64(v) + } else { // must be constraints.Signed + // See [google.golang.org/protobuf/encoding/protowire.EncodeZigZag] + return uint64(int64(v)<<1) ^ uint64(int64(v)>>63) + } +} + +// decodeZigZag decodes an unsigned integer as an integer ensuring that +// negative integers near zero still have a near zero positive value. +// For unsigned integers, it returns the value verbatim. +func decodeZigZag[T constraints.Integer](v uint64) T { + var zero T + if ^zero >= 0 { // must be constraints.Unsigned + return T(v) + } else { // must be constraints.Signed + // See [google.golang.org/protobuf/encoding/protowire.DecodeZigZag] + return T(int64(v>>1) ^ int64(v)<<63>>63) + } +} diff --git a/util/set/intset_test.go b/util/set/intset_test.go new file mode 100644 index 0000000000000..9523fe88db127 --- /dev/null +++ b/util/set/intset_test.go @@ -0,0 +1,174 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package set + +import ( + "maps" + "math" + "slices" + "testing" + + "golang.org/x/exp/constraints" +) + +func TestIntSet(t *testing.T) { + t.Run("Int64", func(t *testing.T) { + ss := make(Set[int64]) + var si IntSet[int64] + intValues(t, ss, si) + deleteInt(t, ss, &si, -5) + deleteInt(t, ss, &si, 2) + deleteInt(t, ss, &si, 75) + intValues(t, ss, si) + addInt(t, ss, &si, 2) + addInt(t, ss, &si, 75) + addInt(t, ss, &si, 75) + addInt(t, ss, &si, -3) + addInt(t, ss, &si, -3) + addInt(t, ss, &si, -3) + addInt(t, ss, &si, math.MinInt64) + addInt(t, ss, &si, 8) + intValues(t, ss, si) + addInt(t, ss, &si, 77) + addInt(t, ss, &si, 76) + addInt(t, ss, &si, 76) + addInt(t, ss, &si, 76) + intValues(t, ss, si) + addInt(t, ss, &si, -5) + addInt(t, ss, &si, 7) + addInt(t, ss, &si, -83) + addInt(t, ss, &si, math.MaxInt64) + intValues(t, ss, si) + deleteInt(t, ss, &si, -5) + deleteInt(t, ss, &si, 2) + deleteInt(t, ss, &si, 75) + intValues(t, ss, si) + deleteInt(t, ss, &si, math.MinInt64) + deleteInt(t, ss, &si, math.MaxInt64) + intValues(t, ss, si) + }) + + t.Run("Uint64", func(t *testing.T) { + ss := make(Set[uint64]) + var si IntSet[uint64] + intValues(t, ss, si) + deleteInt(t, ss, &si, 5) + deleteInt(t, ss, &si, 2) + deleteInt(t, ss, &si, 75) + intValues(t, ss, si) + addInt(t, ss, &si, 2) + addInt(t, ss, &si, 75) + addInt(t, ss, &si, 75) + addInt(t, ss, &si, 3) + addInt(t, ss, &si, 3) + addInt(t, ss, &si, 8) + intValues(t, ss, si) + addInt(t, ss, &si, 77) + addInt(t, ss, &si, 76) + addInt(t, ss, &si, 76) + addInt(t, ss, &si, 76) + intValues(t, ss, si) + addInt(t, ss, &si, 5) + addInt(t, ss, &si, 7) + addInt(t, ss, &si, 83) + addInt(t, ss, &si, math.MaxInt64) + intValues(t, ss, si) + deleteInt(t, ss, &si, 5) + deleteInt(t, ss, &si, 2) + deleteInt(t, ss, &si, 75) + intValues(t, ss, si) + deleteInt(t, ss, &si, math.MaxInt64) + intValues(t, ss, si) + }) +} + +func intValues[T constraints.Integer](t testing.TB, ss Set[T], si IntSet[T]) { + got := slices.Collect(maps.Keys(ss)) + slices.Sort(got) + want := slices.Collect(si.Values()) + slices.Sort(want) + if !slices.Equal(got, want) { + t.Fatalf("Values mismatch:\n\tgot %v\n\twant %v", got, want) + } + if got, want := si.Len(), ss.Len(); got != want { + t.Fatalf("Len() = %v, want %v", got, want) + } +} + +func addInt[T constraints.Integer](t testing.TB, ss Set[T], si *IntSet[T], v T) { + t.Helper() + if got, want := si.Contains(v), ss.Contains(v); got != want { + t.Fatalf("Contains(%v) = %v, want %v", v, got, want) + } + ss.Add(v) + si.Add(v) + if !si.Contains(v) { + t.Fatalf("Contains(%v) = false, want true", v) + } + if got, want := si.Len(), ss.Len(); got != want { + t.Fatalf("Len() = %v, want %v", got, want) + } +} + +func deleteInt[T constraints.Integer](t testing.TB, ss Set[T], si *IntSet[T], v T) { + t.Helper() + if got, want := si.Contains(v), ss.Contains(v); got != want { + t.Fatalf("Contains(%v) = %v, want %v", v, got, want) + } + ss.Delete(v) + si.Delete(v) + if si.Contains(v) { + t.Fatalf("Contains(%v) = true, want false", v) + } + if got, want := si.Len(), ss.Len(); got != want { + t.Fatalf("Len() = %v, want %v", got, want) + } +} + +func TestZigZag(t *testing.T) { + t.Run("Int64", func(t *testing.T) { + for _, tt := range []struct { + decoded int64 + encoded uint64 + }{ + {math.MinInt64, math.MaxUint64}, + {-2, 3}, + {-1, 1}, + {0, 0}, + {1, 2}, + {2, 4}, + {math.MaxInt64, math.MaxUint64 - 1}, + } { + encoded := encodeZigZag(tt.decoded) + if encoded != tt.encoded { + t.Errorf("encodeZigZag(%v) = %v, want %v", tt.decoded, encoded, tt.encoded) + } + decoded := decodeZigZag[int64](tt.encoded) + if decoded != tt.decoded { + t.Errorf("decodeZigZag(%v) = %v, want %v", tt.encoded, decoded, tt.decoded) + } + } + }) + t.Run("Uint64", func(t *testing.T) { + for _, tt := range []struct { + decoded uint64 + encoded uint64 + }{ + {0, 0}, + {1, 1}, + {2, 2}, + {math.MaxInt64, math.MaxInt64}, + {math.MaxUint64, math.MaxUint64}, + } { + encoded := encodeZigZag(tt.decoded) + if encoded != tt.encoded { + t.Errorf("encodeZigZag(%v) = %v, want %v", tt.decoded, encoded, tt.encoded) + } + decoded := decodeZigZag[uint64](tt.encoded) + if decoded != tt.decoded { + t.Errorf("decodeZigZag(%v) = %v, want %v", tt.encoded, decoded, tt.decoded) + } + } + }) +} From 19faaff95c6f32a2fae26f003b467fb623962d09 Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Tue, 22 Jul 2025 16:23:51 -0400 Subject: [PATCH 241/263] cmd/tailscale/cli: revert key for web config for services to FQDN (#16627) This commit reverts the key of Web field in ipn.ServiceConfig to use FQDN instead of service name for the host part of HostPort. This change is because k8s operator already build base on the assumption of the part being FQDN. We don't want to break the code with dependency. Fixes tailscale/corp#30695 Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- cmd/tailscale/cli/serve_legacy.go | 2 +- cmd/tailscale/cli/serve_legacy_test.go | 1 + cmd/tailscale/cli/serve_v2.go | 40 ++++++++++++++------------ cmd/tailscale/cli/serve_v2_test.go | 37 ++++++++++++------------ cmd/tsidp/tsidp.go | 2 +- ipn/ipnlocal/serve.go | 4 ++- ipn/serve.go | 10 +++---- 7 files changed, 51 insertions(+), 45 deletions(-) diff --git a/cmd/tailscale/cli/serve_legacy.go b/cmd/tailscale/cli/serve_legacy.go index 7c79f7f7bc972..1a05d0543f58e 100644 --- a/cmd/tailscale/cli/serve_legacy.go +++ b/cmd/tailscale/cli/serve_legacy.go @@ -363,7 +363,7 @@ func (e *serveEnv) handleWebServe(ctx context.Context, srvPort uint16, useTLS bo return errHelp } - sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS) + sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS, noService.String()) if !reflect.DeepEqual(cursc, sc) { if err := e.lc.SetServeConfig(ctx, sc); err != nil { diff --git a/cmd/tailscale/cli/serve_legacy_test.go b/cmd/tailscale/cli/serve_legacy_test.go index 6b053fbd774ba..1ea76e72ca818 100644 --- a/cmd/tailscale/cli/serve_legacy_test.go +++ b/cmd/tailscale/cli/serve_legacy_test.go @@ -876,6 +876,7 @@ var fakeStatus = &ipnstate.Status{ tailcfg.CapabilityFunnelPorts + "?ports=443,8443": nil, }, }, + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, } func (lc *fakeLocalServeClient) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) { diff --git a/cmd/tailscale/cli/serve_v2.go b/cmd/tailscale/cli/serve_v2.go index 8832a232d0f4e..056bfabb0a202 100644 --- a/cmd/tailscale/cli/serve_v2.go +++ b/cmd/tailscale/cli/serve_v2.go @@ -331,6 +331,7 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { return fmt.Errorf("getting client status: %w", err) } dnsName := strings.TrimSuffix(st.Self.DNSName, ".") + magicDNSSuffix := st.CurrentTailnet.MagicDNSSuffix // set parent serve config to always be persisted // at the top level, but a nested config might be @@ -394,7 +395,7 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { var msg string if turnOff { // only unset serve when trying to unset with type and port flags. - err = e.unsetServe(sc, st, dnsName, srvType, srvPort, mount) + err = e.unsetServe(sc, dnsName, srvType, srvPort, mount, magicDNSSuffix) } else { if err := e.validateConfig(parentSC, srvPort, srvType, svcName); err != nil { return err @@ -406,7 +407,7 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { if len(args) > 0 { target = args[0] } - err = e.setServe(sc, dnsName, srvType, srvPort, mount, target, funnel) + err = e.setServe(sc, dnsName, srvType, srvPort, mount, target, funnel, magicDNSSuffix) msg = e.messageForPort(sc, st, dnsName, srvType, srvPort) } if err != nil { @@ -585,12 +586,12 @@ func serveFromPortHandler(tcp *ipn.TCPPortHandler) serveType { } } -func (e *serveEnv) setServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string, target string, allowFunnel bool) error { +func (e *serveEnv) setServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string, target string, allowFunnel bool, mds string) error { // update serve config based on the type switch srvType { case serveTypeHTTPS, serveTypeHTTP: useTLS := srvType == serveTypeHTTPS - err := e.applyWebServe(sc, dnsName, srvPort, useTLS, mount, target) + err := e.applyWebServe(sc, dnsName, srvPort, useTLS, mount, target, mds) if err != nil { return fmt.Errorf("failed apply web serve: %w", err) } @@ -643,11 +644,10 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN var webConfig *ipn.WebServerConfig var tcpHandler *ipn.TCPPortHandler ips := st.TailscaleIPs + magicDNSSuffix := st.CurrentTailnet.MagicDNSSuffix host := dnsName - displayedHost := dnsName if forService { - displayedHost = strings.Join([]string{svcName.WithoutPrefix(), st.CurrentTailnet.MagicDNSSuffix}, ".") - host = svcName.WithoutPrefix() + host = strings.Join([]string{svcName.WithoutPrefix(), magicDNSSuffix}, ".") } hp := ipn.HostPort(net.JoinHostPort(host, strconv.Itoa(int(srvPort)))) @@ -687,7 +687,7 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN output.WriteString("\n\n") svc := sc.Services[svcName] if srvType == serveTypeTUN && svc.Tun { - output.WriteString(fmt.Sprintf(msgRunningTunService, displayedHost)) + output.WriteString(fmt.Sprintf(msgRunningTunService, host)) output.WriteString("\n") output.WriteString(fmt.Sprintf(msgDisableServiceTun, dnsName)) output.WriteString("\n") @@ -716,7 +716,7 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN }) for _, m := range mounts { t, d := srvTypeAndDesc(webConfig.Handlers[m]) - output.WriteString(fmt.Sprintf("%s://%s%s%s\n", scheme, displayedHost, portPart, m)) + output.WriteString(fmt.Sprintf("%s://%s%s%s\n", scheme, host, portPart, m)) output.WriteString(fmt.Sprintf("%s %-5s %s\n\n", "|--", t, d)) } } else if tcpHandler != nil { @@ -726,7 +726,7 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN tlsStatus = "TLS terminated" } - output.WriteString(fmt.Sprintf("|-- tcp://%s:%d (%s)\n", displayedHost, srvPort, tlsStatus)) + output.WriteString(fmt.Sprintf("|-- tcp://%s:%d (%s)\n", host, srvPort, tlsStatus)) for _, a := range ips { ipp := net.JoinHostPort(a.String(), strconv.Itoa(int(srvPort))) output.WriteString(fmt.Sprintf("|-- tcp://%s\n", ipp)) @@ -755,7 +755,7 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN return output.String() } -func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, useTLS bool, mount, target string) error { +func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, useTLS bool, mount, target string, mds string) error { h := new(ipn.HTTPHandler) switch { case strings.HasPrefix(target, "text:"): @@ -797,7 +797,7 @@ func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort ui return errors.New("cannot serve web; already serving TCP") } - sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS) + sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS, mds) return nil } @@ -850,11 +850,12 @@ func (e *serveEnv) applyFunnel(sc *ipn.ServeConfig, dnsName string, srvPort uint } // unsetServe removes the serve config for the given serve port. -// dnsName is a FQDN or a serviceName (with `svc:` prefix). -func (e *serveEnv) unsetServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName string, srvType serveType, srvPort uint16, mount string) error { +// dnsName is a FQDN or a serviceName (with `svc:` prefix). mds +// is the Magic DNS suffix, which is used to recreate serve's host. +func (e *serveEnv) unsetServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string, mds string) error { switch srvType { case serveTypeHTTPS, serveTypeHTTP: - err := e.removeWebServe(sc, st, dnsName, srvPort, mount) + err := e.removeWebServe(sc, dnsName, srvPort, mount, mds) if err != nil { return fmt.Errorf("failed to remove web serve: %w", err) } @@ -1010,8 +1011,9 @@ func isLegacyInvocation(subcmd serveMode, args []string) (string, bool) { // removeWebServe removes a web handler from the serve config // and removes funnel if no remaining mounts exist for the serve port. // The srvPort argument is the serving port and the mount argument is -// the mount point or registered path to remove. -func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName string, srvPort uint16, mount string) error { +// the mount point or registered path to remove. mds is the Magic DNS suffix, +// which is used to recreate serve's host. +func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, mount string, mds string) error { if sc == nil { return nil } @@ -1026,7 +1028,7 @@ func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN if svc == nil { return errors.New("service does not exist") } - hostName = svcName.WithoutPrefix() + hostName = strings.Join([]string{svcName.WithoutPrefix(), mds}, ".") webServeMap = svc.Web } @@ -1063,7 +1065,7 @@ func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN } if forService { - sc.RemoveServiceWebHandler(st, svcName, srvPort, mounts) + sc.RemoveServiceWebHandler(svcName, hostName, srvPort, mounts) } else { sc.RemoveWebHandler(dnsName, srvPort, mounts, true) } diff --git a/cmd/tailscale/cli/serve_v2_test.go b/cmd/tailscale/cli/serve_v2_test.go index 2ba0b3f8434c8..95bf5b1012f8c 100644 --- a/cmd/tailscale/cli/serve_v2_test.go +++ b/cmd/tailscale/cli/serve_v2_test.go @@ -1299,7 +1299,7 @@ func TestMessageForPort(t *testing.T) { "foo.test.ts.net:443": true, }, }, - status: &ipnstate.Status{}, + status: &ipnstate.Status{CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}}, dnsName: "foo.test.ts.net", srvType: serveTypeHTTPS, srvPort: 443, @@ -1328,7 +1328,7 @@ func TestMessageForPort(t *testing.T) { }, }, }, - status: &ipnstate.Status{}, + status: &ipnstate.Status{CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}}, dnsName: "foo.test.ts.net", srvType: serveTypeHTTP, srvPort: 80, @@ -1352,7 +1352,7 @@ func TestMessageForPort(t *testing.T) { 80: {HTTP: true}, }, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "foo:80": { + "foo.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, @@ -1396,7 +1396,7 @@ func TestMessageForPort(t *testing.T) { 80: {HTTP: true}, }, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, @@ -1440,7 +1440,7 @@ func TestMessageForPort(t *testing.T) { 2200: {HTTPS: true}, }, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "foo:2200": { + "foo.test.ts.net:2200": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, @@ -1670,6 +1670,7 @@ func TestIsLegacyInvocation(t *testing.T) { func TestSetServe(t *testing.T) { e := &serveEnv{} + magicDNSSuffix := "test.ts.net" tests := []struct { name string desc string @@ -1816,7 +1817,7 @@ func TestSetServe(t *testing.T) { "svc:bar": { TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, @@ -1834,7 +1835,7 @@ func TestSetServe(t *testing.T) { "svc:bar": { TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, @@ -1853,7 +1854,7 @@ func TestSetServe(t *testing.T) { "svc:bar": { TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3001"}, }, @@ -1871,7 +1872,7 @@ func TestSetServe(t *testing.T) { "svc:bar": { TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, @@ -1893,12 +1894,12 @@ func TestSetServe(t *testing.T) { 88: {HTTP: true}, }, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, }, - "bar:88": { + "bar.test.ts.net:88": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3001"}, }, @@ -1916,7 +1917,7 @@ func TestSetServe(t *testing.T) { "svc:bar": { TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, @@ -1937,7 +1938,7 @@ func TestSetServe(t *testing.T) { 80: {HTTP: true}, }, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, "/added": {Proxy: "http://localhost:3001"}, @@ -1965,7 +1966,7 @@ func TestSetServe(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := e.setServe(tt.cfg, tt.dnsName, tt.srvType, tt.srvPort, tt.mountPath, tt.target, tt.allowFunnel) + err := e.setServe(tt.cfg, tt.dnsName, tt.srvType, tt.srvPort, tt.mountPath, tt.target, tt.allowFunnel, magicDNSSuffix) if err != nil && !tt.expectErr { t.Fatalf("got error: %v; did not expect error.", err) } @@ -2030,7 +2031,7 @@ func TestUnsetServe(t *testing.T) { 80: {HTTP: true}, }, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, @@ -2124,7 +2125,7 @@ func TestUnsetServe(t *testing.T) { 80: {HTTP: true}, }, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, @@ -2199,7 +2200,7 @@ func TestUnsetServe(t *testing.T) { 80: {HTTP: true}, }, Web: map[ipn.HostPort]*ipn.WebServerConfig{ - "bar:80": { + "bar.test.ts.net:80": { Handlers: map[string]*ipn.HTTPHandler{ "/": {Proxy: "http://localhost:3000"}, }, @@ -2224,7 +2225,7 @@ func TestUnsetServe(t *testing.T) { if tt.setServeEnv { e = tt.serveEnv } - err := e.unsetServe(tt.cfg, tt.st, tt.dnsName, tt.srvType, tt.srvPort, tt.mount) + err := e.unsetServe(tt.cfg, tt.dnsName, tt.srvType, tt.srvPort, tt.mount, tt.st.CurrentTailnet.MagicDNSSuffix) if err != nil && !tt.expectErr { t.Fatalf("got error: %v; did not expect error.", err) } diff --git a/cmd/tsidp/tsidp.go b/cmd/tsidp/tsidp.go index 6a0c2d89e685e..8df68cd744148 100644 --- a/cmd/tsidp/tsidp.go +++ b/cmd/tsidp/tsidp.go @@ -270,7 +270,7 @@ func serveOnLocalTailscaled(ctx context.Context, lc *local.Client, st *ipnstate. foregroundSc.SetFunnel(serverURL, dstPort, shouldFunnel) foregroundSc.SetWebHandler(&ipn.HTTPHandler{ Proxy: fmt.Sprintf("https://%s", net.JoinHostPort(serverURL, strconv.Itoa(int(dstPort)))), - }, serverURL, uint16(*flagPort), "/", true) + }, serverURL, uint16(*flagPort), "/", true, st.CurrentTailnet.MagicDNSSuffix) err = lc.SetServeConfig(ctx, sc) if err != nil { return nil, watcherChan, fmt.Errorf("could not set serve config: %v", err) diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index 28262251c6880..36738b88119f5 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -1014,7 +1014,9 @@ func (b *LocalBackend) webServerConfig(hostname string, forVIPService tailcfg.Se return c, false } if forVIPService != "" { - key := ipn.HostPort(net.JoinHostPort(forVIPService.WithoutPrefix(), fmt.Sprintf("%d", port))) + magicDNSSuffix := b.currentNode().NetMap().MagicDNSSuffix() + fqdn := strings.Join([]string{forVIPService.WithoutPrefix(), magicDNSSuffix}, ".") + key := ipn.HostPort(net.JoinHostPort(fqdn, fmt.Sprintf("%d", port))) return b.serveConfig.FindServiceWeb(forVIPService, key) } key := ipn.HostPort(net.JoinHostPort(hostname, fmt.Sprintf("%d", port))) diff --git a/ipn/serve.go b/ipn/serve.go index fae0ad5d6568a..a0f1334d7d150 100644 --- a/ipn/serve.go +++ b/ipn/serve.go @@ -343,8 +343,9 @@ func (sc *ServeConfig) FindConfig(port uint16) (*ServeConfig, bool) { // SetWebHandler sets the given HTTPHandler at the specified host, port, // and mount in the serve config. sc.TCP is also updated to reflect web // serving usage of the given port. The st argument is needed when setting -// a web handler for a service, otherwise it can be nil. -func (sc *ServeConfig) SetWebHandler(handler *HTTPHandler, host string, port uint16, mount string, useTLS bool) { +// a web handler for a service, otherwise it can be nil. mds is the Magic DNS +// suffix, which is used to recreate serve's host. +func (sc *ServeConfig) SetWebHandler(handler *HTTPHandler, host string, port uint16, mount string, useTLS bool, mds string) { if sc == nil { sc = new(ServeConfig) } @@ -353,7 +354,7 @@ func (sc *ServeConfig) SetWebHandler(handler *HTTPHandler, host string, port uin webServerMap := &sc.Web hostName := host if svcName := tailcfg.AsServiceName(host); svcName != "" { - hostName = svcName.WithoutPrefix() + hostName = strings.Join([]string{svcName.WithoutPrefix(), mds}, ".") svc, ok := sc.Services[svcName] if !ok { svc = new(ServiceConfig) @@ -464,8 +465,7 @@ func (sc *ServeConfig) RemoveWebHandler(host string, port uint16, mounts []strin // RemoveServiceWebHandler deletes the web handlers at all of the given mount points // for the provided host and port in the serve config for the given service. -func (sc *ServeConfig) RemoveServiceWebHandler(st *ipnstate.Status, svcName tailcfg.ServiceName, port uint16, mounts []string) { - hostName := svcName.WithoutPrefix() +func (sc *ServeConfig) RemoveServiceWebHandler(svcName tailcfg.ServiceName, hostName string, port uint16, mounts []string) { hp := HostPort(net.JoinHostPort(hostName, strconv.Itoa(int(port)))) svc, ok := sc.Services[svcName] From 729d6532ff356101b3e34c28b3c5ce9a186af44e Mon Sep 17 00:00:00 2001 From: Simon Law Date: Tue, 22 Jul 2025 13:54:28 -0700 Subject: [PATCH 242/263] tailcfg: add Hostinfo.ExitNodeID to report the selected exit node (#16625) When a client selects a particular exit node, Control may use that as a signal for deciding other routes. This patch causes the client to report whenever the current exit node changes, through tailcfg.Hostinfo.ExitNodeID. It relies on a properly set ipn.Prefs.ExitNodeID, which should already be resolved by `tailscale set`. Updates tailscale/corp#30536 Signed-off-by: Simon Law --- ipn/ipnlocal/local.go | 9 +++++ ipn/ipnlocal/local_test.go | 67 +++++++++++++++++++++++++++++--------- tailcfg/tailcfg.go | 4 ++- tailcfg/tailcfg_clone.go | 1 + tailcfg/tailcfg_test.go | 16 +++++++++ tailcfg/tailcfg_view.go | 2 ++ 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 8665a88c4f867..ce0f4f6873d74 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -5612,6 +5612,11 @@ func (b *LocalBackend) applyPrefsToHostinfoLocked(hi *tailcfg.Hostinfo, prefs ip // WireIngress. hi.WireIngress = b.shouldWireInactiveIngressLocked() hi.AppConnector.Set(prefs.AppConnector().Advertise) + + // The [tailcfg.Hostinfo.ExitNodeID] field tells control which exit node + // was selected, if any. Since [LocalBackend.resolveExitNodeIPLocked] + // has already run, there is no need to consult [ipn.Prefs.ExitNodeIP]. + hi.ExitNodeID = prefs.ExitNodeID() } // enterState transitions the backend into newState, updating internal @@ -6136,6 +6141,10 @@ func (b *LocalBackend) resolveExitNode() (changed bool) { }); err != nil { b.logf("failed to save exit node changes: %v", err) } + + // Send the resolved exit node to Control via Hostinfo. + b.hostinfo.ExitNodeID = prefs.ExitNodeID + b.sendToLocked(ipn.Notify{Prefs: ptr.To(prefs.View())}, allClients) return true } diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 13681fc0430ea..da6fc8b4a5725 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -614,19 +614,20 @@ func TestConfigureExitNode(t *testing.T) { } tests := []struct { - name string - prefs ipn.Prefs - netMap *netmap.NetworkMap - report *netcheck.Report - changePrefs *ipn.MaskedPrefs - useExitNodeEnabled *bool - exitNodeIDPolicy *tailcfg.StableNodeID - exitNodeIPPolicy *netip.Addr - exitNodeAllowedIDs []tailcfg.StableNodeID // nil if all IDs are allowed for auto exit nodes - exitNodeAllowOverride bool // whether [syspolicy.AllowExitNodeOverride] should be set to true - wantChangePrefsErr error // if non-nil, the error we expect from [LocalBackend.EditPrefsAs] - wantPrefs ipn.Prefs - wantExitNodeToggleErr error // if non-nil, the error we expect from [LocalBackend.SetUseExitNodeEnabled] + name string + prefs ipn.Prefs + netMap *netmap.NetworkMap + report *netcheck.Report + changePrefs *ipn.MaskedPrefs + useExitNodeEnabled *bool + exitNodeIDPolicy *tailcfg.StableNodeID + exitNodeIPPolicy *netip.Addr + exitNodeAllowedIDs []tailcfg.StableNodeID // nil if all IDs are allowed for auto exit nodes + exitNodeAllowOverride bool // whether [syspolicy.AllowExitNodeOverride] should be set to true + wantChangePrefsErr error // if non-nil, the error we expect from [LocalBackend.EditPrefsAs] + wantPrefs ipn.Prefs + wantExitNodeToggleErr error // if non-nil, the error we expect from [LocalBackend.SetUseExitNodeEnabled] + wantHostinfoExitNodeID *tailcfg.StableNodeID }{ { name: "exit-node-id-via-prefs", // set exit node ID via prefs @@ -643,6 +644,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "exit-node-ip-via-prefs", // set exit node IP via prefs (should be resolved to an ID) @@ -659,6 +661,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "auto-exit-node-via-prefs/any", // set auto exit node via prefs @@ -676,6 +679,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "auto-exit-node-via-prefs/set-exit-node-id-via-prefs", // setting exit node ID explicitly should disable auto exit node @@ -695,6 +699,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode2.StableID(), AutoExitNode: "", // should be unset }, + wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), }, { name: "auto-exit-node-via-prefs/any/no-report", // set auto exit node via prefs, but no report means we can't resolve the exit node ID @@ -711,6 +716,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: unresolvedExitNodeID, // cannot resolve; traffic will be dropped AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(unresolvedExitNodeID), }, { name: "auto-exit-node-via-prefs/any/no-netmap", // similarly, but without a netmap (no exit node should be selected) @@ -727,6 +733,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: unresolvedExitNodeID, // cannot resolve; traffic will be dropped AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(unresolvedExitNodeID), }, { name: "auto-exit-node-via-prefs/foo", // set auto exit node via prefs with an unknown/unsupported expression @@ -744,6 +751,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" AutoExitNode: "foo", }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "auto-exit-node-via-prefs/off", // toggle the exit node off after it was set to "any" @@ -763,6 +771,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "", InternalExitNodePrior: "auto:any", }, + wantHostinfoExitNodeID: ptr.To(tailcfg.StableNodeID("")), }, { name: "auto-exit-node-via-prefs/on", // toggle the exit node on @@ -779,6 +788,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "any", InternalExitNodePrior: "auto:any", }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "id-via-policy", // set exit node ID via syspolicy @@ -791,6 +801,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "id-via-policy/cannot-override-via-prefs/by-id", // syspolicy should take precedence over prefs @@ -809,7 +820,8 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, - wantChangePrefsErr: errManagedByPolicy, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantChangePrefsErr: errManagedByPolicy, }, { name: "id-via-policy/cannot-override-via-prefs/by-ip", // syspolicy should take precedence over prefs @@ -828,7 +840,8 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, - wantChangePrefsErr: errManagedByPolicy, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantChangePrefsErr: errManagedByPolicy, }, { name: "id-via-policy/cannot-override-via-prefs/by-auto-expr", // syspolicy should take precedence over prefs @@ -860,6 +873,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode2.StableID(), }, + wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), }, { name: "auto-any-via-policy", // set auto exit node via syspolicy (an exit node should be selected) @@ -874,6 +888,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "auto-any-via-policy/no-report", // set auto exit node via syspolicy without a netcheck report (no exit node should be selected) @@ -888,6 +903,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: unresolvedExitNodeID, AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(unresolvedExitNodeID), }, { name: "auto-any-via-policy/no-netmap", // similarly, but without a netmap (no exit node should be selected) @@ -902,6 +918,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: unresolvedExitNodeID, AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(unresolvedExitNodeID), }, { name: "auto-any-via-policy/no-netmap/with-existing", // set auto exit node via syspolicy without a netmap, but with a previously set exit node ID @@ -918,6 +935,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode2.StableID(), AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), }, { name: "auto-any-via-policy/no-netmap/with-allowed-existing", // same, but now with a syspolicy setting that explicitly allows the existing exit node ID @@ -936,6 +954,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode2.StableID(), AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), }, { name: "auto-any-via-policy/no-netmap/with-disallowed-existing", // same, but now with a syspolicy setting that does not allow the existing exit node ID @@ -954,6 +973,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: unresolvedExitNodeID, // we don't have a netmap yet, and the current exit node ID is not allowed; block traffic AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(unresolvedExitNodeID), }, { name: "auto-any-via-policy/with-netmap/with-allowed-existing", // same, but now with a syspolicy setting that does not allow the existing exit node ID @@ -972,6 +992,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode2.StableID(), // we have a netmap; switch to the best allowed exit node AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), }, { name: "auto-any-via-policy/with-netmap/switch-to-better", // if all exit nodes are allowed, switch to the best one once we have a netmap @@ -987,6 +1008,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), // switch to the best exit node AutoExitNode: "any", }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "auto-foo-via-policy", // set auto exit node via syspolicy with an unknown/unsupported expression @@ -1001,6 +1023,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" AutoExitNode: "foo", }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "auto-foo-via-edit-prefs", // set auto exit node via EditPrefs with an unknown/unsupported expression @@ -1018,6 +1041,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" AutoExitNode: "foo", }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "auto-any-via-policy/toggle-off", // cannot toggle off the exit node if it was set via syspolicy @@ -1035,6 +1059,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "any", InternalExitNodePrior: "", }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "auto-any-via-policy/allow-override/change", // changing the exit node is allowed by [syspolicy.AllowExitNodeOverride] @@ -1056,6 +1081,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode2.StableID(), // overridden by user AutoExitNode: "", // cleared, as we are setting the exit node ID explicitly }, + wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), }, { name: "auto-any-via-policy/allow-override/clear", // clearing the exit node ID is not allowed by [syspolicy.AllowExitNodeOverride] @@ -1079,6 +1105,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "any", InternalExitNodePrior: "", }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "auto-any-via-policy/allow-override/toggle-off", // similarly, toggling off the exit node is not allowed even with [syspolicy.AllowExitNodeOverride] @@ -1097,6 +1124,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "any", InternalExitNodePrior: "", }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, { name: "auto-any-via-initial-prefs/no-netmap/clear-auto-exit-node", @@ -1117,6 +1145,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "", // cleared ExitNodeID: "", // has never been resolved, so it should be cleared as well }, + wantHostinfoExitNodeID: ptr.To(tailcfg.StableNodeID("")), }, { name: "auto-any-via-initial-prefs/with-netmap/clear-auto-exit-node", @@ -1137,6 +1166,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "", // cleared ExitNodeID: exitNode1.StableID(), // a resolved exit node ID should be retained }, + wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), }, } syspolicy.RegisterWellKnownSettingsForTest(t) @@ -1197,6 +1227,13 @@ func TestConfigureExitNode(t *testing.T) { if diff := cmp.Diff(&tt.wantPrefs, lb.Prefs().AsStruct(), opts...); diff != "" { t.Errorf("Prefs(+got -want): %v", diff) } + + // And check Hostinfo. + if tt.wantHostinfoExitNodeID != nil { + if got := lb.hostinfo.ExitNodeID; got != *tt.wantHostinfoExitNodeID { + t.Errorf("Hostinfo.ExitNodeID got %v, want %v", got, *tt.wantHostinfoExitNodeID) + } + } }) } } diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 550914b96e31a..307b39f93903c 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -166,7 +166,8 @@ type CapabilityVersion int // - 119: 2025-07-10: Client uses Hostinfo.Location.Priority to prioritize one route over another. // - 120: 2025-07-15: Client understands peer relay disco messages, and implements peer client and relay server functions // - 121: 2025-07-19: Client understands peer relay endpoint alloc with [disco.AllocateUDPRelayEndpointRequest] & [disco.AllocateUDPRelayEndpointResponse] -const CurrentCapabilityVersion CapabilityVersion = 121 +// - 122: 2025-07-21: Client sends Hostinfo.ExitNodeID to report which exit node it has selected, if any. +const CurrentCapabilityVersion CapabilityVersion = 122 // ID is an integer ID for a user, node, or login allocated by the // control plane. @@ -875,6 +876,7 @@ type Hostinfo struct { UserspaceRouter opt.Bool `json:",omitempty"` // if the client's subnet router is running in userspace (netstack) mode AppConnector opt.Bool `json:",omitempty"` // if the client is running the app-connector service ServicesHash string `json:",omitempty"` // opaque hash of the most recent list of tailnet services, change in hash indicates config should be fetched via c2n + ExitNodeID StableNodeID `json:",omitzero"` // the client’s selected exit node, empty when unselected. // Location represents geographical location data about a // Tailscale host. Location is optional and only set if diff --git a/tailcfg/tailcfg_clone.go b/tailcfg/tailcfg_clone.go index 412e1f38d18bc..95f8905b84e69 100644 --- a/tailcfg/tailcfg_clone.go +++ b/tailcfg/tailcfg_clone.go @@ -186,6 +186,7 @@ var _HostinfoCloneNeedsRegeneration = Hostinfo(struct { UserspaceRouter opt.Bool AppConnector opt.Bool ServicesHash string + ExitNodeID StableNodeID Location *Location TPM *TPMInfo StateEncrypted opt.Bool diff --git a/tailcfg/tailcfg_test.go b/tailcfg/tailcfg_test.go index 833314df8fd6d..addd2330ba239 100644 --- a/tailcfg/tailcfg_test.go +++ b/tailcfg/tailcfg_test.go @@ -67,6 +67,7 @@ func TestHostinfoEqual(t *testing.T) { "UserspaceRouter", "AppConnector", "ServicesHash", + "ExitNodeID", "Location", "TPM", "StateEncrypted", @@ -273,6 +274,21 @@ func TestHostinfoEqual(t *testing.T) { &Hostinfo{IngressEnabled: true}, false, }, + { + &Hostinfo{ExitNodeID: "stable-exit"}, + &Hostinfo{ExitNodeID: "stable-exit"}, + true, + }, + { + &Hostinfo{ExitNodeID: ""}, + &Hostinfo{}, + true, + }, + { + &Hostinfo{ExitNodeID: ""}, + &Hostinfo{ExitNodeID: "stable-exit"}, + false, + }, } for i, tt := range tests { got := tt.a.Equal(tt.b) diff --git a/tailcfg/tailcfg_view.go b/tailcfg/tailcfg_view.go index 7e82cd871c64a..c407800210a5e 100644 --- a/tailcfg/tailcfg_view.go +++ b/tailcfg/tailcfg_view.go @@ -300,6 +300,7 @@ func (v HostinfoView) Userspace() opt.Bool { return v.ж.User func (v HostinfoView) UserspaceRouter() opt.Bool { return v.ж.UserspaceRouter } func (v HostinfoView) AppConnector() opt.Bool { return v.ж.AppConnector } func (v HostinfoView) ServicesHash() string { return v.ж.ServicesHash } +func (v HostinfoView) ExitNodeID() StableNodeID { return v.ж.ExitNodeID } func (v HostinfoView) Location() LocationView { return v.ж.Location.View() } func (v HostinfoView) TPM() views.ValuePointer[TPMInfo] { return views.ValuePointerOf(v.ж.TPM) } @@ -345,6 +346,7 @@ var _HostinfoViewNeedsRegeneration = Hostinfo(struct { UserspaceRouter opt.Bool AppConnector opt.Bool ServicesHash string + ExitNodeID StableNodeID Location *Location TPM *TPMInfo StateEncrypted opt.Bool From 1ae6a97a7313b3412dc89618efffad3181a07997 Mon Sep 17 00:00:00 2001 From: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> Date: Tue, 22 Jul 2025 21:13:25 -0400 Subject: [PATCH 243/263] cmd/tailscale/cli: add advertise command to advertise a node as service proxy to tailnet (#16620) This commit adds a advertise subcommand for tailscale serve, that would declare the node as a service proxy for a service. This command only adds the service to node's list of advertised service, but doesn't modify the list of services currently advertised. Fixes tailscale/corp#28016 Signed-off-by: KevinLiang10 <37811973+KevinLiang10@users.noreply.github.com> --- cmd/tailscale/cli/serve_v2.go | 33 ++++++++++++++++++++++++++---- cmd/tailscale/cli/serve_v2_test.go | 14 ++++++------- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/cmd/tailscale/cli/serve_v2.go b/cmd/tailscale/cli/serve_v2.go index 056bfabb0a202..91a23697035a8 100644 --- a/cmd/tailscale/cli/serve_v2.go +++ b/cmd/tailscale/cli/serve_v2.go @@ -220,6 +220,16 @@ func newServeV2Command(e *serveEnv, subcmd serveMode) *ffcli.Command { LongHelp: "Remove all handlers configured for the specified service.", Exec: e.runServeClear, }, + { + Name: "advertise", + ShortUsage: fmt.Sprintf("tailscale %s advertise ", info.Name), + ShortHelp: "Advertise this node as a service proxy to the tailnet", + LongHelp: "Advertise this node as a service proxy to the tailnet. This command is used\n" + + "to make the current node be considered as a service host for a service. This is\n" + + "useful to bring a service back after it has been drained. (i.e. after running \n" + + "`tailscale serve drain `). This is not needed if you are using `tailscale serve` to initialize a service.", + Exec: e.runServeAdvertise, + }, }, } } @@ -401,7 +411,7 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { return err } if forService { - e.addServiceToPrefs(ctx, svcName.String()) + e.addServiceToPrefs(ctx, svcName) } target := "" if len(args) > 0 { @@ -442,16 +452,16 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } } -func (e *serveEnv) addServiceToPrefs(ctx context.Context, serviceName string) error { +func (e *serveEnv) addServiceToPrefs(ctx context.Context, serviceName tailcfg.ServiceName) error { prefs, err := e.lc.GetPrefs(ctx) if err != nil { return fmt.Errorf("error getting prefs: %w", err) } advertisedServices := prefs.AdvertiseServices - if slices.Contains(advertisedServices, serviceName) { + if slices.Contains(advertisedServices, serviceName.String()) { return nil // already advertised } - advertisedServices = append(advertisedServices, serviceName) + advertisedServices = append(advertisedServices, serviceName.String()) _, err = e.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ AdvertiseServicesSet: true, Prefs: ipn.Prefs{ @@ -526,6 +536,21 @@ func (e *serveEnv) runServeClear(ctx context.Context, args []string) error { return e.lc.SetServeConfig(ctx, sc) } +func (e *serveEnv) runServeAdvertise(ctx context.Context, args []string) error { + if len(args) == 0 { + return fmt.Errorf("error: missing service name argument") + } + if len(args) != 1 { + fmt.Fprintf(Stderr, "error: invalid number of arguments\n\n") + return errHelp + } + svc := tailcfg.ServiceName(args[0]) + if err := svc.Validate(); err != nil { + return fmt.Errorf("invalid service name: %w", err) + } + return e.addServiceToPrefs(ctx, svc) +} + const backgroundExistsMsg = "background configuration already exists, use `tailscale %s --%s=%d off` to remove the existing configuration" // validateConfig checks if the serve config is valid to serve the type wanted on the port. diff --git a/cmd/tailscale/cli/serve_v2_test.go b/cmd/tailscale/cli/serve_v2_test.go index 95bf5b1012f8c..1deeaf3eaa9b5 100644 --- a/cmd/tailscale/cli/serve_v2_test.go +++ b/cmd/tailscale/cli/serve_v2_test.go @@ -1167,24 +1167,24 @@ func TestCleanURLPath(t *testing.T) { func TestAddServiceToPrefs(t *testing.T) { tests := []struct { name string - dnsName string + svcName tailcfg.ServiceName startServices []string expected []string }{ { name: "add service to empty prefs", - dnsName: "svc:foo", + svcName: "svc:foo", expected: []string{"svc:foo"}, }, { name: "add service to existing prefs", - dnsName: "svc:bar", + svcName: "svc:bar", startServices: []string{"svc:foo"}, expected: []string{"svc:foo", "svc:bar"}, }, { name: "add existing service to prefs", - dnsName: "svc:foo", + svcName: "svc:foo", startServices: []string{"svc:foo"}, expected: []string{"svc:foo"}, }, @@ -1200,12 +1200,12 @@ func TestAddServiceToPrefs(t *testing.T) { }, }) e := &serveEnv{lc: lc, bg: bgBoolFlag{true, false}} - err := e.addServiceToPrefs(ctx, tt.dnsName) + err := e.addServiceToPrefs(ctx, tt.svcName) if err != nil { - t.Fatalf("addServiceToPrefs(%q) returned unexpected error: %v", tt.dnsName, err) + t.Fatalf("addServiceToPrefs(%q) returned unexpected error: %v", tt.svcName, err) } if !slices.Equal(lc.prefs.AdvertiseServices, tt.expected) { - t.Errorf("addServiceToPrefs(%q) = %v, want %v", tt.dnsName, lc.prefs.AdvertiseServices, tt.expected) + t.Errorf("addServiceToPrefs(%q) = %v, want %v", tt.svcName, lc.prefs.AdvertiseServices, tt.expected) } }) } From f1f334b23d4891d5195442b4581e72febff17de4 Mon Sep 17 00:00:00 2001 From: Mike O'Driscoll Date: Wed, 23 Jul 2025 11:25:05 -0400 Subject: [PATCH 244/263] flake.lock/go.mod.sri: update flake version info (#16631) Update nixpkgs-unstable to include newer golang to satisfy go.mod requirement of 1.24.4 Update vendor hash to current. Updates #15015 Signed-off-by: Mike O'Driscoll --- .github/workflows/update-flake.yml | 2 +- flake.lock | 6 +++--- flake.nix | 3 ++- go.mod.sri | 2 +- shell.nix | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/update-flake.yml b/.github/workflows/update-flake.yml index 61a09cea1c990..1968c68302d37 100644 --- a/.github/workflows/update-flake.yml +++ b/.github/workflows/update-flake.yml @@ -8,7 +8,7 @@ on: - main paths: - go.mod - - .github/workflows/update-flakes.yml + - .github/workflows/update-flake.yml workflow_dispatch: concurrency: diff --git a/flake.lock b/flake.lock index 05b0f303e6433..87f234e3ecab1 100644 --- a/flake.lock +++ b/flake.lock @@ -36,11 +36,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1743938762, - "narHash": "sha256-UgFYn8sGv9B8PoFpUfCa43CjMZBl1x/ShQhRDHBFQdI=", + "lastModified": 1753151930, + "narHash": "sha256-XSQy6wRKHhRe//iVY5lS/ZpI/Jn6crWI8fQzl647wCg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "74a40410369a1c35ee09b8a1abee6f4acbedc059", + "rev": "83e677f31c84212343f4cc553bab85c2efcad60a", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 2f920bfd40ba5..17d263a8dd3c9 100644 --- a/flake.nix +++ b/flake.nix @@ -130,4 +130,5 @@ in flake-utils.lib.eachDefaultSystem (system: flakeForSystem nixpkgs system); } -# nix-direnv cache busting line: sha256-av4kr09rjNRmag94ziNjJuI/cg8b8lAD3Tk24t/ezH4= +# nix-direnv cache busting line: sha256-4QTSspHLYJfzlontQ7msXyOB5gzq7ZwSvWmKuYY5klA= + diff --git a/go.mod.sri b/go.mod.sri index 6c8357e0468ba..845086191699a 100644 --- a/go.mod.sri +++ b/go.mod.sri @@ -1 +1 @@ -sha256-av4kr09rjNRmag94ziNjJuI/cg8b8lAD3Tk24t/ezH4= \ No newline at end of file +sha256-4QTSspHLYJfzlontQ7msXyOB5gzq7ZwSvWmKuYY5klA= diff --git a/shell.nix b/shell.nix index bb8eacb67ee18..2eb5b441a2d87 100644 --- a/shell.nix +++ b/shell.nix @@ -16,4 +16,4 @@ ) { src = ./.; }).shellNix -# nix-direnv cache busting line: sha256-av4kr09rjNRmag94ziNjJuI/cg8b8lAD3Tk24t/ezH4= +# nix-direnv cache busting line: sha256-4QTSspHLYJfzlontQ7msXyOB5gzq7ZwSvWmKuYY5klA= From 1ef8fbf4705637ee73c46300566e3df56c4885e4 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Wed, 23 Jul 2025 11:50:42 -0700 Subject: [PATCH 245/263] ipn/ipnlocal: send Hostinfo after resolveExitNode for "auto:any" (#16632) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In #16625, I introduced a mechanism for sending the selected exit node to Control via tailcfg.Hostinfo.ExitNodeID as part of the MapRequest. @nickkhyl pointed out that LocalBackend.doSetHostinfoFilterServices needs to be triggered in order to actually send this update. This patch adds that command. It also prevents the client from sending "auto:any" in that field, because that’s not a real exit node ID. This patch also fills in some missing checks in TestConfigureExitNode. Updates tailscale/corp#30536 Signed-off-by: Simon Law --- ipn/ipnlocal/local.go | 40 +++++++++++++++++----- ipn/ipnlocal/local_test.go | 69 +++++++++++++++++++------------------- 2 files changed, 66 insertions(+), 43 deletions(-) diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index ce0f4f6873d74..7154b942c1690 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -5614,9 +5614,22 @@ func (b *LocalBackend) applyPrefsToHostinfoLocked(hi *tailcfg.Hostinfo, prefs ip hi.AppConnector.Set(prefs.AppConnector().Advertise) // The [tailcfg.Hostinfo.ExitNodeID] field tells control which exit node - // was selected, if any. Since [LocalBackend.resolveExitNodeIPLocked] - // has already run, there is no need to consult [ipn.Prefs.ExitNodeIP]. - hi.ExitNodeID = prefs.ExitNodeID() + // was selected, if any. + // + // If auto exit node is enabled (via [ipn.Prefs.AutoExitNode] or + // [syspolicy.ExitNodeID]), or an exit node is specified by ExitNodeIP + // instead of ExitNodeID , and we don't yet have enough info to resolve + // it (usually due to missing netmap or net report), then ExitNodeID in + // the prefs may be invalid (typically, [unresolvedExitNodeID]) until + // the netmap is available. + // + // In this case, we shouldn't update the Hostinfo with the bogus + // ExitNodeID here; [LocalBackend.ResolveExitNode] will be called once + // the netmap and/or net report have been received to both pick the exit + // node and notify control of the change. + if sid := prefs.ExitNodeID(); sid != unresolvedExitNodeID { + hi.ExitNodeID = prefs.ExitNodeID() + } } // enterState transitions the backend into newState, updating internal @@ -6117,9 +6130,10 @@ func (b *LocalBackend) RefreshExitNode() { } } -// resolveExitNode determines which exit node to use based on the current -// prefs and netmap. It updates the exit node ID in the prefs if needed, -// sends a notification to clients, and returns true if the exit node has changed. +// resolveExitNode determines which exit node to use based on the current prefs +// and netmap. It updates the exit node ID in the prefs if needed, updates the +// exit node ID in the hostinfo if needed, sends a notification to clients, and +// returns true if the exit node has changed. // // It is the caller's responsibility to reconfigure routes and actually // start using the selected exit node, if needed. @@ -6142,8 +6156,18 @@ func (b *LocalBackend) resolveExitNode() (changed bool) { b.logf("failed to save exit node changes: %v", err) } - // Send the resolved exit node to Control via Hostinfo. - b.hostinfo.ExitNodeID = prefs.ExitNodeID + // Send the resolved exit node to control via [tailcfg.Hostinfo]. + // [LocalBackend.applyPrefsToHostinfoLocked] usually sets the Hostinfo, + // but it deferred until this point because there was a bogus ExitNodeID + // in the prefs. + // + // TODO(sfllaw): Mutating b.hostinfo here is undesirable, mutating + // in-place doubly so. + sid := prefs.ExitNodeID + if sid != unresolvedExitNodeID && b.hostinfo.ExitNodeID != sid { + b.hostinfo.ExitNodeID = sid + b.goTracker.Go(b.doSetHostinfoFilterServices) + } b.sendToLocked(ipn.Notify{Prefs: ptr.To(prefs.View())}, allClients) return true diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index da6fc8b4a5725..dd2837022f064 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -627,7 +627,7 @@ func TestConfigureExitNode(t *testing.T) { wantChangePrefsErr error // if non-nil, the error we expect from [LocalBackend.EditPrefsAs] wantPrefs ipn.Prefs wantExitNodeToggleErr error // if non-nil, the error we expect from [LocalBackend.SetUseExitNodeEnabled] - wantHostinfoExitNodeID *tailcfg.StableNodeID + wantHostinfoExitNodeID tailcfg.StableNodeID }{ { name: "exit-node-id-via-prefs", // set exit node ID via prefs @@ -644,7 +644,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "exit-node-ip-via-prefs", // set exit node IP via prefs (should be resolved to an ID) @@ -661,7 +661,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "auto-exit-node-via-prefs/any", // set auto exit node via prefs @@ -679,7 +679,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "auto-exit-node-via-prefs/set-exit-node-id-via-prefs", // setting exit node ID explicitly should disable auto exit node @@ -699,7 +699,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode2.StableID(), AutoExitNode: "", // should be unset }, - wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), + wantHostinfoExitNodeID: exitNode2.StableID(), }, { name: "auto-exit-node-via-prefs/any/no-report", // set auto exit node via prefs, but no report means we can't resolve the exit node ID @@ -716,7 +716,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: unresolvedExitNodeID, // cannot resolve; traffic will be dropped AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(unresolvedExitNodeID), + wantHostinfoExitNodeID: "", }, { name: "auto-exit-node-via-prefs/any/no-netmap", // similarly, but without a netmap (no exit node should be selected) @@ -733,7 +733,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: unresolvedExitNodeID, // cannot resolve; traffic will be dropped AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(unresolvedExitNodeID), + wantHostinfoExitNodeID: "", }, { name: "auto-exit-node-via-prefs/foo", // set auto exit node via prefs with an unknown/unsupported expression @@ -751,7 +751,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" AutoExitNode: "foo", }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "auto-exit-node-via-prefs/off", // toggle the exit node off after it was set to "any" @@ -771,7 +771,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "", InternalExitNodePrior: "auto:any", }, - wantHostinfoExitNodeID: ptr.To(tailcfg.StableNodeID("")), + wantHostinfoExitNodeID: "", }, { name: "auto-exit-node-via-prefs/on", // toggle the exit node on @@ -788,7 +788,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "any", InternalExitNodePrior: "auto:any", }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "id-via-policy", // set exit node ID via syspolicy @@ -801,7 +801,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "id-via-policy/cannot-override-via-prefs/by-id", // syspolicy should take precedence over prefs @@ -820,7 +820,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), wantChangePrefsErr: errManagedByPolicy, }, { @@ -840,7 +840,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), wantChangePrefsErr: errManagedByPolicy, }, { @@ -860,7 +860,8 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode1.StableID(), }, - wantChangePrefsErr: errManagedByPolicy, + wantHostinfoExitNodeID: exitNode1.StableID(), + wantChangePrefsErr: errManagedByPolicy, }, { name: "ip-via-policy", // set exit node IP via syspolicy (should be resolved to an ID) @@ -873,7 +874,7 @@ func TestConfigureExitNode(t *testing.T) { ControlURL: controlURL, ExitNodeID: exitNode2.StableID(), }, - wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), + wantHostinfoExitNodeID: exitNode2.StableID(), }, { name: "auto-any-via-policy", // set auto exit node via syspolicy (an exit node should be selected) @@ -888,7 +889,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "auto-any-via-policy/no-report", // set auto exit node via syspolicy without a netcheck report (no exit node should be selected) @@ -903,7 +904,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: unresolvedExitNodeID, AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(unresolvedExitNodeID), + wantHostinfoExitNodeID: "", }, { name: "auto-any-via-policy/no-netmap", // similarly, but without a netmap (no exit node should be selected) @@ -918,7 +919,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: unresolvedExitNodeID, AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(unresolvedExitNodeID), + wantHostinfoExitNodeID: "", }, { name: "auto-any-via-policy/no-netmap/with-existing", // set auto exit node via syspolicy without a netmap, but with a previously set exit node ID @@ -935,7 +936,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode2.StableID(), AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), + wantHostinfoExitNodeID: exitNode2.StableID(), }, { name: "auto-any-via-policy/no-netmap/with-allowed-existing", // same, but now with a syspolicy setting that explicitly allows the existing exit node ID @@ -954,7 +955,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode2.StableID(), AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), + wantHostinfoExitNodeID: exitNode2.StableID(), }, { name: "auto-any-via-policy/no-netmap/with-disallowed-existing", // same, but now with a syspolicy setting that does not allow the existing exit node ID @@ -973,7 +974,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: unresolvedExitNodeID, // we don't have a netmap yet, and the current exit node ID is not allowed; block traffic AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(unresolvedExitNodeID), + wantHostinfoExitNodeID: "", }, { name: "auto-any-via-policy/with-netmap/with-allowed-existing", // same, but now with a syspolicy setting that does not allow the existing exit node ID @@ -992,7 +993,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode2.StableID(), // we have a netmap; switch to the best allowed exit node AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), + wantHostinfoExitNodeID: exitNode2.StableID(), }, { name: "auto-any-via-policy/with-netmap/switch-to-better", // if all exit nodes are allowed, switch to the best one once we have a netmap @@ -1008,7 +1009,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), // switch to the best exit node AutoExitNode: "any", }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "auto-foo-via-policy", // set auto exit node via syspolicy with an unknown/unsupported expression @@ -1023,7 +1024,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" AutoExitNode: "foo", }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "auto-foo-via-edit-prefs", // set auto exit node via EditPrefs with an unknown/unsupported expression @@ -1041,7 +1042,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" AutoExitNode: "foo", }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "auto-any-via-policy/toggle-off", // cannot toggle off the exit node if it was set via syspolicy @@ -1059,7 +1060,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "any", InternalExitNodePrior: "", }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "auto-any-via-policy/allow-override/change", // changing the exit node is allowed by [syspolicy.AllowExitNodeOverride] @@ -1081,7 +1082,7 @@ func TestConfigureExitNode(t *testing.T) { ExitNodeID: exitNode2.StableID(), // overridden by user AutoExitNode: "", // cleared, as we are setting the exit node ID explicitly }, - wantHostinfoExitNodeID: ptr.To(exitNode2.StableID()), + wantHostinfoExitNodeID: exitNode2.StableID(), }, { name: "auto-any-via-policy/allow-override/clear", // clearing the exit node ID is not allowed by [syspolicy.AllowExitNodeOverride] @@ -1105,7 +1106,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "any", InternalExitNodePrior: "", }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "auto-any-via-policy/allow-override/toggle-off", // similarly, toggling off the exit node is not allowed even with [syspolicy.AllowExitNodeOverride] @@ -1124,7 +1125,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "any", InternalExitNodePrior: "", }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, { name: "auto-any-via-initial-prefs/no-netmap/clear-auto-exit-node", @@ -1145,7 +1146,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "", // cleared ExitNodeID: "", // has never been resolved, so it should be cleared as well }, - wantHostinfoExitNodeID: ptr.To(tailcfg.StableNodeID("")), + wantHostinfoExitNodeID: "", }, { name: "auto-any-via-initial-prefs/with-netmap/clear-auto-exit-node", @@ -1166,7 +1167,7 @@ func TestConfigureExitNode(t *testing.T) { AutoExitNode: "", // cleared ExitNodeID: exitNode1.StableID(), // a resolved exit node ID should be retained }, - wantHostinfoExitNodeID: ptr.To(exitNode1.StableID()), + wantHostinfoExitNodeID: exitNode1.StableID(), }, } syspolicy.RegisterWellKnownSettingsForTest(t) @@ -1229,10 +1230,8 @@ func TestConfigureExitNode(t *testing.T) { } // And check Hostinfo. - if tt.wantHostinfoExitNodeID != nil { - if got := lb.hostinfo.ExitNodeID; got != *tt.wantHostinfoExitNodeID { - t.Errorf("Hostinfo.ExitNodeID got %v, want %v", got, *tt.wantHostinfoExitNodeID) - } + if got := lb.hostinfo.ExitNodeID; got != tt.wantHostinfoExitNodeID { + t.Errorf("Hostinfo.ExitNodeID got %s, want %s", got, tt.wantHostinfoExitNodeID) } }) } From 179745b83ed6d687bdc9d501ccdbfdec1cb3f9d7 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 23 Jul 2025 12:30:04 -0700 Subject: [PATCH 246/263] wgengine/magicsock: update discoInfo docs (#16638) discoInfo is also used for holding peer relay server disco keys. Updates #cleanup Signed-off-by: Jordan Whited --- wgengine/magicsock/magicsock.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index ee0ee40ca1d13..fb7f5edcbd14f 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -3907,14 +3907,18 @@ type epAddrEndpointCache struct { } // discoInfo is the info and state for the DiscoKey -// in the Conn.discoInfo map key. +// in the [Conn.discoInfo] and [relayManager.discoInfoByServerDisco] map keys. +// +// When the disco protocol is used to handshake with a peer relay server, the +// corresponding discoInfo is held in [relayManager.discoInfoByServerDisco] +// instead of [Conn.discoInfo]. // // Note that a DiscoKey does not necessarily map to exactly one // node. In the case of shared nodes and users switching accounts, two // nodes in the NetMap may legitimately have the same DiscoKey. As // such, no fields in here should be considered node-specific. type discoInfo struct { - // discoKey is the same as the Conn.discoInfo map key, + // discoKey is the same as the corresponding map key, // just so you can pass around a *discoInfo alone. // Not modified once initialized. discoKey key.DiscoPublic @@ -3925,11 +3929,13 @@ type discoInfo struct { // sharedKey is the precomputed key for communication with the // peer that has the DiscoKey used to look up this *discoInfo in - // Conn.discoInfo. + // the corresponding map. // Not modified once initialized. sharedKey key.DiscoShared - // Mutable fields follow, owned by Conn.mu: + // Mutable fields follow, owned by [Conn.mu]. These are irrelevant when + // discoInfo is a peer relay server disco key in the + // [relayManager.discoInfoByServerDisco] map: // lastPingFrom is the src of a ping for discoKey. lastPingFrom epAddr From c87f44b687e4b549d30fe420d45bfeebf47e5cd1 Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Tue, 22 Jul 2025 18:57:24 -0500 Subject: [PATCH 247/263] cmd/tailscale/cli: use DNS name instead of Location to hide Mullvad exit nodes from status output Previously, we used a non-nil Location as an indicator that a peer is a Mullvad exit node. However, this is not, or no longer, reliable, since regular exit nodes may also have a non-nil Location, such as when traffic steering is enabled for a tailnet. In this PR, we update the plaintext `tailscale status` output to omit only Mullvad exit nodes, rather than all exit nodes with a non-nil Location. The JSON output remains unchanged and continues to include all peers. Updates tailscale/corp#30614 Signed-off-by: Nick Khyl --- cmd/tailscale/cli/status.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/tailscale/cli/status.go b/cmd/tailscale/cli/status.go index 39e6f9fbdfd8a..726606109aa15 100644 --- a/cmd/tailscale/cli/status.go +++ b/cmd/tailscale/cli/status.go @@ -70,6 +70,8 @@ var statusArgs struct { peers bool // in CLI mode, show status of peer machines } +const mullvadTCD = "mullvad.ts.net." + func runStatus(ctx context.Context, args []string) error { if len(args) > 0 { return errors.New("unexpected non-flag arguments to 'tailscale status'") @@ -212,9 +214,8 @@ func runStatus(ctx context.Context, args []string) error { if ps.ShareeNode { continue } - if ps.Location != nil && ps.ExitNodeOption && !ps.ExitNode { - // Location based exit nodes are only shown with the - // `exit-node list` command. + if ps.ExitNodeOption && !ps.ExitNode && strings.HasSuffix(ps.DNSName, mullvadTCD) { + // Mullvad exit nodes are only shown with the `exit-node list` command. locBasedExitNode = true continue } From 758dfe72035fcb2ce00e6ea7693e6f1a11eb135c Mon Sep 17 00:00:00 2001 From: Aaron Klotz Date: Thu, 24 Jul 2025 11:56:43 -0600 Subject: [PATCH 248/263] VERSION.txt: this is v1.86.0 Signed-off-by: Aaron Klotz --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index f288d11142d11..b7844a6ffdcb8 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.85.0 +1.86.0 From fdcff402fbed97be663d097fc16fb42211f18f80 Mon Sep 17 00:00:00 2001 From: Aaron Klotz Date: Fri, 25 Jul 2025 11:54:40 -0600 Subject: [PATCH 249/263] VERSION.txt: this is v1.86.1 Signed-off-by: Aaron Klotz --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index b7844a6ffdcb8..e52a44e47822a 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.86.0 +1.86.1 From 91d65e03e85b0d64b7483e5d2fd00c62ea25400c Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Mon, 28 Jul 2025 14:11:30 +0100 Subject: [PATCH 250/263] k8s-operator: handle multiple WebSocket frames per read (#16678) (#16679) Cherry picks bug fix #16678 and flake fix #16680 onto the 1.86 release branch. When kubectl starts an interactive attach session, it sends 2 resize messages in quick succession. It seems that particularly in HTTP mode, we often receive both of these WebSocket frames from the underlying connection in a single read. However, our parser currently assumes 0-1 frames per read, and leaves the second frame in the read buffer until the next read from the underlying connection. It doesn't take long after that before we end up failing to skip a control message as we normally should, and then we parse a control message as though it will have a stream ID (part of the Kubernetes protocol) and error out. Instead, we should keep parsing frames from the read buffer for as long as we're able to parse complete frames, so this commit refactors the messages parsing logic into a loop based on the contents of the read buffer being non-empty. k/k staging/src/k8s.io/kubectl/pkg/cmd/attach/attach.go for full details of the resize messages. There are at least a couple more multiple-frame read edge cases we should handle, but this commit is very conservatively fixing a single observed issue to make it a low-risk candidate for cherry picking. Updates #13358 Change-Id: Iafb91ad1cbeed9c5231a1525d4563164fc1f002f Signed-off-by: Tom Proctor --- k8s-operator/api-proxy/proxy.go | 6 +- k8s-operator/sessionrecording/hijacker.go | 1 - k8s-operator/sessionrecording/ws/conn.go | 97 ++++++++++--------- k8s-operator/sessionrecording/ws/conn_test.go | 68 +++++++++---- k8s-operator/sessionrecording/ws/message.go | 4 +- 5 files changed, 109 insertions(+), 67 deletions(-) diff --git a/k8s-operator/api-proxy/proxy.go b/k8s-operator/api-proxy/proxy.go index c648e1622537d..ff0373270b2c0 100644 --- a/k8s-operator/api-proxy/proxy.go +++ b/k8s-operator/api-proxy/proxy.go @@ -114,8 +114,9 @@ func (ap *APIServerProxy) Run(ctx context.Context) error { mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/attach", ap.serveAttachWS) ap.hs = &http.Server{ - Handler: mux, - ErrorLog: zap.NewStdLog(ap.log.Desugar()), + Handler: mux, + ErrorLog: zap.NewStdLog(ap.log.Desugar()), + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), } mode := "noauth" @@ -140,7 +141,6 @@ func (ap *APIServerProxy) Run(ctx context.Context) error { GetCertificate: ap.lc.GetCertificate, NextProtos: []string{"http/1.1"}, } - ap.hs.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) } else { var err error tsLn, err = ap.ts.Listen("tcp", ":80") diff --git a/k8s-operator/sessionrecording/hijacker.go b/k8s-operator/sessionrecording/hijacker.go index 675a9b1ddacc6..f816a8483f273 100644 --- a/k8s-operator/sessionrecording/hijacker.go +++ b/k8s-operator/sessionrecording/hijacker.go @@ -236,7 +236,6 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, if err := lc.Close(); err != nil { h.log.Infof("error closing recorder connections: %v", err) } - return }() return lc, nil } diff --git a/k8s-operator/sessionrecording/ws/conn.go b/k8s-operator/sessionrecording/ws/conn.go index 0d8aefaace52e..a34379658caa2 100644 --- a/k8s-operator/sessionrecording/ws/conn.go +++ b/k8s-operator/sessionrecording/ws/conn.go @@ -148,6 +148,8 @@ func (c *conn) Read(b []byte) (int, error) { return 0, nil } + // TODO(tomhjp): If we get multiple frames in a single Read with different + // types, we may parse the second frame with the wrong type. typ := messageType(opcode(b)) if (typ == noOpcode && c.readMsgIsIncomplete()) || c.readBufHasIncompleteFragment() { // subsequent fragment if typ, err = c.curReadMsgType(); err != nil { @@ -157,6 +159,8 @@ func (c *conn) Read(b []byte) (int, error) { // A control message can not be fragmented and we are not interested in // these messages. Just return. + // TODO(tomhjp): If we get multiple frames in a single Read, we may skip + // some non-control messages. if isControlMessage(typ) { return n, nil } @@ -169,62 +173,65 @@ func (c *conn) Read(b []byte) (int, error) { return n, nil } - readMsg := &message{typ: typ} // start a new message... - // ... or pick up an already started one if the previous fragment was not final. - if c.readMsgIsIncomplete() || c.readBufHasIncompleteFragment() { - readMsg = c.currentReadMsg - } - if _, err := c.readBuf.Write(b[:n]); err != nil { return 0, fmt.Errorf("[unexpected] error writing message contents to read buffer: %w", err) } - ok, err := readMsg.Parse(c.readBuf.Bytes(), c.log) - if err != nil { - return 0, fmt.Errorf("error parsing message: %v", err) - } - if !ok { // incomplete fragment - return n, nil - } - c.readBuf.Next(len(readMsg.raw)) - - if readMsg.isFinalized && !c.readMsgIsIncomplete() { - // we want to send stream resize messages for terminal sessions - // Stream IDs for websocket streams are static. - // https://github.com/kubernetes/client-go/blob/v0.30.0-rc.1/tools/remotecommand/websocket.go#L218 - if readMsg.streamID.Load() == remotecommand.StreamResize && c.hasTerm { - var msg tsrecorder.ResizeMsg - if err = json.Unmarshal(readMsg.payload, &msg); err != nil { - return 0, fmt.Errorf("error umarshalling resize message: %w", err) - } + for c.readBuf.Len() != 0 { + readMsg := &message{typ: typ} // start a new message... + // ... or pick up an already started one if the previous fragment was not final. + if c.readMsgIsIncomplete() { + readMsg = c.currentReadMsg + } - c.ch.Width = msg.Width - c.ch.Height = msg.Height - - var isInitialResize bool - c.writeCastHeaderOnce.Do(func() { - isInitialResize = true - // If this is a session with a terminal attached, - // we must wait for the terminal width and - // height to be parsed from a resize message - // before sending CastHeader, else tsrecorder - // will not be able to play this recording. - err = c.rec.WriteCastHeader(c.ch) - close(c.initialCastHeaderSent) - }) - if err != nil { - return 0, fmt.Errorf("error writing CastHeader: %w", err) - } + ok, err := readMsg.Parse(c.readBuf.Bytes(), c.log) + if err != nil { + return 0, fmt.Errorf("error parsing message: %v", err) + } + if !ok { // incomplete fragment + return n, nil + } + c.readBuf.Next(len(readMsg.raw)) + + if readMsg.isFinalized && !c.readMsgIsIncomplete() { + // we want to send stream resize messages for terminal sessions + // Stream IDs for websocket streams are static. + // https://github.com/kubernetes/client-go/blob/v0.30.0-rc.1/tools/remotecommand/websocket.go#L218 + if readMsg.streamID.Load() == remotecommand.StreamResize && c.hasTerm { + var msg tsrecorder.ResizeMsg + if err = json.Unmarshal(readMsg.payload, &msg); err != nil { + return 0, fmt.Errorf("error umarshalling resize message: %w", err) + } + + c.ch.Width = msg.Width + c.ch.Height = msg.Height + + var isInitialResize bool + c.writeCastHeaderOnce.Do(func() { + isInitialResize = true + // If this is a session with a terminal attached, + // we must wait for the terminal width and + // height to be parsed from a resize message + // before sending CastHeader, else tsrecorder + // will not be able to play this recording. + err = c.rec.WriteCastHeader(c.ch) + close(c.initialCastHeaderSent) + }) + if err != nil { + return 0, fmt.Errorf("error writing CastHeader: %w", err) + } - if !isInitialResize { - if err := c.rec.WriteResize(msg.Height, msg.Width); err != nil { - return 0, fmt.Errorf("error writing resize message: %w", err) + if !isInitialResize { + if err := c.rec.WriteResize(msg.Height, msg.Width); err != nil { + return 0, fmt.Errorf("error writing resize message: %w", err) + } } } } + + c.currentReadMsg = readMsg } - c.currentReadMsg = readMsg return n, nil } diff --git a/k8s-operator/sessionrecording/ws/conn_test.go b/k8s-operator/sessionrecording/ws/conn_test.go index f29154c622602..f2fd4ea55f554 100644 --- a/k8s-operator/sessionrecording/ws/conn_test.go +++ b/k8s-operator/sessionrecording/ws/conn_test.go @@ -9,6 +9,7 @@ import ( "context" "fmt" "reflect" + "runtime/debug" "testing" "time" @@ -58,15 +59,39 @@ func Test_conn_Read(t *testing.T) { wantCastHeaderHeight: 20, }, { - name: "two_reads_resize_message", - inputs: [][]byte{{0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x3a, 0x32, 0x30, 0x7d}}, + name: "resize_data_frame_two_in_one_read", + inputs: [][]byte{ + fmt.Appendf(nil, "%s%s", + append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...), + append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...), + ), + }, + wantRecorded: append(fakes.AsciinemaCastHeaderMsg(t, 10, 20), fakes.AsciinemaCastResizeMsg(t, 10, 20)...), + wantCastHeaderWidth: 10, + wantCastHeaderHeight: 20, + }, + { + name: "two_reads_resize_message", + inputs: [][]byte{ + // op, len, stream ID, `{"width` + {0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, + // op, len, stream ID, `:10,"height":20}` + {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x3a, 0x32, 0x30, 0x7d}, + }, wantCastHeaderWidth: 10, wantCastHeaderHeight: 20, wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), }, { - name: "three_reads_resize_message_with_split_fragment", - inputs: [][]byte{{0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74}, {0x22, 0x3a, 0x32, 0x30, 0x7d}}, + name: "three_reads_resize_message_with_split_fragment", + inputs: [][]byte{ + // op, len, stream ID, `{"width"` + {0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, + // op, len, stream ID, `:10,"height` + {0x00, 0x0c, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74}, + // op, len, stream ID, `":20}` + {0x80, 0x06, 0x4, 0x22, 0x3a, 0x32, 0x30, 0x7d}, + }, wantCastHeaderWidth: 10, wantCastHeaderHeight: 20, wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), @@ -260,19 +285,28 @@ func Test_conn_WriteRand(t *testing.T) { sr := &fakes.TestSessionRecorder{} rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar()) for i := range 100 { - tc := &fakes.TestConn{} - c := &conn{ - Conn: tc, - log: zl.Sugar(), - rec: rec, - } - bb := fakes.RandomBytes(t) - for j, input := range bb { - f := func() { - c.Write(input) + t.Run(fmt.Sprintf("test_%d", i), func(t *testing.T) { + tc := &fakes.TestConn{} + c := &conn{ + Conn: tc, + log: zl.Sugar(), + rec: rec, + + ctx: context.Background(), // ctx must be non-nil. + initialCastHeaderSent: make(chan struct{}), } - testPanic(t, f, fmt.Sprintf("[%d %d] Write: panic parsing input of length %d first bytes %b current write message %+#v", i, j, len(input), firstBytes(input), c.currentWriteMsg)) - } + // Never block for random data. + c.writeCastHeaderOnce.Do(func() { + close(c.initialCastHeaderSent) + }) + bb := fakes.RandomBytes(t) + for j, input := range bb { + f := func() { + c.Write(input) + } + testPanic(t, f, fmt.Sprintf("[%d %d] Write: panic parsing input of length %d first bytes %b current write message %+#v", i, j, len(input), firstBytes(input), c.currentWriteMsg)) + } + }) } } @@ -280,7 +314,7 @@ func testPanic(t *testing.T, f func(), msg string) { t.Helper() defer func() { if r := recover(); r != nil { - t.Fatal(msg, r) + t.Fatal(msg, r, string(debug.Stack())) } }() f() diff --git a/k8s-operator/sessionrecording/ws/message.go b/k8s-operator/sessionrecording/ws/message.go index 713febec76ae8..35667ae21a5d0 100644 --- a/k8s-operator/sessionrecording/ws/message.go +++ b/k8s-operator/sessionrecording/ws/message.go @@ -7,10 +7,10 @@ package ws import ( "encoding/binary" + "errors" "fmt" "sync/atomic" - "github.com/pkg/errors" "go.uber.org/zap" "golang.org/x/net/websocket" @@ -139,6 +139,8 @@ func (msg *message) Parse(b []byte, log *zap.SugaredLogger) (bool, error) { return false, errors.New("[unexpected] received a message fragment with no stream ID") } + // Stream ID will be one of the constants from: + // https://github.com/kubernetes/kubernetes/blob/f9ed14bf9b1119a2e091f4b487a3b54930661034/staging/src/k8s.io/apimachinery/pkg/util/remotecommand/constants.go#L57-L64 streamID := uint32(msgPayload[0]) if !isInitialFragment && msg.streamID.Load() != streamID { return false, fmt.Errorf("[unexpected] received message fragments with mismatched streamIDs %d and %d", msg.streamID.Load(), streamID) From 4123469edfaf6cf3c8609952e83ba32993876f1d Mon Sep 17 00:00:00 2001 From: Nick Khyl Date: Mon, 28 Jul 2025 12:23:40 -0500 Subject: [PATCH 251/263] util/syspolicy/setting: use a custom marshaler for time.Duration jsonv2 now returns an error when you marshal or unmarshal a time.Duration without an explicit format flag. This is an intentional, temporary choice until the default [time.Duration] representation is decided (see golang/go#71631). setting.Snapshot can hold time.Duration values inside a map[string]any, so the jsonv2 update breaks marshaling. In this PR, we start using a custom marshaler until that decision is made or golang/go#71664 lets us specify the format explicitly. This fixes `tailscale syspolicy list` failing when KeyExpirationNotice or any other time.Duration policy setting is configured. Fixes #16683 Signed-off-by: Nick Khyl (cherry picked from commit 4df02bbb486d07b0ad23f59c4cb3675ab691e79b) --- util/syspolicy/setting/snapshot.go | 21 ++++++++++++++++++++- util/syspolicy/setting/snapshot_test.go | 12 ++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/util/syspolicy/setting/snapshot.go b/util/syspolicy/setting/snapshot.go index 087325a04c6f1..3a40785dce9de 100644 --- a/util/syspolicy/setting/snapshot.go +++ b/util/syspolicy/setting/snapshot.go @@ -9,6 +9,7 @@ import ( "maps" "slices" "strings" + "time" jsonv2 "github.com/go-json-experiment/json" "github.com/go-json-experiment/json/jsontext" @@ -152,6 +153,24 @@ var ( _ jsonv2.UnmarshalerFrom = (*Snapshot)(nil) ) +// As of 2025-07-28, jsonv2 no longer has a default representation for [time.Duration], +// so we need to provide a custom marshaler. +// +// This is temporary until the decision on the default representation is made +// (see https://github.com/golang/go/issues/71631#issuecomment-2981670799). +// +// In the future, we might either use the default representation (if compatible with +// [time.Duration.String]) or specify something like json.WithFormat[time.Duration]("units") +// when golang/go#71664 is implemented. +// +// TODO(nickkhyl): revisit this when the decision on the default [time.Duration] +// representation is made in golang/go#71631 and/or golang/go#71664 is implemented. +var formatDurationAsUnits = jsonv2.JoinOptions( + jsonv2.WithMarshalers(jsonv2.MarshalToFunc(func(e *jsontext.Encoder, t time.Duration) error { + return e.WriteToken(jsontext.String(t.String())) + })), +) + // MarshalJSONTo implements [jsonv2.MarshalerTo]. func (s *Snapshot) MarshalJSONTo(out *jsontext.Encoder) error { data := &snapshotJSON{} @@ -159,7 +178,7 @@ func (s *Snapshot) MarshalJSONTo(out *jsontext.Encoder) error { data.Summary = s.summary data.Settings = s.m } - return jsonv2.MarshalEncode(out, data) + return jsonv2.MarshalEncode(out, data, formatDurationAsUnits) } // UnmarshalJSONFrom implements [jsonv2.UnmarshalerFrom]. diff --git a/util/syspolicy/setting/snapshot_test.go b/util/syspolicy/setting/snapshot_test.go index d41b362f06976..19f014acaa831 100644 --- a/util/syspolicy/setting/snapshot_test.go +++ b/util/syspolicy/setting/snapshot_test.go @@ -491,6 +491,18 @@ func TestMarshalUnmarshalSnapshot(t *testing.T) { snapshot: NewSnapshot(map[Key]RawItem{"ListPolicy": RawItemOf([]string{"Value1", "Value2"})}), wantJSON: `{"Settings": {"ListPolicy": {"Value": ["Value1", "Value2"]}}}`, }, + { + name: "Duration/Zero", + snapshot: NewSnapshot(map[Key]RawItem{"DurationPolicy": RawItemOf(time.Duration(0))}), + wantJSON: `{"Settings": {"DurationPolicy": {"Value": "0s"}}}`, + wantBack: NewSnapshot(map[Key]RawItem{"DurationPolicy": RawItemOf("0s")}), + }, + { + name: "Duration/NonZero", + snapshot: NewSnapshot(map[Key]RawItem{"DurationPolicy": RawItemOf(2 * time.Hour)}), + wantJSON: `{"Settings": {"DurationPolicy": {"Value": "2h0m0s"}}}`, + wantBack: NewSnapshot(map[Key]RawItem{"DurationPolicy": RawItemOf("2h0m0s")}), + }, { name: "Empty/With-Summary", snapshot: NewSnapshot( From 9c7305074a29d7357411c12fc91659736052ab28 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Mon, 28 Jul 2025 09:01:41 -0700 Subject: [PATCH 252/263] net/portmapper: avert a panic when a mapping is not available (#16686) Ideally when we attempt to create a new port mapping, we should not return without error when no mapping is available. We already log these cases as unexpected, so this change is just to avoiding panicking dispatch on the invalid result in those cases. We still separately need to fix the underlying control flow. Updates #16662 Change-Id: I51e8a116b922b49eda45e31cd27f6b89dd51abc8 Signed-off-by: M. J. Fromberger (cherry picked from commit 5ce3845a021b8384814f8279546af80e9fddbf39) --- net/portmapper/portmapper.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/net/portmapper/portmapper.go b/net/portmapper/portmapper.go index 1c6c7634bf34a..c82fbf9da112a 100644 --- a/net/portmapper/portmapper.go +++ b/net/portmapper/portmapper.go @@ -507,6 +507,13 @@ func (c *Client) createMapping() { c.logf("createOrGetMapping: %v", err) } return + } else if mapping == nil { + return + + // TODO(creachadair): This was already logged in createOrGetMapping. + // It really should not happen at all, but we will need to untangle + // the control flow to eliminate that possibility. Meanwhile, this + // mitigates a panic downstream, cf. #16662. } if c.updates != nil { c.updates.Publish(Mapping{ From 50a476fbc439f500945cb69c6d11caa246bff5fb Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 28 Jul 2025 19:28:27 -0700 Subject: [PATCH 253/263] wgengine/magicsock: fix magicsock deadlock around Conn.NoteRecvActivity (#16687) (#16696) Updates #16651 Updates tailscale/corp#30836 (cherry picked from commit a9f3fd1c67ca427aceee708f319a0a12df6a5de8) Signed-off-by: Jordan Whited --- tailcfg/tailcfg.go | 3 ++- wgengine/magicsock/magicsock.go | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 307b39f93903c..5e3c4e5720a92 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -167,7 +167,8 @@ type CapabilityVersion int // - 120: 2025-07-15: Client understands peer relay disco messages, and implements peer client and relay server functions // - 121: 2025-07-19: Client understands peer relay endpoint alloc with [disco.AllocateUDPRelayEndpointRequest] & [disco.AllocateUDPRelayEndpointResponse] // - 122: 2025-07-21: Client sends Hostinfo.ExitNodeID to report which exit node it has selected, if any. -const CurrentCapabilityVersion CapabilityVersion = 122 +// - 123: 2025-07-28: fix deadlock regression from cryptokey routing change (issue #16651) +const CurrentCapabilityVersion CapabilityVersion = 123 // ID is an integer ID for a user, node, or login allocated by the // control plane. diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index fb7f5edcbd14f..d2835aed34547 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -4119,8 +4119,11 @@ func (le *lazyEndpoint) InitiationMessagePublicKey(peerPublicKey [32]byte) { return } le.c.mu.Lock() - defer le.c.mu.Unlock() ep, ok := le.c.peerMap.endpointForNodeKey(pubKey) + // [Conn.mu] must not be held while [Conn.noteRecvActivity] is called, which + // [endpoint.noteRecvActivity] can end up calling. See + // [Options.NoteRecvActivity] docs. + le.c.mu.Unlock() if !ok { return } From a277abcae82c985eb6294f6899785643fcc4e5ed Mon Sep 17 00:00:00 2001 From: Tom Meadows Date: Tue, 29 Jul 2025 11:31:16 +0100 Subject: [PATCH 254/263] k8s-operator: adding session type to cast header (#16660) (#16689) Updates #16490 Signed-off-by: chaosinthecrd --- k8s-operator/sessionrecording/hijacker.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/k8s-operator/sessionrecording/hijacker.go b/k8s-operator/sessionrecording/hijacker.go index f816a8483f273..789a9fdb9f6a3 100644 --- a/k8s-operator/sessionrecording/hijacker.go +++ b/k8s-operator/sessionrecording/hijacker.go @@ -184,9 +184,10 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, SrcNode: strings.TrimSuffix(h.who.Node.Name, "."), SrcNodeID: h.who.Node.StableID, Kubernetes: &sessionrecording.Kubernetes{ - PodName: h.pod, - Namespace: h.ns, - Container: container, + PodName: h.pod, + Namespace: h.ns, + Container: container, + SessionType: string(h.sessionType), }, } if !h.who.Node.IsTagged() { From d72494bac7a2fb6b6a01715cfc5bcc903dbd7594 Mon Sep 17 00:00:00 2001 From: Aaron Klotz Date: Tue, 29 Jul 2025 10:56:20 -0600 Subject: [PATCH 255/263] VERSION.txt: this is v1.86.2 Signed-off-by: Aaron Klotz --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index e52a44e47822a..2568a54800d86 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.86.1 +1.86.2 From d6116ea41805ff7dff2aea5907e63941c7f274fa Mon Sep 17 00:00:00 2001 From: kari-ts <135075563+kari-ts@users.noreply.github.com> Date: Fri, 1 Aug 2025 15:10:00 -0700 Subject: [PATCH 256/263] feature/taildrop: do not use m.opts.Dir for Android (#16316) In Android, we are prompting the user to select a Taildrop directory when they first receive a Taildrop: we block writes on Taildrop dir selection. This means that we cannot use Dir inside managerOptions, since the http request would not get the new Taildrop extension. This PR removes, in the Android case, the reliance on m.opts.Dir, and instead has FileOps hold the correct directory. This expands FileOps to be the Taildrop interface for all file system operations. Updates tailscale/corp#29211 Signed-off-by: kari-ts restore tstest --- feature/taildrop/delete.go | 52 +++--- feature/taildrop/delete_test.go | 34 ++-- feature/taildrop/ext.go | 67 +++----- feature/taildrop/fileops.go | 41 +++++ feature/taildrop/fileops_fs.go | 221 ++++++++++++++++++++++++ feature/taildrop/paths.go | 2 +- feature/taildrop/peerapi_test.go | 41 +++-- feature/taildrop/resume.go | 28 ++-- feature/taildrop/resume_test.go | 9 +- feature/taildrop/retrieve.go | 116 +++++++------ feature/taildrop/send.go | 270 ++++-------------------------- feature/taildrop/send_test.go | 131 ++++----------- feature/taildrop/taildrop.go | 83 +++------ feature/taildrop/taildrop_test.go | 56 +++---- 14 files changed, 555 insertions(+), 596 deletions(-) create mode 100644 feature/taildrop/fileops.go create mode 100644 feature/taildrop/fileops_fs.go diff --git a/feature/taildrop/delete.go b/feature/taildrop/delete.go index e9c8d7f1c90fa..0b7259879f941 100644 --- a/feature/taildrop/delete.go +++ b/feature/taildrop/delete.go @@ -6,9 +6,7 @@ package taildrop import ( "container/list" "context" - "io/fs" "os" - "path/filepath" "strings" "sync" "time" @@ -28,7 +26,6 @@ const deleteDelay = time.Hour type fileDeleter struct { logf logger.Logf clock tstime.DefaultClock - dir string event func(string) // called for certain events; for testing only mu sync.Mutex @@ -39,6 +36,7 @@ type fileDeleter struct { group syncs.WaitGroup shutdownCtx context.Context shutdown context.CancelFunc + fs FileOps // must be used for all filesystem operations } // deleteFile is a specific file to delete after deleteDelay. @@ -50,15 +48,14 @@ type deleteFile struct { func (d *fileDeleter) Init(m *manager, eventHook func(string)) { d.logf = m.opts.Logf d.clock = m.opts.Clock - d.dir = m.opts.Dir d.event = eventHook + d.fs = m.opts.fileOps d.byName = make(map[string]*list.Element) d.emptySignal = make(chan struct{}) d.shutdownCtx, d.shutdown = context.WithCancel(context.Background()) // From a cold-start, load the list of partial and deleted files. - // // Only run this if we have ever received at least one file // to avoid ever touching the taildrop directory on systems (e.g., MacOS) // that pop up a security dialog window upon first access. @@ -71,38 +68,45 @@ func (d *fileDeleter) Init(m *manager, eventHook func(string)) { d.group.Go(func() { d.event("start full-scan") defer d.event("end full-scan") - rangeDir(d.dir, func(de fs.DirEntry) bool { + + if d.fs == nil { + d.logf("deleter: nil FileOps") + } + + files, err := d.fs.ListFiles() + if err != nil { + d.logf("deleter: ListDir error: %v", err) + return + } + for _, filename := range files { switch { case d.shutdownCtx.Err() != nil: - return false // terminate early - case !de.Type().IsRegular(): - return true - case strings.HasSuffix(de.Name(), partialSuffix): + return // terminate early + case strings.HasSuffix(filename, partialSuffix): // Only enqueue the file for deletion if there is no active put. - nameID := strings.TrimSuffix(de.Name(), partialSuffix) + nameID := strings.TrimSuffix(filename, partialSuffix) if i := strings.LastIndexByte(nameID, '.'); i > 0 { key := incomingFileKey{clientID(nameID[i+len("."):]), nameID[:i]} m.incomingFiles.LoadFunc(key, func(_ *incomingFile, loaded bool) { if !loaded { - d.Insert(de.Name()) + d.Insert(filename) } }) } else { - d.Insert(de.Name()) + d.Insert(filename) } - case strings.HasSuffix(de.Name(), deletedSuffix): + case strings.HasSuffix(filename, deletedSuffix): // Best-effort immediate deletion of deleted files. - name := strings.TrimSuffix(de.Name(), deletedSuffix) - if os.Remove(filepath.Join(d.dir, name)) == nil { - if os.Remove(filepath.Join(d.dir, de.Name())) == nil { - break + name := strings.TrimSuffix(filename, deletedSuffix) + if d.fs.Remove(name) == nil { + if d.fs.Remove(filename) == nil { + continue } } - // Otherwise, enqueue the file for later deletion. - d.Insert(de.Name()) + // Otherwise enqueue for later deletion. + d.Insert(filename) } - return true - }) + } }) } @@ -149,13 +153,13 @@ func (d *fileDeleter) waitAndDelete(wait time.Duration) { // Delete the expired file. if name, ok := strings.CutSuffix(file.name, deletedSuffix); ok { - if err := os.Remove(filepath.Join(d.dir, name)); err != nil && !os.IsNotExist(err) { + if err := d.fs.Remove(name); err != nil && !os.IsNotExist(err) { d.logf("could not delete: %v", redactError(err)) failed = append(failed, elem) continue } } - if err := os.Remove(filepath.Join(d.dir, file.name)); err != nil && !os.IsNotExist(err) { + if err := d.fs.Remove(file.name); err != nil && !os.IsNotExist(err) { d.logf("could not delete: %v", redactError(err)) failed = append(failed, elem) continue diff --git a/feature/taildrop/delete_test.go b/feature/taildrop/delete_test.go index 7a58de55c2492..36950f58288cb 100644 --- a/feature/taildrop/delete_test.go +++ b/feature/taildrop/delete_test.go @@ -5,7 +5,6 @@ package taildrop import ( "os" - "path/filepath" "slices" "testing" "time" @@ -20,11 +19,20 @@ import ( func TestDeleter(t *testing.T) { dir := t.TempDir() - must.Do(touchFile(filepath.Join(dir, "foo.partial"))) - must.Do(touchFile(filepath.Join(dir, "bar.partial"))) - must.Do(touchFile(filepath.Join(dir, "fizz"))) - must.Do(touchFile(filepath.Join(dir, "fizz.deleted"))) - must.Do(touchFile(filepath.Join(dir, "buzz.deleted"))) // lacks a matching "buzz" file + var m manager + var fd fileDeleter + m.opts.Logf = t.Logf + m.opts.Clock = tstime.DefaultClock{Clock: tstest.NewClock(tstest.ClockOpts{ + Start: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), + })} + m.opts.State = must.Get(mem.New(nil, "")) + m.opts.fileOps, _ = newFileOps(dir) + + must.Do(m.touchFile("foo.partial")) + must.Do(m.touchFile("bar.partial")) + must.Do(m.touchFile("fizz")) + must.Do(m.touchFile("fizz.deleted")) + must.Do(m.touchFile("buzz.deleted")) // lacks a matching "buzz" file checkDirectory := func(want ...string) { t.Helper() @@ -69,12 +77,10 @@ func TestDeleter(t *testing.T) { } eventHook := func(event string) { eventsChan <- event } - var m manager - var fd fileDeleter m.opts.Logf = t.Logf m.opts.Clock = tstime.DefaultClock{Clock: clock} - m.opts.Dir = dir m.opts.State = must.Get(mem.New(nil, "")) + m.opts.fileOps, _ = newFileOps(dir) must.Do(m.opts.State.WriteState(ipn.TaildropReceivedKey, []byte{1})) fd.Init(&m, eventHook) defer fd.Shutdown() @@ -100,17 +106,17 @@ func TestDeleter(t *testing.T) { checkEvents("end waitAndDelete") checkDirectory() - must.Do(touchFile(filepath.Join(dir, "one.partial"))) + must.Do(m.touchFile("one.partial")) insert("one.partial") checkEvents("start waitAndDelete") advance(deleteDelay / 4) - must.Do(touchFile(filepath.Join(dir, "two.partial"))) + must.Do(m.touchFile("two.partial")) insert("two.partial") advance(deleteDelay / 4) - must.Do(touchFile(filepath.Join(dir, "three.partial"))) + must.Do(m.touchFile("three.partial")) insert("three.partial") advance(deleteDelay / 4) - must.Do(touchFile(filepath.Join(dir, "four.partial"))) + must.Do(m.touchFile("four.partial")) insert("four.partial") advance(deleteDelay / 4) @@ -145,8 +151,8 @@ func TestDeleterInitWithoutTaildrop(t *testing.T) { var m manager var fd fileDeleter m.opts.Logf = t.Logf - m.opts.Dir = t.TempDir() m.opts.State = must.Get(mem.New(nil, "")) + m.opts.fileOps, _ = newFileOps(t.TempDir()) fd.Init(&m, func(event string) { t.Errorf("unexpected event: %v", event) }) fd.Shutdown() } diff --git a/feature/taildrop/ext.go b/feature/taildrop/ext.go index c11fe3af427a1..f8f45b53fae26 100644 --- a/feature/taildrop/ext.go +++ b/feature/taildrop/ext.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "maps" - "os" "path/filepath" "runtime" "slices" @@ -75,7 +74,7 @@ type Extension struct { // FileOps abstracts platform-specific file operations needed for file transfers. // This is currently being used for Android to use the Storage Access Framework. - FileOps FileOps + fileOps FileOps nodeBackendForTest ipnext.NodeBackend // if non-nil, pretend we're this node state for tests @@ -89,30 +88,6 @@ type Extension struct { outgoingFiles map[string]*ipn.OutgoingFile } -// safDirectoryPrefix is used to determine if the directory is managed via SAF. -const SafDirectoryPrefix = "content://" - -// PutMode controls how Manager.PutFile writes files to storage. -// -// PutModeDirect – write files directly to a filesystem path (default). -// PutModeAndroidSAF – use Android’s Storage Access Framework (SAF), where -// the OS manages the underlying directory permissions. -type PutMode int - -const ( - PutModeDirect PutMode = iota - PutModeAndroidSAF -) - -// FileOps defines platform-specific file operations. -type FileOps interface { - OpenFileWriter(filename string) (io.WriteCloser, string, error) - - // RenamePartialFile finalizes a partial file. - // It returns the new SAF URI as a string and an error. - RenamePartialFile(partialUri, targetDirUri, targetName string) (string, error) -} - func (e *Extension) Name() string { return "taildrop" } @@ -176,23 +151,34 @@ func (e *Extension) onChangeProfile(profile ipn.LoginProfileView, _ ipn.PrefsVie return } - // If we have a netmap, create a taildrop manager. - fileRoot, isDirectFileMode := e.fileRoot(uid, activeLogin) - if fileRoot == "" { - e.logf("no Taildrop directory configured") - } - mode := PutModeDirect - if e.directFileRoot != "" && strings.HasPrefix(e.directFileRoot, SafDirectoryPrefix) { - mode = PutModeAndroidSAF + // Use the provided [FileOps] implementation (typically for SAF access on Android), + // or create an [fsFileOps] instance rooted at fileRoot. + // + // A non-nil [FileOps] also implies that we are in DirectFileMode. + fops := e.fileOps + isDirectFileMode := fops != nil + if fops == nil { + var fileRoot string + if fileRoot, isDirectFileMode = e.fileRoot(uid, activeLogin); fileRoot == "" { + e.logf("no Taildrop directory configured") + e.setMgrLocked(nil) + return + } + + var err error + if fops, err = newFileOps(fileRoot); err != nil { + e.logf("taildrop: cannot create FileOps: %v", err) + e.setMgrLocked(nil) + return + } } + e.setMgrLocked(managerOptions{ Logf: e.logf, Clock: tstime.DefaultClock{Clock: e.sb.Clock()}, State: e.stateStore, - Dir: fileRoot, DirectFileMode: isDirectFileMode, - FileOps: e.FileOps, - Mode: mode, + fileOps: fops, SendFileNotify: e.sendFileNotify, }.New()) } @@ -221,12 +207,7 @@ func (e *Extension) fileRoot(uid tailcfg.UserID, activeLogin string) (root strin baseDir := fmt.Sprintf("%s-uid-%d", strings.ReplaceAll(activeLogin, "@", "-"), uid) - dir := filepath.Join(varRoot, "files", baseDir) - if err := os.MkdirAll(dir, 0700); err != nil { - e.logf("Taildrop disabled; error making directory: %v", err) - return "", false - } - return dir, false + return filepath.Join(varRoot, "files", baseDir), false } // hasCapFileSharing reports whether the current node has the file sharing diff --git a/feature/taildrop/fileops.go b/feature/taildrop/fileops.go new file mode 100644 index 0000000000000..14f76067a8094 --- /dev/null +++ b/feature/taildrop/fileops.go @@ -0,0 +1,41 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package taildrop + +import ( + "io" + "io/fs" + "os" +) + +// FileOps abstracts over both local‐FS paths and Android SAF URIs. +type FileOps interface { + // OpenWriter creates or truncates a file named relative to the receiver's root, + // seeking to the specified offset. If the file does not exist, it is created with mode perm + // on platforms that support it. + // + // It returns an [io.WriteCloser] and the file's absolute path, or an error. + // This call may block. Callers should avoid holding locks when calling OpenWriter. + OpenWriter(name string, offset int64, perm os.FileMode) (wc io.WriteCloser, path string, err error) + + // Remove deletes a file or directory relative to the receiver's root. + // It returns [io.ErrNotExist] if the file or directory does not exist. + Remove(name string) error + + // Rename atomically renames oldPath to a new file named newName, + // returning the full new path or an error. + Rename(oldPath, newName string) (newPath string, err error) + + // ListFiles returns just the basenames of all regular files + // in the root directory. + ListFiles() ([]string, error) + + // Stat returns the FileInfo for the given name or an error. + Stat(name string) (fs.FileInfo, error) + + // OpenReader opens the given basename for the given name or an error. + OpenReader(name string) (io.ReadCloser, error) +} + +var newFileOps func(dir string) (FileOps, error) diff --git a/feature/taildrop/fileops_fs.go b/feature/taildrop/fileops_fs.go new file mode 100644 index 0000000000000..4fecbe4af6bbb --- /dev/null +++ b/feature/taildrop/fileops_fs.go @@ -0,0 +1,221 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause +//go:build !android + +package taildrop + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "sync" + "unicode/utf8" +) + +var renameMu sync.Mutex + +// fsFileOps implements FileOps using the local filesystem rooted at a directory. +// It is used on non-Android platforms. +type fsFileOps struct{ rootDir string } + +func init() { + newFileOps = func(dir string) (FileOps, error) { + if dir == "" { + return nil, errors.New("rootDir cannot be empty") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("mkdir %q: %w", dir, err) + } + return fsFileOps{rootDir: dir}, nil + } +} + +func (f fsFileOps) OpenWriter(name string, offset int64, perm os.FileMode) (io.WriteCloser, string, error) { + path, err := joinDir(f.rootDir, name) + if err != nil { + return nil, "", err + } + if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, "", err + } + fi, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, perm) + if err != nil { + return nil, "", err + } + if offset != 0 { + curr, err := fi.Seek(0, io.SeekEnd) + if err != nil { + fi.Close() + return nil, "", err + } + if offset < 0 || offset > curr { + fi.Close() + return nil, "", fmt.Errorf("offset %d out of range", offset) + } + if _, err := fi.Seek(offset, io.SeekStart); err != nil { + fi.Close() + return nil, "", err + } + if err := fi.Truncate(offset); err != nil { + fi.Close() + return nil, "", err + } + } + return fi, path, nil +} + +func (f fsFileOps) Remove(name string) error { + path, err := joinDir(f.rootDir, name) + if err != nil { + return err + } + return os.Remove(path) +} + +// Rename moves the partial file into its final name. +// newName must be a base name (not absolute or containing path separators). +// It will retry up to 10 times, de-dup same-checksum files, etc. +func (f fsFileOps) Rename(oldPath, newName string) (newPath string, err error) { + var dst string + if filepath.IsAbs(newName) || strings.ContainsRune(newName, os.PathSeparator) { + return "", fmt.Errorf("invalid newName %q: must not be an absolute path or contain path separators", newName) + } + + dst = filepath.Join(f.rootDir, newName) + + if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { + return "", err + } + + st, err := os.Stat(oldPath) + if err != nil { + return "", err + } + wantSize := st.Size() + + const maxRetries = 10 + for i := 0; i < maxRetries; i++ { + renameMu.Lock() + fi, statErr := os.Stat(dst) + // Atomically rename the partial file as the destination file if it doesn't exist. + // Otherwise, it returns the length of the current destination file. + // The operation is atomic. + if os.IsNotExist(statErr) { + err = os.Rename(oldPath, dst) + renameMu.Unlock() + if err != nil { + return "", err + } + return dst, nil + } + if statErr != nil { + renameMu.Unlock() + return "", statErr + } + gotSize := fi.Size() + renameMu.Unlock() + + // Avoid the final rename if a destination file has the same contents. + // + // Note: this is best effort and copying files from iOS from the Media Library + // results in processing on the iOS side which means the size and shas of the + // same file can be different. + if gotSize == wantSize { + sumP, err := sha256File(oldPath) + if err != nil { + return "", err + } + sumD, err := sha256File(dst) + if err != nil { + return "", err + } + if bytes.Equal(sumP[:], sumD[:]) { + if err := os.Remove(oldPath); err != nil { + return "", err + } + return dst, nil + } + } + + // Choose a new destination filename and try again. + dst = filepath.Join(filepath.Dir(dst), nextFilename(filepath.Base(dst))) + } + + return "", fmt.Errorf("too many retries trying to rename %q to %q", oldPath, newName) +} + +// sha256File computes the SHA‑256 of a file. +func sha256File(path string) (sum [sha256.Size]byte, _ error) { + f, err := os.Open(path) + if err != nil { + return sum, err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return sum, err + } + copy(sum[:], h.Sum(nil)) + return sum, nil +} + +func (f fsFileOps) ListFiles() ([]string, error) { + entries, err := os.ReadDir(f.rootDir) + if err != nil { + return nil, err + } + var names []string + for _, e := range entries { + if e.Type().IsRegular() { + names = append(names, e.Name()) + } + } + return names, nil +} + +func (f fsFileOps) Stat(name string) (fs.FileInfo, error) { + path, err := joinDir(f.rootDir, name) + if err != nil { + return nil, err + } + return os.Stat(path) +} + +func (f fsFileOps) OpenReader(name string) (io.ReadCloser, error) { + path, err := joinDir(f.rootDir, name) + if err != nil { + return nil, err + } + return os.Open(path) +} + +// joinDir is like [filepath.Join] but returns an error if baseName is too long, +// is a relative path instead of a basename, or is otherwise invalid or unsafe for incoming files. +func joinDir(dir, baseName string) (string, error) { + if !utf8.ValidString(baseName) || + strings.TrimSpace(baseName) != baseName || + len(baseName) > 255 { + return "", ErrInvalidFileName + } + // TODO: validate unicode normalization form too? Varies by platform. + clean := path.Clean(baseName) + if clean != baseName || clean == "." || clean == ".." { + return "", ErrInvalidFileName + } + for _, r := range baseName { + if !validFilenameRune(r) { + return "", ErrInvalidFileName + } + } + if !filepath.IsLocal(baseName) { + return "", ErrInvalidFileName + } + return filepath.Join(dir, baseName), nil +} diff --git a/feature/taildrop/paths.go b/feature/taildrop/paths.go index 22d01160cff8e..79dc37d8f0699 100644 --- a/feature/taildrop/paths.go +++ b/feature/taildrop/paths.go @@ -21,7 +21,7 @@ func (e *Extension) SetDirectFileRoot(root string) { // SetFileOps sets the platform specific file operations. This is used // to call Android's Storage Access Framework APIs. func (e *Extension) SetFileOps(fileOps FileOps) { - e.FileOps = fileOps + e.fileOps = fileOps } func (e *Extension) setPlatformDefaultDirectFileRoot() { diff --git a/feature/taildrop/peerapi_test.go b/feature/taildrop/peerapi_test.go index 1a003b6eddca7..6339973544453 100644 --- a/feature/taildrop/peerapi_test.go +++ b/feature/taildrop/peerapi_test.go @@ -24,6 +24,7 @@ import ( "tailscale.com/tstest" "tailscale.com/tstime" "tailscale.com/types/logger" + "tailscale.com/util/must" ) // peerAPIHandler serves the PeerAPI for a source specific client. @@ -93,7 +94,16 @@ func bodyContains(sub string) check { func fileHasSize(name string, size int) check { return func(t *testing.T, e *peerAPITestEnv) { - root := e.taildrop.Dir() + fsImpl, ok := e.taildrop.opts.fileOps.(*fsFileOps) + if !ok { + t.Skip("fileHasSize only supported on fsFileOps backend") + return + } + root := fsImpl.rootDir + if root == "" { + t.Errorf("no rootdir; can't check whether %q has size %v", name, size) + return + } if root == "" { t.Errorf("no rootdir; can't check whether %q has size %v", name, size) return @@ -109,12 +119,12 @@ func fileHasSize(name string, size int) check { func fileHasContents(name string, want string) check { return func(t *testing.T, e *peerAPITestEnv) { - root := e.taildrop.Dir() - if root == "" { - t.Errorf("no rootdir; can't check contents of %q", name) + fsImpl, ok := e.taildrop.opts.fileOps.(*fsFileOps) + if !ok { + t.Skip("fileHasContents only supported on fsFileOps backend") return } - path := filepath.Join(root, name) + path := filepath.Join(fsImpl.rootDir, name) got, err := os.ReadFile(path) if err != nil { t.Errorf("fileHasContents: %v", err) @@ -172,9 +182,10 @@ func TestHandlePeerAPI(t *testing.T) { reqs: []*http.Request{httptest.NewRequest("PUT", "/v0/put/foo", nil)}, checks: checks( httpStatus(http.StatusForbidden), - bodyContains("Taildrop disabled; no storage directory"), + bodyContains("Taildrop disabled"), ), }, + { name: "bad_method", isSelf: true, @@ -471,14 +482,18 @@ func TestHandlePeerAPI(t *testing.T) { selfNode.CapMap = tailcfg.NodeCapMap{tailcfg.CapabilityDebug: nil} } var rootDir string + var fo FileOps if !tt.omitRoot { - rootDir = t.TempDir() + var err error + if fo, err = newFileOps(t.TempDir()); err != nil { + t.Fatalf("newFileOps: %v", err) + } } var e peerAPITestEnv e.taildrop = managerOptions{ - Logf: e.logBuf.Logf, - Dir: rootDir, + Logf: e.logBuf.Logf, + fileOps: fo, }.New() ext := &fakeExtension{ @@ -490,9 +505,7 @@ func TestHandlePeerAPI(t *testing.T) { e.ph = &peerAPIHandler{ isSelf: tt.isSelf, selfNode: selfNode.View(), - peerNode: (&tailcfg.Node{ - ComputedName: "some-peer-name", - }).View(), + peerNode: (&tailcfg.Node{ComputedName: "some-peer-name"}).View(), } for _, req := range tt.reqs { e.rr = httptest.NewRecorder() @@ -526,8 +539,8 @@ func TestHandlePeerAPI(t *testing.T) { func TestFileDeleteRace(t *testing.T) { dir := t.TempDir() taildropMgr := managerOptions{ - Logf: t.Logf, - Dir: dir, + Logf: t.Logf, + fileOps: must.Get(newFileOps(dir)), }.New() ph := &peerAPIHandler{ diff --git a/feature/taildrop/resume.go b/feature/taildrop/resume.go index 211a1ff6b68dd..20ef527a6da55 100644 --- a/feature/taildrop/resume.go +++ b/feature/taildrop/resume.go @@ -9,7 +9,6 @@ import ( "encoding/hex" "fmt" "io" - "io/fs" "os" "strings" ) @@ -51,19 +50,20 @@ func (cs *checksum) UnmarshalText(b []byte) error { // PartialFiles returns a list of partial files in [Handler.Dir] // that were sent (or is actively being sent) by the provided id. -func (m *manager) PartialFiles(id clientID) (ret []string, err error) { - if m == nil || m.opts.Dir == "" { +func (m *manager) PartialFiles(id clientID) ([]string, error) { + if m == nil || m.opts.fileOps == nil { return nil, ErrNoTaildrop } - suffix := id.partialSuffix() - if err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { - if name := de.Name(); strings.HasSuffix(name, suffix) { - ret = append(ret, name) + files, err := m.opts.fileOps.ListFiles() + if err != nil { + return nil, redactError(err) + } + var ret []string + for _, filename := range files { + if strings.HasSuffix(filename, suffix) { + ret = append(ret, filename) } - return true - }); err != nil { - return ret, redactError(err) } return ret, nil } @@ -73,17 +73,13 @@ func (m *manager) PartialFiles(id clientID) (ret []string, err error) { // It returns (BlockChecksum{}, io.EOF) when the stream is complete. // It is the caller's responsibility to call close. func (m *manager) HashPartialFile(id clientID, baseName string) (next func() (blockChecksum, error), close func() error, err error) { - if m == nil || m.opts.Dir == "" { + if m == nil || m.opts.fileOps == nil { return nil, nil, ErrNoTaildrop } noopNext := func() (blockChecksum, error) { return blockChecksum{}, io.EOF } noopClose := func() error { return nil } - dstFile, err := joinDir(m.opts.Dir, baseName) - if err != nil { - return nil, nil, err - } - f, err := os.Open(dstFile + id.partialSuffix()) + f, err := m.opts.fileOps.OpenReader(baseName + id.partialSuffix()) if err != nil { if os.IsNotExist(err) { return noopNext, noopClose, nil diff --git a/feature/taildrop/resume_test.go b/feature/taildrop/resume_test.go index dac3c657bfb58..4e59d401dcc53 100644 --- a/feature/taildrop/resume_test.go +++ b/feature/taildrop/resume_test.go @@ -8,6 +8,7 @@ import ( "io" "math/rand" "os" + "path/filepath" "testing" "testing/iotest" @@ -19,7 +20,9 @@ func TestResume(t *testing.T) { defer func() { blockSize = oldBlockSize }() blockSize = 256 - m := managerOptions{Logf: t.Logf, Dir: t.TempDir()}.New() + dir := t.TempDir() + + m := managerOptions{Logf: t.Logf, fileOps: must.Get(newFileOps(dir))}.New() defer m.Shutdown() rn := rand.New(rand.NewSource(0)) @@ -37,7 +40,7 @@ func TestResume(t *testing.T) { must.Do(close()) // Windows wants the file handle to be closed to rename it. must.Get(m.PutFile("", "foo", r, offset, -1)) - got := must.Get(os.ReadFile(must.Get(joinDir(m.opts.Dir, "foo")))) + got := must.Get(os.ReadFile(filepath.Join(dir, "foo"))) if !bytes.Equal(got, want) { t.Errorf("content mismatches") } @@ -66,7 +69,7 @@ func TestResume(t *testing.T) { t.Fatalf("too many iterations to complete the test") } } - got := must.Get(os.ReadFile(must.Get(joinDir(m.opts.Dir, "bar")))) + got := must.Get(os.ReadFile(filepath.Join(dir, "bar"))) if !bytes.Equal(got, want) { t.Errorf("content mismatches") } diff --git a/feature/taildrop/retrieve.go b/feature/taildrop/retrieve.go index 6fb97519363bc..b048a1b3b5f9d 100644 --- a/feature/taildrop/retrieve.go +++ b/feature/taildrop/retrieve.go @@ -9,19 +9,19 @@ import ( "io" "io/fs" "os" - "path/filepath" "runtime" "sort" "time" "tailscale.com/client/tailscale/apitype" "tailscale.com/logtail/backoff" + "tailscale.com/util/set" ) // HasFilesWaiting reports whether any files are buffered in [Handler.Dir]. // This always returns false when [Handler.DirectFileMode] is false. -func (m *manager) HasFilesWaiting() (has bool) { - if m == nil || m.opts.Dir == "" || m.opts.DirectFileMode { +func (m *manager) HasFilesWaiting() bool { + if m == nil || m.opts.fileOps == nil || m.opts.DirectFileMode { return false } @@ -30,63 +30,66 @@ func (m *manager) HasFilesWaiting() (has bool) { // has-files-or-not values as the macOS/iOS client might // in the future use+delete the files directly. So only // keep this negative cache. - totalReceived := m.totalReceived.Load() - if totalReceived == m.emptySince.Load() { + total := m.totalReceived.Load() + if total == m.emptySince.Load() { return false } - // Check whether there is at least one one waiting file. - err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { - name := de.Name() - if isPartialOrDeleted(name) || !de.Type().IsRegular() { - return true + files, err := m.opts.fileOps.ListFiles() + if err != nil { + return false + } + + // Build a set of filenames present in Dir + fileSet := set.Of(files...) + + for _, filename := range files { + if isPartialOrDeleted(filename) { + continue } - _, err := os.Stat(filepath.Join(m.opts.Dir, name+deletedSuffix)) - if os.IsNotExist(err) { - has = true - return false + if fileSet.Contains(filename + deletedSuffix) { + continue // already handled } + // Found at least one downloadable file return true - }) - - // If there are no more waiting files, record totalReceived as emptySince - // so that we can short-circuit the expensive directory traversal - // if no files have been received after the start of this call. - if err == nil && !has { - m.emptySince.Store(totalReceived) } - return has + + // No waiting files → update negative‑result cache + m.emptySince.Store(total) + return false } // WaitingFiles returns the list of files that have been sent by a // peer that are waiting in [Handler.Dir]. // This always returns nil when [Handler.DirectFileMode] is false. -func (m *manager) WaitingFiles() (ret []apitype.WaitingFile, err error) { - if m == nil || m.opts.Dir == "" { +func (m *manager) WaitingFiles() ([]apitype.WaitingFile, error) { + if m == nil || m.opts.fileOps == nil { return nil, ErrNoTaildrop } if m.opts.DirectFileMode { return nil, nil } - if err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { - name := de.Name() - if isPartialOrDeleted(name) || !de.Type().IsRegular() { - return true + names, err := m.opts.fileOps.ListFiles() + if err != nil { + return nil, redactError(err) + } + var ret []apitype.WaitingFile + for _, name := range names { + if isPartialOrDeleted(name) { + continue } - _, err := os.Stat(filepath.Join(m.opts.Dir, name+deletedSuffix)) - if os.IsNotExist(err) { - fi, err := de.Info() - if err != nil { - return true - } - ret = append(ret, apitype.WaitingFile{ - Name: filepath.Base(name), - Size: fi.Size(), - }) + // A corresponding .deleted marker means the file was already handled. + if _, err := m.opts.fileOps.Stat(name + deletedSuffix); err == nil { + continue } - return true - }); err != nil { - return nil, redactError(err) + fi, err := m.opts.fileOps.Stat(name) + if err != nil { + continue + } + ret = append(ret, apitype.WaitingFile{ + Name: name, + Size: fi.Size(), + }) } sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) return ret, nil @@ -95,21 +98,18 @@ func (m *manager) WaitingFiles() (ret []apitype.WaitingFile, err error) { // DeleteFile deletes a file of the given baseName from [Handler.Dir]. // This method is only allowed when [Handler.DirectFileMode] is false. func (m *manager) DeleteFile(baseName string) error { - if m == nil || m.opts.Dir == "" { + if m == nil || m.opts.fileOps == nil { return ErrNoTaildrop } if m.opts.DirectFileMode { return errors.New("deletes not allowed in direct mode") } - path, err := joinDir(m.opts.Dir, baseName) - if err != nil { - return err - } + var bo *backoff.Backoff logf := m.opts.Logf t0 := m.opts.Clock.Now() for { - err := os.Remove(path) + err := m.opts.fileOps.Remove(baseName) if err != nil && !os.IsNotExist(err) { err = redactError(err) // Put a retry loop around deletes on Windows. @@ -129,7 +129,7 @@ func (m *manager) DeleteFile(baseName string) error { bo.BackOff(context.Background(), err) continue } - if err := touchFile(path + deletedSuffix); err != nil { + if err := m.touchFile(baseName + deletedSuffix); err != nil { logf("peerapi: failed to leave deleted marker: %v", err) } m.deleter.Insert(baseName + deletedSuffix) @@ -141,35 +141,31 @@ func (m *manager) DeleteFile(baseName string) error { } } -func touchFile(path string) error { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) +func (m *manager) touchFile(name string) error { + wc, _, err := m.opts.fileOps.OpenWriter(name /* offset= */, 0, 0666) if err != nil { return redactError(err) } - return f.Close() + return wc.Close() } // OpenFile opens a file of the given baseName from [Handler.Dir]. // This method is only allowed when [Handler.DirectFileMode] is false. func (m *manager) OpenFile(baseName string) (rc io.ReadCloser, size int64, err error) { - if m == nil || m.opts.Dir == "" { + if m == nil || m.opts.fileOps == nil { return nil, 0, ErrNoTaildrop } if m.opts.DirectFileMode { return nil, 0, errors.New("opens not allowed in direct mode") } - path, err := joinDir(m.opts.Dir, baseName) - if err != nil { - return nil, 0, err - } - if _, err := os.Stat(path + deletedSuffix); err == nil { - return nil, 0, redactError(&fs.PathError{Op: "open", Path: path, Err: fs.ErrNotExist}) + if _, err := m.opts.fileOps.Stat(baseName + deletedSuffix); err == nil { + return nil, 0, redactError(&fs.PathError{Op: "open", Path: baseName, Err: fs.ErrNotExist}) } - f, err := os.Open(path) + f, err := m.opts.fileOps.OpenReader(baseName) if err != nil { return nil, 0, redactError(err) } - fi, err := f.Stat() + fi, err := m.opts.fileOps.Stat(baseName) if err != nil { f.Close() return nil, 0, redactError(err) diff --git a/feature/taildrop/send.go b/feature/taildrop/send.go index 59a1701da6f0d..32ba5f6f0d644 100644 --- a/feature/taildrop/send.go +++ b/feature/taildrop/send.go @@ -4,11 +4,8 @@ package taildrop import ( - "crypto/sha256" "fmt" "io" - "os" - "path/filepath" "sync" "time" @@ -73,9 +70,10 @@ func (f *incomingFile) Write(p []byte) (n int, err error) { // specific partial file. This allows the client to determine whether to resume // a partial file. While resuming, PutFile may be called again with a non-zero // offset to specify where to resume receiving data at. -func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, length int64) (int64, error) { +func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, length int64) (fileLength int64, err error) { + switch { - case m == nil || m.opts.Dir == "": + case m == nil || m.opts.fileOps == nil: return 0, ErrNoTaildrop case !envknob.CanTaildrop(): return 0, ErrNoTaildrop @@ -83,47 +81,47 @@ func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, len return 0, ErrNotAccessible } - //Compute dstPath & avoid mid‑upload deletion - var dstPath string - if m.opts.Mode == PutModeDirect { - var err error - dstPath, err = joinDir(m.opts.Dir, baseName) + if err := validateBaseName(baseName); err != nil { + return 0, err + } + + // and make sure we don't delete it while uploading: + m.deleter.Remove(baseName) + + // Create (if not already) the partial file with read-write permissions. + partialName := baseName + id.partialSuffix() + wc, partialPath, err := m.opts.fileOps.OpenWriter(partialName, offset, 0o666) + if err != nil { + return 0, m.redactAndLogError("Create", err) + } + defer func() { + wc.Close() if err != nil { - return 0, err + m.deleter.Insert(partialName) // mark partial file for eventual deletion } - } else { - // In SAF mode, we simply use the baseName as the destination "path" - // (the actual directory is managed by SAF). - dstPath = baseName - } - m.deleter.Remove(filepath.Base(dstPath)) // avoid deleting the partial file while receiving + }() // Check whether there is an in-progress transfer for the file. - partialFileKey := incomingFileKey{id, baseName} - inFile, loaded := m.incomingFiles.LoadOrInit(partialFileKey, func() *incomingFile { - return &incomingFile{ + inFileKey := incomingFileKey{id, baseName} + inFile, loaded := m.incomingFiles.LoadOrInit(inFileKey, func() *incomingFile { + inFile := &incomingFile{ clock: m.opts.Clock, started: m.opts.Clock.Now(), size: length, sendFileNotify: m.opts.SendFileNotify, } + if m.opts.DirectFileMode { + inFile.partialPath = partialPath + } + return inFile }) + + inFile.w = wc + if loaded { return 0, ErrFileExists } - defer m.incomingFiles.Delete(partialFileKey) - - // Open writer & populate inFile paths - wc, partialPath, err := m.openWriterAndPaths(id, m.opts.Mode, inFile, baseName, dstPath, offset) - if err != nil { - return 0, m.redactAndLogError("Create", err) - } - defer func() { - wc.Close() - if err != nil { - m.deleter.Insert(filepath.Base(partialPath)) // mark partial file for eventual deletion - } - }() + defer m.incomingFiles.Delete(inFileKey) // Record that we have started to receive at least one file. // This is used by the deleter upon a cold-start to scan the directory @@ -148,220 +146,26 @@ func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, len return 0, m.redactAndLogError("Close", err) } - fileLength := offset + copyLength + fileLength = offset + copyLength inFile.mu.Lock() inFile.done = true inFile.mu.Unlock() - // Finalize rename - switch m.opts.Mode { - case PutModeDirect: - var finalDst string - finalDst, err = m.finalizeDirect(inFile, partialPath, dstPath, fileLength) - if err != nil { - return 0, m.redactAndLogError("Rename", err) - } - inFile.finalPath = finalDst - - case PutModeAndroidSAF: - if err = m.finalizeSAF(partialPath, baseName); err != nil { - return 0, m.redactAndLogError("Rename", err) - } + // 6) Finalize (rename/move) the partial into place via FileOps.Rename + finalPath, err := m.opts.fileOps.Rename(partialPath, baseName) + if err != nil { + return 0, m.redactAndLogError("Rename", err) } + inFile.finalPath = finalPath m.totalReceived.Add(1) m.opts.SendFileNotify() return fileLength, nil } -// openWriterAndPaths opens the correct writer, seeks/truncates if needed, -// and sets inFile.partialPath & inFile.finalPath for later cleanup/rename. -// The caller is responsible for closing the file on completion. -func (m *manager) openWriterAndPaths( - id clientID, - mode PutMode, - inFile *incomingFile, - baseName string, - dstPath string, - offset int64, -) (wc io.WriteCloser, partialPath string, err error) { - switch mode { - - case PutModeDirect: - partialPath = dstPath + id.partialSuffix() - f, err := os.OpenFile(partialPath, os.O_CREATE|os.O_RDWR, 0o666) - if err != nil { - return nil, "", m.redactAndLogError("Create", err) - } - if offset != 0 { - curr, err := f.Seek(0, io.SeekEnd) - if err != nil { - f.Close() - return nil, "", m.redactAndLogError("Seek", err) - } - if offset < 0 || offset > curr { - f.Close() - return nil, "", m.redactAndLogError("Seek", fmt.Errorf("offset %d out of range", offset)) - } - if _, err := f.Seek(offset, io.SeekStart); err != nil { - f.Close() - return nil, "", m.redactAndLogError("Seek", err) - } - if err := f.Truncate(offset); err != nil { - f.Close() - return nil, "", m.redactAndLogError("Truncate", err) - } - } - inFile.w = f - wc = f - inFile.partialPath = partialPath - inFile.finalPath = dstPath - return wc, partialPath, nil - - case PutModeAndroidSAF: - if m.opts.FileOps == nil { - return nil, "", m.redactAndLogError("Create (SAF)", fmt.Errorf("missing FileOps")) - } - writer, uri, err := m.opts.FileOps.OpenFileWriter(baseName) - if err != nil { - return nil, "", m.redactAndLogError("Create (SAF)", fmt.Errorf("failed to open file for writing via SAF")) - } - if writer == nil || uri == "" { - return nil, "", fmt.Errorf("invalid SAF writer or URI") - } - // SAF mode does not support resuming, so enforce offset == 0. - if offset != 0 { - writer.Close() - return nil, "", m.redactAndLogError("Seek", fmt.Errorf("resuming is not supported in SAF mode")) - } - inFile.w = writer - wc = writer - partialPath = uri - inFile.partialPath = uri - inFile.finalPath = baseName - return wc, partialPath, nil - - default: - return nil, "", fmt.Errorf("unsupported PutMode: %v", mode) - } -} - -// finalizeDirect atomically renames or dedups the partial file, retrying -// under new names up to 10 times. It returns the final path that succeeded. -func (m *manager) finalizeDirect( - inFile *incomingFile, - partialPath string, - initialDst string, - fileLength int64, -) (string, error) { - var ( - once sync.Once - cachedSum [sha256.Size]byte - cacheErr error - computeSum = func() ([sha256.Size]byte, error) { - once.Do(func() { cachedSum, cacheErr = sha256File(partialPath) }) - return cachedSum, cacheErr - } - ) - - dstPath := initialDst - const maxRetries = 10 - for i := 0; i < maxRetries; i++ { - // Atomically rename the partial file as the destination file if it doesn't exist. - // Otherwise, it returns the length of the current destination file. - // The operation is atomic. - lengthOnDisk, err := func() (int64, error) { - m.renameMu.Lock() - defer m.renameMu.Unlock() - fi, statErr := os.Stat(dstPath) - if os.IsNotExist(statErr) { - // dst missing → rename partial into place - return -1, os.Rename(partialPath, dstPath) - } - if statErr != nil { - return -1, statErr - } - return fi.Size(), nil - }() - if err != nil { - return "", err - } - if lengthOnDisk < 0 { - // successfully moved - inFile.finalPath = dstPath - return dstPath, nil - } - - // Avoid the final rename if a destination file has the same contents. - // - // Note: this is best effort and copying files from iOS from the Media Library - // results in processing on the iOS side which means the size and shas of the - // same file can be different. - if lengthOnDisk == fileLength { - partSum, err := computeSum() - if err != nil { - return "", err - } - dstSum, err := sha256File(dstPath) - if err != nil { - return "", err - } - if partSum == dstSum { - // same content → drop the partial - if err := os.Remove(partialPath); err != nil { - return "", err - } - inFile.finalPath = dstPath - return dstPath, nil - } - } - - // Choose a new destination filename and try again. - dstPath = nextFilename(dstPath) - } - - return "", fmt.Errorf("too many retries trying to rename a partial file %q", initialDst) -} - -// finalizeSAF retries RenamePartialFile up to 10 times, generating a new -// name on each failure until the SAF URI changes. -func (m *manager) finalizeSAF( - partialPath, finalName string, -) error { - if m.opts.FileOps == nil { - return fmt.Errorf("missing FileOps for SAF finalize") - } - const maxTries = 10 - name := finalName - for i := 0; i < maxTries; i++ { - newURI, err := m.opts.FileOps.RenamePartialFile(partialPath, m.opts.Dir, name) - if err != nil { - return err - } - if newURI != "" && newURI != name { - return nil - } - name = nextFilename(name) - } - return fmt.Errorf("failed to finalize SAF file after %d retries", maxTries) -} - func (m *manager) redactAndLogError(stage string, err error) error { err = redactError(err) m.opts.Logf("put %s error: %v", stage, err) return err } - -func sha256File(file string) (out [sha256.Size]byte, err error) { - h := sha256.New() - f, err := os.Open(file) - if err != nil { - return out, err - } - defer f.Close() - if _, err := io.Copy(h, f); err != nil { - return out, err - } - return [sha256.Size]byte(h.Sum(nil)), nil -} diff --git a/feature/taildrop/send_test.go b/feature/taildrop/send_test.go index 8edb704172fc5..9ffa5fccc0a36 100644 --- a/feature/taildrop/send_test.go +++ b/feature/taildrop/send_test.go @@ -4,123 +4,64 @@ package taildrop import ( - "bytes" - "fmt" - "io" "os" "path/filepath" + "strings" "testing" "tailscale.com/tstime" + "tailscale.com/util/must" ) -// nopWriteCloser is a no-op io.WriteCloser wrapping a bytes.Buffer. -type nopWriteCloser struct{ *bytes.Buffer } - -func (nwc nopWriteCloser) Close() error { return nil } - -// mockFileOps implements just enough of the FileOps interface for SAF tests. -type mockFileOps struct { - writes *bytes.Buffer - renameOK bool -} - -func (m *mockFileOps) OpenFileWriter(name string) (io.WriteCloser, string, error) { - m.writes = new(bytes.Buffer) - return nopWriteCloser{m.writes}, "uri://" + name + ".partial", nil -} - -func (m *mockFileOps) RenamePartialFile(partialPath, dir, finalName string) (string, error) { - if !m.renameOK { - m.renameOK = true - return "uri://" + finalName, nil - } - return "", io.ErrUnexpectedEOF -} - func TestPutFile(t *testing.T) { const content = "hello, world" tests := []struct { - name string - mode PutMode - setup func(t *testing.T) (*manager, string, *mockFileOps) - wantFile string + name string + directFileMode bool }{ - { - name: "PutModeDirect", - mode: PutModeDirect, - setup: func(t *testing.T) (*manager, string, *mockFileOps) { - dir := t.TempDir() - opts := managerOptions{ - Logf: t.Logf, - Clock: tstime.DefaultClock{}, - State: nil, - Dir: dir, - Mode: PutModeDirect, - DirectFileMode: true, - SendFileNotify: func() {}, - } - mgr := opts.New() - return mgr, dir, nil - }, - wantFile: "file.txt", - }, - { - name: "PutModeAndroidSAF", - mode: PutModeAndroidSAF, - setup: func(t *testing.T) (*manager, string, *mockFileOps) { - // SAF still needs a non-empty Dir to pass the guard. - dir := t.TempDir() - mops := &mockFileOps{} - opts := managerOptions{ - Logf: t.Logf, - Clock: tstime.DefaultClock{}, - State: nil, - Dir: dir, - Mode: PutModeAndroidSAF, - FileOps: mops, - DirectFileMode: true, - SendFileNotify: func() {}, - } - mgr := opts.New() - return mgr, dir, mops - }, - wantFile: "file.txt", - }, + {"DirectFileMode", true}, + {"NonDirectFileMode", false}, } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - mgr, dir, mops := tc.setup(t) - id := clientID(fmt.Sprint(0)) - reader := bytes.NewReader([]byte(content)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + mgr := managerOptions{ + Logf: t.Logf, + Clock: tstime.DefaultClock{}, + State: nil, + fileOps: must.Get(newFileOps(dir)), + DirectFileMode: tt.directFileMode, + SendFileNotify: func() {}, + }.New() - n, err := mgr.PutFile(id, "file.txt", reader, 0, int64(len(content))) + id := clientID("0") + n, err := mgr.PutFile(id, "file.txt", strings.NewReader(content), 0, int64(len(content))) if err != nil { - t.Fatalf("PutFile(%s) error: %v", tc.name, err) + t.Fatalf("PutFile error: %v", err) } if n != int64(len(content)) { t.Errorf("wrote %d bytes; want %d", n, len(content)) } - switch tc.mode { - case PutModeDirect: - path := filepath.Join(dir, tc.wantFile) - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("ReadFile error: %v", err) - } - if got := string(data); got != content { - t.Errorf("file contents = %q; want %q", got, content) - } + path := filepath.Join(dir, "file.txt") - case PutModeAndroidSAF: - if mops.writes == nil { - t.Fatal("SAF writer was never created") - } - if got := mops.writes.String(); got != content { - t.Errorf("SAF writes = %q; want %q", got, content) + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile %q: %v", path, err) + } + if string(got) != content { + t.Errorf("file contents = %q; want %q", string(got), content) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.Contains(entry.Name(), ".partial") { + t.Errorf("unexpected partial file left behind: %s", entry.Name()) } } }) diff --git a/feature/taildrop/taildrop.go b/feature/taildrop/taildrop.go index 2dfa415bbf0cc..6c3deaed1b538 100644 --- a/feature/taildrop/taildrop.go +++ b/feature/taildrop/taildrop.go @@ -12,8 +12,6 @@ package taildrop import ( "errors" "hash/adler32" - "io" - "io/fs" "os" "path" "path/filepath" @@ -21,7 +19,6 @@ import ( "sort" "strconv" "strings" - "sync" "sync/atomic" "unicode" "unicode/utf8" @@ -72,11 +69,6 @@ type managerOptions struct { Clock tstime.DefaultClock // may be nil State ipn.StateStore // may be nil - // Dir is the directory to store received files. - // This main either be the final location for the files - // or just a temporary staging directory (see DirectFileMode). - Dir string - // DirectFileMode reports whether we are writing files // directly to a download directory, rather than writing them to // a temporary staging directory. @@ -91,9 +83,10 @@ type managerOptions struct { // copy them out, and then delete them. DirectFileMode bool - FileOps FileOps - - Mode PutMode + // FileOps abstracts platform-specific file operations needed for file transfers. + // Android's implementation uses the Storage Access Framework, and other platforms + // use fsFileOps. + fileOps FileOps // SendFileNotify is called periodically while a file is actively // receiving the contents for the file. There is a final call @@ -111,9 +104,6 @@ type manager struct { // deleter managers asynchronous deletion of files. deleter fileDeleter - // renameMu is used to protect os.Rename calls so that they are atomic. - renameMu sync.Mutex - // totalReceived counts the cumulative total of received files. totalReceived atomic.Int64 // emptySince specifies that there were no waiting files @@ -137,11 +127,6 @@ func (opts managerOptions) New() *manager { return m } -// Dir returns the directory. -func (m *manager) Dir() string { - return m.opts.Dir -} - // Shutdown shuts down the Manager. // It blocks until all spawned goroutines have stopped running. func (m *manager) Shutdown() { @@ -172,57 +157,29 @@ func isPartialOrDeleted(s string) bool { return strings.HasSuffix(s, deletedSuffix) || strings.HasSuffix(s, partialSuffix) } -func joinDir(dir, baseName string) (fullPath string, err error) { - if !utf8.ValidString(baseName) { - return "", ErrInvalidFileName - } - if strings.TrimSpace(baseName) != baseName { - return "", ErrInvalidFileName - } - if len(baseName) > 255 { - return "", ErrInvalidFileName +func validateBaseName(name string) error { + if !utf8.ValidString(name) || + strings.TrimSpace(name) != name || + len(name) > 255 { + return ErrInvalidFileName } // TODO: validate unicode normalization form too? Varies by platform. - clean := path.Clean(baseName) - if clean != baseName || - clean == "." || clean == ".." || - isPartialOrDeleted(clean) { - return "", ErrInvalidFileName + clean := path.Clean(name) + if clean != name || clean == "." || clean == ".." { + return ErrInvalidFileName } - for _, r := range baseName { + if isPartialOrDeleted(name) { + return ErrInvalidFileName + } + for _, r := range name { if !validFilenameRune(r) { - return "", ErrInvalidFileName + return ErrInvalidFileName } } - if !filepath.IsLocal(baseName) { - return "", ErrInvalidFileName - } - return filepath.Join(dir, baseName), nil -} - -// rangeDir iterates over the contents of a directory, calling fn for each entry. -// It continues iterating while fn returns true. -// It reports the number of entries seen. -func rangeDir(dir string, fn func(fs.DirEntry) bool) error { - f, err := os.Open(dir) - if err != nil { - return err - } - defer f.Close() - for { - des, err := f.ReadDir(10) - for _, de := range des { - if !fn(de) { - return nil - } - } - if err != nil { - if err == io.EOF { - return nil - } - return err - } + if !filepath.IsLocal(name) { + return ErrInvalidFileName } + return nil } // IncomingFiles returns a list of active incoming files. diff --git a/feature/taildrop/taildrop_test.go b/feature/taildrop/taildrop_test.go index da0bd2f430579..0d77273f0aab0 100644 --- a/feature/taildrop/taildrop_test.go +++ b/feature/taildrop/taildrop_test.go @@ -4,40 +4,10 @@ package taildrop import ( - "path/filepath" "strings" "testing" ) -func TestJoinDir(t *testing.T) { - dir := t.TempDir() - tests := []struct { - in string - want string // just relative to m.Dir - wantOk bool - }{ - {"", "", false}, - {"foo", "foo", true}, - {"./foo", "", false}, - {"../foo", "", false}, - {"foo/bar", "", false}, - {"😋", "😋", true}, - {"\xde\xad\xbe\xef", "", false}, - {"foo.partial", "", false}, - {"foo.deleted", "", false}, - {strings.Repeat("a", 1024), "", false}, - {"foo:bar", "", false}, - } - for _, tt := range tests { - got, gotErr := joinDir(dir, tt.in) - got, _ = filepath.Rel(dir, got) - gotOk := gotErr == nil - if got != tt.want || gotOk != tt.wantOk { - t.Errorf("joinDir(%q) = (%v, %v), want (%v, %v)", tt.in, got, gotOk, tt.want, tt.wantOk) - } - } -} - func TestNextFilename(t *testing.T) { tests := []struct { in string @@ -67,3 +37,29 @@ func TestNextFilename(t *testing.T) { } } } + +func TestValidateBaseName(t *testing.T) { + tests := []struct { + in string + wantOk bool + }{ + {"", false}, + {"foo", true}, + {"./foo", false}, + {"../foo", false}, + {"foo/bar", false}, + {"😋", true}, + {"\xde\xad\xbe\xef", false}, + {"foo.partial", false}, + {"foo.deleted", false}, + {strings.Repeat("a", 1024), false}, + {"foo:bar", false}, + } + for _, tt := range tests { + err := validateBaseName(tt.in) + gotOk := err == nil + if gotOk != tt.wantOk { + t.Errorf("validateBaseName(%q) = %v, wantOk = %v", tt.in, err, tt.wantOk) + } + } +} From 55027d40fb4832384e434db48e78b32646a73ad5 Mon Sep 17 00:00:00 2001 From: kari-ts Date: Wed, 6 Aug 2025 10:21:13 -0700 Subject: [PATCH 257/263] Revert "feature/taildrop: do not use m.opts.Dir for Android (#16316)" This reverts commit d6116ea41805ff7dff2aea5907e63941c7f274fa. --- feature/taildrop/delete.go | 52 +++--- feature/taildrop/delete_test.go | 34 ++-- feature/taildrop/ext.go | 67 +++++--- feature/taildrop/fileops.go | 41 ----- feature/taildrop/fileops_fs.go | 221 ------------------------ feature/taildrop/paths.go | 2 +- feature/taildrop/peerapi_test.go | 41 ++--- feature/taildrop/resume.go | 28 ++-- feature/taildrop/resume_test.go | 9 +- feature/taildrop/retrieve.go | 116 ++++++------- feature/taildrop/send.go | 270 ++++++++++++++++++++++++++---- feature/taildrop/send_test.go | 131 +++++++++++---- feature/taildrop/taildrop.go | 83 ++++++--- feature/taildrop/taildrop_test.go | 56 ++++--- 14 files changed, 596 insertions(+), 555 deletions(-) delete mode 100644 feature/taildrop/fileops.go delete mode 100644 feature/taildrop/fileops_fs.go diff --git a/feature/taildrop/delete.go b/feature/taildrop/delete.go index 0b7259879f941..e9c8d7f1c90fa 100644 --- a/feature/taildrop/delete.go +++ b/feature/taildrop/delete.go @@ -6,7 +6,9 @@ package taildrop import ( "container/list" "context" + "io/fs" "os" + "path/filepath" "strings" "sync" "time" @@ -26,6 +28,7 @@ const deleteDelay = time.Hour type fileDeleter struct { logf logger.Logf clock tstime.DefaultClock + dir string event func(string) // called for certain events; for testing only mu sync.Mutex @@ -36,7 +39,6 @@ type fileDeleter struct { group syncs.WaitGroup shutdownCtx context.Context shutdown context.CancelFunc - fs FileOps // must be used for all filesystem operations } // deleteFile is a specific file to delete after deleteDelay. @@ -48,14 +50,15 @@ type deleteFile struct { func (d *fileDeleter) Init(m *manager, eventHook func(string)) { d.logf = m.opts.Logf d.clock = m.opts.Clock + d.dir = m.opts.Dir d.event = eventHook - d.fs = m.opts.fileOps d.byName = make(map[string]*list.Element) d.emptySignal = make(chan struct{}) d.shutdownCtx, d.shutdown = context.WithCancel(context.Background()) // From a cold-start, load the list of partial and deleted files. + // // Only run this if we have ever received at least one file // to avoid ever touching the taildrop directory on systems (e.g., MacOS) // that pop up a security dialog window upon first access. @@ -68,45 +71,38 @@ func (d *fileDeleter) Init(m *manager, eventHook func(string)) { d.group.Go(func() { d.event("start full-scan") defer d.event("end full-scan") - - if d.fs == nil { - d.logf("deleter: nil FileOps") - } - - files, err := d.fs.ListFiles() - if err != nil { - d.logf("deleter: ListDir error: %v", err) - return - } - for _, filename := range files { + rangeDir(d.dir, func(de fs.DirEntry) bool { switch { case d.shutdownCtx.Err() != nil: - return // terminate early - case strings.HasSuffix(filename, partialSuffix): + return false // terminate early + case !de.Type().IsRegular(): + return true + case strings.HasSuffix(de.Name(), partialSuffix): // Only enqueue the file for deletion if there is no active put. - nameID := strings.TrimSuffix(filename, partialSuffix) + nameID := strings.TrimSuffix(de.Name(), partialSuffix) if i := strings.LastIndexByte(nameID, '.'); i > 0 { key := incomingFileKey{clientID(nameID[i+len("."):]), nameID[:i]} m.incomingFiles.LoadFunc(key, func(_ *incomingFile, loaded bool) { if !loaded { - d.Insert(filename) + d.Insert(de.Name()) } }) } else { - d.Insert(filename) + d.Insert(de.Name()) } - case strings.HasSuffix(filename, deletedSuffix): + case strings.HasSuffix(de.Name(), deletedSuffix): // Best-effort immediate deletion of deleted files. - name := strings.TrimSuffix(filename, deletedSuffix) - if d.fs.Remove(name) == nil { - if d.fs.Remove(filename) == nil { - continue + name := strings.TrimSuffix(de.Name(), deletedSuffix) + if os.Remove(filepath.Join(d.dir, name)) == nil { + if os.Remove(filepath.Join(d.dir, de.Name())) == nil { + break } } - // Otherwise enqueue for later deletion. - d.Insert(filename) + // Otherwise, enqueue the file for later deletion. + d.Insert(de.Name()) } - } + return true + }) }) } @@ -153,13 +149,13 @@ func (d *fileDeleter) waitAndDelete(wait time.Duration) { // Delete the expired file. if name, ok := strings.CutSuffix(file.name, deletedSuffix); ok { - if err := d.fs.Remove(name); err != nil && !os.IsNotExist(err) { + if err := os.Remove(filepath.Join(d.dir, name)); err != nil && !os.IsNotExist(err) { d.logf("could not delete: %v", redactError(err)) failed = append(failed, elem) continue } } - if err := d.fs.Remove(file.name); err != nil && !os.IsNotExist(err) { + if err := os.Remove(filepath.Join(d.dir, file.name)); err != nil && !os.IsNotExist(err) { d.logf("could not delete: %v", redactError(err)) failed = append(failed, elem) continue diff --git a/feature/taildrop/delete_test.go b/feature/taildrop/delete_test.go index 36950f58288cb..7a58de55c2492 100644 --- a/feature/taildrop/delete_test.go +++ b/feature/taildrop/delete_test.go @@ -5,6 +5,7 @@ package taildrop import ( "os" + "path/filepath" "slices" "testing" "time" @@ -19,20 +20,11 @@ import ( func TestDeleter(t *testing.T) { dir := t.TempDir() - var m manager - var fd fileDeleter - m.opts.Logf = t.Logf - m.opts.Clock = tstime.DefaultClock{Clock: tstest.NewClock(tstest.ClockOpts{ - Start: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), - })} - m.opts.State = must.Get(mem.New(nil, "")) - m.opts.fileOps, _ = newFileOps(dir) - - must.Do(m.touchFile("foo.partial")) - must.Do(m.touchFile("bar.partial")) - must.Do(m.touchFile("fizz")) - must.Do(m.touchFile("fizz.deleted")) - must.Do(m.touchFile("buzz.deleted")) // lacks a matching "buzz" file + must.Do(touchFile(filepath.Join(dir, "foo.partial"))) + must.Do(touchFile(filepath.Join(dir, "bar.partial"))) + must.Do(touchFile(filepath.Join(dir, "fizz"))) + must.Do(touchFile(filepath.Join(dir, "fizz.deleted"))) + must.Do(touchFile(filepath.Join(dir, "buzz.deleted"))) // lacks a matching "buzz" file checkDirectory := func(want ...string) { t.Helper() @@ -77,10 +69,12 @@ func TestDeleter(t *testing.T) { } eventHook := func(event string) { eventsChan <- event } + var m manager + var fd fileDeleter m.opts.Logf = t.Logf m.opts.Clock = tstime.DefaultClock{Clock: clock} + m.opts.Dir = dir m.opts.State = must.Get(mem.New(nil, "")) - m.opts.fileOps, _ = newFileOps(dir) must.Do(m.opts.State.WriteState(ipn.TaildropReceivedKey, []byte{1})) fd.Init(&m, eventHook) defer fd.Shutdown() @@ -106,17 +100,17 @@ func TestDeleter(t *testing.T) { checkEvents("end waitAndDelete") checkDirectory() - must.Do(m.touchFile("one.partial")) + must.Do(touchFile(filepath.Join(dir, "one.partial"))) insert("one.partial") checkEvents("start waitAndDelete") advance(deleteDelay / 4) - must.Do(m.touchFile("two.partial")) + must.Do(touchFile(filepath.Join(dir, "two.partial"))) insert("two.partial") advance(deleteDelay / 4) - must.Do(m.touchFile("three.partial")) + must.Do(touchFile(filepath.Join(dir, "three.partial"))) insert("three.partial") advance(deleteDelay / 4) - must.Do(m.touchFile("four.partial")) + must.Do(touchFile(filepath.Join(dir, "four.partial"))) insert("four.partial") advance(deleteDelay / 4) @@ -151,8 +145,8 @@ func TestDeleterInitWithoutTaildrop(t *testing.T) { var m manager var fd fileDeleter m.opts.Logf = t.Logf + m.opts.Dir = t.TempDir() m.opts.State = must.Get(mem.New(nil, "")) - m.opts.fileOps, _ = newFileOps(t.TempDir()) fd.Init(&m, func(event string) { t.Errorf("unexpected event: %v", event) }) fd.Shutdown() } diff --git a/feature/taildrop/ext.go b/feature/taildrop/ext.go index f8f45b53fae26..c11fe3af427a1 100644 --- a/feature/taildrop/ext.go +++ b/feature/taildrop/ext.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "maps" + "os" "path/filepath" "runtime" "slices" @@ -74,7 +75,7 @@ type Extension struct { // FileOps abstracts platform-specific file operations needed for file transfers. // This is currently being used for Android to use the Storage Access Framework. - fileOps FileOps + FileOps FileOps nodeBackendForTest ipnext.NodeBackend // if non-nil, pretend we're this node state for tests @@ -88,6 +89,30 @@ type Extension struct { outgoingFiles map[string]*ipn.OutgoingFile } +// safDirectoryPrefix is used to determine if the directory is managed via SAF. +const SafDirectoryPrefix = "content://" + +// PutMode controls how Manager.PutFile writes files to storage. +// +// PutModeDirect – write files directly to a filesystem path (default). +// PutModeAndroidSAF – use Android’s Storage Access Framework (SAF), where +// the OS manages the underlying directory permissions. +type PutMode int + +const ( + PutModeDirect PutMode = iota + PutModeAndroidSAF +) + +// FileOps defines platform-specific file operations. +type FileOps interface { + OpenFileWriter(filename string) (io.WriteCloser, string, error) + + // RenamePartialFile finalizes a partial file. + // It returns the new SAF URI as a string and an error. + RenamePartialFile(partialUri, targetDirUri, targetName string) (string, error) +} + func (e *Extension) Name() string { return "taildrop" } @@ -151,34 +176,23 @@ func (e *Extension) onChangeProfile(profile ipn.LoginProfileView, _ ipn.PrefsVie return } - // Use the provided [FileOps] implementation (typically for SAF access on Android), - // or create an [fsFileOps] instance rooted at fileRoot. - // - // A non-nil [FileOps] also implies that we are in DirectFileMode. - fops := e.fileOps - isDirectFileMode := fops != nil - if fops == nil { - var fileRoot string - if fileRoot, isDirectFileMode = e.fileRoot(uid, activeLogin); fileRoot == "" { - e.logf("no Taildrop directory configured") - e.setMgrLocked(nil) - return - } - - var err error - if fops, err = newFileOps(fileRoot); err != nil { - e.logf("taildrop: cannot create FileOps: %v", err) - e.setMgrLocked(nil) - return - } + // If we have a netmap, create a taildrop manager. + fileRoot, isDirectFileMode := e.fileRoot(uid, activeLogin) + if fileRoot == "" { + e.logf("no Taildrop directory configured") + } + mode := PutModeDirect + if e.directFileRoot != "" && strings.HasPrefix(e.directFileRoot, SafDirectoryPrefix) { + mode = PutModeAndroidSAF } - e.setMgrLocked(managerOptions{ Logf: e.logf, Clock: tstime.DefaultClock{Clock: e.sb.Clock()}, State: e.stateStore, + Dir: fileRoot, DirectFileMode: isDirectFileMode, - fileOps: fops, + FileOps: e.FileOps, + Mode: mode, SendFileNotify: e.sendFileNotify, }.New()) } @@ -207,7 +221,12 @@ func (e *Extension) fileRoot(uid tailcfg.UserID, activeLogin string) (root strin baseDir := fmt.Sprintf("%s-uid-%d", strings.ReplaceAll(activeLogin, "@", "-"), uid) - return filepath.Join(varRoot, "files", baseDir), false + dir := filepath.Join(varRoot, "files", baseDir) + if err := os.MkdirAll(dir, 0700); err != nil { + e.logf("Taildrop disabled; error making directory: %v", err) + return "", false + } + return dir, false } // hasCapFileSharing reports whether the current node has the file sharing diff --git a/feature/taildrop/fileops.go b/feature/taildrop/fileops.go deleted file mode 100644 index 14f76067a8094..0000000000000 --- a/feature/taildrop/fileops.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package taildrop - -import ( - "io" - "io/fs" - "os" -) - -// FileOps abstracts over both local‐FS paths and Android SAF URIs. -type FileOps interface { - // OpenWriter creates or truncates a file named relative to the receiver's root, - // seeking to the specified offset. If the file does not exist, it is created with mode perm - // on platforms that support it. - // - // It returns an [io.WriteCloser] and the file's absolute path, or an error. - // This call may block. Callers should avoid holding locks when calling OpenWriter. - OpenWriter(name string, offset int64, perm os.FileMode) (wc io.WriteCloser, path string, err error) - - // Remove deletes a file or directory relative to the receiver's root. - // It returns [io.ErrNotExist] if the file or directory does not exist. - Remove(name string) error - - // Rename atomically renames oldPath to a new file named newName, - // returning the full new path or an error. - Rename(oldPath, newName string) (newPath string, err error) - - // ListFiles returns just the basenames of all regular files - // in the root directory. - ListFiles() ([]string, error) - - // Stat returns the FileInfo for the given name or an error. - Stat(name string) (fs.FileInfo, error) - - // OpenReader opens the given basename for the given name or an error. - OpenReader(name string) (io.ReadCloser, error) -} - -var newFileOps func(dir string) (FileOps, error) diff --git a/feature/taildrop/fileops_fs.go b/feature/taildrop/fileops_fs.go deleted file mode 100644 index 4fecbe4af6bbb..0000000000000 --- a/feature/taildrop/fileops_fs.go +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause -//go:build !android - -package taildrop - -import ( - "bytes" - "crypto/sha256" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path" - "path/filepath" - "strings" - "sync" - "unicode/utf8" -) - -var renameMu sync.Mutex - -// fsFileOps implements FileOps using the local filesystem rooted at a directory. -// It is used on non-Android platforms. -type fsFileOps struct{ rootDir string } - -func init() { - newFileOps = func(dir string) (FileOps, error) { - if dir == "" { - return nil, errors.New("rootDir cannot be empty") - } - if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, fmt.Errorf("mkdir %q: %w", dir, err) - } - return fsFileOps{rootDir: dir}, nil - } -} - -func (f fsFileOps) OpenWriter(name string, offset int64, perm os.FileMode) (io.WriteCloser, string, error) { - path, err := joinDir(f.rootDir, name) - if err != nil { - return nil, "", err - } - if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return nil, "", err - } - fi, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, perm) - if err != nil { - return nil, "", err - } - if offset != 0 { - curr, err := fi.Seek(0, io.SeekEnd) - if err != nil { - fi.Close() - return nil, "", err - } - if offset < 0 || offset > curr { - fi.Close() - return nil, "", fmt.Errorf("offset %d out of range", offset) - } - if _, err := fi.Seek(offset, io.SeekStart); err != nil { - fi.Close() - return nil, "", err - } - if err := fi.Truncate(offset); err != nil { - fi.Close() - return nil, "", err - } - } - return fi, path, nil -} - -func (f fsFileOps) Remove(name string) error { - path, err := joinDir(f.rootDir, name) - if err != nil { - return err - } - return os.Remove(path) -} - -// Rename moves the partial file into its final name. -// newName must be a base name (not absolute or containing path separators). -// It will retry up to 10 times, de-dup same-checksum files, etc. -func (f fsFileOps) Rename(oldPath, newName string) (newPath string, err error) { - var dst string - if filepath.IsAbs(newName) || strings.ContainsRune(newName, os.PathSeparator) { - return "", fmt.Errorf("invalid newName %q: must not be an absolute path or contain path separators", newName) - } - - dst = filepath.Join(f.rootDir, newName) - - if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { - return "", err - } - - st, err := os.Stat(oldPath) - if err != nil { - return "", err - } - wantSize := st.Size() - - const maxRetries = 10 - for i := 0; i < maxRetries; i++ { - renameMu.Lock() - fi, statErr := os.Stat(dst) - // Atomically rename the partial file as the destination file if it doesn't exist. - // Otherwise, it returns the length of the current destination file. - // The operation is atomic. - if os.IsNotExist(statErr) { - err = os.Rename(oldPath, dst) - renameMu.Unlock() - if err != nil { - return "", err - } - return dst, nil - } - if statErr != nil { - renameMu.Unlock() - return "", statErr - } - gotSize := fi.Size() - renameMu.Unlock() - - // Avoid the final rename if a destination file has the same contents. - // - // Note: this is best effort and copying files from iOS from the Media Library - // results in processing on the iOS side which means the size and shas of the - // same file can be different. - if gotSize == wantSize { - sumP, err := sha256File(oldPath) - if err != nil { - return "", err - } - sumD, err := sha256File(dst) - if err != nil { - return "", err - } - if bytes.Equal(sumP[:], sumD[:]) { - if err := os.Remove(oldPath); err != nil { - return "", err - } - return dst, nil - } - } - - // Choose a new destination filename and try again. - dst = filepath.Join(filepath.Dir(dst), nextFilename(filepath.Base(dst))) - } - - return "", fmt.Errorf("too many retries trying to rename %q to %q", oldPath, newName) -} - -// sha256File computes the SHA‑256 of a file. -func sha256File(path string) (sum [sha256.Size]byte, _ error) { - f, err := os.Open(path) - if err != nil { - return sum, err - } - defer f.Close() - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return sum, err - } - copy(sum[:], h.Sum(nil)) - return sum, nil -} - -func (f fsFileOps) ListFiles() ([]string, error) { - entries, err := os.ReadDir(f.rootDir) - if err != nil { - return nil, err - } - var names []string - for _, e := range entries { - if e.Type().IsRegular() { - names = append(names, e.Name()) - } - } - return names, nil -} - -func (f fsFileOps) Stat(name string) (fs.FileInfo, error) { - path, err := joinDir(f.rootDir, name) - if err != nil { - return nil, err - } - return os.Stat(path) -} - -func (f fsFileOps) OpenReader(name string) (io.ReadCloser, error) { - path, err := joinDir(f.rootDir, name) - if err != nil { - return nil, err - } - return os.Open(path) -} - -// joinDir is like [filepath.Join] but returns an error if baseName is too long, -// is a relative path instead of a basename, or is otherwise invalid or unsafe for incoming files. -func joinDir(dir, baseName string) (string, error) { - if !utf8.ValidString(baseName) || - strings.TrimSpace(baseName) != baseName || - len(baseName) > 255 { - return "", ErrInvalidFileName - } - // TODO: validate unicode normalization form too? Varies by platform. - clean := path.Clean(baseName) - if clean != baseName || clean == "." || clean == ".." { - return "", ErrInvalidFileName - } - for _, r := range baseName { - if !validFilenameRune(r) { - return "", ErrInvalidFileName - } - } - if !filepath.IsLocal(baseName) { - return "", ErrInvalidFileName - } - return filepath.Join(dir, baseName), nil -} diff --git a/feature/taildrop/paths.go b/feature/taildrop/paths.go index 79dc37d8f0699..22d01160cff8e 100644 --- a/feature/taildrop/paths.go +++ b/feature/taildrop/paths.go @@ -21,7 +21,7 @@ func (e *Extension) SetDirectFileRoot(root string) { // SetFileOps sets the platform specific file operations. This is used // to call Android's Storage Access Framework APIs. func (e *Extension) SetFileOps(fileOps FileOps) { - e.fileOps = fileOps + e.FileOps = fileOps } func (e *Extension) setPlatformDefaultDirectFileRoot() { diff --git a/feature/taildrop/peerapi_test.go b/feature/taildrop/peerapi_test.go index 6339973544453..1a003b6eddca7 100644 --- a/feature/taildrop/peerapi_test.go +++ b/feature/taildrop/peerapi_test.go @@ -24,7 +24,6 @@ import ( "tailscale.com/tstest" "tailscale.com/tstime" "tailscale.com/types/logger" - "tailscale.com/util/must" ) // peerAPIHandler serves the PeerAPI for a source specific client. @@ -94,16 +93,7 @@ func bodyContains(sub string) check { func fileHasSize(name string, size int) check { return func(t *testing.T, e *peerAPITestEnv) { - fsImpl, ok := e.taildrop.opts.fileOps.(*fsFileOps) - if !ok { - t.Skip("fileHasSize only supported on fsFileOps backend") - return - } - root := fsImpl.rootDir - if root == "" { - t.Errorf("no rootdir; can't check whether %q has size %v", name, size) - return - } + root := e.taildrop.Dir() if root == "" { t.Errorf("no rootdir; can't check whether %q has size %v", name, size) return @@ -119,12 +109,12 @@ func fileHasSize(name string, size int) check { func fileHasContents(name string, want string) check { return func(t *testing.T, e *peerAPITestEnv) { - fsImpl, ok := e.taildrop.opts.fileOps.(*fsFileOps) - if !ok { - t.Skip("fileHasContents only supported on fsFileOps backend") + root := e.taildrop.Dir() + if root == "" { + t.Errorf("no rootdir; can't check contents of %q", name) return } - path := filepath.Join(fsImpl.rootDir, name) + path := filepath.Join(root, name) got, err := os.ReadFile(path) if err != nil { t.Errorf("fileHasContents: %v", err) @@ -182,10 +172,9 @@ func TestHandlePeerAPI(t *testing.T) { reqs: []*http.Request{httptest.NewRequest("PUT", "/v0/put/foo", nil)}, checks: checks( httpStatus(http.StatusForbidden), - bodyContains("Taildrop disabled"), + bodyContains("Taildrop disabled; no storage directory"), ), }, - { name: "bad_method", isSelf: true, @@ -482,18 +471,14 @@ func TestHandlePeerAPI(t *testing.T) { selfNode.CapMap = tailcfg.NodeCapMap{tailcfg.CapabilityDebug: nil} } var rootDir string - var fo FileOps if !tt.omitRoot { - var err error - if fo, err = newFileOps(t.TempDir()); err != nil { - t.Fatalf("newFileOps: %v", err) - } + rootDir = t.TempDir() } var e peerAPITestEnv e.taildrop = managerOptions{ - Logf: e.logBuf.Logf, - fileOps: fo, + Logf: e.logBuf.Logf, + Dir: rootDir, }.New() ext := &fakeExtension{ @@ -505,7 +490,9 @@ func TestHandlePeerAPI(t *testing.T) { e.ph = &peerAPIHandler{ isSelf: tt.isSelf, selfNode: selfNode.View(), - peerNode: (&tailcfg.Node{ComputedName: "some-peer-name"}).View(), + peerNode: (&tailcfg.Node{ + ComputedName: "some-peer-name", + }).View(), } for _, req := range tt.reqs { e.rr = httptest.NewRecorder() @@ -539,8 +526,8 @@ func TestHandlePeerAPI(t *testing.T) { func TestFileDeleteRace(t *testing.T) { dir := t.TempDir() taildropMgr := managerOptions{ - Logf: t.Logf, - fileOps: must.Get(newFileOps(dir)), + Logf: t.Logf, + Dir: dir, }.New() ph := &peerAPIHandler{ diff --git a/feature/taildrop/resume.go b/feature/taildrop/resume.go index 20ef527a6da55..211a1ff6b68dd 100644 --- a/feature/taildrop/resume.go +++ b/feature/taildrop/resume.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "fmt" "io" + "io/fs" "os" "strings" ) @@ -50,20 +51,19 @@ func (cs *checksum) UnmarshalText(b []byte) error { // PartialFiles returns a list of partial files in [Handler.Dir] // that were sent (or is actively being sent) by the provided id. -func (m *manager) PartialFiles(id clientID) ([]string, error) { - if m == nil || m.opts.fileOps == nil { +func (m *manager) PartialFiles(id clientID) (ret []string, err error) { + if m == nil || m.opts.Dir == "" { return nil, ErrNoTaildrop } + suffix := id.partialSuffix() - files, err := m.opts.fileOps.ListFiles() - if err != nil { - return nil, redactError(err) - } - var ret []string - for _, filename := range files { - if strings.HasSuffix(filename, suffix) { - ret = append(ret, filename) + if err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { + if name := de.Name(); strings.HasSuffix(name, suffix) { + ret = append(ret, name) } + return true + }); err != nil { + return ret, redactError(err) } return ret, nil } @@ -73,13 +73,17 @@ func (m *manager) PartialFiles(id clientID) ([]string, error) { // It returns (BlockChecksum{}, io.EOF) when the stream is complete. // It is the caller's responsibility to call close. func (m *manager) HashPartialFile(id clientID, baseName string) (next func() (blockChecksum, error), close func() error, err error) { - if m == nil || m.opts.fileOps == nil { + if m == nil || m.opts.Dir == "" { return nil, nil, ErrNoTaildrop } noopNext := func() (blockChecksum, error) { return blockChecksum{}, io.EOF } noopClose := func() error { return nil } - f, err := m.opts.fileOps.OpenReader(baseName + id.partialSuffix()) + dstFile, err := joinDir(m.opts.Dir, baseName) + if err != nil { + return nil, nil, err + } + f, err := os.Open(dstFile + id.partialSuffix()) if err != nil { if os.IsNotExist(err) { return noopNext, noopClose, nil diff --git a/feature/taildrop/resume_test.go b/feature/taildrop/resume_test.go index 4e59d401dcc53..dac3c657bfb58 100644 --- a/feature/taildrop/resume_test.go +++ b/feature/taildrop/resume_test.go @@ -8,7 +8,6 @@ import ( "io" "math/rand" "os" - "path/filepath" "testing" "testing/iotest" @@ -20,9 +19,7 @@ func TestResume(t *testing.T) { defer func() { blockSize = oldBlockSize }() blockSize = 256 - dir := t.TempDir() - - m := managerOptions{Logf: t.Logf, fileOps: must.Get(newFileOps(dir))}.New() + m := managerOptions{Logf: t.Logf, Dir: t.TempDir()}.New() defer m.Shutdown() rn := rand.New(rand.NewSource(0)) @@ -40,7 +37,7 @@ func TestResume(t *testing.T) { must.Do(close()) // Windows wants the file handle to be closed to rename it. must.Get(m.PutFile("", "foo", r, offset, -1)) - got := must.Get(os.ReadFile(filepath.Join(dir, "foo"))) + got := must.Get(os.ReadFile(must.Get(joinDir(m.opts.Dir, "foo")))) if !bytes.Equal(got, want) { t.Errorf("content mismatches") } @@ -69,7 +66,7 @@ func TestResume(t *testing.T) { t.Fatalf("too many iterations to complete the test") } } - got := must.Get(os.ReadFile(filepath.Join(dir, "bar"))) + got := must.Get(os.ReadFile(must.Get(joinDir(m.opts.Dir, "bar")))) if !bytes.Equal(got, want) { t.Errorf("content mismatches") } diff --git a/feature/taildrop/retrieve.go b/feature/taildrop/retrieve.go index b048a1b3b5f9d..6fb97519363bc 100644 --- a/feature/taildrop/retrieve.go +++ b/feature/taildrop/retrieve.go @@ -9,19 +9,19 @@ import ( "io" "io/fs" "os" + "path/filepath" "runtime" "sort" "time" "tailscale.com/client/tailscale/apitype" "tailscale.com/logtail/backoff" - "tailscale.com/util/set" ) // HasFilesWaiting reports whether any files are buffered in [Handler.Dir]. // This always returns false when [Handler.DirectFileMode] is false. -func (m *manager) HasFilesWaiting() bool { - if m == nil || m.opts.fileOps == nil || m.opts.DirectFileMode { +func (m *manager) HasFilesWaiting() (has bool) { + if m == nil || m.opts.Dir == "" || m.opts.DirectFileMode { return false } @@ -30,66 +30,63 @@ func (m *manager) HasFilesWaiting() bool { // has-files-or-not values as the macOS/iOS client might // in the future use+delete the files directly. So only // keep this negative cache. - total := m.totalReceived.Load() - if total == m.emptySince.Load() { + totalReceived := m.totalReceived.Load() + if totalReceived == m.emptySince.Load() { return false } - files, err := m.opts.fileOps.ListFiles() - if err != nil { - return false - } - - // Build a set of filenames present in Dir - fileSet := set.Of(files...) - - for _, filename := range files { - if isPartialOrDeleted(filename) { - continue + // Check whether there is at least one one waiting file. + err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { + name := de.Name() + if isPartialOrDeleted(name) || !de.Type().IsRegular() { + return true } - if fileSet.Contains(filename + deletedSuffix) { - continue // already handled + _, err := os.Stat(filepath.Join(m.opts.Dir, name+deletedSuffix)) + if os.IsNotExist(err) { + has = true + return false } - // Found at least one downloadable file return true - } + }) - // No waiting files → update negative‑result cache - m.emptySince.Store(total) - return false + // If there are no more waiting files, record totalReceived as emptySince + // so that we can short-circuit the expensive directory traversal + // if no files have been received after the start of this call. + if err == nil && !has { + m.emptySince.Store(totalReceived) + } + return has } // WaitingFiles returns the list of files that have been sent by a // peer that are waiting in [Handler.Dir]. // This always returns nil when [Handler.DirectFileMode] is false. -func (m *manager) WaitingFiles() ([]apitype.WaitingFile, error) { - if m == nil || m.opts.fileOps == nil { +func (m *manager) WaitingFiles() (ret []apitype.WaitingFile, err error) { + if m == nil || m.opts.Dir == "" { return nil, ErrNoTaildrop } if m.opts.DirectFileMode { return nil, nil } - names, err := m.opts.fileOps.ListFiles() - if err != nil { - return nil, redactError(err) - } - var ret []apitype.WaitingFile - for _, name := range names { - if isPartialOrDeleted(name) { - continue + if err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { + name := de.Name() + if isPartialOrDeleted(name) || !de.Type().IsRegular() { + return true } - // A corresponding .deleted marker means the file was already handled. - if _, err := m.opts.fileOps.Stat(name + deletedSuffix); err == nil { - continue - } - fi, err := m.opts.fileOps.Stat(name) - if err != nil { - continue + _, err := os.Stat(filepath.Join(m.opts.Dir, name+deletedSuffix)) + if os.IsNotExist(err) { + fi, err := de.Info() + if err != nil { + return true + } + ret = append(ret, apitype.WaitingFile{ + Name: filepath.Base(name), + Size: fi.Size(), + }) } - ret = append(ret, apitype.WaitingFile{ - Name: name, - Size: fi.Size(), - }) + return true + }); err != nil { + return nil, redactError(err) } sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) return ret, nil @@ -98,18 +95,21 @@ func (m *manager) WaitingFiles() ([]apitype.WaitingFile, error) { // DeleteFile deletes a file of the given baseName from [Handler.Dir]. // This method is only allowed when [Handler.DirectFileMode] is false. func (m *manager) DeleteFile(baseName string) error { - if m == nil || m.opts.fileOps == nil { + if m == nil || m.opts.Dir == "" { return ErrNoTaildrop } if m.opts.DirectFileMode { return errors.New("deletes not allowed in direct mode") } - + path, err := joinDir(m.opts.Dir, baseName) + if err != nil { + return err + } var bo *backoff.Backoff logf := m.opts.Logf t0 := m.opts.Clock.Now() for { - err := m.opts.fileOps.Remove(baseName) + err := os.Remove(path) if err != nil && !os.IsNotExist(err) { err = redactError(err) // Put a retry loop around deletes on Windows. @@ -129,7 +129,7 @@ func (m *manager) DeleteFile(baseName string) error { bo.BackOff(context.Background(), err) continue } - if err := m.touchFile(baseName + deletedSuffix); err != nil { + if err := touchFile(path + deletedSuffix); err != nil { logf("peerapi: failed to leave deleted marker: %v", err) } m.deleter.Insert(baseName + deletedSuffix) @@ -141,31 +141,35 @@ func (m *manager) DeleteFile(baseName string) error { } } -func (m *manager) touchFile(name string) error { - wc, _, err := m.opts.fileOps.OpenWriter(name /* offset= */, 0, 0666) +func touchFile(path string) error { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) if err != nil { return redactError(err) } - return wc.Close() + return f.Close() } // OpenFile opens a file of the given baseName from [Handler.Dir]. // This method is only allowed when [Handler.DirectFileMode] is false. func (m *manager) OpenFile(baseName string) (rc io.ReadCloser, size int64, err error) { - if m == nil || m.opts.fileOps == nil { + if m == nil || m.opts.Dir == "" { return nil, 0, ErrNoTaildrop } if m.opts.DirectFileMode { return nil, 0, errors.New("opens not allowed in direct mode") } - if _, err := m.opts.fileOps.Stat(baseName + deletedSuffix); err == nil { - return nil, 0, redactError(&fs.PathError{Op: "open", Path: baseName, Err: fs.ErrNotExist}) + path, err := joinDir(m.opts.Dir, baseName) + if err != nil { + return nil, 0, err + } + if _, err := os.Stat(path + deletedSuffix); err == nil { + return nil, 0, redactError(&fs.PathError{Op: "open", Path: path, Err: fs.ErrNotExist}) } - f, err := m.opts.fileOps.OpenReader(baseName) + f, err := os.Open(path) if err != nil { return nil, 0, redactError(err) } - fi, err := m.opts.fileOps.Stat(baseName) + fi, err := f.Stat() if err != nil { f.Close() return nil, 0, redactError(err) diff --git a/feature/taildrop/send.go b/feature/taildrop/send.go index 32ba5f6f0d644..59a1701da6f0d 100644 --- a/feature/taildrop/send.go +++ b/feature/taildrop/send.go @@ -4,8 +4,11 @@ package taildrop import ( + "crypto/sha256" "fmt" "io" + "os" + "path/filepath" "sync" "time" @@ -70,10 +73,9 @@ func (f *incomingFile) Write(p []byte) (n int, err error) { // specific partial file. This allows the client to determine whether to resume // a partial file. While resuming, PutFile may be called again with a non-zero // offset to specify where to resume receiving data at. -func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, length int64) (fileLength int64, err error) { - +func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, length int64) (int64, error) { switch { - case m == nil || m.opts.fileOps == nil: + case m == nil || m.opts.Dir == "": return 0, ErrNoTaildrop case !envknob.CanTaildrop(): return 0, ErrNoTaildrop @@ -81,47 +83,47 @@ func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, len return 0, ErrNotAccessible } - if err := validateBaseName(baseName); err != nil { - return 0, err - } - - // and make sure we don't delete it while uploading: - m.deleter.Remove(baseName) - - // Create (if not already) the partial file with read-write permissions. - partialName := baseName + id.partialSuffix() - wc, partialPath, err := m.opts.fileOps.OpenWriter(partialName, offset, 0o666) - if err != nil { - return 0, m.redactAndLogError("Create", err) - } - defer func() { - wc.Close() + //Compute dstPath & avoid mid‑upload deletion + var dstPath string + if m.opts.Mode == PutModeDirect { + var err error + dstPath, err = joinDir(m.opts.Dir, baseName) if err != nil { - m.deleter.Insert(partialName) // mark partial file for eventual deletion + return 0, err } - }() + } else { + // In SAF mode, we simply use the baseName as the destination "path" + // (the actual directory is managed by SAF). + dstPath = baseName + } + m.deleter.Remove(filepath.Base(dstPath)) // avoid deleting the partial file while receiving // Check whether there is an in-progress transfer for the file. - inFileKey := incomingFileKey{id, baseName} - inFile, loaded := m.incomingFiles.LoadOrInit(inFileKey, func() *incomingFile { - inFile := &incomingFile{ + partialFileKey := incomingFileKey{id, baseName} + inFile, loaded := m.incomingFiles.LoadOrInit(partialFileKey, func() *incomingFile { + return &incomingFile{ clock: m.opts.Clock, started: m.opts.Clock.Now(), size: length, sendFileNotify: m.opts.SendFileNotify, } - if m.opts.DirectFileMode { - inFile.partialPath = partialPath - } - return inFile }) - - inFile.w = wc - if loaded { return 0, ErrFileExists } - defer m.incomingFiles.Delete(inFileKey) + defer m.incomingFiles.Delete(partialFileKey) + + // Open writer & populate inFile paths + wc, partialPath, err := m.openWriterAndPaths(id, m.opts.Mode, inFile, baseName, dstPath, offset) + if err != nil { + return 0, m.redactAndLogError("Create", err) + } + defer func() { + wc.Close() + if err != nil { + m.deleter.Insert(filepath.Base(partialPath)) // mark partial file for eventual deletion + } + }() // Record that we have started to receive at least one file. // This is used by the deleter upon a cold-start to scan the directory @@ -146,26 +148,220 @@ func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, len return 0, m.redactAndLogError("Close", err) } - fileLength = offset + copyLength + fileLength := offset + copyLength inFile.mu.Lock() inFile.done = true inFile.mu.Unlock() - // 6) Finalize (rename/move) the partial into place via FileOps.Rename - finalPath, err := m.opts.fileOps.Rename(partialPath, baseName) - if err != nil { - return 0, m.redactAndLogError("Rename", err) + // Finalize rename + switch m.opts.Mode { + case PutModeDirect: + var finalDst string + finalDst, err = m.finalizeDirect(inFile, partialPath, dstPath, fileLength) + if err != nil { + return 0, m.redactAndLogError("Rename", err) + } + inFile.finalPath = finalDst + + case PutModeAndroidSAF: + if err = m.finalizeSAF(partialPath, baseName); err != nil { + return 0, m.redactAndLogError("Rename", err) + } } - inFile.finalPath = finalPath m.totalReceived.Add(1) m.opts.SendFileNotify() return fileLength, nil } +// openWriterAndPaths opens the correct writer, seeks/truncates if needed, +// and sets inFile.partialPath & inFile.finalPath for later cleanup/rename. +// The caller is responsible for closing the file on completion. +func (m *manager) openWriterAndPaths( + id clientID, + mode PutMode, + inFile *incomingFile, + baseName string, + dstPath string, + offset int64, +) (wc io.WriteCloser, partialPath string, err error) { + switch mode { + + case PutModeDirect: + partialPath = dstPath + id.partialSuffix() + f, err := os.OpenFile(partialPath, os.O_CREATE|os.O_RDWR, 0o666) + if err != nil { + return nil, "", m.redactAndLogError("Create", err) + } + if offset != 0 { + curr, err := f.Seek(0, io.SeekEnd) + if err != nil { + f.Close() + return nil, "", m.redactAndLogError("Seek", err) + } + if offset < 0 || offset > curr { + f.Close() + return nil, "", m.redactAndLogError("Seek", fmt.Errorf("offset %d out of range", offset)) + } + if _, err := f.Seek(offset, io.SeekStart); err != nil { + f.Close() + return nil, "", m.redactAndLogError("Seek", err) + } + if err := f.Truncate(offset); err != nil { + f.Close() + return nil, "", m.redactAndLogError("Truncate", err) + } + } + inFile.w = f + wc = f + inFile.partialPath = partialPath + inFile.finalPath = dstPath + return wc, partialPath, nil + + case PutModeAndroidSAF: + if m.opts.FileOps == nil { + return nil, "", m.redactAndLogError("Create (SAF)", fmt.Errorf("missing FileOps")) + } + writer, uri, err := m.opts.FileOps.OpenFileWriter(baseName) + if err != nil { + return nil, "", m.redactAndLogError("Create (SAF)", fmt.Errorf("failed to open file for writing via SAF")) + } + if writer == nil || uri == "" { + return nil, "", fmt.Errorf("invalid SAF writer or URI") + } + // SAF mode does not support resuming, so enforce offset == 0. + if offset != 0 { + writer.Close() + return nil, "", m.redactAndLogError("Seek", fmt.Errorf("resuming is not supported in SAF mode")) + } + inFile.w = writer + wc = writer + partialPath = uri + inFile.partialPath = uri + inFile.finalPath = baseName + return wc, partialPath, nil + + default: + return nil, "", fmt.Errorf("unsupported PutMode: %v", mode) + } +} + +// finalizeDirect atomically renames or dedups the partial file, retrying +// under new names up to 10 times. It returns the final path that succeeded. +func (m *manager) finalizeDirect( + inFile *incomingFile, + partialPath string, + initialDst string, + fileLength int64, +) (string, error) { + var ( + once sync.Once + cachedSum [sha256.Size]byte + cacheErr error + computeSum = func() ([sha256.Size]byte, error) { + once.Do(func() { cachedSum, cacheErr = sha256File(partialPath) }) + return cachedSum, cacheErr + } + ) + + dstPath := initialDst + const maxRetries = 10 + for i := 0; i < maxRetries; i++ { + // Atomically rename the partial file as the destination file if it doesn't exist. + // Otherwise, it returns the length of the current destination file. + // The operation is atomic. + lengthOnDisk, err := func() (int64, error) { + m.renameMu.Lock() + defer m.renameMu.Unlock() + fi, statErr := os.Stat(dstPath) + if os.IsNotExist(statErr) { + // dst missing → rename partial into place + return -1, os.Rename(partialPath, dstPath) + } + if statErr != nil { + return -1, statErr + } + return fi.Size(), nil + }() + if err != nil { + return "", err + } + if lengthOnDisk < 0 { + // successfully moved + inFile.finalPath = dstPath + return dstPath, nil + } + + // Avoid the final rename if a destination file has the same contents. + // + // Note: this is best effort and copying files from iOS from the Media Library + // results in processing on the iOS side which means the size and shas of the + // same file can be different. + if lengthOnDisk == fileLength { + partSum, err := computeSum() + if err != nil { + return "", err + } + dstSum, err := sha256File(dstPath) + if err != nil { + return "", err + } + if partSum == dstSum { + // same content → drop the partial + if err := os.Remove(partialPath); err != nil { + return "", err + } + inFile.finalPath = dstPath + return dstPath, nil + } + } + + // Choose a new destination filename and try again. + dstPath = nextFilename(dstPath) + } + + return "", fmt.Errorf("too many retries trying to rename a partial file %q", initialDst) +} + +// finalizeSAF retries RenamePartialFile up to 10 times, generating a new +// name on each failure until the SAF URI changes. +func (m *manager) finalizeSAF( + partialPath, finalName string, +) error { + if m.opts.FileOps == nil { + return fmt.Errorf("missing FileOps for SAF finalize") + } + const maxTries = 10 + name := finalName + for i := 0; i < maxTries; i++ { + newURI, err := m.opts.FileOps.RenamePartialFile(partialPath, m.opts.Dir, name) + if err != nil { + return err + } + if newURI != "" && newURI != name { + return nil + } + name = nextFilename(name) + } + return fmt.Errorf("failed to finalize SAF file after %d retries", maxTries) +} + func (m *manager) redactAndLogError(stage string, err error) error { err = redactError(err) m.opts.Logf("put %s error: %v", stage, err) return err } + +func sha256File(file string) (out [sha256.Size]byte, err error) { + h := sha256.New() + f, err := os.Open(file) + if err != nil { + return out, err + } + defer f.Close() + if _, err := io.Copy(h, f); err != nil { + return out, err + } + return [sha256.Size]byte(h.Sum(nil)), nil +} diff --git a/feature/taildrop/send_test.go b/feature/taildrop/send_test.go index 9ffa5fccc0a36..8edb704172fc5 100644 --- a/feature/taildrop/send_test.go +++ b/feature/taildrop/send_test.go @@ -4,64 +4,123 @@ package taildrop import ( + "bytes" + "fmt" + "io" "os" "path/filepath" - "strings" "testing" "tailscale.com/tstime" - "tailscale.com/util/must" ) +// nopWriteCloser is a no-op io.WriteCloser wrapping a bytes.Buffer. +type nopWriteCloser struct{ *bytes.Buffer } + +func (nwc nopWriteCloser) Close() error { return nil } + +// mockFileOps implements just enough of the FileOps interface for SAF tests. +type mockFileOps struct { + writes *bytes.Buffer + renameOK bool +} + +func (m *mockFileOps) OpenFileWriter(name string) (io.WriteCloser, string, error) { + m.writes = new(bytes.Buffer) + return nopWriteCloser{m.writes}, "uri://" + name + ".partial", nil +} + +func (m *mockFileOps) RenamePartialFile(partialPath, dir, finalName string) (string, error) { + if !m.renameOK { + m.renameOK = true + return "uri://" + finalName, nil + } + return "", io.ErrUnexpectedEOF +} + func TestPutFile(t *testing.T) { const content = "hello, world" tests := []struct { - name string - directFileMode bool + name string + mode PutMode + setup func(t *testing.T) (*manager, string, *mockFileOps) + wantFile string }{ - {"DirectFileMode", true}, - {"NonDirectFileMode", false}, + { + name: "PutModeDirect", + mode: PutModeDirect, + setup: func(t *testing.T) (*manager, string, *mockFileOps) { + dir := t.TempDir() + opts := managerOptions{ + Logf: t.Logf, + Clock: tstime.DefaultClock{}, + State: nil, + Dir: dir, + Mode: PutModeDirect, + DirectFileMode: true, + SendFileNotify: func() {}, + } + mgr := opts.New() + return mgr, dir, nil + }, + wantFile: "file.txt", + }, + { + name: "PutModeAndroidSAF", + mode: PutModeAndroidSAF, + setup: func(t *testing.T) (*manager, string, *mockFileOps) { + // SAF still needs a non-empty Dir to pass the guard. + dir := t.TempDir() + mops := &mockFileOps{} + opts := managerOptions{ + Logf: t.Logf, + Clock: tstime.DefaultClock{}, + State: nil, + Dir: dir, + Mode: PutModeAndroidSAF, + FileOps: mops, + DirectFileMode: true, + SendFileNotify: func() {}, + } + mgr := opts.New() + return mgr, dir, mops + }, + wantFile: "file.txt", + }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dir := t.TempDir() - mgr := managerOptions{ - Logf: t.Logf, - Clock: tstime.DefaultClock{}, - State: nil, - fileOps: must.Get(newFileOps(dir)), - DirectFileMode: tt.directFileMode, - SendFileNotify: func() {}, - }.New() + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + mgr, dir, mops := tc.setup(t) + id := clientID(fmt.Sprint(0)) + reader := bytes.NewReader([]byte(content)) - id := clientID("0") - n, err := mgr.PutFile(id, "file.txt", strings.NewReader(content), 0, int64(len(content))) + n, err := mgr.PutFile(id, "file.txt", reader, 0, int64(len(content))) if err != nil { - t.Fatalf("PutFile error: %v", err) + t.Fatalf("PutFile(%s) error: %v", tc.name, err) } if n != int64(len(content)) { t.Errorf("wrote %d bytes; want %d", n, len(content)) } - path := filepath.Join(dir, "file.txt") - - got, err := os.ReadFile(path) - if err != nil { - t.Fatalf("ReadFile %q: %v", path, err) - } - if string(got) != content { - t.Errorf("file contents = %q; want %q", string(got), content) - } + switch tc.mode { + case PutModeDirect: + path := filepath.Join(dir, tc.wantFile) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile error: %v", err) + } + if got := string(data); got != content { + t.Errorf("file contents = %q; want %q", got, content) + } - entries, err := os.ReadDir(dir) - if err != nil { - t.Fatal(err) - } - for _, entry := range entries { - if strings.Contains(entry.Name(), ".partial") { - t.Errorf("unexpected partial file left behind: %s", entry.Name()) + case PutModeAndroidSAF: + if mops.writes == nil { + t.Fatal("SAF writer was never created") + } + if got := mops.writes.String(); got != content { + t.Errorf("SAF writes = %q; want %q", got, content) } } }) diff --git a/feature/taildrop/taildrop.go b/feature/taildrop/taildrop.go index 6c3deaed1b538..2dfa415bbf0cc 100644 --- a/feature/taildrop/taildrop.go +++ b/feature/taildrop/taildrop.go @@ -12,6 +12,8 @@ package taildrop import ( "errors" "hash/adler32" + "io" + "io/fs" "os" "path" "path/filepath" @@ -19,6 +21,7 @@ import ( "sort" "strconv" "strings" + "sync" "sync/atomic" "unicode" "unicode/utf8" @@ -69,6 +72,11 @@ type managerOptions struct { Clock tstime.DefaultClock // may be nil State ipn.StateStore // may be nil + // Dir is the directory to store received files. + // This main either be the final location for the files + // or just a temporary staging directory (see DirectFileMode). + Dir string + // DirectFileMode reports whether we are writing files // directly to a download directory, rather than writing them to // a temporary staging directory. @@ -83,10 +91,9 @@ type managerOptions struct { // copy them out, and then delete them. DirectFileMode bool - // FileOps abstracts platform-specific file operations needed for file transfers. - // Android's implementation uses the Storage Access Framework, and other platforms - // use fsFileOps. - fileOps FileOps + FileOps FileOps + + Mode PutMode // SendFileNotify is called periodically while a file is actively // receiving the contents for the file. There is a final call @@ -104,6 +111,9 @@ type manager struct { // deleter managers asynchronous deletion of files. deleter fileDeleter + // renameMu is used to protect os.Rename calls so that they are atomic. + renameMu sync.Mutex + // totalReceived counts the cumulative total of received files. totalReceived atomic.Int64 // emptySince specifies that there were no waiting files @@ -127,6 +137,11 @@ func (opts managerOptions) New() *manager { return m } +// Dir returns the directory. +func (m *manager) Dir() string { + return m.opts.Dir +} + // Shutdown shuts down the Manager. // It blocks until all spawned goroutines have stopped running. func (m *manager) Shutdown() { @@ -157,29 +172,57 @@ func isPartialOrDeleted(s string) bool { return strings.HasSuffix(s, deletedSuffix) || strings.HasSuffix(s, partialSuffix) } -func validateBaseName(name string) error { - if !utf8.ValidString(name) || - strings.TrimSpace(name) != name || - len(name) > 255 { - return ErrInvalidFileName +func joinDir(dir, baseName string) (fullPath string, err error) { + if !utf8.ValidString(baseName) { + return "", ErrInvalidFileName } - // TODO: validate unicode normalization form too? Varies by platform. - clean := path.Clean(name) - if clean != name || clean == "." || clean == ".." { - return ErrInvalidFileName + if strings.TrimSpace(baseName) != baseName { + return "", ErrInvalidFileName + } + if len(baseName) > 255 { + return "", ErrInvalidFileName } - if isPartialOrDeleted(name) { - return ErrInvalidFileName + // TODO: validate unicode normalization form too? Varies by platform. + clean := path.Clean(baseName) + if clean != baseName || + clean == "." || clean == ".." || + isPartialOrDeleted(clean) { + return "", ErrInvalidFileName } - for _, r := range name { + for _, r := range baseName { if !validFilenameRune(r) { - return ErrInvalidFileName + return "", ErrInvalidFileName } } - if !filepath.IsLocal(name) { - return ErrInvalidFileName + if !filepath.IsLocal(baseName) { + return "", ErrInvalidFileName + } + return filepath.Join(dir, baseName), nil +} + +// rangeDir iterates over the contents of a directory, calling fn for each entry. +// It continues iterating while fn returns true. +// It reports the number of entries seen. +func rangeDir(dir string, fn func(fs.DirEntry) bool) error { + f, err := os.Open(dir) + if err != nil { + return err + } + defer f.Close() + for { + des, err := f.ReadDir(10) + for _, de := range des { + if !fn(de) { + return nil + } + } + if err != nil { + if err == io.EOF { + return nil + } + return err + } } - return nil } // IncomingFiles returns a list of active incoming files. diff --git a/feature/taildrop/taildrop_test.go b/feature/taildrop/taildrop_test.go index 0d77273f0aab0..da0bd2f430579 100644 --- a/feature/taildrop/taildrop_test.go +++ b/feature/taildrop/taildrop_test.go @@ -4,10 +4,40 @@ package taildrop import ( + "path/filepath" "strings" "testing" ) +func TestJoinDir(t *testing.T) { + dir := t.TempDir() + tests := []struct { + in string + want string // just relative to m.Dir + wantOk bool + }{ + {"", "", false}, + {"foo", "foo", true}, + {"./foo", "", false}, + {"../foo", "", false}, + {"foo/bar", "", false}, + {"😋", "😋", true}, + {"\xde\xad\xbe\xef", "", false}, + {"foo.partial", "", false}, + {"foo.deleted", "", false}, + {strings.Repeat("a", 1024), "", false}, + {"foo:bar", "", false}, + } + for _, tt := range tests { + got, gotErr := joinDir(dir, tt.in) + got, _ = filepath.Rel(dir, got) + gotOk := gotErr == nil + if got != tt.want || gotOk != tt.wantOk { + t.Errorf("joinDir(%q) = (%v, %v), want (%v, %v)", tt.in, got, gotOk, tt.want, tt.wantOk) + } + } +} + func TestNextFilename(t *testing.T) { tests := []struct { in string @@ -37,29 +67,3 @@ func TestNextFilename(t *testing.T) { } } } - -func TestValidateBaseName(t *testing.T) { - tests := []struct { - in string - wantOk bool - }{ - {"", false}, - {"foo", true}, - {"./foo", false}, - {"../foo", false}, - {"foo/bar", false}, - {"😋", true}, - {"\xde\xad\xbe\xef", false}, - {"foo.partial", false}, - {"foo.deleted", false}, - {strings.Repeat("a", 1024), false}, - {"foo:bar", false}, - } - for _, tt := range tests { - err := validateBaseName(tt.in) - gotOk := err == nil - if gotOk != tt.wantOk { - t.Errorf("validateBaseName(%q) = %v, wantOk = %v", tt.in, err, tt.wantOk) - } - } -} From 2589be2fdf739e701fed2859f9b76f08e6c4ed48 Mon Sep 17 00:00:00 2001 From: kari-ts <135075563+kari-ts@users.noreply.github.com> Date: Wed, 6 Aug 2025 10:35:05 -0700 Subject: [PATCH 258/263] feature/taildrop: do not use m.opts.Dir for Android (#16316) (#16789) In Android, we are prompting the user to select a Taildrop directory when they first receive a Taildrop: we block writes on Taildrop dir selection. This means that we cannot use Dir inside managerOptions, since the http request would not get the new Taildrop extension. This PR removes, in the Android case, the reliance on m.opts.Dir, and instead has FileOps hold the correct directory. This expands FileOps to be the Taildrop interface for all file system operations. Updates tailscale/corp#29211 Signed-off-by: kari-ts restore tstest (cherry picked from commit d897d809d649b312a3f87d01d9f9426d518cdced) --- feature/taildrop/delete.go | 52 +++--- feature/taildrop/delete_test.go | 34 ++-- feature/taildrop/ext.go | 67 +++----- feature/taildrop/fileops.go | 41 +++++ feature/taildrop/fileops_fs.go | 221 ++++++++++++++++++++++++ feature/taildrop/paths.go | 2 +- feature/taildrop/peerapi_test.go | 41 +++-- feature/taildrop/resume.go | 28 ++-- feature/taildrop/resume_test.go | 9 +- feature/taildrop/retrieve.go | 116 +++++++------ feature/taildrop/send.go | 270 ++++-------------------------- feature/taildrop/send_test.go | 131 ++++----------- feature/taildrop/taildrop.go | 83 +++------ feature/taildrop/taildrop_test.go | 56 +++---- 14 files changed, 555 insertions(+), 596 deletions(-) create mode 100644 feature/taildrop/fileops.go create mode 100644 feature/taildrop/fileops_fs.go diff --git a/feature/taildrop/delete.go b/feature/taildrop/delete.go index e9c8d7f1c90fa..0b7259879f941 100644 --- a/feature/taildrop/delete.go +++ b/feature/taildrop/delete.go @@ -6,9 +6,7 @@ package taildrop import ( "container/list" "context" - "io/fs" "os" - "path/filepath" "strings" "sync" "time" @@ -28,7 +26,6 @@ const deleteDelay = time.Hour type fileDeleter struct { logf logger.Logf clock tstime.DefaultClock - dir string event func(string) // called for certain events; for testing only mu sync.Mutex @@ -39,6 +36,7 @@ type fileDeleter struct { group syncs.WaitGroup shutdownCtx context.Context shutdown context.CancelFunc + fs FileOps // must be used for all filesystem operations } // deleteFile is a specific file to delete after deleteDelay. @@ -50,15 +48,14 @@ type deleteFile struct { func (d *fileDeleter) Init(m *manager, eventHook func(string)) { d.logf = m.opts.Logf d.clock = m.opts.Clock - d.dir = m.opts.Dir d.event = eventHook + d.fs = m.opts.fileOps d.byName = make(map[string]*list.Element) d.emptySignal = make(chan struct{}) d.shutdownCtx, d.shutdown = context.WithCancel(context.Background()) // From a cold-start, load the list of partial and deleted files. - // // Only run this if we have ever received at least one file // to avoid ever touching the taildrop directory on systems (e.g., MacOS) // that pop up a security dialog window upon first access. @@ -71,38 +68,45 @@ func (d *fileDeleter) Init(m *manager, eventHook func(string)) { d.group.Go(func() { d.event("start full-scan") defer d.event("end full-scan") - rangeDir(d.dir, func(de fs.DirEntry) bool { + + if d.fs == nil { + d.logf("deleter: nil FileOps") + } + + files, err := d.fs.ListFiles() + if err != nil { + d.logf("deleter: ListDir error: %v", err) + return + } + for _, filename := range files { switch { case d.shutdownCtx.Err() != nil: - return false // terminate early - case !de.Type().IsRegular(): - return true - case strings.HasSuffix(de.Name(), partialSuffix): + return // terminate early + case strings.HasSuffix(filename, partialSuffix): // Only enqueue the file for deletion if there is no active put. - nameID := strings.TrimSuffix(de.Name(), partialSuffix) + nameID := strings.TrimSuffix(filename, partialSuffix) if i := strings.LastIndexByte(nameID, '.'); i > 0 { key := incomingFileKey{clientID(nameID[i+len("."):]), nameID[:i]} m.incomingFiles.LoadFunc(key, func(_ *incomingFile, loaded bool) { if !loaded { - d.Insert(de.Name()) + d.Insert(filename) } }) } else { - d.Insert(de.Name()) + d.Insert(filename) } - case strings.HasSuffix(de.Name(), deletedSuffix): + case strings.HasSuffix(filename, deletedSuffix): // Best-effort immediate deletion of deleted files. - name := strings.TrimSuffix(de.Name(), deletedSuffix) - if os.Remove(filepath.Join(d.dir, name)) == nil { - if os.Remove(filepath.Join(d.dir, de.Name())) == nil { - break + name := strings.TrimSuffix(filename, deletedSuffix) + if d.fs.Remove(name) == nil { + if d.fs.Remove(filename) == nil { + continue } } - // Otherwise, enqueue the file for later deletion. - d.Insert(de.Name()) + // Otherwise enqueue for later deletion. + d.Insert(filename) } - return true - }) + } }) } @@ -149,13 +153,13 @@ func (d *fileDeleter) waitAndDelete(wait time.Duration) { // Delete the expired file. if name, ok := strings.CutSuffix(file.name, deletedSuffix); ok { - if err := os.Remove(filepath.Join(d.dir, name)); err != nil && !os.IsNotExist(err) { + if err := d.fs.Remove(name); err != nil && !os.IsNotExist(err) { d.logf("could not delete: %v", redactError(err)) failed = append(failed, elem) continue } } - if err := os.Remove(filepath.Join(d.dir, file.name)); err != nil && !os.IsNotExist(err) { + if err := d.fs.Remove(file.name); err != nil && !os.IsNotExist(err) { d.logf("could not delete: %v", redactError(err)) failed = append(failed, elem) continue diff --git a/feature/taildrop/delete_test.go b/feature/taildrop/delete_test.go index 7a58de55c2492..36950f58288cb 100644 --- a/feature/taildrop/delete_test.go +++ b/feature/taildrop/delete_test.go @@ -5,7 +5,6 @@ package taildrop import ( "os" - "path/filepath" "slices" "testing" "time" @@ -20,11 +19,20 @@ import ( func TestDeleter(t *testing.T) { dir := t.TempDir() - must.Do(touchFile(filepath.Join(dir, "foo.partial"))) - must.Do(touchFile(filepath.Join(dir, "bar.partial"))) - must.Do(touchFile(filepath.Join(dir, "fizz"))) - must.Do(touchFile(filepath.Join(dir, "fizz.deleted"))) - must.Do(touchFile(filepath.Join(dir, "buzz.deleted"))) // lacks a matching "buzz" file + var m manager + var fd fileDeleter + m.opts.Logf = t.Logf + m.opts.Clock = tstime.DefaultClock{Clock: tstest.NewClock(tstest.ClockOpts{ + Start: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), + })} + m.opts.State = must.Get(mem.New(nil, "")) + m.opts.fileOps, _ = newFileOps(dir) + + must.Do(m.touchFile("foo.partial")) + must.Do(m.touchFile("bar.partial")) + must.Do(m.touchFile("fizz")) + must.Do(m.touchFile("fizz.deleted")) + must.Do(m.touchFile("buzz.deleted")) // lacks a matching "buzz" file checkDirectory := func(want ...string) { t.Helper() @@ -69,12 +77,10 @@ func TestDeleter(t *testing.T) { } eventHook := func(event string) { eventsChan <- event } - var m manager - var fd fileDeleter m.opts.Logf = t.Logf m.opts.Clock = tstime.DefaultClock{Clock: clock} - m.opts.Dir = dir m.opts.State = must.Get(mem.New(nil, "")) + m.opts.fileOps, _ = newFileOps(dir) must.Do(m.opts.State.WriteState(ipn.TaildropReceivedKey, []byte{1})) fd.Init(&m, eventHook) defer fd.Shutdown() @@ -100,17 +106,17 @@ func TestDeleter(t *testing.T) { checkEvents("end waitAndDelete") checkDirectory() - must.Do(touchFile(filepath.Join(dir, "one.partial"))) + must.Do(m.touchFile("one.partial")) insert("one.partial") checkEvents("start waitAndDelete") advance(deleteDelay / 4) - must.Do(touchFile(filepath.Join(dir, "two.partial"))) + must.Do(m.touchFile("two.partial")) insert("two.partial") advance(deleteDelay / 4) - must.Do(touchFile(filepath.Join(dir, "three.partial"))) + must.Do(m.touchFile("three.partial")) insert("three.partial") advance(deleteDelay / 4) - must.Do(touchFile(filepath.Join(dir, "four.partial"))) + must.Do(m.touchFile("four.partial")) insert("four.partial") advance(deleteDelay / 4) @@ -145,8 +151,8 @@ func TestDeleterInitWithoutTaildrop(t *testing.T) { var m manager var fd fileDeleter m.opts.Logf = t.Logf - m.opts.Dir = t.TempDir() m.opts.State = must.Get(mem.New(nil, "")) + m.opts.fileOps, _ = newFileOps(t.TempDir()) fd.Init(&m, func(event string) { t.Errorf("unexpected event: %v", event) }) fd.Shutdown() } diff --git a/feature/taildrop/ext.go b/feature/taildrop/ext.go index c11fe3af427a1..f8f45b53fae26 100644 --- a/feature/taildrop/ext.go +++ b/feature/taildrop/ext.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "maps" - "os" "path/filepath" "runtime" "slices" @@ -75,7 +74,7 @@ type Extension struct { // FileOps abstracts platform-specific file operations needed for file transfers. // This is currently being used for Android to use the Storage Access Framework. - FileOps FileOps + fileOps FileOps nodeBackendForTest ipnext.NodeBackend // if non-nil, pretend we're this node state for tests @@ -89,30 +88,6 @@ type Extension struct { outgoingFiles map[string]*ipn.OutgoingFile } -// safDirectoryPrefix is used to determine if the directory is managed via SAF. -const SafDirectoryPrefix = "content://" - -// PutMode controls how Manager.PutFile writes files to storage. -// -// PutModeDirect – write files directly to a filesystem path (default). -// PutModeAndroidSAF – use Android’s Storage Access Framework (SAF), where -// the OS manages the underlying directory permissions. -type PutMode int - -const ( - PutModeDirect PutMode = iota - PutModeAndroidSAF -) - -// FileOps defines platform-specific file operations. -type FileOps interface { - OpenFileWriter(filename string) (io.WriteCloser, string, error) - - // RenamePartialFile finalizes a partial file. - // It returns the new SAF URI as a string and an error. - RenamePartialFile(partialUri, targetDirUri, targetName string) (string, error) -} - func (e *Extension) Name() string { return "taildrop" } @@ -176,23 +151,34 @@ func (e *Extension) onChangeProfile(profile ipn.LoginProfileView, _ ipn.PrefsVie return } - // If we have a netmap, create a taildrop manager. - fileRoot, isDirectFileMode := e.fileRoot(uid, activeLogin) - if fileRoot == "" { - e.logf("no Taildrop directory configured") - } - mode := PutModeDirect - if e.directFileRoot != "" && strings.HasPrefix(e.directFileRoot, SafDirectoryPrefix) { - mode = PutModeAndroidSAF + // Use the provided [FileOps] implementation (typically for SAF access on Android), + // or create an [fsFileOps] instance rooted at fileRoot. + // + // A non-nil [FileOps] also implies that we are in DirectFileMode. + fops := e.fileOps + isDirectFileMode := fops != nil + if fops == nil { + var fileRoot string + if fileRoot, isDirectFileMode = e.fileRoot(uid, activeLogin); fileRoot == "" { + e.logf("no Taildrop directory configured") + e.setMgrLocked(nil) + return + } + + var err error + if fops, err = newFileOps(fileRoot); err != nil { + e.logf("taildrop: cannot create FileOps: %v", err) + e.setMgrLocked(nil) + return + } } + e.setMgrLocked(managerOptions{ Logf: e.logf, Clock: tstime.DefaultClock{Clock: e.sb.Clock()}, State: e.stateStore, - Dir: fileRoot, DirectFileMode: isDirectFileMode, - FileOps: e.FileOps, - Mode: mode, + fileOps: fops, SendFileNotify: e.sendFileNotify, }.New()) } @@ -221,12 +207,7 @@ func (e *Extension) fileRoot(uid tailcfg.UserID, activeLogin string) (root strin baseDir := fmt.Sprintf("%s-uid-%d", strings.ReplaceAll(activeLogin, "@", "-"), uid) - dir := filepath.Join(varRoot, "files", baseDir) - if err := os.MkdirAll(dir, 0700); err != nil { - e.logf("Taildrop disabled; error making directory: %v", err) - return "", false - } - return dir, false + return filepath.Join(varRoot, "files", baseDir), false } // hasCapFileSharing reports whether the current node has the file sharing diff --git a/feature/taildrop/fileops.go b/feature/taildrop/fileops.go new file mode 100644 index 0000000000000..14f76067a8094 --- /dev/null +++ b/feature/taildrop/fileops.go @@ -0,0 +1,41 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package taildrop + +import ( + "io" + "io/fs" + "os" +) + +// FileOps abstracts over both local‐FS paths and Android SAF URIs. +type FileOps interface { + // OpenWriter creates or truncates a file named relative to the receiver's root, + // seeking to the specified offset. If the file does not exist, it is created with mode perm + // on platforms that support it. + // + // It returns an [io.WriteCloser] and the file's absolute path, or an error. + // This call may block. Callers should avoid holding locks when calling OpenWriter. + OpenWriter(name string, offset int64, perm os.FileMode) (wc io.WriteCloser, path string, err error) + + // Remove deletes a file or directory relative to the receiver's root. + // It returns [io.ErrNotExist] if the file or directory does not exist. + Remove(name string) error + + // Rename atomically renames oldPath to a new file named newName, + // returning the full new path or an error. + Rename(oldPath, newName string) (newPath string, err error) + + // ListFiles returns just the basenames of all regular files + // in the root directory. + ListFiles() ([]string, error) + + // Stat returns the FileInfo for the given name or an error. + Stat(name string) (fs.FileInfo, error) + + // OpenReader opens the given basename for the given name or an error. + OpenReader(name string) (io.ReadCloser, error) +} + +var newFileOps func(dir string) (FileOps, error) diff --git a/feature/taildrop/fileops_fs.go b/feature/taildrop/fileops_fs.go new file mode 100644 index 0000000000000..4fecbe4af6bbb --- /dev/null +++ b/feature/taildrop/fileops_fs.go @@ -0,0 +1,221 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause +//go:build !android + +package taildrop + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "sync" + "unicode/utf8" +) + +var renameMu sync.Mutex + +// fsFileOps implements FileOps using the local filesystem rooted at a directory. +// It is used on non-Android platforms. +type fsFileOps struct{ rootDir string } + +func init() { + newFileOps = func(dir string) (FileOps, error) { + if dir == "" { + return nil, errors.New("rootDir cannot be empty") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("mkdir %q: %w", dir, err) + } + return fsFileOps{rootDir: dir}, nil + } +} + +func (f fsFileOps) OpenWriter(name string, offset int64, perm os.FileMode) (io.WriteCloser, string, error) { + path, err := joinDir(f.rootDir, name) + if err != nil { + return nil, "", err + } + if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, "", err + } + fi, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, perm) + if err != nil { + return nil, "", err + } + if offset != 0 { + curr, err := fi.Seek(0, io.SeekEnd) + if err != nil { + fi.Close() + return nil, "", err + } + if offset < 0 || offset > curr { + fi.Close() + return nil, "", fmt.Errorf("offset %d out of range", offset) + } + if _, err := fi.Seek(offset, io.SeekStart); err != nil { + fi.Close() + return nil, "", err + } + if err := fi.Truncate(offset); err != nil { + fi.Close() + return nil, "", err + } + } + return fi, path, nil +} + +func (f fsFileOps) Remove(name string) error { + path, err := joinDir(f.rootDir, name) + if err != nil { + return err + } + return os.Remove(path) +} + +// Rename moves the partial file into its final name. +// newName must be a base name (not absolute or containing path separators). +// It will retry up to 10 times, de-dup same-checksum files, etc. +func (f fsFileOps) Rename(oldPath, newName string) (newPath string, err error) { + var dst string + if filepath.IsAbs(newName) || strings.ContainsRune(newName, os.PathSeparator) { + return "", fmt.Errorf("invalid newName %q: must not be an absolute path or contain path separators", newName) + } + + dst = filepath.Join(f.rootDir, newName) + + if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { + return "", err + } + + st, err := os.Stat(oldPath) + if err != nil { + return "", err + } + wantSize := st.Size() + + const maxRetries = 10 + for i := 0; i < maxRetries; i++ { + renameMu.Lock() + fi, statErr := os.Stat(dst) + // Atomically rename the partial file as the destination file if it doesn't exist. + // Otherwise, it returns the length of the current destination file. + // The operation is atomic. + if os.IsNotExist(statErr) { + err = os.Rename(oldPath, dst) + renameMu.Unlock() + if err != nil { + return "", err + } + return dst, nil + } + if statErr != nil { + renameMu.Unlock() + return "", statErr + } + gotSize := fi.Size() + renameMu.Unlock() + + // Avoid the final rename if a destination file has the same contents. + // + // Note: this is best effort and copying files from iOS from the Media Library + // results in processing on the iOS side which means the size and shas of the + // same file can be different. + if gotSize == wantSize { + sumP, err := sha256File(oldPath) + if err != nil { + return "", err + } + sumD, err := sha256File(dst) + if err != nil { + return "", err + } + if bytes.Equal(sumP[:], sumD[:]) { + if err := os.Remove(oldPath); err != nil { + return "", err + } + return dst, nil + } + } + + // Choose a new destination filename and try again. + dst = filepath.Join(filepath.Dir(dst), nextFilename(filepath.Base(dst))) + } + + return "", fmt.Errorf("too many retries trying to rename %q to %q", oldPath, newName) +} + +// sha256File computes the SHA‑256 of a file. +func sha256File(path string) (sum [sha256.Size]byte, _ error) { + f, err := os.Open(path) + if err != nil { + return sum, err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return sum, err + } + copy(sum[:], h.Sum(nil)) + return sum, nil +} + +func (f fsFileOps) ListFiles() ([]string, error) { + entries, err := os.ReadDir(f.rootDir) + if err != nil { + return nil, err + } + var names []string + for _, e := range entries { + if e.Type().IsRegular() { + names = append(names, e.Name()) + } + } + return names, nil +} + +func (f fsFileOps) Stat(name string) (fs.FileInfo, error) { + path, err := joinDir(f.rootDir, name) + if err != nil { + return nil, err + } + return os.Stat(path) +} + +func (f fsFileOps) OpenReader(name string) (io.ReadCloser, error) { + path, err := joinDir(f.rootDir, name) + if err != nil { + return nil, err + } + return os.Open(path) +} + +// joinDir is like [filepath.Join] but returns an error if baseName is too long, +// is a relative path instead of a basename, or is otherwise invalid or unsafe for incoming files. +func joinDir(dir, baseName string) (string, error) { + if !utf8.ValidString(baseName) || + strings.TrimSpace(baseName) != baseName || + len(baseName) > 255 { + return "", ErrInvalidFileName + } + // TODO: validate unicode normalization form too? Varies by platform. + clean := path.Clean(baseName) + if clean != baseName || clean == "." || clean == ".." { + return "", ErrInvalidFileName + } + for _, r := range baseName { + if !validFilenameRune(r) { + return "", ErrInvalidFileName + } + } + if !filepath.IsLocal(baseName) { + return "", ErrInvalidFileName + } + return filepath.Join(dir, baseName), nil +} diff --git a/feature/taildrop/paths.go b/feature/taildrop/paths.go index 22d01160cff8e..79dc37d8f0699 100644 --- a/feature/taildrop/paths.go +++ b/feature/taildrop/paths.go @@ -21,7 +21,7 @@ func (e *Extension) SetDirectFileRoot(root string) { // SetFileOps sets the platform specific file operations. This is used // to call Android's Storage Access Framework APIs. func (e *Extension) SetFileOps(fileOps FileOps) { - e.FileOps = fileOps + e.fileOps = fileOps } func (e *Extension) setPlatformDefaultDirectFileRoot() { diff --git a/feature/taildrop/peerapi_test.go b/feature/taildrop/peerapi_test.go index 1a003b6eddca7..6339973544453 100644 --- a/feature/taildrop/peerapi_test.go +++ b/feature/taildrop/peerapi_test.go @@ -24,6 +24,7 @@ import ( "tailscale.com/tstest" "tailscale.com/tstime" "tailscale.com/types/logger" + "tailscale.com/util/must" ) // peerAPIHandler serves the PeerAPI for a source specific client. @@ -93,7 +94,16 @@ func bodyContains(sub string) check { func fileHasSize(name string, size int) check { return func(t *testing.T, e *peerAPITestEnv) { - root := e.taildrop.Dir() + fsImpl, ok := e.taildrop.opts.fileOps.(*fsFileOps) + if !ok { + t.Skip("fileHasSize only supported on fsFileOps backend") + return + } + root := fsImpl.rootDir + if root == "" { + t.Errorf("no rootdir; can't check whether %q has size %v", name, size) + return + } if root == "" { t.Errorf("no rootdir; can't check whether %q has size %v", name, size) return @@ -109,12 +119,12 @@ func fileHasSize(name string, size int) check { func fileHasContents(name string, want string) check { return func(t *testing.T, e *peerAPITestEnv) { - root := e.taildrop.Dir() - if root == "" { - t.Errorf("no rootdir; can't check contents of %q", name) + fsImpl, ok := e.taildrop.opts.fileOps.(*fsFileOps) + if !ok { + t.Skip("fileHasContents only supported on fsFileOps backend") return } - path := filepath.Join(root, name) + path := filepath.Join(fsImpl.rootDir, name) got, err := os.ReadFile(path) if err != nil { t.Errorf("fileHasContents: %v", err) @@ -172,9 +182,10 @@ func TestHandlePeerAPI(t *testing.T) { reqs: []*http.Request{httptest.NewRequest("PUT", "/v0/put/foo", nil)}, checks: checks( httpStatus(http.StatusForbidden), - bodyContains("Taildrop disabled; no storage directory"), + bodyContains("Taildrop disabled"), ), }, + { name: "bad_method", isSelf: true, @@ -471,14 +482,18 @@ func TestHandlePeerAPI(t *testing.T) { selfNode.CapMap = tailcfg.NodeCapMap{tailcfg.CapabilityDebug: nil} } var rootDir string + var fo FileOps if !tt.omitRoot { - rootDir = t.TempDir() + var err error + if fo, err = newFileOps(t.TempDir()); err != nil { + t.Fatalf("newFileOps: %v", err) + } } var e peerAPITestEnv e.taildrop = managerOptions{ - Logf: e.logBuf.Logf, - Dir: rootDir, + Logf: e.logBuf.Logf, + fileOps: fo, }.New() ext := &fakeExtension{ @@ -490,9 +505,7 @@ func TestHandlePeerAPI(t *testing.T) { e.ph = &peerAPIHandler{ isSelf: tt.isSelf, selfNode: selfNode.View(), - peerNode: (&tailcfg.Node{ - ComputedName: "some-peer-name", - }).View(), + peerNode: (&tailcfg.Node{ComputedName: "some-peer-name"}).View(), } for _, req := range tt.reqs { e.rr = httptest.NewRecorder() @@ -526,8 +539,8 @@ func TestHandlePeerAPI(t *testing.T) { func TestFileDeleteRace(t *testing.T) { dir := t.TempDir() taildropMgr := managerOptions{ - Logf: t.Logf, - Dir: dir, + Logf: t.Logf, + fileOps: must.Get(newFileOps(dir)), }.New() ph := &peerAPIHandler{ diff --git a/feature/taildrop/resume.go b/feature/taildrop/resume.go index 211a1ff6b68dd..20ef527a6da55 100644 --- a/feature/taildrop/resume.go +++ b/feature/taildrop/resume.go @@ -9,7 +9,6 @@ import ( "encoding/hex" "fmt" "io" - "io/fs" "os" "strings" ) @@ -51,19 +50,20 @@ func (cs *checksum) UnmarshalText(b []byte) error { // PartialFiles returns a list of partial files in [Handler.Dir] // that were sent (or is actively being sent) by the provided id. -func (m *manager) PartialFiles(id clientID) (ret []string, err error) { - if m == nil || m.opts.Dir == "" { +func (m *manager) PartialFiles(id clientID) ([]string, error) { + if m == nil || m.opts.fileOps == nil { return nil, ErrNoTaildrop } - suffix := id.partialSuffix() - if err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { - if name := de.Name(); strings.HasSuffix(name, suffix) { - ret = append(ret, name) + files, err := m.opts.fileOps.ListFiles() + if err != nil { + return nil, redactError(err) + } + var ret []string + for _, filename := range files { + if strings.HasSuffix(filename, suffix) { + ret = append(ret, filename) } - return true - }); err != nil { - return ret, redactError(err) } return ret, nil } @@ -73,17 +73,13 @@ func (m *manager) PartialFiles(id clientID) (ret []string, err error) { // It returns (BlockChecksum{}, io.EOF) when the stream is complete. // It is the caller's responsibility to call close. func (m *manager) HashPartialFile(id clientID, baseName string) (next func() (blockChecksum, error), close func() error, err error) { - if m == nil || m.opts.Dir == "" { + if m == nil || m.opts.fileOps == nil { return nil, nil, ErrNoTaildrop } noopNext := func() (blockChecksum, error) { return blockChecksum{}, io.EOF } noopClose := func() error { return nil } - dstFile, err := joinDir(m.opts.Dir, baseName) - if err != nil { - return nil, nil, err - } - f, err := os.Open(dstFile + id.partialSuffix()) + f, err := m.opts.fileOps.OpenReader(baseName + id.partialSuffix()) if err != nil { if os.IsNotExist(err) { return noopNext, noopClose, nil diff --git a/feature/taildrop/resume_test.go b/feature/taildrop/resume_test.go index dac3c657bfb58..4e59d401dcc53 100644 --- a/feature/taildrop/resume_test.go +++ b/feature/taildrop/resume_test.go @@ -8,6 +8,7 @@ import ( "io" "math/rand" "os" + "path/filepath" "testing" "testing/iotest" @@ -19,7 +20,9 @@ func TestResume(t *testing.T) { defer func() { blockSize = oldBlockSize }() blockSize = 256 - m := managerOptions{Logf: t.Logf, Dir: t.TempDir()}.New() + dir := t.TempDir() + + m := managerOptions{Logf: t.Logf, fileOps: must.Get(newFileOps(dir))}.New() defer m.Shutdown() rn := rand.New(rand.NewSource(0)) @@ -37,7 +40,7 @@ func TestResume(t *testing.T) { must.Do(close()) // Windows wants the file handle to be closed to rename it. must.Get(m.PutFile("", "foo", r, offset, -1)) - got := must.Get(os.ReadFile(must.Get(joinDir(m.opts.Dir, "foo")))) + got := must.Get(os.ReadFile(filepath.Join(dir, "foo"))) if !bytes.Equal(got, want) { t.Errorf("content mismatches") } @@ -66,7 +69,7 @@ func TestResume(t *testing.T) { t.Fatalf("too many iterations to complete the test") } } - got := must.Get(os.ReadFile(must.Get(joinDir(m.opts.Dir, "bar")))) + got := must.Get(os.ReadFile(filepath.Join(dir, "bar"))) if !bytes.Equal(got, want) { t.Errorf("content mismatches") } diff --git a/feature/taildrop/retrieve.go b/feature/taildrop/retrieve.go index 6fb97519363bc..b048a1b3b5f9d 100644 --- a/feature/taildrop/retrieve.go +++ b/feature/taildrop/retrieve.go @@ -9,19 +9,19 @@ import ( "io" "io/fs" "os" - "path/filepath" "runtime" "sort" "time" "tailscale.com/client/tailscale/apitype" "tailscale.com/logtail/backoff" + "tailscale.com/util/set" ) // HasFilesWaiting reports whether any files are buffered in [Handler.Dir]. // This always returns false when [Handler.DirectFileMode] is false. -func (m *manager) HasFilesWaiting() (has bool) { - if m == nil || m.opts.Dir == "" || m.opts.DirectFileMode { +func (m *manager) HasFilesWaiting() bool { + if m == nil || m.opts.fileOps == nil || m.opts.DirectFileMode { return false } @@ -30,63 +30,66 @@ func (m *manager) HasFilesWaiting() (has bool) { // has-files-or-not values as the macOS/iOS client might // in the future use+delete the files directly. So only // keep this negative cache. - totalReceived := m.totalReceived.Load() - if totalReceived == m.emptySince.Load() { + total := m.totalReceived.Load() + if total == m.emptySince.Load() { return false } - // Check whether there is at least one one waiting file. - err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { - name := de.Name() - if isPartialOrDeleted(name) || !de.Type().IsRegular() { - return true + files, err := m.opts.fileOps.ListFiles() + if err != nil { + return false + } + + // Build a set of filenames present in Dir + fileSet := set.Of(files...) + + for _, filename := range files { + if isPartialOrDeleted(filename) { + continue } - _, err := os.Stat(filepath.Join(m.opts.Dir, name+deletedSuffix)) - if os.IsNotExist(err) { - has = true - return false + if fileSet.Contains(filename + deletedSuffix) { + continue // already handled } + // Found at least one downloadable file return true - }) - - // If there are no more waiting files, record totalReceived as emptySince - // so that we can short-circuit the expensive directory traversal - // if no files have been received after the start of this call. - if err == nil && !has { - m.emptySince.Store(totalReceived) } - return has + + // No waiting files → update negative‑result cache + m.emptySince.Store(total) + return false } // WaitingFiles returns the list of files that have been sent by a // peer that are waiting in [Handler.Dir]. // This always returns nil when [Handler.DirectFileMode] is false. -func (m *manager) WaitingFiles() (ret []apitype.WaitingFile, err error) { - if m == nil || m.opts.Dir == "" { +func (m *manager) WaitingFiles() ([]apitype.WaitingFile, error) { + if m == nil || m.opts.fileOps == nil { return nil, ErrNoTaildrop } if m.opts.DirectFileMode { return nil, nil } - if err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { - name := de.Name() - if isPartialOrDeleted(name) || !de.Type().IsRegular() { - return true + names, err := m.opts.fileOps.ListFiles() + if err != nil { + return nil, redactError(err) + } + var ret []apitype.WaitingFile + for _, name := range names { + if isPartialOrDeleted(name) { + continue } - _, err := os.Stat(filepath.Join(m.opts.Dir, name+deletedSuffix)) - if os.IsNotExist(err) { - fi, err := de.Info() - if err != nil { - return true - } - ret = append(ret, apitype.WaitingFile{ - Name: filepath.Base(name), - Size: fi.Size(), - }) + // A corresponding .deleted marker means the file was already handled. + if _, err := m.opts.fileOps.Stat(name + deletedSuffix); err == nil { + continue } - return true - }); err != nil { - return nil, redactError(err) + fi, err := m.opts.fileOps.Stat(name) + if err != nil { + continue + } + ret = append(ret, apitype.WaitingFile{ + Name: name, + Size: fi.Size(), + }) } sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) return ret, nil @@ -95,21 +98,18 @@ func (m *manager) WaitingFiles() (ret []apitype.WaitingFile, err error) { // DeleteFile deletes a file of the given baseName from [Handler.Dir]. // This method is only allowed when [Handler.DirectFileMode] is false. func (m *manager) DeleteFile(baseName string) error { - if m == nil || m.opts.Dir == "" { + if m == nil || m.opts.fileOps == nil { return ErrNoTaildrop } if m.opts.DirectFileMode { return errors.New("deletes not allowed in direct mode") } - path, err := joinDir(m.opts.Dir, baseName) - if err != nil { - return err - } + var bo *backoff.Backoff logf := m.opts.Logf t0 := m.opts.Clock.Now() for { - err := os.Remove(path) + err := m.opts.fileOps.Remove(baseName) if err != nil && !os.IsNotExist(err) { err = redactError(err) // Put a retry loop around deletes on Windows. @@ -129,7 +129,7 @@ func (m *manager) DeleteFile(baseName string) error { bo.BackOff(context.Background(), err) continue } - if err := touchFile(path + deletedSuffix); err != nil { + if err := m.touchFile(baseName + deletedSuffix); err != nil { logf("peerapi: failed to leave deleted marker: %v", err) } m.deleter.Insert(baseName + deletedSuffix) @@ -141,35 +141,31 @@ func (m *manager) DeleteFile(baseName string) error { } } -func touchFile(path string) error { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) +func (m *manager) touchFile(name string) error { + wc, _, err := m.opts.fileOps.OpenWriter(name /* offset= */, 0, 0666) if err != nil { return redactError(err) } - return f.Close() + return wc.Close() } // OpenFile opens a file of the given baseName from [Handler.Dir]. // This method is only allowed when [Handler.DirectFileMode] is false. func (m *manager) OpenFile(baseName string) (rc io.ReadCloser, size int64, err error) { - if m == nil || m.opts.Dir == "" { + if m == nil || m.opts.fileOps == nil { return nil, 0, ErrNoTaildrop } if m.opts.DirectFileMode { return nil, 0, errors.New("opens not allowed in direct mode") } - path, err := joinDir(m.opts.Dir, baseName) - if err != nil { - return nil, 0, err - } - if _, err := os.Stat(path + deletedSuffix); err == nil { - return nil, 0, redactError(&fs.PathError{Op: "open", Path: path, Err: fs.ErrNotExist}) + if _, err := m.opts.fileOps.Stat(baseName + deletedSuffix); err == nil { + return nil, 0, redactError(&fs.PathError{Op: "open", Path: baseName, Err: fs.ErrNotExist}) } - f, err := os.Open(path) + f, err := m.opts.fileOps.OpenReader(baseName) if err != nil { return nil, 0, redactError(err) } - fi, err := f.Stat() + fi, err := m.opts.fileOps.Stat(baseName) if err != nil { f.Close() return nil, 0, redactError(err) diff --git a/feature/taildrop/send.go b/feature/taildrop/send.go index 59a1701da6f0d..32ba5f6f0d644 100644 --- a/feature/taildrop/send.go +++ b/feature/taildrop/send.go @@ -4,11 +4,8 @@ package taildrop import ( - "crypto/sha256" "fmt" "io" - "os" - "path/filepath" "sync" "time" @@ -73,9 +70,10 @@ func (f *incomingFile) Write(p []byte) (n int, err error) { // specific partial file. This allows the client to determine whether to resume // a partial file. While resuming, PutFile may be called again with a non-zero // offset to specify where to resume receiving data at. -func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, length int64) (int64, error) { +func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, length int64) (fileLength int64, err error) { + switch { - case m == nil || m.opts.Dir == "": + case m == nil || m.opts.fileOps == nil: return 0, ErrNoTaildrop case !envknob.CanTaildrop(): return 0, ErrNoTaildrop @@ -83,47 +81,47 @@ func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, len return 0, ErrNotAccessible } - //Compute dstPath & avoid mid‑upload deletion - var dstPath string - if m.opts.Mode == PutModeDirect { - var err error - dstPath, err = joinDir(m.opts.Dir, baseName) + if err := validateBaseName(baseName); err != nil { + return 0, err + } + + // and make sure we don't delete it while uploading: + m.deleter.Remove(baseName) + + // Create (if not already) the partial file with read-write permissions. + partialName := baseName + id.partialSuffix() + wc, partialPath, err := m.opts.fileOps.OpenWriter(partialName, offset, 0o666) + if err != nil { + return 0, m.redactAndLogError("Create", err) + } + defer func() { + wc.Close() if err != nil { - return 0, err + m.deleter.Insert(partialName) // mark partial file for eventual deletion } - } else { - // In SAF mode, we simply use the baseName as the destination "path" - // (the actual directory is managed by SAF). - dstPath = baseName - } - m.deleter.Remove(filepath.Base(dstPath)) // avoid deleting the partial file while receiving + }() // Check whether there is an in-progress transfer for the file. - partialFileKey := incomingFileKey{id, baseName} - inFile, loaded := m.incomingFiles.LoadOrInit(partialFileKey, func() *incomingFile { - return &incomingFile{ + inFileKey := incomingFileKey{id, baseName} + inFile, loaded := m.incomingFiles.LoadOrInit(inFileKey, func() *incomingFile { + inFile := &incomingFile{ clock: m.opts.Clock, started: m.opts.Clock.Now(), size: length, sendFileNotify: m.opts.SendFileNotify, } + if m.opts.DirectFileMode { + inFile.partialPath = partialPath + } + return inFile }) + + inFile.w = wc + if loaded { return 0, ErrFileExists } - defer m.incomingFiles.Delete(partialFileKey) - - // Open writer & populate inFile paths - wc, partialPath, err := m.openWriterAndPaths(id, m.opts.Mode, inFile, baseName, dstPath, offset) - if err != nil { - return 0, m.redactAndLogError("Create", err) - } - defer func() { - wc.Close() - if err != nil { - m.deleter.Insert(filepath.Base(partialPath)) // mark partial file for eventual deletion - } - }() + defer m.incomingFiles.Delete(inFileKey) // Record that we have started to receive at least one file. // This is used by the deleter upon a cold-start to scan the directory @@ -148,220 +146,26 @@ func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, len return 0, m.redactAndLogError("Close", err) } - fileLength := offset + copyLength + fileLength = offset + copyLength inFile.mu.Lock() inFile.done = true inFile.mu.Unlock() - // Finalize rename - switch m.opts.Mode { - case PutModeDirect: - var finalDst string - finalDst, err = m.finalizeDirect(inFile, partialPath, dstPath, fileLength) - if err != nil { - return 0, m.redactAndLogError("Rename", err) - } - inFile.finalPath = finalDst - - case PutModeAndroidSAF: - if err = m.finalizeSAF(partialPath, baseName); err != nil { - return 0, m.redactAndLogError("Rename", err) - } + // 6) Finalize (rename/move) the partial into place via FileOps.Rename + finalPath, err := m.opts.fileOps.Rename(partialPath, baseName) + if err != nil { + return 0, m.redactAndLogError("Rename", err) } + inFile.finalPath = finalPath m.totalReceived.Add(1) m.opts.SendFileNotify() return fileLength, nil } -// openWriterAndPaths opens the correct writer, seeks/truncates if needed, -// and sets inFile.partialPath & inFile.finalPath for later cleanup/rename. -// The caller is responsible for closing the file on completion. -func (m *manager) openWriterAndPaths( - id clientID, - mode PutMode, - inFile *incomingFile, - baseName string, - dstPath string, - offset int64, -) (wc io.WriteCloser, partialPath string, err error) { - switch mode { - - case PutModeDirect: - partialPath = dstPath + id.partialSuffix() - f, err := os.OpenFile(partialPath, os.O_CREATE|os.O_RDWR, 0o666) - if err != nil { - return nil, "", m.redactAndLogError("Create", err) - } - if offset != 0 { - curr, err := f.Seek(0, io.SeekEnd) - if err != nil { - f.Close() - return nil, "", m.redactAndLogError("Seek", err) - } - if offset < 0 || offset > curr { - f.Close() - return nil, "", m.redactAndLogError("Seek", fmt.Errorf("offset %d out of range", offset)) - } - if _, err := f.Seek(offset, io.SeekStart); err != nil { - f.Close() - return nil, "", m.redactAndLogError("Seek", err) - } - if err := f.Truncate(offset); err != nil { - f.Close() - return nil, "", m.redactAndLogError("Truncate", err) - } - } - inFile.w = f - wc = f - inFile.partialPath = partialPath - inFile.finalPath = dstPath - return wc, partialPath, nil - - case PutModeAndroidSAF: - if m.opts.FileOps == nil { - return nil, "", m.redactAndLogError("Create (SAF)", fmt.Errorf("missing FileOps")) - } - writer, uri, err := m.opts.FileOps.OpenFileWriter(baseName) - if err != nil { - return nil, "", m.redactAndLogError("Create (SAF)", fmt.Errorf("failed to open file for writing via SAF")) - } - if writer == nil || uri == "" { - return nil, "", fmt.Errorf("invalid SAF writer or URI") - } - // SAF mode does not support resuming, so enforce offset == 0. - if offset != 0 { - writer.Close() - return nil, "", m.redactAndLogError("Seek", fmt.Errorf("resuming is not supported in SAF mode")) - } - inFile.w = writer - wc = writer - partialPath = uri - inFile.partialPath = uri - inFile.finalPath = baseName - return wc, partialPath, nil - - default: - return nil, "", fmt.Errorf("unsupported PutMode: %v", mode) - } -} - -// finalizeDirect atomically renames or dedups the partial file, retrying -// under new names up to 10 times. It returns the final path that succeeded. -func (m *manager) finalizeDirect( - inFile *incomingFile, - partialPath string, - initialDst string, - fileLength int64, -) (string, error) { - var ( - once sync.Once - cachedSum [sha256.Size]byte - cacheErr error - computeSum = func() ([sha256.Size]byte, error) { - once.Do(func() { cachedSum, cacheErr = sha256File(partialPath) }) - return cachedSum, cacheErr - } - ) - - dstPath := initialDst - const maxRetries = 10 - for i := 0; i < maxRetries; i++ { - // Atomically rename the partial file as the destination file if it doesn't exist. - // Otherwise, it returns the length of the current destination file. - // The operation is atomic. - lengthOnDisk, err := func() (int64, error) { - m.renameMu.Lock() - defer m.renameMu.Unlock() - fi, statErr := os.Stat(dstPath) - if os.IsNotExist(statErr) { - // dst missing → rename partial into place - return -1, os.Rename(partialPath, dstPath) - } - if statErr != nil { - return -1, statErr - } - return fi.Size(), nil - }() - if err != nil { - return "", err - } - if lengthOnDisk < 0 { - // successfully moved - inFile.finalPath = dstPath - return dstPath, nil - } - - // Avoid the final rename if a destination file has the same contents. - // - // Note: this is best effort and copying files from iOS from the Media Library - // results in processing on the iOS side which means the size and shas of the - // same file can be different. - if lengthOnDisk == fileLength { - partSum, err := computeSum() - if err != nil { - return "", err - } - dstSum, err := sha256File(dstPath) - if err != nil { - return "", err - } - if partSum == dstSum { - // same content → drop the partial - if err := os.Remove(partialPath); err != nil { - return "", err - } - inFile.finalPath = dstPath - return dstPath, nil - } - } - - // Choose a new destination filename and try again. - dstPath = nextFilename(dstPath) - } - - return "", fmt.Errorf("too many retries trying to rename a partial file %q", initialDst) -} - -// finalizeSAF retries RenamePartialFile up to 10 times, generating a new -// name on each failure until the SAF URI changes. -func (m *manager) finalizeSAF( - partialPath, finalName string, -) error { - if m.opts.FileOps == nil { - return fmt.Errorf("missing FileOps for SAF finalize") - } - const maxTries = 10 - name := finalName - for i := 0; i < maxTries; i++ { - newURI, err := m.opts.FileOps.RenamePartialFile(partialPath, m.opts.Dir, name) - if err != nil { - return err - } - if newURI != "" && newURI != name { - return nil - } - name = nextFilename(name) - } - return fmt.Errorf("failed to finalize SAF file after %d retries", maxTries) -} - func (m *manager) redactAndLogError(stage string, err error) error { err = redactError(err) m.opts.Logf("put %s error: %v", stage, err) return err } - -func sha256File(file string) (out [sha256.Size]byte, err error) { - h := sha256.New() - f, err := os.Open(file) - if err != nil { - return out, err - } - defer f.Close() - if _, err := io.Copy(h, f); err != nil { - return out, err - } - return [sha256.Size]byte(h.Sum(nil)), nil -} diff --git a/feature/taildrop/send_test.go b/feature/taildrop/send_test.go index 8edb704172fc5..9ffa5fccc0a36 100644 --- a/feature/taildrop/send_test.go +++ b/feature/taildrop/send_test.go @@ -4,123 +4,64 @@ package taildrop import ( - "bytes" - "fmt" - "io" "os" "path/filepath" + "strings" "testing" "tailscale.com/tstime" + "tailscale.com/util/must" ) -// nopWriteCloser is a no-op io.WriteCloser wrapping a bytes.Buffer. -type nopWriteCloser struct{ *bytes.Buffer } - -func (nwc nopWriteCloser) Close() error { return nil } - -// mockFileOps implements just enough of the FileOps interface for SAF tests. -type mockFileOps struct { - writes *bytes.Buffer - renameOK bool -} - -func (m *mockFileOps) OpenFileWriter(name string) (io.WriteCloser, string, error) { - m.writes = new(bytes.Buffer) - return nopWriteCloser{m.writes}, "uri://" + name + ".partial", nil -} - -func (m *mockFileOps) RenamePartialFile(partialPath, dir, finalName string) (string, error) { - if !m.renameOK { - m.renameOK = true - return "uri://" + finalName, nil - } - return "", io.ErrUnexpectedEOF -} - func TestPutFile(t *testing.T) { const content = "hello, world" tests := []struct { - name string - mode PutMode - setup func(t *testing.T) (*manager, string, *mockFileOps) - wantFile string + name string + directFileMode bool }{ - { - name: "PutModeDirect", - mode: PutModeDirect, - setup: func(t *testing.T) (*manager, string, *mockFileOps) { - dir := t.TempDir() - opts := managerOptions{ - Logf: t.Logf, - Clock: tstime.DefaultClock{}, - State: nil, - Dir: dir, - Mode: PutModeDirect, - DirectFileMode: true, - SendFileNotify: func() {}, - } - mgr := opts.New() - return mgr, dir, nil - }, - wantFile: "file.txt", - }, - { - name: "PutModeAndroidSAF", - mode: PutModeAndroidSAF, - setup: func(t *testing.T) (*manager, string, *mockFileOps) { - // SAF still needs a non-empty Dir to pass the guard. - dir := t.TempDir() - mops := &mockFileOps{} - opts := managerOptions{ - Logf: t.Logf, - Clock: tstime.DefaultClock{}, - State: nil, - Dir: dir, - Mode: PutModeAndroidSAF, - FileOps: mops, - DirectFileMode: true, - SendFileNotify: func() {}, - } - mgr := opts.New() - return mgr, dir, mops - }, - wantFile: "file.txt", - }, + {"DirectFileMode", true}, + {"NonDirectFileMode", false}, } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - mgr, dir, mops := tc.setup(t) - id := clientID(fmt.Sprint(0)) - reader := bytes.NewReader([]byte(content)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + mgr := managerOptions{ + Logf: t.Logf, + Clock: tstime.DefaultClock{}, + State: nil, + fileOps: must.Get(newFileOps(dir)), + DirectFileMode: tt.directFileMode, + SendFileNotify: func() {}, + }.New() - n, err := mgr.PutFile(id, "file.txt", reader, 0, int64(len(content))) + id := clientID("0") + n, err := mgr.PutFile(id, "file.txt", strings.NewReader(content), 0, int64(len(content))) if err != nil { - t.Fatalf("PutFile(%s) error: %v", tc.name, err) + t.Fatalf("PutFile error: %v", err) } if n != int64(len(content)) { t.Errorf("wrote %d bytes; want %d", n, len(content)) } - switch tc.mode { - case PutModeDirect: - path := filepath.Join(dir, tc.wantFile) - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("ReadFile error: %v", err) - } - if got := string(data); got != content { - t.Errorf("file contents = %q; want %q", got, content) - } + path := filepath.Join(dir, "file.txt") - case PutModeAndroidSAF: - if mops.writes == nil { - t.Fatal("SAF writer was never created") - } - if got := mops.writes.String(); got != content { - t.Errorf("SAF writes = %q; want %q", got, content) + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile %q: %v", path, err) + } + if string(got) != content { + t.Errorf("file contents = %q; want %q", string(got), content) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.Contains(entry.Name(), ".partial") { + t.Errorf("unexpected partial file left behind: %s", entry.Name()) } } }) diff --git a/feature/taildrop/taildrop.go b/feature/taildrop/taildrop.go index 2dfa415bbf0cc..6c3deaed1b538 100644 --- a/feature/taildrop/taildrop.go +++ b/feature/taildrop/taildrop.go @@ -12,8 +12,6 @@ package taildrop import ( "errors" "hash/adler32" - "io" - "io/fs" "os" "path" "path/filepath" @@ -21,7 +19,6 @@ import ( "sort" "strconv" "strings" - "sync" "sync/atomic" "unicode" "unicode/utf8" @@ -72,11 +69,6 @@ type managerOptions struct { Clock tstime.DefaultClock // may be nil State ipn.StateStore // may be nil - // Dir is the directory to store received files. - // This main either be the final location for the files - // or just a temporary staging directory (see DirectFileMode). - Dir string - // DirectFileMode reports whether we are writing files // directly to a download directory, rather than writing them to // a temporary staging directory. @@ -91,9 +83,10 @@ type managerOptions struct { // copy them out, and then delete them. DirectFileMode bool - FileOps FileOps - - Mode PutMode + // FileOps abstracts platform-specific file operations needed for file transfers. + // Android's implementation uses the Storage Access Framework, and other platforms + // use fsFileOps. + fileOps FileOps // SendFileNotify is called periodically while a file is actively // receiving the contents for the file. There is a final call @@ -111,9 +104,6 @@ type manager struct { // deleter managers asynchronous deletion of files. deleter fileDeleter - // renameMu is used to protect os.Rename calls so that they are atomic. - renameMu sync.Mutex - // totalReceived counts the cumulative total of received files. totalReceived atomic.Int64 // emptySince specifies that there were no waiting files @@ -137,11 +127,6 @@ func (opts managerOptions) New() *manager { return m } -// Dir returns the directory. -func (m *manager) Dir() string { - return m.opts.Dir -} - // Shutdown shuts down the Manager. // It blocks until all spawned goroutines have stopped running. func (m *manager) Shutdown() { @@ -172,57 +157,29 @@ func isPartialOrDeleted(s string) bool { return strings.HasSuffix(s, deletedSuffix) || strings.HasSuffix(s, partialSuffix) } -func joinDir(dir, baseName string) (fullPath string, err error) { - if !utf8.ValidString(baseName) { - return "", ErrInvalidFileName - } - if strings.TrimSpace(baseName) != baseName { - return "", ErrInvalidFileName - } - if len(baseName) > 255 { - return "", ErrInvalidFileName +func validateBaseName(name string) error { + if !utf8.ValidString(name) || + strings.TrimSpace(name) != name || + len(name) > 255 { + return ErrInvalidFileName } // TODO: validate unicode normalization form too? Varies by platform. - clean := path.Clean(baseName) - if clean != baseName || - clean == "." || clean == ".." || - isPartialOrDeleted(clean) { - return "", ErrInvalidFileName + clean := path.Clean(name) + if clean != name || clean == "." || clean == ".." { + return ErrInvalidFileName } - for _, r := range baseName { + if isPartialOrDeleted(name) { + return ErrInvalidFileName + } + for _, r := range name { if !validFilenameRune(r) { - return "", ErrInvalidFileName + return ErrInvalidFileName } } - if !filepath.IsLocal(baseName) { - return "", ErrInvalidFileName - } - return filepath.Join(dir, baseName), nil -} - -// rangeDir iterates over the contents of a directory, calling fn for each entry. -// It continues iterating while fn returns true. -// It reports the number of entries seen. -func rangeDir(dir string, fn func(fs.DirEntry) bool) error { - f, err := os.Open(dir) - if err != nil { - return err - } - defer f.Close() - for { - des, err := f.ReadDir(10) - for _, de := range des { - if !fn(de) { - return nil - } - } - if err != nil { - if err == io.EOF { - return nil - } - return err - } + if !filepath.IsLocal(name) { + return ErrInvalidFileName } + return nil } // IncomingFiles returns a list of active incoming files. diff --git a/feature/taildrop/taildrop_test.go b/feature/taildrop/taildrop_test.go index da0bd2f430579..0d77273f0aab0 100644 --- a/feature/taildrop/taildrop_test.go +++ b/feature/taildrop/taildrop_test.go @@ -4,40 +4,10 @@ package taildrop import ( - "path/filepath" "strings" "testing" ) -func TestJoinDir(t *testing.T) { - dir := t.TempDir() - tests := []struct { - in string - want string // just relative to m.Dir - wantOk bool - }{ - {"", "", false}, - {"foo", "foo", true}, - {"./foo", "", false}, - {"../foo", "", false}, - {"foo/bar", "", false}, - {"😋", "😋", true}, - {"\xde\xad\xbe\xef", "", false}, - {"foo.partial", "", false}, - {"foo.deleted", "", false}, - {strings.Repeat("a", 1024), "", false}, - {"foo:bar", "", false}, - } - for _, tt := range tests { - got, gotErr := joinDir(dir, tt.in) - got, _ = filepath.Rel(dir, got) - gotOk := gotErr == nil - if got != tt.want || gotOk != tt.wantOk { - t.Errorf("joinDir(%q) = (%v, %v), want (%v, %v)", tt.in, got, gotOk, tt.want, tt.wantOk) - } - } -} - func TestNextFilename(t *testing.T) { tests := []struct { in string @@ -67,3 +37,29 @@ func TestNextFilename(t *testing.T) { } } } + +func TestValidateBaseName(t *testing.T) { + tests := []struct { + in string + wantOk bool + }{ + {"", false}, + {"foo", true}, + {"./foo", false}, + {"../foo", false}, + {"foo/bar", false}, + {"😋", true}, + {"\xde\xad\xbe\xef", false}, + {"foo.partial", false}, + {"foo.deleted", false}, + {strings.Repeat("a", 1024), false}, + {"foo:bar", false}, + } + for _, tt := range tests { + err := validateBaseName(tt.in) + gotOk := err == nil + if gotOk != tt.wantOk { + t.Errorf("validateBaseName(%q) = %v, wantOk = %v", tt.in, err, tt.wantOk) + } + } +} From 2b42f22a8f69e39069f380f6e4ed76d728115d52 Mon Sep 17 00:00:00 2001 From: Jonathan Nobels Date: Thu, 7 Aug 2025 12:20:18 -0400 Subject: [PATCH 259/263] VERSION.txt: this is v1.86.3 Signed-off-by: Jonathan Nobels --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 2568a54800d86..891ff04379c0e 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.86.2 +1.86.3 From 51c11a864b1241d1cf1a736fbc94b0f8c76da563 Mon Sep 17 00:00:00 2001 From: Jonathan Nobels Date: Thu, 7 Aug 2025 12:46:29 -0400 Subject: [PATCH 260/263] VERSION.txt: this is v1.86.4 Signed-off-by: Jonathan Nobels --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 891ff04379c0e..3c9665e69be0d 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.86.3 +1.86.4 From 56f738a25d6778e103c47748bf256417624c1ebf Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Fri, 22 Aug 2025 17:00:55 +0100 Subject: [PATCH 261/263] cmd/k8s-proxy,k8s-operator: fix serve config for userspace mode (#16919) (#16924) The serve code leaves it up to the system's DNS resolver and netstack to figure out how to reach the proxy destination. Combined with k8s-proxy running in userspace mode, this means we can't rely on MagicDNS being available or tailnet IPs being routable. I'd like to implement that as a feature for serve in userspace mode, but for now the safer fix to get kube-apiserver ProxyGroups consistently working in all environments is to switch to using localhost as the proxy target instead. This has a small knock-on in the code that does WhoIs lookups, which now needs to check the X-Forwarded-For header that serve populates to get the correct tailnet IP to look up, because the request's remote address will be loopback. Fixes #16920 Change-Id: I869ddcaf93102da50e66071bb00114cc1acc1288 (cherry picked from commit 3eeecb4c7f340a43ee133c85985111cf0e00e537) Signed-off-by: Tom Proctor --- cmd/k8s-proxy/k8s-proxy.go | 2 +- k8s-operator/api-proxy/proxy.go | 30 +++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go index 448bbe3971c0d..7a77072140568 100644 --- a/cmd/k8s-proxy/k8s-proxy.go +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -453,7 +453,7 @@ func setServeConfig(ctx context.Context, lc *local.Client, cm *certs.CertManager serviceHostPort: { Handlers: map[string]*ipn.HTTPHandler{ "/": { - Proxy: fmt.Sprintf("http://%s:80", strings.TrimSuffix(status.Self.DNSName, ".")), + Proxy: "http://localhost:80", }, }, }, diff --git a/k8s-operator/api-proxy/proxy.go b/k8s-operator/api-proxy/proxy.go index ff0373270b2c0..a0f2f930b8067 100644 --- a/k8s-operator/api-proxy/proxy.go +++ b/k8s-operator/api-proxy/proxy.go @@ -123,11 +123,11 @@ func (ap *APIServerProxy) Run(ctx context.Context) error { if ap.authMode { mode = "auth" } - var tsLn net.Listener + var proxyLn net.Listener var serve func(ln net.Listener) error if ap.https { var err error - tsLn, err = ap.ts.Listen("tcp", ":443") + proxyLn, err = ap.ts.Listen("tcp", ":443") if err != nil { return fmt.Errorf("could not listen on :443: %w", err) } @@ -143,7 +143,7 @@ func (ap *APIServerProxy) Run(ctx context.Context) error { } } else { var err error - tsLn, err = ap.ts.Listen("tcp", ":80") + proxyLn, err = net.Listen("tcp", "localhost:80") if err != nil { return fmt.Errorf("could not listen on :80: %w", err) } @@ -152,8 +152,8 @@ func (ap *APIServerProxy) Run(ctx context.Context) error { errs := make(chan error) go func() { - ap.log.Infof("API server proxy in %s mode is listening on tailnet addresses %s", mode, tsLn.Addr()) - if err := serve(tsLn); err != nil && err != http.ErrServerClosed { + ap.log.Infof("API server proxy in %s mode is listening on %s", mode, proxyLn.Addr()) + if err := serve(proxyLn); err != nil && err != http.ErrServerClosed { errs <- fmt.Errorf("error serving: %w", err) } }() @@ -179,7 +179,7 @@ type APIServerProxy struct { rp *httputil.ReverseProxy authMode bool // Whether to run with impersonation using caller's tailnet identity. - https bool // Whether to serve on https for the device hostname; true for k8s-operator, false for k8s-proxy. + https bool // Whether to serve on https for the device hostname; true for k8s-operator, false (and localhost) for k8s-proxy. ts *tsnet.Server hs *http.Server upstreamURL *url.URL @@ -317,7 +317,23 @@ func (ap *APIServerProxy) addImpersonationHeadersAsRequired(r *http.Request) { } func (ap *APIServerProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) { - return ap.lc.WhoIs(r.Context(), r.RemoteAddr) + who, remoteErr := ap.lc.WhoIs(r.Context(), r.RemoteAddr) + if remoteErr == nil { + ap.log.Debugf("WhoIs from remote addr: %s", r.RemoteAddr) + return who, nil + } + + var fwdErr error + fwdFor := r.Header.Get("X-Forwarded-For") + if fwdFor != "" && !ap.https { + who, fwdErr = ap.lc.WhoIs(r.Context(), fwdFor) + if fwdErr == nil { + ap.log.Debugf("WhoIs from X-Forwarded-For header: %s", fwdFor) + return who, nil + } + } + + return nil, errors.Join(remoteErr, fwdErr) } func (ap *APIServerProxy) authError(w http.ResponseWriter, err error) { From db392aed39630023f969e1961fcbced785d09358 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Fri, 22 Aug 2025 17:19:28 +0100 Subject: [PATCH 262/263] VERSION.txt: this is v1.86.5 Signed-off-by: Tom Proctor --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 3c9665e69be0d..8f22cfee44fb8 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.86.4 +1.86.5 From 4518c6f215cdf21b215de10d2c26d080bb755910 Mon Sep 17 00:00:00 2001 From: ChandonPierre Date: Wed, 3 Sep 2025 07:46:41 -0400 Subject: [PATCH 263/263] chore(ci): remove additional upstream workflows --- .github/workflows/natlab-integrationtest.yml | 27 ---------------- .../workflows/request-dataplane-review.yml | 31 ------------------- 2 files changed, 58 deletions(-) delete mode 100644 .github/workflows/natlab-integrationtest.yml delete mode 100644 .github/workflows/request-dataplane-review.yml diff --git a/.github/workflows/natlab-integrationtest.yml b/.github/workflows/natlab-integrationtest.yml deleted file mode 100644 index 99d58717b7beb..0000000000000 --- a/.github/workflows/natlab-integrationtest.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Run some natlab integration tests. -# See https://github.com/tailscale/tailscale/issues/13038 -name: "natlab-integrationtest" - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -on: - pull_request: - paths: - - "tstest/integration/nat/nat_test.go" -jobs: - natlab-integrationtest: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Install qemu - run: | - sudo rm /var/lib/man-db/auto-update - sudo apt-get -y update - sudo apt-get -y remove man-db - sudo apt-get install -y qemu-system-x86 qemu-utils - - name: Run natlab integration tests - run: | - ./tool/go test -v -run=^TestEasyEasy$ -timeout=3m -count=1 ./tstest/integration/nat --run-vm-tests diff --git a/.github/workflows/request-dataplane-review.yml b/.github/workflows/request-dataplane-review.yml deleted file mode 100644 index 836fef6fbce7c..0000000000000 --- a/.github/workflows/request-dataplane-review.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: request-dataplane-review - -on: - pull_request: - branches: - - "*" - paths: - - ".github/workflows/request-dataplane-review.yml" - - "**/*derp*" - - "**/derp*/**" - -jobs: - request-dataplane-review: - name: Request Dataplane Review - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Get access token - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 - id: generate-token - with: - # Get token for app: https://github.com/apps/change-visibility-bot - app-id: ${{ secrets.VISIBILITY_BOT_APP_ID }} - private-key: ${{ secrets.VISIBILITY_BOT_APP_PRIVATE_KEY }} - - name: Add reviewers - env: - GH_TOKEN: ${{ steps.generate-token.outputs.token }} - url: ${{ github.event.pull_request.html_url }} - run: | - gh pr edit "$url" --add-reviewer tailscale/dataplane