diff --git a/.github/workflows/natlab-integrationtest.yml b/.github/workflows/natlab-integrationtest.yml deleted file mode 100644 index 1de74cdaa45f8..0000000000000 --- a/.github/workflows/natlab-integrationtest.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Run some natlab integration tests. -# See https://github.com/tailscale/tailscale/issues/13038 -name: "natlab-integrationtest" - -concurrency: - group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -on: - pull_request: - paths: - - "tstest/integration/nat/nat_test.go" -jobs: - natlab-integrationtest: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Install qemu - run: | - sudo rm /var/lib/man-db/auto-update - sudo apt-get -y update - sudo apt-get -y remove man-db - sudo apt-get install -y qemu-system-x86 qemu-utils - - name: Run natlab integration tests - run: | - ./tool/go test -v -run=^TestEasyEasy$ -timeout=3m -count=1 ./tstest/integration/nat --run-vm-tests diff --git a/.gitignore b/.gitignore index 47d2bbe959ae1..3941fd06ef6d5 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,6 @@ client/web/build/assets *.xcworkspacedata /tstest/tailmac/bin /tstest/tailmac/build + +# Ignore personal IntelliJ settings +.idea/ diff --git a/Dockerfile b/Dockerfile index 015022e49fc28..fbc0d1194ffc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,15 @@ # Tailscale images are currently built using https://github.com/tailscale/mkctr, # and the build script can be found in ./build_docker.sh. # +# If you want to build local images for testing, you can use make. +# +# To build a Tailscale image and push to the local docker registry: +# +# $ REPO=local/tailscale TAGS=v0.0.1 PLATFORM=local make publishdevimage +# +# To build a Tailscale image and push to a remote docker registry: +# +# $ REPO=//tailscale TAGS=v0.0.1 make publishdevimage # # This Dockerfile includes all the tailscale binaries. # diff --git a/Makefile b/Makefile index c30818c965b77..55e55f209575c 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,8 @@ updatedeps: ## Update depaware deps tailscale.com/cmd/tailscale \ tailscale.com/cmd/derper \ tailscale.com/cmd/k8s-operator \ - tailscale.com/cmd/stund + tailscale.com/cmd/stund \ + tailscale.com/cmd/tsidp PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --update -goos=linux,darwin,windows,android,ios --internal \ tailscale.com/tsnet @@ -34,7 +35,8 @@ depaware: ## Run depaware checks tailscale.com/cmd/tailscale \ tailscale.com/cmd/derper \ tailscale.com/cmd/k8s-operator \ - tailscale.com/cmd/stund + tailscale.com/cmd/stund \ + tailscale.com/cmd/tsidp PATH="$$(./tool/go env GOROOT)/bin:$$PATH" ./tool/go run github.com/tailscale/depaware --check --goos=linux,darwin,windows,android,ios --internal \ tailscale.com/tsnet @@ -62,7 +64,7 @@ buildmultiarchimage: ## Build (and optionally push) multiarch docker image check: staticcheck vet depaware buildwindows build386 buildlinuxarm buildwasm ## Perform basic checks and compilation tests staticcheck: ## Run staticcheck.io checks - ./tool/go run honnef.co/go/tools/cmd/staticcheck -- $$(./tool/go list ./... | grep -v tempfork) + ./tool/go run honnef.co/go/tools/cmd/staticcheck -- $$(./tool/go run ./tool/listpkgs --ignore-3p ./...) kube-generate-all: kube-generate-deepcopy ## Refresh generated files for Tailscale Kubernetes Operator ./tool/go generate ./cmd/k8s-operator @@ -90,34 +92,42 @@ pushspk: spk ## Push and install synology package on ${SYNO_HOST} host scp tailscale.spk root@${SYNO_HOST}: ssh root@${SYNO_HOST} /usr/syno/bin/synopkg install tailscale.spk -publishdevimage: ## Build and publish tailscale image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1) +.PHONY: check-image-repo +check-image-repo: + @if [ -z "$(REPO)" ]; then \ + echo "REPO=... required; e.g. REPO=ghcr.io/$$USER/tailscale" >&2; \ + exit 1; \ + fi + @for repo in tailscale/tailscale ghcr.io/tailscale/tailscale \ + tailscale/k8s-operator ghcr.io/tailscale/k8s-operator \ + tailscale/k8s-nameserver ghcr.io/tailscale/k8s-nameserver \ + tailscale/tsidp ghcr.io/tailscale/tsidp \ + tailscale/k8s-proxy ghcr.io/tailscale/k8s-proxy; do \ + if [ "$(REPO)" = "$$repo" ]; then \ + echo "REPO=... must not be $$repo" >&2; \ + exit 1; \ + fi; \ + done + +publishdevimage: check-image-repo ## Build and publish tailscale image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=client ./build_docker.sh -publishdevoperator: ## Build and publish k8s-operator image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-operator" || (echo "REPO=... must not be tailscale/k8s-operator" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-operator" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-operator" && exit 1) +publishdevoperator: check-image-repo ## Build and publish k8s-operator image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-operator ./build_docker.sh -publishdevnameserver: ## Build and publish k8s-nameserver image to location specified by ${REPO} - @test -n "${REPO}" || (echo "REPO=... required; e.g. REPO=ghcr.io/${USER}/tailscale" && exit 1) - @test "${REPO}" != "tailscale/tailscale" || (echo "REPO=... must not be tailscale/tailscale" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/tailscale" || (echo "REPO=... must not be ghcr.io/tailscale/tailscale" && exit 1) - @test "${REPO}" != "tailscale/k8s-nameserver" || (echo "REPO=... must not be tailscale/k8s-nameserver" && exit 1) - @test "${REPO}" != "ghcr.io/tailscale/k8s-nameserver" || (echo "REPO=... must not be ghcr.io/tailscale/k8s-nameserver" && exit 1) +publishdevnameserver: check-image-repo ## Build and publish k8s-nameserver image to location specified by ${REPO} TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-nameserver ./build_docker.sh +publishdevtsidp: check-image-repo ## Build and publish tsidp image to location specified by ${REPO} + TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=tsidp ./build_docker.sh + +publishdevproxy: check-image-repo ## Build and publish k8s-proxy image to location specified by ${REPO} + TAGS="${TAGS}" REPOS=${REPO} PLATFORM=${PLATFORM} PUSH=true TARGET=k8s-proxy ./build_docker.sh + .PHONY: sshintegrationtest sshintegrationtest: ## Run the SSH integration tests in various Docker containers - @GOOS=linux GOARCH=amd64 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ - GOOS=linux GOARCH=amd64 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \ + @GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go test -tags integrationtest -c ./ssh/tailssh -o ssh/tailssh/testcontainers/tailssh.test && \ + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 ./tool/go build -o ssh/tailssh/testcontainers/tailscaled ./cmd/tailscaled && \ echo "Testing on ubuntu:focal" && docker build --build-arg="BASE=ubuntu:focal" -t ssh-ubuntu-focal ssh/tailssh/testcontainers && \ echo "Testing on ubuntu:jammy" && docker build --build-arg="BASE=ubuntu:jammy" -t ssh-ubuntu-jammy ssh/tailssh/testcontainers && \ echo "Testing on ubuntu:noble" && docker build --build-arg="BASE=ubuntu:noble" -t ssh-ubuntu-noble ssh/tailssh/testcontainers && \ diff --git a/VERSION.txt b/VERSION.txt index ffc3ee716ed63..8f22cfee44fb8 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.84.3 +1.86.5 diff --git a/atomicfile/atomicfile.go b/atomicfile/atomicfile.go index b3c8c93da2af9..9cae9bb750fa8 100644 --- a/atomicfile/atomicfile.go +++ b/atomicfile/atomicfile.go @@ -48,5 +48,9 @@ func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { if err := f.Close(); err != nil { return err } - return rename(tmpName, filename) + return Rename(tmpName, filename) } + +// Rename srcFile to dstFile, similar to [os.Rename] but preserving file +// attributes and ACLs on Windows. +func Rename(srcFile, dstFile string) error { return rename(srcFile, dstFile) } diff --git a/build_docker.sh b/build_docker.sh index 15105c2ef5541..bdeaa8659b805 100755 --- a/build_docker.sh +++ b/build_docker.sh @@ -6,6 +6,16 @@ # hash of this repository as produced by ./cmd/mkversion. # This is the image build mechanim used to build the official Tailscale # container images. +# +# If you want to build local images for testing, you can use make, which provides few convenience wrappers around this script. +# +# To build a Tailscale image and push to the local docker registry: + +# $ REPO=local/tailscale TAGS=v0.0.1 PLATFORM=local make publishdevimage +# +# To build a Tailscale image and push to a remote docker registry: +# +# $ REPO=//tailscale TAGS=v0.0.1 make publishdevimage set -eu @@ -90,6 +100,42 @@ case "$TARGET" in --annotations="${ANNOTATIONS}" \ /usr/local/bin/k8s-nameserver ;; + tsidp) + DEFAULT_REPOS="tailscale/tsidp" + REPOS="${REPOS:-${DEFAULT_REPOS}}" + go run github.com/tailscale/mkctr \ + --gopaths="tailscale.com/cmd/tsidp:/usr/local/bin/tsidp" \ + --ldflags=" \ + -X tailscale.com/version.longStamp=${VERSION_LONG} \ + -X tailscale.com/version.shortStamp=${VERSION_SHORT} \ + -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \ + --base="${BASE}" \ + --tags="${TAGS}" \ + --gotags="ts_package_container" \ + --repos="${REPOS}" \ + --push="${PUSH}" \ + --target="${PLATFORM}" \ + --annotations="${ANNOTATIONS}" \ + /usr/local/bin/tsidp + ;; + k8s-proxy) + DEFAULT_REPOS="tailscale/k8s-proxy" + REPOS="${REPOS:-${DEFAULT_REPOS}}" + go run github.com/tailscale/mkctr \ + --gopaths="tailscale.com/cmd/k8s-proxy:/usr/local/bin/k8s-proxy" \ + --ldflags=" \ + -X tailscale.com/version.longStamp=${VERSION_LONG} \ + -X tailscale.com/version.shortStamp=${VERSION_SHORT} \ + -X tailscale.com/version.gitCommitStamp=${VERSION_GIT_HASH}" \ + --base="${BASE}" \ + --tags="${TAGS}" \ + --gotags="ts_kube,ts_package_container" \ + --repos="${REPOS}" \ + --push="${PUSH}" \ + --target="${PLATFORM}" \ + --annotations="${ANNOTATIONS}" \ + /usr/local/bin/k8s-proxy + ;; *) echo "unknown target: $TARGET" exit 1 diff --git a/chirp/chirp_test.go b/chirp/chirp_test.go index 2549c163fd819..a57ef224b2c1b 100644 --- a/chirp/chirp_test.go +++ b/chirp/chirp_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package chirp import ( diff --git a/client/local/local.go b/client/local/local.go index 0e4d495d3dd18..55d14f95eee5a 100644 --- a/client/local/local.go +++ b/client/local/local.go @@ -1,12 +1,11 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build go1.22 - // Package local contains a Go client for the Tailscale LocalAPI. package local import ( + "bufio" "bytes" "cmp" "context" @@ -16,6 +15,7 @@ import ( "errors" "fmt" "io" + "iter" "net" "net/http" "net/http/httptrace" @@ -42,6 +42,7 @@ import ( "tailscale.com/types/dnstype" "tailscale.com/types/key" "tailscale.com/types/tkatype" + "tailscale.com/util/eventbus" "tailscale.com/util/syspolicy/setting" ) @@ -397,6 +398,23 @@ func (lc *Client) IncrementCounter(ctx context.Context, name string, delta int) return err } +// IncrementGauge increments the value of a Tailscale daemon's gauge +// metric by the given delta. If the metric has yet to exist, a new gauge +// metric is created and initialized to delta. The delta value can be negative. +func (lc *Client) IncrementGauge(ctx context.Context, name string, delta int) error { + type metricUpdate struct { + Name string `json:"name"` + Type string `json:"type"` + Value int `json:"value"` // amount to increment by + } + _, err := lc.send(ctx, "POST", "/localapi/v0/upload-client-metrics", 200, jsonBody([]metricUpdate{{ + Name: name, + Type: "gauge", + Value: delta, + }})) + return err +} + // TailDaemonLogs returns a stream the Tailscale daemon's logs as they arrive. // Close the context to stop the stream. func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) { @@ -414,6 +432,50 @@ func (lc *Client) TailDaemonLogs(ctx context.Context) (io.Reader, error) { return res.Body, nil } +// EventBusGraph returns a graph of active publishers and subscribers in the eventbus +// as a [eventbus.DebugTopics] +func (lc *Client) EventBusGraph(ctx context.Context) ([]byte, error) { + return lc.get200(ctx, "/localapi/v0/debug-bus-graph") +} + +// StreamBusEvents returns an iterator of Tailscale bus events as they arrive. +// Each pair is a valid event and a nil error, or a zero event a non-nil error. +// In case of error, the iterator ends after the pair reporting the error. +// Iteration stops if ctx ends. +func (lc *Client) StreamBusEvents(ctx context.Context) iter.Seq2[eventbus.DebugEvent, error] { + return func(yield func(eventbus.DebugEvent, error) bool) { + req, err := http.NewRequestWithContext(ctx, "GET", + "http://"+apitype.LocalAPIHost+"/localapi/v0/debug-bus-events", nil) + if err != nil { + yield(eventbus.DebugEvent{}, err) + return + } + res, err := lc.doLocalRequestNiceError(req) + if err != nil { + yield(eventbus.DebugEvent{}, err) + return + } + if res.StatusCode != http.StatusOK { + yield(eventbus.DebugEvent{}, errors.New(res.Status)) + return + } + defer res.Body.Close() + dec := json.NewDecoder(bufio.NewReader(res.Body)) + for { + var evt eventbus.DebugEvent + if err := dec.Decode(&evt); err == io.EOF { + return + } else if err != nil { + yield(eventbus.DebugEvent{}, err) + return + } + if !yield(evt, nil) { + return + } + } + } +} + // Pprof returns a pprof profile of the Tailscale daemon. func (lc *Client) Pprof(ctx context.Context, pprofType string, sec int) ([]byte, error) { var secArg string @@ -788,6 +850,25 @@ func (lc *Client) CheckUDPGROForwarding(ctx context.Context) error { return nil } +// CheckReversePathFiltering asks the local Tailscale daemon whether strict +// reverse path filtering is enabled, which would break exit node usage on Linux. +func (lc *Client) CheckReversePathFiltering(ctx context.Context) error { + body, err := lc.get200(ctx, "/localapi/v0/check-reverse-path-filtering") + if err != nil { + return err + } + var jres struct { + Warning string + } + if err := json.Unmarshal(body, &jres); err != nil { + return fmt.Errorf("invalid JSON from check-reverse-path-filtering: %w", err) + } + if jres.Warning != "" { + return errors.New(jres.Warning) + } + return nil +} + // SetUDPGROForwarding enables UDP GRO forwarding for the main interface of this // node. This can be done to improve performance of tailnet nodes acting as exit // nodes or subnet routers. diff --git a/client/systray/systray.go b/client/systray/systray.go index 195a157fb1386..76c93ae18e781 100644 --- a/client/systray/systray.go +++ b/client/systray/systray.go @@ -61,7 +61,8 @@ func (menu *Menu) Run() { case <-menu.bgCtx.Done(): } }() - go menu.lc.IncrementCounter(menu.bgCtx, "systray_start", 1) + go menu.lc.IncrementGauge(menu.bgCtx, "systray_running", 1) + defer menu.lc.IncrementGauge(menu.bgCtx, "systray_running", -1) systray.Run(menu.onReady, menu.onExit) } @@ -127,7 +128,7 @@ func init() { desktop := strings.ToLower(os.Getenv("XDG_CURRENT_DESKTOP")) switch desktop { - case "gnome": + case "gnome", "ubuntu:gnome": // GNOME expands submenus downward in the main menu, rather than flyouts to the side. // Either as a result of that or another limitation, there seems to be a maximum depth of submenus. // Mullvad countries that have a city submenu are not being rendered, and so can't be selected. diff --git a/client/web/src/api.ts b/client/web/src/api.ts index 9414e2d5d7e16..e780c76459dfd 100644 --- a/client/web/src/api.ts +++ b/client/web/src/api.ts @@ -249,7 +249,6 @@ export function useAPI() { return api } -let csrfToken: string let synoToken: string | undefined // required for synology API requests let unraidCsrfToken: string | undefined // required for unraid POST requests (#8062) @@ -298,12 +297,10 @@ export function apiFetch( headers: { Accept: "application/json", "Content-Type": contentType, - "X-CSRF-Token": csrfToken, }, body: body, }) .then((r) => { - updateCsrfToken(r) if (!r.ok) { return r.text().then((err) => { throw new Error(err) @@ -322,13 +319,6 @@ export function apiFetch( }) } -function updateCsrfToken(r: Response) { - const tok = r.headers.get("X-CSRF-Token") - if (tok) { - csrfToken = tok - } -} - export function setSynoToken(token?: string) { synoToken = token } diff --git a/client/web/web.go b/client/web/web.go index 6eccdadcfdb65..f3158cd1f6ff5 100644 --- a/client/web/web.go +++ b/client/web/web.go @@ -6,7 +6,6 @@ package web import ( "context" - "crypto/rand" "encoding/json" "errors" "fmt" @@ -14,14 +13,14 @@ import ( "log" "net/http" "net/netip" + "net/url" "os" "path" - "path/filepath" + "slices" "strings" "sync" "time" - "github.com/gorilla/csrf" "tailscale.com/client/local" "tailscale.com/client/tailscale/apitype" "tailscale.com/clientupdate" @@ -60,6 +59,12 @@ type Server struct { cgiMode bool pathPrefix string + // originOverride is the origin that the web UI is accessible from. + // This value is used in the fallback CSRF checks when Sec-Fetch-Site is not + // available. In this case the application will compare Host and Origin + // header values to determine if the request is from the same origin. + originOverride string + apiHandler http.Handler // serves api endpoints; csrf-protected assetsHandler http.Handler // serves frontend assets assetsCleanup func() // called from Server.Shutdown @@ -150,6 +155,9 @@ type ServerOpts struct { // as completed. // This field is required for ManageServerMode mode. WaitAuthURL func(ctx context.Context, id string, src tailcfg.NodeID) (*tailcfg.WebClientAuthResponse, error) + + // OriginOverride specifies the origin that the web UI will be accessible from if hosted behind a reverse proxy or CGI. + OriginOverride string } // NewServer constructs a new Tailscale web client server. @@ -169,15 +177,16 @@ func NewServer(opts ServerOpts) (s *Server, err error) { opts.LocalClient = &local.Client{} } s = &Server{ - mode: opts.Mode, - logf: opts.Logf, - devMode: envknob.Bool("TS_DEBUG_WEB_CLIENT_DEV"), - lc: opts.LocalClient, - cgiMode: opts.CGIMode, - pathPrefix: opts.PathPrefix, - timeNow: opts.TimeNow, - newAuthURL: opts.NewAuthURL, - waitAuthURL: opts.WaitAuthURL, + mode: opts.Mode, + logf: opts.Logf, + devMode: envknob.Bool("TS_DEBUG_WEB_CLIENT_DEV"), + lc: opts.LocalClient, + cgiMode: opts.CGIMode, + pathPrefix: opts.PathPrefix, + timeNow: opts.TimeNow, + newAuthURL: opts.NewAuthURL, + waitAuthURL: opts.WaitAuthURL, + originOverride: opts.OriginOverride, } if opts.PathPrefix != "" { // Enforce that path prefix always has a single leading '/' @@ -205,7 +214,7 @@ func NewServer(opts ServerOpts) (s *Server, err error) { var metric string s.apiHandler, metric = s.modeAPIHandler(s.mode) - s.apiHandler = s.withCSRF(s.apiHandler) + s.apiHandler = s.csrfProtect(s.apiHandler) // Don't block startup on reporting metric. // Report in separate go routine with 5 second timeout. @@ -218,23 +227,64 @@ func NewServer(opts ServerOpts) (s *Server, err error) { return s, nil } -func (s *Server) withCSRF(h http.Handler) http.Handler { - csrfProtect := csrf.Protect(s.csrfKey(), csrf.Secure(false)) +func (s *Server) csrfProtect(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // CSRF is not required for GET, HEAD, or OPTIONS requests. + if slices.Contains([]string{"GET", "HEAD", "OPTIONS"}, r.Method) { + h.ServeHTTP(w, r) + return + } - // ref https://github.com/tailscale/tailscale/pull/14822 - // signal to the CSRF middleware that the request is being served over - // plaintext HTTP to skip TLS-only header checks. - withSetPlaintext := func(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - r = csrf.PlaintextHTTPRequest(r) + // first attempt to use Sec-Fetch-Site header (sent by all modern + // browsers to "potentially trustworthy" origins i.e. localhost or those + // served over HTTPS) + secFetchSite := r.Header.Get("Sec-Fetch-Site") + if secFetchSite == "same-origin" { h.ServeHTTP(w, r) - }) - } + return + } else if secFetchSite != "" { + http.Error(w, fmt.Sprintf("CSRF request denied with Sec-Fetch-Site %q", secFetchSite), http.StatusForbidden) + return + } + + // if Sec-Fetch-Site is not available we presume we are operating over HTTP. + // We fall back to comparing the Origin & Host headers. + + // use the Host header to determine the expected origin + // (use the override if set to allow for reverse proxying) + host := r.Host + if host == "" { + http.Error(w, "CSRF request denied with no Host header", http.StatusForbidden) + return + } + if s.originOverride != "" { + host = s.originOverride + } + + originHeader := r.Header.Get("Origin") + if originHeader == "" { + http.Error(w, "CSRF request denied with no Origin header", http.StatusForbidden) + return + } + parsedOrigin, err := url.Parse(originHeader) + if err != nil { + http.Error(w, fmt.Sprintf("CSRF request denied with invalid Origin %q", r.Header.Get("Origin")), http.StatusForbidden) + return + } + origin := parsedOrigin.Host + if origin == "" { + http.Error(w, "CSRF request denied with no host in the Origin header", http.StatusForbidden) + return + } + + if origin != host { + http.Error(w, fmt.Sprintf("CSRF request denied with mismatched Origin %q and Host %q", origin, host), http.StatusForbidden) + return + } + + h.ServeHTTP(w, r) - // NB: the order of the withSetPlaintext and csrfProtect calls is important - // to ensure that we signal to the CSRF middleware that the request is being - // served over plaintext HTTP and not over TLS as it presumes by default. - return withSetPlaintext(csrfProtect(h)) + }) } func (s *Server) modeAPIHandler(mode ServerMode) (http.Handler, string) { @@ -452,7 +502,6 @@ func (s *Server) authorizeRequest(w http.ResponseWriter, r *http.Request) (ok bo // It should only be called by Server.ServeHTTP, via Server.apiHandler, // which protects the handler using gorilla csrf. func (s *Server) serveLoginAPI(w http.ResponseWriter, r *http.Request) { - w.Header().Set("X-CSRF-Token", csrf.Token(r)) switch { case r.URL.Path == "/api/data" && r.Method == httpm.GET: s.serveGetNodeData(w, r) @@ -575,7 +624,6 @@ func (s *Server) serveAPI(w http.ResponseWriter, r *http.Request) { } } - w.Header().Set("X-CSRF-Token", csrf.Token(r)) path := strings.TrimPrefix(r.URL.Path, "/api") switch { case path == "/data" && r.Method == httpm.GET: @@ -1276,37 +1324,6 @@ func (s *Server) proxyRequestToLocalAPI(w http.ResponseWriter, r *http.Request) } } -// csrfKey returns a key that can be used for CSRF protection. -// If an error occurs during key creation, the error is logged and the active process terminated. -// If the server is running in CGI mode, the key is cached to disk and reused between requests. -// If an error occurs during key storage, the error is logged and the active process terminated. -func (s *Server) csrfKey() []byte { - csrfFile := filepath.Join(os.TempDir(), "tailscale-web-csrf.key") - - // if running in CGI mode, try to read from disk, but ignore errors - if s.cgiMode { - key, _ := os.ReadFile(csrfFile) - if len(key) == 32 { - return key - } - } - - // create a new key - key := make([]byte, 32) - if _, err := rand.Read(key); err != nil { - log.Fatalf("error generating CSRF key: %v", err) - } - - // if running in CGI mode, try to write the newly created key to disk, and exit if it fails. - if s.cgiMode { - if err := os.WriteFile(csrfFile, key, 0600); err != nil { - log.Fatalf("unable to store CSRF key: %v", err) - } - } - - return key -} - // enforcePrefix returns a HandlerFunc that enforces a given path prefix is used in requests, // then strips it before invoking h. // Unlike http.StripPrefix, it does not return a 404 if the prefix is not present. diff --git a/client/web/web_test.go b/client/web/web_test.go index 2a6bc787ac396..12dbb5c79b13a 100644 --- a/client/web/web_test.go +++ b/client/web/web_test.go @@ -11,7 +11,6 @@ import ( "fmt" "io" "net/http" - "net/http/cookiejar" "net/http/httptest" "net/netip" "net/url" @@ -21,14 +20,12 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/gorilla/csrf" "tailscale.com/client/local" "tailscale.com/client/tailscale/apitype" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/net/memnet" "tailscale.com/tailcfg" - "tailscale.com/tstest/nettest" "tailscale.com/types/views" "tailscale.com/util/httpm" ) @@ -1492,81 +1489,99 @@ func mockWaitAuthURL(_ context.Context, id string, src tailcfg.NodeID) (*tailcfg } func TestCSRFProtect(t *testing.T) { - s := &Server{} - - mux := http.NewServeMux() - mux.HandleFunc("GET /test/csrf-token", func(w http.ResponseWriter, r *http.Request) { - token := csrf.Token(r) - _, err := io.WriteString(w, token) - if err != nil { - t.Fatal(err) - } - }) - mux.HandleFunc("POST /test/csrf-protected", func(w http.ResponseWriter, r *http.Request) { - _, err := io.WriteString(w, "ok") - if err != nil { - t.Fatal(err) - } - }) - h := s.withCSRF(mux) - ser := nettest.NewHTTPServer(nettest.GetNetwork(t), h) - defer ser.Close() - - jar, err := cookiejar.New(nil) - if err != nil { - t.Fatalf("unable to construct cookie jar: %v", err) + tests := []struct { + name string + method string + secFetchSite string + host string + origin string + originOverride string + wantError bool + }{ + { + name: "GET requests with no header are allowed", + method: "GET", + }, + { + name: "POST requests with same-origin are allowed", + method: "POST", + secFetchSite: "same-origin", + }, + { + name: "POST requests with cross-site are not allowed", + method: "POST", + secFetchSite: "cross-site", + wantError: true, + }, + { + name: "POST requests with unknown sec-fetch-site values are not allowed", + method: "POST", + secFetchSite: "new-unknown-value", + wantError: true, + }, + { + name: "POST requests with none are not allowed", + method: "POST", + secFetchSite: "none", + wantError: true, + }, + { + name: "POST requests with no sec-fetch-site header but matching host and origin are allowed", + method: "POST", + host: "example.com", + origin: "https://example.com", + }, + { + name: "POST requests with no sec-fetch-site and non-matching host and origin are not allowed", + method: "POST", + host: "example.com", + origin: "https://example.net", + wantError: true, + }, + { + name: "POST requests with no sec-fetch-site and and origin that matches the override are allowed", + method: "POST", + originOverride: "example.net", + host: "internal.example.foo", // Host can be changed by reverse proxies + origin: "http://example.net", + }, } - client := ser.Client() - client.Jar = jar - - // make GET request to populate cookie jar - resp, err := client.Get(ser.URL + "/test/csrf-token") - if err != nil { - t.Fatalf("unable to make request: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status: %v", resp.Status) - } - tokenBytes, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatalf("unable to read body: %v", err) - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "OK") + }) - csrfToken := strings.TrimSpace(string(tokenBytes)) - if csrfToken == "" { - t.Fatal("empty csrf token") - } + s := &Server{ + originOverride: tt.originOverride, + } + withCSRF := s.csrfProtect(handler) - // make a POST request without the CSRF header; ensure it fails - resp, err = client.Post(ser.URL+"/test/csrf-protected", "text/plain", nil) - if err != nil { - t.Fatalf("unable to make request: %v", err) - } - if resp.StatusCode != http.StatusForbidden { - t.Fatalf("unexpected status: %v", resp.Status) - } + r := httptest.NewRequest(tt.method, "http://example.com/", nil) + if tt.secFetchSite != "" { + r.Header.Set("Sec-Fetch-Site", tt.secFetchSite) + } + if tt.host != "" { + r.Host = tt.host + } + if tt.origin != "" { + r.Header.Set("Origin", tt.origin) + } - // make a POST request with the CSRF header; ensure it succeeds - req, err := http.NewRequest("POST", ser.URL+"/test/csrf-protected", nil) - if err != nil { - t.Fatalf("error building request: %v", err) - } - req.Header.Set("X-CSRF-Token", csrfToken) - resp, err = client.Do(req) - if err != nil { - t.Fatalf("unable to make request: %v", err) - } - if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected status: %v", resp.Status) - } - defer resp.Body.Close() - out, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatalf("unable to read body: %v", err) - } - if string(out) != "ok" { - t.Fatalf("unexpected body: %q", out) + w := httptest.NewRecorder() + withCSRF.ServeHTTP(w, r) + res := w.Result() + defer res.Body.Close() + if tt.wantError { + if res.StatusCode != http.StatusForbidden { + t.Errorf("expected status forbidden, got %v", res.StatusCode) + } + return + } + if res.StatusCode != http.StatusOK { + t.Errorf("expected status ok, got %v", res.StatusCode) + } + }) } } diff --git a/cmd/cloner/cloner_test.go b/cmd/cloner/cloner_test.go index d8d5df3cb040c..cf1063714afda 100644 --- a/cmd/cloner/cloner_test.go +++ b/cmd/cloner/cloner_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package main import ( diff --git a/cmd/containerboot/healthz.go b/cmd/containerboot/healthz.go deleted file mode 100644 index d6a64a37c4ac5..0000000000000 --- a/cmd/containerboot/healthz.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -//go:build linux - -package main - -import ( - "fmt" - "log" - "net/http" - "sync" - - "tailscale.com/kube/kubetypes" -) - -// healthz is a simple health check server, if enabled it returns 200 OK if -// this tailscale node currently has at least one tailnet IP address else -// returns 503. -type healthz struct { - sync.Mutex - hasAddrs bool - podIPv4 string -} - -func (h *healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) { - h.Lock() - defer h.Unlock() - - if h.hasAddrs { - w.Header().Add(kubetypes.PodIPv4Header, h.podIPv4) - if _, err := w.Write([]byte("ok")); err != nil { - http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError) - } - } else { - http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable) - } -} - -func (h *healthz) update(healthy bool) { - h.Lock() - defer h.Unlock() - - if h.hasAddrs != healthy { - log.Println("Setting healthy", healthy) - } - h.hasAddrs = healthy -} - -// registerHealthHandlers registers a simple health handler at /healthz. -// A containerized tailscale instance is considered healthy if -// it has at least one tailnet IP address. -func registerHealthHandlers(mux *http.ServeMux, podIPv4 string) *healthz { - h := &healthz{podIPv4: podIPv4} - mux.Handle("GET /healthz", h) - return h -} diff --git a/cmd/containerboot/kube.go b/cmd/containerboot/kube.go index 0a2dfa1bf342f..d4a974e6f3a24 100644 --- a/cmd/containerboot/kube.go +++ b/cmd/containerboot/kube.go @@ -18,12 +18,15 @@ import ( "time" "tailscale.com/ipn" + "tailscale.com/kube/egressservices" + "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" "tailscale.com/logtail/backoff" "tailscale.com/tailcfg" "tailscale.com/types/logger" + "tailscale.com/util/set" ) // kubeClient is a wrapper around Tailscale's internal kube client that knows how to talk to the kube API server. We use @@ -117,20 +120,39 @@ func (kc *kubeClient) deleteAuthKey(ctx context.Context) error { return nil } -// storeCapVerUID stores the current capability version of tailscale and, if provided, UID of the Pod in the tailscale -// state Secret. -// These two fields are used by the Kubernetes Operator to observe the current capability version of tailscaled running in this container. -func (kc *kubeClient) storeCapVerUID(ctx context.Context, podUID string) error { - capVerS := fmt.Sprintf("%d", tailcfg.CurrentCapabilityVersion) - d := map[string][]byte{ - kubetypes.KeyCapVer: []byte(capVerS), +// resetContainerbootState resets state from previous runs of containerboot to +// ensure the operator doesn't use stale state when a Pod is first recreated. +func (kc *kubeClient) resetContainerbootState(ctx context.Context, podUID string) error { + existingSecret, err := kc.GetSecret(ctx, kc.stateSecret) + if err != nil { + return fmt.Errorf("failed to read state Secret %q to reset state: %w", kc.stateSecret, err) + } + + s := &kubeapi.Secret{ + Data: map[string][]byte{ + kubetypes.KeyCapVer: fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion), + }, } if podUID != "" { - d[kubetypes.KeyPodUID] = []byte(podUID) + s.Data[kubetypes.KeyPodUID] = []byte(podUID) } - s := &kubeapi.Secret{ - Data: d, + + toClear := set.SetOf([]string{ + kubetypes.KeyDeviceID, + kubetypes.KeyDeviceFQDN, + kubetypes.KeyDeviceIPs, + kubetypes.KeyHTTPSEndpoint, + egressservices.KeyEgressServices, + ingressservices.IngressConfigKey, + }) + for key := range existingSecret.Data { + if toClear.Contains(key) { + // It's fine to leave the key in place as a debugging breadcrumb, + // it should get a new value soon. + s.Data[key] = nil + } } + return kc.StrategicMergePatchSecret(ctx, kc.stateSecret, s, "tailscale-container") } diff --git a/cmd/containerboot/kube_test.go b/cmd/containerboot/kube_test.go index 413971bc6df23..c33714ed12ace 100644 --- a/cmd/containerboot/kube_test.go +++ b/cmd/containerboot/kube_test.go @@ -8,13 +8,18 @@ package main import ( "context" "errors" + "fmt" "testing" "time" "github.com/google/go-cmp/cmp" "tailscale.com/ipn" + "tailscale.com/kube/egressservices" + "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" ) func TestSetupKube(t *testing.T) { @@ -238,3 +243,78 @@ func TestWaitForConsistentState(t *testing.T) { t.Fatalf("expected nil, got %v", err) } } + +func TestResetContainerbootState(t *testing.T) { + capver := fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion) + for name, tc := range map[string]struct { + podUID string + initial map[string][]byte + expected map[string][]byte + }{ + "empty_initial": { + podUID: "1234", + initial: map[string][]byte{}, + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: []byte("1234"), + }, + }, + "empty_initial_no_pod_uid": { + initial: map[string][]byte{}, + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + }, + }, + "only_relevant_keys_updated": { + podUID: "1234", + initial: map[string][]byte{ + kubetypes.KeyCapVer: []byte("1"), + kubetypes.KeyPodUID: []byte("5678"), + kubetypes.KeyDeviceID: []byte("device-id"), + kubetypes.KeyDeviceFQDN: []byte("device-fqdn"), + kubetypes.KeyDeviceIPs: []byte(`["192.0.2.1"]`), + kubetypes.KeyHTTPSEndpoint: []byte("https://example.com"), + egressservices.KeyEgressServices: []byte("egress-services"), + ingressservices.IngressConfigKey: []byte("ingress-config"), + "_current-profile": []byte("current-profile"), + "_machinekey": []byte("machine-key"), + "_profiles": []byte("profiles"), + "_serve_e0ce": []byte("serve-e0ce"), + "profile-e0ce": []byte("profile-e0ce"), + }, + expected: map[string][]byte{ + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: []byte("1234"), + // Cleared keys. + kubetypes.KeyDeviceID: nil, + kubetypes.KeyDeviceFQDN: nil, + kubetypes.KeyDeviceIPs: nil, + kubetypes.KeyHTTPSEndpoint: nil, + egressservices.KeyEgressServices: nil, + ingressservices.IngressConfigKey: nil, + // Tailscaled keys not included in patch. + }, + }, + } { + t.Run(name, func(t *testing.T) { + var actual map[string][]byte + kc := &kubeClient{stateSecret: "foo", Client: &kubeclient.FakeClient{ + GetSecretImpl: func(context.Context, string) (*kubeapi.Secret, error) { + return &kubeapi.Secret{ + Data: tc.initial, + }, nil + }, + StrategicMergePatchSecretImpl: func(ctx context.Context, name string, secret *kubeapi.Secret, _ string) error { + actual = secret.Data + return nil + }, + }} + if err := kc.resetContainerbootState(context.Background(), tc.podUID); err != nil { + t.Fatalf("resetContainerbootState() error = %v", err) + } + if diff := cmp.Diff(tc.expected, actual); diff != "" { + t.Errorf("resetContainerbootState() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/cmd/containerboot/main.go b/cmd/containerboot/main.go index 9543308975b79..f056d26f3c2c0 100644 --- a/cmd/containerboot/main.go +++ b/cmd/containerboot/main.go @@ -121,7 +121,10 @@ import ( "tailscale.com/client/tailscale" "tailscale.com/ipn" kubeutils "tailscale.com/k8s-operator" + healthz "tailscale.com/kube/health" "tailscale.com/kube/kubetypes" + "tailscale.com/kube/metrics" + "tailscale.com/kube/services" "tailscale.com/tailcfg" "tailscale.com/types/logger" "tailscale.com/types/ptr" @@ -188,6 +191,14 @@ func run() error { if err := cfg.setupKube(bootCtx, kc); err != nil { return fmt.Errorf("error setting up for running on Kubernetes: %w", err) } + // Clear out any state from previous runs of containerboot. Check + // hasKubeStateStore because although we know we're in kube, that + // doesn't guarantee the state store is properly configured. + if hasKubeStateStore(cfg) { + if err := kc.resetContainerbootState(bootCtx, cfg.PodUID); err != nil { + return fmt.Errorf("error clearing previous state from Secret: %w", err) + } + } } client, daemonProcess, err := startTailscaled(bootCtx, cfg) @@ -202,7 +213,7 @@ func run() error { ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) defer cancel() - if err := ensureServicesNotAdvertised(ctx, client); err != nil { + if err := services.EnsureServicesNotAdvertised(ctx, client, log.Printf); err != nil { log.Printf("Error ensuring services are not advertised: %v", err) } @@ -223,13 +234,13 @@ func run() error { } defer killTailscaled() - var healthCheck *healthz + var healthCheck *healthz.Healthz ep := &egressProxy{} if cfg.HealthCheckAddrPort != "" { mux := http.NewServeMux() log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort) - healthCheck = registerHealthHandlers(mux, cfg.PodIPv4) + healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, log.Printf) close := runHTTPServer(mux, cfg.HealthCheckAddrPort) defer close() @@ -240,12 +251,12 @@ func run() error { if cfg.localMetricsEnabled() { log.Printf("Running metrics endpoint at %s/metrics", cfg.LocalAddrPort) - registerMetricsHandlers(mux, client, cfg.DebugAddrPort) + metrics.RegisterMetricsHandlers(mux, client, cfg.DebugAddrPort) } if cfg.localHealthEnabled() { log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort) - healthCheck = registerHealthHandlers(mux, cfg.PodIPv4) + healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, log.Printf) } if cfg.egressSvcsTerminateEPEnabled() { @@ -367,11 +378,6 @@ authLoop: if err := client.SetServeConfig(ctx, new(ipn.ServeConfig)); err != nil { return fmt.Errorf("failed to unset serve config: %w", err) } - if hasKubeStateStore(cfg) { - if err := kc.storeHTTPSEndpoint(ctx, ""); err != nil { - return fmt.Errorf("failed to update HTTPS endpoint in tailscale state: %w", err) - } - } } if hasKubeStateStore(cfg) && isTwoStepConfigAuthOnce(cfg) { @@ -384,12 +390,6 @@ authLoop: } } - if hasKubeStateStore(cfg) { - if err := kc.storeCapVerUID(ctx, cfg.PodUID); err != nil { - return fmt.Errorf("storing capability version and UID: %w", err) - } - } - w, err = client.WatchIPNBus(ctx, ipn.NotifyInitialNetMap|ipn.NotifyInitialState) if err != nil { return fmt.Errorf("rewatching tailscaled for updates after auth: %w", err) @@ -440,8 +440,8 @@ authLoop: ) // egressSvcsErrorChan will get an error sent to it if this containerboot instance is configured to expose 1+ // egress services in HA mode and errored. - var egressSvcsErrorChan = make(chan error) - var ingressSvcsErrorChan = make(chan error) + egressSvcsErrorChan := make(chan error) + ingressSvcsErrorChan := make(chan error) defer t.Stop() // resetTimer resets timer for when to next attempt to resolve the DNS // name for the proxy configured with TS_EXPERIMENTAL_DEST_DNS_NAME. The @@ -646,7 +646,7 @@ runLoop: } if healthCheck != nil { - healthCheck.update(len(addrs) != 0) + healthCheck.Update(len(addrs) != 0) } if cfg.ServeConfigPath != "" { diff --git a/cmd/containerboot/main_test.go b/cmd/containerboot/main_test.go index c7293c77a4afa..96feef682af5b 100644 --- a/cmd/containerboot/main_test.go +++ b/cmd/containerboot/main_test.go @@ -460,6 +460,7 @@ func TestContainerBoot(t *testing.T) { Env: map[string]string{ "KUBERNETES_SERVICE_HOST": env.kube.Host, "KUBERNETES_SERVICE_PORT_HTTPS": env.kube.Port, + "POD_UID": "some-pod-uid", }, KubeSecret: map[string]string{ "authkey": "tskey-key", @@ -471,17 +472,20 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: "some-pod-uid", }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, + kubetypes.KeyPodUID: "some-pod-uid", }, }, }, @@ -554,7 +558,8 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscaled --socket=/tmp/tailscaled.sock --state=kube:tailscale --statedir=/tmp --tun=userspace-networking", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { @@ -565,7 +570,8 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { @@ -574,10 +580,10 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock set --accept-dns=false", }, WantKubeSecret: map[string]string{ - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, }, }, @@ -599,17 +605,18 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, }, { @@ -624,11 +631,11 @@ func TestContainerBoot(t *testing.T) { }, }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "new-name.test.ts.net", - "device_id": "newID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "new-name.test.ts.net", + "device_id": "newID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, }, }, @@ -912,18 +919,19 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { Notify: runningNotify, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "https_endpoint": "no-https", - "tailscale_capver": capver, + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + "https_endpoint": "no-https", + kubetypes.KeyCapVer: capver, }, }, }, @@ -947,7 +955,8 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, EndpointStatuses: map[string]int{ egressSvcTerminateURL(env.localAddrPort): 200, @@ -956,12 +965,12 @@ func TestContainerBoot(t *testing.T) { { Notify: runningNotify, WantKubeSecret: map[string]string{ - "egress-services": mustBase64(t, egressStatus), - "authkey": "tskey-key", - "device_fqdn": "test-node.test.ts.net", - "device_id": "myID", - "device_ips": `["100.64.0.1"]`, - "tailscale_capver": capver, + "egress-services": string(mustJSON(t, egressStatus)), + "authkey": "tskey-key", + "device_fqdn": "test-node.test.ts.net", + "device_id": "myID", + "device_ips": `["100.64.0.1"]`, + kubetypes.KeyCapVer: capver, }, EndpointStatuses: map[string]int{ egressSvcTerminateURL(env.localAddrPort): 200, @@ -1002,7 +1011,8 @@ func TestContainerBoot(t *testing.T) { "/usr/bin/tailscale --socket=/tmp/tailscaled.sock up --accept-dns=false --authkey=tskey-key", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", + "authkey": "tskey-key", + kubetypes.KeyCapVer: capver, }, }, { @@ -1016,10 +1026,11 @@ func TestContainerBoot(t *testing.T) { // Missing "_current-profile" key. }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "_machinekey": "foo", - "_profiles": "foo", - "profile-baff": "foo", + "authkey": "tskey-key", + "_machinekey": "foo", + "_profiles": "foo", + "profile-baff": "foo", + kubetypes.KeyCapVer: capver, }, WantLog: "Waiting for tailscaled to finish writing state to Secret \"tailscale\"", }, @@ -1029,11 +1040,12 @@ func TestContainerBoot(t *testing.T) { "_current-profile": "foo", }, WantKubeSecret: map[string]string{ - "authkey": "tskey-key", - "_machinekey": "foo", - "_profiles": "foo", - "profile-baff": "foo", - "_current-profile": "foo", + "authkey": "tskey-key", + "_machinekey": "foo", + "_profiles": "foo", + "profile-baff": "foo", + "_current-profile": "foo", + kubetypes.KeyCapVer: capver, }, WantLog: "HTTP server at [::]:9002 closed", WantExitCode: ptr.To(0), @@ -1061,7 +1073,7 @@ func TestContainerBoot(t *testing.T) { fmt.Sprintf("TS_TEST_SOCKET=%s", env.lapi.Path), fmt.Sprintf("TS_SOCKET=%s", env.runningSockPath), fmt.Sprintf("TS_TEST_ONLY_ROOT=%s", env.d), - fmt.Sprint("TS_TEST_FAKE_NETFILTER=true"), + "TS_TEST_FAKE_NETFILTER=true", } for k, v := range tc.Env { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) @@ -1489,10 +1501,7 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { } switch r.Header.Get("Content-Type") { case "application/json-patch+json": - req := []struct { - Op string `json:"op"` - Path string `json:"path"` - }{} + req := []kubeclient.JSONPatch{} if err := json.Unmarshal(bs, &req); err != nil { panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) } @@ -1503,23 +1512,20 @@ func (k *kubeServer) serveSecret(w http.ResponseWriter, r *http.Request) { panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) } delete(k.secret, strings.TrimPrefix(op.Path, "/data/")) - case "replace": + case "add", "replace": path, ok := strings.CutPrefix(op.Path, "/data/") if !ok { panic(fmt.Sprintf("unsupported json-patch path %q", op.Path)) } - req := make([]kubeclient.JSONPatch, 0) - if err := json.Unmarshal(bs, &req); err != nil { - panic(fmt.Sprintf("json decode failed: %v. Body:\n\n%s", err, string(bs))) + val, ok := op.Value.(string) + if !ok { + panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", op.Value)) } - - for _, patch := range req { - val, ok := patch.Value.(string) - if !ok { - panic(fmt.Sprintf("unsupported json patch value %v: cannot be converted to string", patch.Value)) - } - k.secret[path] = val + v, err := base64.StdEncoding.DecodeString(val) + if err != nil { + panic(fmt.Sprintf("json patch value %q is not base64 encoded: %v", val, err)) } + k.secret[path] = string(v) default: panic(fmt.Sprintf("unsupported json-patch op %q", op.Op)) } diff --git a/cmd/containerboot/serve.go b/cmd/containerboot/serve.go index 37fd497779c75..5fa8e580d5828 100644 --- a/cmd/containerboot/serve.go +++ b/cmd/containerboot/serve.go @@ -19,7 +19,9 @@ import ( "github.com/fsnotify/fsnotify" "tailscale.com/client/local" "tailscale.com/ipn" + "tailscale.com/kube/certs" "tailscale.com/kube/kubetypes" + klc "tailscale.com/kube/localclient" "tailscale.com/types/netmap" ) @@ -52,11 +54,9 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom var certDomain string var prevServeConfig *ipn.ServeConfig - var cm certManager + var cm *certs.CertManager if cfg.CertShareMode == "rw" { - cm = certManager{ - lc: lc, - } + cm = certs.NewCertManager(klc.New(lc), log.Printf) } for { select { @@ -93,7 +93,7 @@ func watchServeConfigChanges(ctx context.Context, cdChanged <-chan bool, certDom if cfg.CertShareMode != "rw" { continue } - if err := cm.ensureCertLoops(ctx, sc); err != nil { + if err := cm.EnsureCertLoops(ctx, sc); err != nil { log.Fatalf("serve proxy: error ensuring cert loops: %v", err) } } diff --git a/cmd/derper/depaware.txt b/cmd/derper/depaware.txt index ca772353079dc..7adbf397f2f4f 100644 --- a/cmd/derper/depaware.txt +++ b/cmd/derper/depaware.txt @@ -12,7 +12,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa github.com/coder/websocket/internal/util from github.com/coder/websocket github.com/coder/websocket/internal/xsync from github.com/coder/websocket L github.com/coreos/go-iptables/iptables from tailscale.com/util/linuxfw - W 💣 github.com/dblohm7/wingoes from tailscale.com/util/winutil + W 💣 github.com/dblohm7/wingoes from tailscale.com/util/winutil+ github.com/fxamacker/cbor/v2 from tailscale.com/tka github.com/go-json-experiment/json from tailscale.com/types/opt+ github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json+ @@ -157,7 +157,7 @@ tailscale.com/cmd/derper dependencies: (generated by github.com/tailscale/depawa 💣 tailscale.com/util/deephash from tailscale.com/util/syspolicy/setting L 💣 tailscale.com/util/dirwalk from tailscale.com/metrics tailscale.com/util/dnsname from tailscale.com/hostinfo+ - tailscale.com/util/eventbus from tailscale.com/net/netmon + tailscale.com/util/eventbus from tailscale.com/net/netmon+ 💣 tailscale.com/util/hashx from tailscale.com/util/deephash tailscale.com/util/httpm from tailscale.com/client/tailscale tailscale.com/util/lineiter from tailscale.com/hostinfo+ diff --git a/cmd/derper/derper.go b/cmd/derper/derper.go index 3c6fda68c4d59..7ea404beb50af 100644 --- a/cmd/derper/derper.go +++ b/cmd/derper/derper.go @@ -68,7 +68,7 @@ var ( runDERP = flag.Bool("derp", true, "whether to run a DERP server. The only reason to set this false is if you're decommissioning a server but want to keep its bootstrap DNS functionality still running.") flagHome = flag.String("home", "", "what to serve at the root path. It may be left empty (the default, for a default homepage), \"blank\" for a blank page, or a URL to redirect to") - meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It should contain some hex string; whitespace is trimmed.") + meshPSKFile = flag.String("mesh-psk-file", defaultMeshPSKFile(), "if non-empty, path to file containing the mesh pre-shared key file. It must be 64 lowercase hexadecimal characters; whitespace is trimmed.") meshWith = flag.String("mesh-with", "", "optional comma-separated list of hostnames to mesh with; the server's own hostname can be in the list. If an entry contains a slash, the second part names a hostname to be used when dialing the target.") secretsURL = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key") secretPrefix = flag.String("secrets-path-prefix", "prod/derp", "setec path prefix for \""+setecMeshKeyName+"\" secret for DERP mesh key") @@ -96,9 +96,6 @@ var ( var ( tlsRequestVersion = &metrics.LabelMap{Label: "version"} tlsActiveVersion = &metrics.LabelMap{Label: "version"} - - // Exactly 64 hexadecimal lowercase digits. - validMeshKey = regexp.MustCompile(`^[0-9a-f]{64}$`) ) const setecMeshKeyName = "meshkey" @@ -159,14 +156,6 @@ func writeNewConfig() config { return cfg } -func checkMeshKey(key string) (string, error) { - key = strings.TrimSpace(key) - if !validMeshKey.MatchString(key) { - return "", errors.New("key must contain exactly 64 hex digits") - } - return key, nil -} - func main() { flag.Parse() if *versionFlag { @@ -246,10 +235,9 @@ func main() { log.Printf("No mesh key configured for --dev mode") } else if meshKey == "" { log.Printf("No mesh key configured") - } else if key, err := checkMeshKey(meshKey); err != nil { + } else if err := s.SetMeshKey(meshKey); err != nil { log.Fatalf("invalid mesh key: %v", err) } else { - s.SetMeshKey(key) log.Println("DERP mesh key configured") } diff --git a/cmd/derper/derper_test.go b/cmd/derper/derper_test.go index 12686ce4eb5f3..6dce1fcdfebdd 100644 --- a/cmd/derper/derper_test.go +++ b/cmd/derper/derper_test.go @@ -138,46 +138,3 @@ func TestTemplate(t *testing.T) { t.Error("Output is missing debug info") } } - -func TestCheckMeshKey(t *testing.T) { - testCases := []struct { - name string - input string - want string - wantErr bool - }{ - { - name: "KeyOkay", - input: "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6", - want: "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6", - wantErr: false, - }, - { - name: "TrimKeyOkay", - input: " f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6 ", - want: "f1ffafffffffffffffffffffffffffffffffffffffffffffffffff2ffffcfff6", - wantErr: false, - }, - { - name: "NotAKey", - input: "zzthisisnotakey", - want: "", - wantErr: true, - }, - } - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - k, err := checkMeshKey(tt.input) - if err != nil && !tt.wantErr { - t.Errorf("unexpected error: %v", err) - } - if k != tt.want && err == nil { - t.Errorf("want: %s doesn't match expected: %s", tt.want, k) - } - - }) - } - -} diff --git a/cmd/derper/mesh.go b/cmd/derper/mesh.go index 1d8e3ef93c8b3..cbb2fa59ac030 100644 --- a/cmd/derper/mesh.go +++ b/cmd/derper/mesh.go @@ -72,6 +72,7 @@ func startMeshWithHost(s *derp.Server, hostTuple string) error { add := func(m derp.PeerPresentMessage) { s.AddPacketForwarder(m.Key, c) } remove := func(m derp.PeerGoneMessage) { s.RemovePacketForwarder(m.Peer, c) } - go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove) + notifyError := func(err error) {} + go c.RunWatchConnectionLoop(context.Background(), s.PublicKey(), logf, add, remove, notifyError) return nil } diff --git a/cmd/derpprobe/derpprobe.go b/cmd/derpprobe/derpprobe.go index 2723a31aea471..25159d649408e 100644 --- a/cmd/derpprobe/derpprobe.go +++ b/cmd/derpprobe/derpprobe.go @@ -5,23 +5,36 @@ package main import ( + "context" "flag" "fmt" "log" "net/http" "os" + "path" + "path/filepath" "sort" "time" + "github.com/tailscale/setec/client/setec" "tailscale.com/prober" "tailscale.com/tsweb" + "tailscale.com/types/key" "tailscale.com/version" // Support for prometheus varz in tsweb _ "tailscale.com/tsweb/promvarz" ) +const meshKeyEnvVar = "TAILSCALE_DERPER_MESH_KEY" +const setecMeshKeyName = "meshkey" + +func defaultSetecCacheDir() string { + return filepath.Join(os.Getenv("HOME"), ".cache", "derper-secrets") +} + var ( + dev = flag.Bool("dev", false, "run in localhost development mode") derpMapURL = flag.String("derp-map", "https://login.tailscale.com/derpmap/default", "URL to DERP map (https:// or file://) or 'local' to use the local tailscaled's DERP map") versionFlag = flag.Bool("version", false, "print version and exit") listen = flag.String("listen", ":8030", "HTTP listen address") @@ -37,6 +50,10 @@ var ( qdPacketsPerSecond = flag.Int("qd-packets-per-second", 0, "if greater than 0, queuing delay will be measured continuously using 260 byte packets (approximate size of a CallMeMaybe packet) sent at this rate per second") qdPacketTimeout = flag.Duration("qd-packet-timeout", 5*time.Second, "queuing delay packets arriving after this period of time from being sent are treated like dropped packets and don't count toward queuing delay timings") regionCodeOrID = flag.String("region-code", "", "probe only this region (e.g. 'lax' or '17'); if left blank, all regions will be probed") + meshPSKFile = flag.String("mesh-psk-file", "", "if non-empty, path to file containing the mesh pre-shared key file. It must be 64 lowercase hexadecimal characters; whitespace is trimmed.") + secretsURL = flag.String("secrets-url", "", "SETEC server URL for secrets retrieval of mesh key") + secretPrefix = flag.String("secrets-path-prefix", "prod/derp", fmt.Sprintf("setec path prefix for \"%s\" secret for DERP mesh key", setecMeshKeyName)) + secretsCacheDir = flag.String("secrets-cache-dir", defaultSetecCacheDir(), "directory to cache setec secrets in (required if --secrets-url is set)") ) func main() { @@ -47,11 +64,16 @@ func main() { } p := prober.New().WithSpread(*spread).WithOnce(*probeOnce).WithMetricNamespace("derpprobe") + meshKey, err := getMeshKey() + if err != nil { + log.Fatalf("failed to get mesh key: %v", err) + } opts := []prober.DERPOpt{ prober.WithMeshProbing(*meshInterval), prober.WithSTUNProbing(*stunInterval), prober.WithTLSProbing(*tlsInterval), prober.WithQueuingDelayProbing(*qdPacketsPerSecond, *qdPacketTimeout), + prober.WithMeshKey(meshKey), } if *bwInterval > 0 { opts = append(opts, prober.WithBandwidthProbing(*bwInterval, *bwSize, *bwTUNIPv4Address)) @@ -99,6 +121,53 @@ func main() { log.Fatal(http.ListenAndServe(*listen, mux)) } +func getMeshKey() (key.DERPMesh, error) { + var meshKey string + + if *dev { + meshKey = os.Getenv(meshKeyEnvVar) + if meshKey == "" { + log.Printf("No mesh key specified for dev via %s\n", meshKeyEnvVar) + } else { + log.Printf("Set mesh key from %s\n", meshKeyEnvVar) + } + } else if *secretsURL != "" { + meshKeySecret := path.Join(*secretPrefix, setecMeshKeyName) + fc, err := setec.NewFileCache(*secretsCacheDir) + if err != nil { + log.Fatalf("NewFileCache: %v", err) + } + log.Printf("Setting up setec store from %q", *secretsURL) + st, err := setec.NewStore(context.Background(), + setec.StoreConfig{ + Client: setec.Client{Server: *secretsURL}, + Secrets: []string{ + meshKeySecret, + }, + Cache: fc, + }) + if err != nil { + log.Fatalf("NewStore: %v", err) + } + meshKey = st.Secret(meshKeySecret).GetString() + log.Println("Got mesh key from setec store") + st.Close() + } else if *meshPSKFile != "" { + b, err := setec.StaticFile(*meshPSKFile) + if err != nil { + log.Fatalf("StaticFile failed to get key: %v", err) + } + log.Println("Got mesh key from static file") + meshKey = b.GetString() + } + if meshKey == "" { + log.Printf("No mesh key found, mesh key is empty") + return key.DERPMesh{}, nil + } + + return key.ParseDERPMesh(meshKey) +} + type overallStatus struct { good, bad []string } diff --git a/cmd/gitops-pusher/gitops-pusher_test.go b/cmd/gitops-pusher/gitops-pusher_test.go index b050761d9832d..e08b06c9cd194 100644 --- a/cmd/gitops-pusher/gitops-pusher_test.go +++ b/cmd/gitops-pusher/gitops-pusher_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package main import ( diff --git a/cmd/k8s-operator/api-server-proxy-pg.go b/cmd/k8s-operator/api-server-proxy-pg.go new file mode 100644 index 0000000000000..252859eb37197 --- /dev/null +++ b/cmd/k8s-operator/api-server-proxy-pg.go @@ -0,0 +1,479 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strings" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/internal/client/tailscale" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" + "tailscale.com/tstime" +) + +const ( + proxyPGFinalizerName = "tailscale.com/kube-apiserver-finalizer" + + // Reasons for KubeAPIServerProxyValid condition. + reasonKubeAPIServerProxyInvalid = "KubeAPIServerProxyInvalid" + reasonKubeAPIServerProxyValid = "KubeAPIServerProxyValid" + + // Reasons for KubeAPIServerProxyConfigured condition. + reasonKubeAPIServerProxyConfigured = "KubeAPIServerProxyConfigured" + reasonKubeAPIServerProxyNoBackends = "KubeAPIServerProxyNoBackends" +) + +// KubeAPIServerTSServiceReconciler reconciles the Tailscale Services required for an +// HA deployment of the API Server Proxy. +type KubeAPIServerTSServiceReconciler struct { + client.Client + recorder record.EventRecorder + logger *zap.SugaredLogger + tsClient tsClient + tsNamespace string + lc localClient + defaultTags []string + operatorID string // stableID of the operator's Tailscale device + + clock tstime.Clock +} + +// Reconcile is the entry point for the controller. +func (r *KubeAPIServerTSServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { + logger := r.logger.With("ProxyGroup", req.Name) + logger.Debugf("starting reconcile") + defer logger.Debugf("reconcile finished") + + pg := new(tsapi.ProxyGroup) + err = r.Get(ctx, req.NamespacedName, pg) + if apierrors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + logger.Debugf("ProxyGroup not found, assuming it was deleted") + return res, nil + } else if err != nil { + return res, fmt.Errorf("failed to get ProxyGroup: %w", err) + } + + serviceName := serviceNameForAPIServerProxy(pg) + logger = logger.With("Tailscale Service", serviceName) + + if markedForDeletion(pg) { + logger.Debugf("ProxyGroup is being deleted, ensuring any created resources are cleaned up") + if err = r.maybeCleanup(ctx, serviceName, pg, logger); err != nil && strings.Contains(err.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error, retrying: %s", err) + return res, nil + } + + return res, err + } + + err = r.maybeProvision(ctx, serviceName, pg, logger) + if err != nil { + if strings.Contains(err.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error, retrying: %s", err) + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +// maybeProvision ensures that a Tailscale Service for this ProxyGroup exists +// and is up to date. +// +// Returns true if the operation resulted in a Tailscale Service update. +func (r *KubeAPIServerTSServiceReconciler) maybeProvision(ctx context.Context, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (err error) { + var dnsName string + oldPGStatus := pg.Status.DeepCopy() + defer func() { + podsAdvertising, podsErr := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName) + if podsErr != nil { + err = errors.Join(err, fmt.Errorf("failed to get number of advertised Pods: %w", podsErr)) + // Continue, updating the status with the best available information. + } + + // Update the ProxyGroup status with the Tailscale Service information + // Update the condition based on how many pods are advertising the service + conditionStatus := metav1.ConditionFalse + conditionReason := reasonKubeAPIServerProxyNoBackends + conditionMessage := fmt.Sprintf("%d/%d proxy backends ready and advertising", podsAdvertising, pgReplicas(pg)) + + pg.Status.URL = "" + if podsAdvertising > 0 { + // At least one pod is advertising the service, consider it configured + conditionStatus = metav1.ConditionTrue + conditionReason = reasonKubeAPIServerProxyConfigured + if dnsName != "" { + pg.Status.URL = "https://" + dnsName + } + } + + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, conditionStatus, conditionReason, conditionMessage, pg.Generation, r.clock, logger) + + if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) { + // An error encountered here should get returned by the Reconcile function. + err = errors.Join(err, r.Client.Status().Update(ctx, pg)) + } + }() + + if !tsoperator.ProxyGroupAvailable(pg) { + return nil + } + + if !slices.Contains(pg.Finalizers, proxyPGFinalizerName) { + // This log line is printed exactly once during initial provisioning, + // because once the finalizer is in place this block gets skipped. So, + // this is a nice place to tell the operator that the high level, + // multi-reconcile operation is underway. + logger.Info("provisioning Tailscale Service for ProxyGroup") + pg.Finalizers = append(pg.Finalizers, proxyPGFinalizerName) + if err := r.Update(ctx, pg); err != nil { + return fmt.Errorf("failed to add finalizer: %w", err) + } + } + + // 1. Check there isn't a Tailscale Service with the same hostname + // already created and not owned by this ProxyGroup. + existingTSSvc, err := r.tsClient.GetVIPService(ctx, serviceName) + if isErrorFeatureFlagNotEnabled(err) { + logger.Warn(msgFeatureFlagNotEnabled) + r.recorder.Event(pg, corev1.EventTypeWarning, warningTailscaleServiceFeatureFlagNotEnabled, msgFeatureFlagNotEnabled) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msgFeatureFlagNotEnabled, pg.Generation, r.clock, logger) + return nil + } + if err != nil && !isErrorTailscaleServiceNotFound(err) { + return fmt.Errorf("error getting Tailscale Service %q: %w", serviceName, err) + } + + updatedAnnotations, err := exclusiveOwnerAnnotations(pg, r.operatorID, existingTSSvc) + if err != nil { + const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different Service name in the ProxyGroup's spec.kubeAPIServer.serviceName field" + msg := fmt.Sprintf("error ensuring exclusive ownership of Tailscale Service %s: %v. %s", serviceName, err, instr) + logger.Warn(msg) + r.recorder.Event(pg, corev1.EventTypeWarning, "InvalidTailscaleService", msg) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, msg, pg.Generation, r.clock, logger) + return nil + } + + // After getting this far, we know the Tailscale Service is valid. + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, reasonKubeAPIServerProxyValid, pg.Generation, r.clock, logger) + + // Service tags are limited to matching the ProxyGroup's tags until we have + // support for querying peer caps for a Service-bound request. + serviceTags := r.defaultTags + if len(pg.Spec.Tags) > 0 { + serviceTags = pg.Spec.Tags.Stringify() + } + + tsSvc := &tailscale.VIPService{ + Name: serviceName, + Tags: serviceTags, + Ports: []string{"tcp:443"}, + Comment: managedTSServiceComment, + Annotations: updatedAnnotations, + } + if existingTSSvc != nil { + tsSvc.Addrs = existingTSSvc.Addrs + } + + // 2. Ensure the Tailscale Service exists and is up to date. + if existingTSSvc == nil || + !slices.Equal(tsSvc.Tags, existingTSSvc.Tags) || + !ownersAreSetAndEqual(tsSvc, existingTSSvc) || + !slices.Equal(tsSvc.Ports, existingTSSvc.Ports) { + logger.Infof("Ensuring Tailscale Service exists and is up to date") + if err := r.tsClient.CreateOrUpdateVIPService(ctx, tsSvc); err != nil { + return fmt.Errorf("error creating Tailscale Service: %w", err) + } + } + + // 3. Ensure that TLS Secret and RBAC exists. + tcd, err := tailnetCertDomain(ctx, r.lc) + if err != nil { + return fmt.Errorf("error determining DNS name base: %w", err) + } + dnsName = serviceName.WithoutPrefix() + "." + tcd + if err = r.ensureCertResources(ctx, pg, dnsName); err != nil { + return fmt.Errorf("error ensuring cert resources: %w", err) + } + + // 4. Configure the Pods to advertise the Tailscale Service. + if err = r.maybeAdvertiseServices(ctx, pg, serviceName, logger); err != nil { + return fmt.Errorf("error updating advertised Tailscale Services: %w", err) + } + + // 5. Clean up any stale Tailscale Services from previous resource versions. + if err = r.maybeDeleteStaleServices(ctx, pg, logger); err != nil { + return fmt.Errorf("failed to delete stale Tailscale Services: %w", err) + } + + return nil +} + +// maybeCleanup ensures that any resources, such as a Tailscale Service created for this Service, are cleaned up when the +// Service is being deleted or is unexposed. The cleanup is safe for a multi-cluster setup- the Tailscale Service is only +// deleted if it does not contain any other owner references. If it does, the cleanup only removes the owner reference +// corresponding to this Service. +func (r *KubeAPIServerTSServiceReconciler) maybeCleanup(ctx context.Context, serviceName tailcfg.ServiceName, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (err error) { + ix := slices.Index(pg.Finalizers, proxyPGFinalizerName) + if ix < 0 { + logger.Debugf("no finalizer, nothing to do") + return nil + } + logger.Infof("Ensuring that Service %q is cleaned up", serviceName) + + defer func() { + if err == nil { + err = r.deleteFinalizer(ctx, pg, logger) + } + }() + + if _, err = cleanupTailscaleService(ctx, r.tsClient, serviceName, r.operatorID, logger); err != nil { + return fmt.Errorf("error deleting Tailscale Service: %w", err) + } + + if err = cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg.Name, serviceName); err != nil { + return fmt.Errorf("failed to clean up cert resources: %w", err) + } + + return nil +} + +// maybeDeleteStaleServices deletes Services that have previously been created for +// this ProxyGroup but are no longer needed. +func (r *KubeAPIServerTSServiceReconciler) maybeDeleteStaleServices(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) error { + serviceName := serviceNameForAPIServerProxy(pg) + + svcs, err := r.tsClient.ListVIPServices(ctx) + if err != nil { + return fmt.Errorf("error listing Tailscale Services: %w", err) + } + + for _, svc := range svcs.VIPServices { + if svc.Name == serviceName { + continue + } + + owners, err := parseOwnerAnnotation(&svc) + if err != nil { + logger.Warnf("error parsing owner annotation for Tailscale Service %s: %v", svc.Name, err) + continue + } + if owners == nil || len(owners.OwnerRefs) != 1 || owners.OwnerRefs[0].OperatorID != r.operatorID { + continue + } + + owner := owners.OwnerRefs[0] + if owner.Resource == nil || owner.Resource.Kind != "ProxyGroup" || owner.Resource.UID != string(pg.UID) { + continue + } + + logger.Infof("Deleting Tailscale Service %s", svc.Name) + if err := r.tsClient.DeleteVIPService(ctx, svc.Name); err != nil && !isErrorTailscaleServiceNotFound(err) { + return fmt.Errorf("error deleting Tailscale Service %s: %w", svc.Name, err) + } + + if err = cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg.Name, svc.Name); err != nil { + return fmt.Errorf("failed to clean up cert resources: %w", err) + } + } + + return nil +} + +func (r *KubeAPIServerTSServiceReconciler) deleteFinalizer(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) error { + pg.Finalizers = slices.DeleteFunc(pg.Finalizers, func(f string) bool { + return f == proxyPGFinalizerName + }) + logger.Debugf("ensure %q finalizer is removed", proxyPGFinalizerName) + + if err := r.Update(ctx, pg); err != nil { + return fmt.Errorf("failed to remove finalizer %q: %w", proxyPGFinalizerName, err) + } + return nil +} + +func (r *KubeAPIServerTSServiceReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string) error { + secret := certSecret(pg.Name, r.tsNamespace, domain, pg) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) { + s.Labels = secret.Labels + }); err != nil { + return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err) + } + role := certSecretRole(pg.Name, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { + r.Labels = role.Labels + r.Rules = role.Rules + }); err != nil { + return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err) + } + rolebinding := certSecretRoleBinding(pg, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) { + rb.Labels = rolebinding.Labels + rb.Subjects = rolebinding.Subjects + rb.RoleRef = rolebinding.RoleRef + }); err != nil { + return fmt.Errorf("failed to create or update RoleBinding %s: %w", rolebinding.Name, err) + } + return nil +} + +func (r *KubeAPIServerTSServiceReconciler) maybeAdvertiseServices(ctx context.Context, pg *tsapi.ProxyGroup, serviceName tailcfg.ServiceName, logger *zap.SugaredLogger) error { + // Get all config Secrets for this ProxyGroup + cfgSecrets := &corev1.SecretList{} + if err := r.List(ctx, cfgSecrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig))); err != nil { + return fmt.Errorf("failed to list config Secrets: %w", err) + } + + // Only advertise a Tailscale Service once the TLS certs required for + // serving it are available. + shouldBeAdvertised, err := hasCerts(ctx, r.Client, r.lc, r.tsNamespace, serviceName) + if err != nil { + return fmt.Errorf("error checking TLS credentials provisioned for Tailscale Service %q: %w", serviceName, err) + } + var advertiseServices []string + if shouldBeAdvertised { + advertiseServices = []string{serviceName.String()} + } + + for _, s := range cfgSecrets.Items { + if len(s.Data[kubetypes.KubeAPIServerConfigFile]) == 0 { + continue + } + + // Parse the existing config. + cfg, err := conf.Load(s.Data[kubetypes.KubeAPIServerConfigFile]) + if err != nil { + return fmt.Errorf("error loading config from Secret %q: %w", s.Name, err) + } + + if cfg.Parsed.APIServerProxy == nil { + return fmt.Errorf("config Secret %q does not contain APIServerProxy config", s.Name) + } + + existingCfgSecret := s.DeepCopy() + + var updated bool + if cfg.Parsed.APIServerProxy.ServiceName == nil || *cfg.Parsed.APIServerProxy.ServiceName != serviceName { + cfg.Parsed.APIServerProxy.ServiceName = &serviceName + updated = true + } + + // Update the services to advertise if required. + if !slices.Equal(cfg.Parsed.AdvertiseServices, advertiseServices) { + cfg.Parsed.AdvertiseServices = advertiseServices + updated = true + } + + if !updated { + continue + } + + // Update the config Secret. + cfgB, err := json.Marshal(conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &cfg.Parsed, + }) + if err != nil { + return err + } + + s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB + if !apiequality.Semantic.DeepEqual(existingCfgSecret, s) { + logger.Debugf("Updating the Tailscale Services in ProxyGroup config Secret %s", s.Name) + if err := r.Update(ctx, &s); err != nil { + return err + } + } + } + + return nil +} + +func serviceNameForAPIServerProxy(pg *tsapi.ProxyGroup) tailcfg.ServiceName { + if pg.Spec.KubeAPIServer != nil && pg.Spec.KubeAPIServer.Hostname != "" { + return tailcfg.ServiceName("svc:" + pg.Spec.KubeAPIServer.Hostname) + } + + return tailcfg.ServiceName("svc:" + pg.Name) +} + +// exclusiveOwnerAnnotations returns the updated annotations required to ensure this +// instance of the operator is the exclusive owner. If the Tailscale Service is not +// nil, but does not contain an owner reference we return an error as this likely means +// that the Service was created by something other than a Tailscale Kubernetes operator. +// We also error if it is already owned by another operator instance, as we do not +// want to load balance a kube-apiserver ProxyGroup across multiple clusters. +func exclusiveOwnerAnnotations(pg *tsapi.ProxyGroup, operatorID string, svc *tailscale.VIPService) (map[string]string, error) { + ref := OwnerRef{ + OperatorID: operatorID, + Resource: &Resource{ + Kind: "ProxyGroup", + Name: pg.Name, + UID: string(pg.UID), + }, + } + if svc == nil { + c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} + json, err := json.Marshal(c) + if err != nil { + return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service's owner annotation contents: %w, please report this", err) + } + return map[string]string{ + ownerAnnotation: string(json), + }, nil + } + o, err := parseOwnerAnnotation(svc) + if err != nil { + return nil, err + } + if o == nil || len(o.OwnerRefs) == 0 { + return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name) + } + if len(o.OwnerRefs) > 1 || o.OwnerRefs[0].OperatorID != operatorID { + return nil, fmt.Errorf("Tailscale Service %s is already owned by other operator(s) and cannot be shared across multiple clusters; configure a difference Service name to continue", svc.Name) + } + if o.OwnerRefs[0].Resource == nil { + return nil, fmt.Errorf("Tailscale Service %s exists, but does not reference an owning resource; not proceeding as this is likely a Service already owned by an Ingress", svc.Name) + } + if o.OwnerRefs[0].Resource.Kind != "ProxyGroup" || o.OwnerRefs[0].Resource.UID != string(pg.UID) { + return nil, fmt.Errorf("Tailscale Service %s is already owned by another resource: %#v; configure a difference Service name to continue", svc.Name, o.OwnerRefs[0].Resource) + } + if o.OwnerRefs[0].Resource.Name != pg.Name { + // ProxyGroup name can be updated in place. + o.OwnerRefs[0].Resource.Name = pg.Name + } + + oBytes, err := json.Marshal(o) + if err != nil { + return nil, err + } + + newAnnots := make(map[string]string, len(svc.Annotations)+1) + maps.Copy(newAnnots, svc.Annotations) + newAnnots[ownerAnnotation] = string(oBytes) + + return newAnnots, nil +} diff --git a/cmd/k8s-operator/api-server-proxy-pg_test.go b/cmd/k8s-operator/api-server-proxy-pg_test.go new file mode 100644 index 0000000000000..dfef63f22ff04 --- /dev/null +++ b/cmd/k8s-operator/api-server-proxy-pg_test.go @@ -0,0 +1,384 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "tailscale.com/internal/client/tailscale" + "tailscale.com/ipn/ipnstate" + tsoperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" + "tailscale.com/tstest" + "tailscale.com/types/opt" + "tailscale.com/types/ptr" +) + +func TestAPIServerProxyReconciler(t *testing.T) { + const ( + pgName = "test-pg" + pgUID = "test-pg-uid" + ns = "operator-ns" + defaultDomain = "test-pg.ts.net" + ) + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgName, + Generation: 1, + UID: pgUID, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + }, + Status: tsapi.ProxyGroupStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.ProxyGroupAvailable), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + }, + }, + }, + } + initialCfg := &conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("test-key"), + APIServerProxy: &conf.APIServerProxyConfig{ + Enabled: opt.NewBool(true), + }, + }, + } + expectedCfg := *initialCfg + initialCfgB, err := json.Marshal(initialCfg) + if err != nil { + t.Fatalf("marshaling initial config: %v", err) + } + pgCfgSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgConfigSecretName(pgName, 0), + Namespace: ns, + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig), + }, + Data: map[string][]byte{ + // Existing config should be preserved. + kubetypes.KubeAPIServerConfigFile: initialCfgB, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg, pgCfgSecret). + WithStatusSubresource(pg). + Build() + expectCfg := func(c *conf.VersionedConfig) { + t.Helper() + cBytes, err := json.Marshal(c) + if err != nil { + t.Fatalf("marshaling expected config: %v", err) + } + pgCfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cBytes + expectEqual(t, fc, pgCfgSecret) + } + + ft := &fakeTSClient{} + ingressTSSvc := &tailscale.VIPService{ + Name: "svc:some-ingress-hostname", + Comment: managedTSServiceComment, + Annotations: map[string]string{ + // No resource field. + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, + }, + Ports: []string{"tcp:443"}, + Tags: []string{"tag:k8s"}, + Addrs: []string{"5.6.7.8"}, + } + ft.CreateOrUpdateVIPService(t.Context(), ingressTSSvc) + + lc := &fakeLocalClient{ + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{ + MagicDNSSuffix: "ts.net", + }, + }, + } + + r := &KubeAPIServerTSServiceReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + tsNamespace: ns, + logger: zap.Must(zap.NewDevelopment()).Sugar(), + recorder: record.NewFakeRecorder(10), + lc: lc, + clock: tstest.NewClock(tstest.ClockOpts{}), + operatorID: "self-id", + } + + // Create a Tailscale Service that will conflict with the initial config. + if err := ft.CreateOrUpdateVIPService(t.Context(), &tailscale.VIPService{ + Name: "svc:" + pgName, + }); err != nil { + t.Fatalf("creating initial Tailscale Service: %v", err) + } + expectReconciled(t, r, "", pgName) + pg.ObjectMeta.Finalizers = []string{proxyPGFinalizerName} + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionFalse, reasonKubeAPIServerProxyInvalid, "", 1, r.clock, r.logger) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + expectMissing[corev1.Secret](t, fc, ns, defaultDomain) + expectMissing[rbacv1.Role](t, fc, ns, defaultDomain) + expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain) + expectEqual(t, fc, pgCfgSecret) // Unchanged. + + // Delete Tailscale Service; should see Service created and valid condition updated to true. + if err := ft.DeleteVIPService(t.Context(), "svc:"+pgName); err != nil { + t.Fatalf("deleting initial Tailscale Service: %v", err) + } + expectReconciled(t, r, "", pgName) + + tsSvc, err := ft.GetVIPService(t.Context(), "svc:"+pgName) + if err != nil { + t.Fatalf("getting Tailscale Service: %v", err) + } + if tsSvc == nil { + t.Fatalf("expected Tailscale Service to be created, but got nil") + } + expectedTSSvc := &tailscale.VIPService{ + Name: "svc:" + pgName, + Comment: managedTSServiceComment, + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"test-pg-uid"}}]}`, + }, + Ports: []string{"tcp:443"}, + Tags: []string{"tag:k8s"}, + Addrs: []string{"5.6.7.8"}, + } + if !reflect.DeepEqual(tsSvc, expectedTSSvc) { + t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc) + } + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.logger) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + expectedCfg.APIServerProxy.ServiceName = ptr.To(tailcfg.ServiceName("svc:" + pgName)) + expectCfg(&expectedCfg) + + expectEqual(t, fc, certSecret(pgName, ns, defaultDomain, pg)) + expectEqual(t, fc, certSecretRole(pgName, ns, defaultDomain)) + expectEqual(t, fc, certSecretRoleBinding(pg, ns, defaultDomain)) + + // Simulate certs being issued; should observe AdvertiseServices config change. + if err := populateTLSSecret(t.Context(), fc, pgName, defaultDomain); err != nil { + t.Fatalf("populating TLS Secret: %v", err) + } + expectReconciled(t, r, "", pgName) + + expectedCfg.AdvertiseServices = []string{"svc:" + pgName} + expectCfg(&expectedCfg) + + expectEqual(t, fc, pg, omitPGStatusConditionMessages) // Unchanged status. + + // Simulate Pod prefs updated with advertised services; should see Configured condition updated to true. + mustCreate(t, fc, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg-0", + Namespace: ns, + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeState), + }, + Data: map[string][]byte{ + "_current-profile": []byte("profile-foo"), + "profile-foo": []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`), + }, + }) + expectReconciled(t, r, "", pgName) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger) + pg.Status.URL = "https://" + defaultDomain + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Rename the Tailscale Service - old one + cert resources should be cleaned up. + updatedServiceName := tailcfg.ServiceName("svc:test-pg-renamed") + updatedDomain := "test-pg-renamed.ts.net" + pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{ + Hostname: updatedServiceName.WithoutPrefix(), + } + mustUpdate(t, fc, "", pgName, func(p *tsapi.ProxyGroup) { + p.Spec.KubeAPIServer = pg.Spec.KubeAPIServer + }) + expectReconciled(t, r, "", pgName) + _, err = ft.GetVIPService(t.Context(), "svc:"+pgName) + if !isErrorTailscaleServiceNotFound(err) { + t.Fatalf("Expected 404, got: %v", err) + } + tsSvc, err = ft.GetVIPService(t.Context(), updatedServiceName) + if err != nil { + t.Fatalf("Expected renamed svc, got error: %v", err) + } + expectedTSSvc.Name = updatedServiceName + if !reflect.DeepEqual(tsSvc, expectedTSSvc) { + t.Fatalf("expected Tailscale Service to be %+v, got %+v", expectedTSSvc, tsSvc) + } + // Check cfg and status reset until TLS certs are available again. + expectedCfg.APIServerProxy.ServiceName = ptr.To(updatedServiceName) + expectedCfg.AdvertiseServices = nil + expectCfg(&expectedCfg) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionFalse, reasonKubeAPIServerProxyNoBackends, "", 1, r.clock, r.logger) + pg.Status.URL = "" + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + expectEqual(t, fc, certSecret(pgName, ns, updatedDomain, pg)) + expectEqual(t, fc, certSecretRole(pgName, ns, updatedDomain)) + expectEqual(t, fc, certSecretRoleBinding(pg, ns, updatedDomain)) + expectMissing[corev1.Secret](t, fc, ns, defaultDomain) + expectMissing[rbacv1.Role](t, fc, ns, defaultDomain) + expectMissing[rbacv1.RoleBinding](t, fc, ns, defaultDomain) + + // Check we get the new hostname in the status once ready. + if err := populateTLSSecret(t.Context(), fc, pgName, updatedDomain); err != nil { + t.Fatalf("populating TLS Secret: %v", err) + } + mustUpdate(t, fc, "operator-ns", "test-pg-0", func(s *corev1.Secret) { + s.Data["profile-foo"] = []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`) + }) + expectReconciled(t, r, "", pgName) + expectedCfg.AdvertiseServices = []string{updatedServiceName.String()} + expectCfg(&expectedCfg) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger) + pg.Status.URL = "https://" + updatedDomain + + // Delete the ProxyGroup and verify Tailscale Service and cert resources are cleaned up. + if err := fc.Delete(t.Context(), pg); err != nil { + t.Fatalf("deleting ProxyGroup: %v", err) + } + expectReconciled(t, r, "", pgName) + expectMissing[corev1.Secret](t, fc, ns, updatedDomain) + expectMissing[rbacv1.Role](t, fc, ns, updatedDomain) + expectMissing[rbacv1.RoleBinding](t, fc, ns, updatedDomain) + _, err = ft.GetVIPService(t.Context(), updatedServiceName) + if !isErrorTailscaleServiceNotFound(err) { + t.Fatalf("Expected 404, got: %v", err) + } + + // Ingress Tailscale Service should not be affected. + svc, err := ft.GetVIPService(t.Context(), ingressTSSvc.Name) + if err != nil { + t.Fatalf("getting ingress Tailscale Service: %v", err) + } + if !reflect.DeepEqual(svc, ingressTSSvc) { + t.Fatalf("expected ingress Tailscale Service to be unmodified %+v, got %+v", ingressTSSvc, svc) + } +} + +func TestExclusiveOwnerAnnotations(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pg1", + UID: "pg1-uid", + }, + } + const ( + selfOperatorID = "self-id" + pg1Owner = `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg1","uid":"pg1-uid"}}]}` + ) + + for name, tc := range map[string]struct { + svc *tailscale.VIPService + wantErr string + }{ + "no_svc": { + svc: nil, + }, + "empty_svc": { + svc: &tailscale.VIPService{}, + wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator", + }, + "already_owner": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: pg1Owner, + }, + }, + }, + "already_owner_name_updated": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"old-pg1-name","uid":"pg1-uid"}}]}`, + }, + }, + }, + "preserves_existing_annotations": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + "existing": "annotation", + ownerAnnotation: pg1Owner, + }, + }, + }, + "owned_by_another_operator": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`, + }, + }, + wantErr: "already owned by other operator(s)", + }, + "owned_by_an_ingress": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, // Ingress doesn't set Resource field (yet). + }, + }, + wantErr: "does not reference an owning resource", + }, + "owned_by_another_pg": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"pg2","uid":"pg2-uid"}}]}`, + }, + }, + wantErr: "already owned by another resource", + }, + } { + t.Run(name, func(t *testing.T) { + got, err := exclusiveOwnerAnnotations(pg, "self-id", tc.svc) + if tc.wantErr != "" { + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("exclusiveOwnerAnnotations() error = %v, wantErr %v", err, tc.wantErr) + } + } else if diff := cmp.Diff(pg1Owner, got[ownerAnnotation]); diff != "" { + t.Errorf("exclusiveOwnerAnnotations() mismatch (-want +got):\n%s", diff) + } + if tc.svc == nil { + return // Don't check annotations being preserved. + } + for k, v := range tc.svc.Annotations { + if k == ownerAnnotation { + continue + } + if got[k] != v { + t.Errorf("exclusiveOwnerAnnotations() did not preserve annotation %q: got %q, want %q", k, got[k], v) + } + } + }) + } +} + +func omitPGStatusConditionMessages(p *tsapi.ProxyGroup) { + for i := range p.Status.Conditions { + // Don't bother validating the message. + p.Status.Conditions[i].Message = "" + } +} diff --git a/cmd/k8s-operator/api-server-proxy.go b/cmd/k8s-operator/api-server-proxy.go new file mode 100644 index 0000000000000..70333d2c48d41 --- /dev/null +++ b/cmd/k8s-operator/api-server-proxy.go @@ -0,0 +1,43 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "fmt" + "log" + "os" + + "tailscale.com/kube/kubetypes" + "tailscale.com/types/ptr" +) + +func parseAPIProxyMode() *kubetypes.APIServerProxyMode { + haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != "" + haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != "" + switch { + case haveAPIProxyEnv && haveAuthProxyEnv: + log.Fatal("AUTH_PROXY (deprecated) and APISERVER_PROXY are mutually exclusive, please unset AUTH_PROXY") + case haveAuthProxyEnv: + var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated + if authProxyEnv { + return ptr.To(kubetypes.APIServerProxyModeAuth) + } + return nil + case haveAPIProxyEnv: + var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth" + switch apiProxyEnv { + case "true": + return ptr.To(kubetypes.APIServerProxyModeAuth) + case "false", "": + return nil + case "noauth": + return ptr.To(kubetypes.APIServerProxyModeNoAuth) + default: + panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv)) + } + } + return nil +} diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index c243036cbabd9..8406a1156fc8f 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -7,6 +7,7 @@ package main import ( "context" + "errors" "fmt" "net/netip" "slices" @@ -14,8 +15,6 @@ import ( "sync" "time" - "errors" - "go.uber.org/zap" xslices "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" @@ -26,6 +25,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -199,6 +199,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge }, ProxyClassName: proxyClass, proxyType: proxyTypeConnector, + LoginServer: a.ssr.loginServer, } if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 { diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index f32fe3282020c..d5829c37fe596 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -80,7 +80,7 @@ func TestConnector(t *testing.T) { app: kubetypes.AppConnector, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Connector status should get updated with the IP/hostname info when available. const hostname = "foo.tailnetxyz.ts.net" @@ -106,7 +106,7 @@ func TestConnector(t *testing.T) { opts.subnetRoutes = "10.40.0.0/14,10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Remove a route. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -114,7 +114,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Remove the subnet router. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -122,7 +122,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Re-add the subnet router. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -132,7 +132,7 @@ func TestConnector(t *testing.T) { }) opts.subnetRoutes = "10.44.0.0/20" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { @@ -176,7 +176,7 @@ func TestConnector(t *testing.T) { app: kubetypes.AppConnector, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Add an exit node. mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { @@ -184,7 +184,7 @@ func TestConnector(t *testing.T) { }) opts.isExitNode = true expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Delete the Connector. if err = fc.Delete(context.Background(), cn); err != nil { @@ -262,7 +262,7 @@ func TestConnectorWithProxyClass(t *testing.T) { app: kubetypes.AppConnector, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. Update Connector to specify a ProxyClass. ProxyClass is not yet // ready, so its configuration is NOT applied to the Connector @@ -271,7 +271,7 @@ func TestConnectorWithProxyClass(t *testing.T) { conn.Spec.ProxyClass = "custom-metadata" }) expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 3. ProxyClass is set to Ready by proxy-class reconciler. Connector // get reconciled and configuration from the ProxyClass is applied to @@ -286,7 +286,7 @@ func TestConnectorWithProxyClass(t *testing.T) { }) opts.proxyClass = pc.Name expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 4. Connector.spec.proxyClass field is unset, Connector gets // reconciled and configuration from the ProxyClass is removed from the @@ -296,7 +296,7 @@ func TestConnectorWithProxyClass(t *testing.T) { }) opts.proxyClass = "" expectReconciled(t, cr, "", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) } func TestConnectorWithAppConnector(t *testing.T) { @@ -352,7 +352,7 @@ func TestConnectorWithAppConnector(t *testing.T) { isAppConnector: true, } expectEqual(t, fc, expectedSecret(t, fc, opts)) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // Connector's ready condition should be set to true cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer") diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index 12fb5cf2e5a65..f810d1b4fd62a 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -144,8 +144,6 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ github.com/google/uuid from github.com/prometheus-community/pro-bing+ - github.com/gorilla/csrf from tailscale.com/client/web - github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L 💣 github.com/illarion/gonotify/v3 from tailscale.com/net/dns L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 @@ -202,7 +200,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+ github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp - github.com/tailscale/hujson from tailscale.com/ipn/conffile + github.com/tailscale/hujson from tailscale.com/ipn/conffile+ L 💣 github.com/tailscale/netlink from tailscale.com/net/routetable+ L 💣 github.com/tailscale/netlink/nl from github.com/tailscale/netlink github.com/tailscale/peercred from tailscale.com/ipn/ipnauth @@ -798,7 +796,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/ipn/ipnext+ tailscale.com/health from tailscale.com/control/controlclient+ - tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ tailscale.com/hostinfo from tailscale.com/client/web+ tailscale.com/internal/client/tailscale from tailscale.com/cmd/k8s-operator tailscale.com/internal/noiseconn from tailscale.com/control/controlclient @@ -824,6 +822,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/k8s-operator/sessionrecording/ws from tailscale.com/k8s-operator/sessionrecording tailscale.com/kube/egressservices from tailscale.com/cmd/k8s-operator tailscale.com/kube/ingressservices from tailscale.com/cmd/k8s-operator + tailscale.com/kube/k8s-proxy/conf from tailscale.com/cmd/k8s-operator tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+ tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore tailscale.com/kube/kubetypes from tailscale.com/cmd/k8s-operator+ @@ -1074,7 +1073,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ crypto/internal/fips140/edwards25519/field from crypto/ecdh+ crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls + crypto/internal/fips140/mlkem from crypto/tls+ crypto/internal/fips140/nistec from crypto/elliptic+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa @@ -1094,6 +1093,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ crypto/internal/randutil from crypto/dsa+ crypto/internal/sysrand from crypto/internal/entropy+ crypto/md5 from crypto/tls+ + LD crypto/mlkem from golang.org/x/crypto/ssh crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls+ crypto/rsa from crypto/tls+ @@ -1112,13 +1112,12 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/tailscale/web-client-prebuilt+ - encoding from encoding/gob+ + encoding from encoding/json+ encoding/asn1 from crypto/x509+ encoding/base32 from github.com/fxamacker/cbor/v2+ encoding/base64 from encoding/json+ encoding/binary from compress/gzip+ encoding/csv from github.com/spf13/pflag - encoding/gob from github.com/gorilla/securecookie encoding/hex from crypto/x509+ encoding/json from expvar+ encoding/pem from crypto/tls+ @@ -1140,7 +1139,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ hash/fnv from google.golang.org/protobuf/internal/detrand hash/maphash from go4.org/mem html from html/template+ - html/template from github.com/gorilla/csrf+ + html/template from tailscale.com/util/eventbus internal/abi from crypto/x509/internal/macos+ internal/asan from internal/runtime/maps+ internal/bisect from internal/godebug @@ -1172,7 +1171,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ internal/runtime/math from internal/runtime/maps+ internal/runtime/sys from crypto/subtle+ L internal/runtime/syscall from runtime+ - internal/saferio from debug/pe+ + W internal/saferio from debug/pe internal/singleflight from net internal/stringslite from embed+ internal/sync from sync+ diff --git a/cmd/k8s-operator/deploy/chart/templates/NOTES.txt b/cmd/k8s-operator/deploy/chart/templates/NOTES.txt new file mode 100644 index 0000000000000..1bee6704616e6 --- /dev/null +++ b/cmd/k8s-operator/deploy/chart/templates/NOTES.txt @@ -0,0 +1,27 @@ +You have successfully installed the Tailscale Kubernetes Operator! + +Once connected, the operator should appear as a device within the Tailscale admin console: +https://login.tailscale.com/admin/machines + +If you have not used the Tailscale operator before, here are some examples to try out: + +* Private Kubernetes API access and authorization using the API server proxy + https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy + +* Private access to cluster Services using an ingress proxy + https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress + +* Private access to the cluster's available subnets using a subnet router + https://tailscale.com/kb/1441/kubernetes-operator-connector + +You can also explore the CRDs, operator, and associated resources within the {{ .Release.Namespace }} namespace: + +$ kubectl explain connector +$ kubectl explain proxygroup +$ kubectl explain proxyclass +$ kubectl explain recorder +$ kubectl explain dnsconfig + +If you're interested to explore what resources were created: + +$ kubectl --namespace={{ .Release.Namespace }} get all -l app.kubernetes.io/managed-by=Helm diff --git a/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml index 072ecf6d22e2f..ad0a6fb66f51e 100644 --- a/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/apiserverproxy-rbac.yaml @@ -1,7 +1,16 @@ # Copyright (c) Tailscale Inc & AUTHORS # SPDX-License-Identifier: BSD-3-Clause -{{ if eq .Values.apiServerProxyConfig.mode "true" }} +# If old setting used, enable both old (operator) and new (ProxyGroup) workflows. +# If new setting used, enable only new workflow. +{{ if or (eq .Values.apiServerProxyConfig.mode "true") + (eq .Values.apiServerProxyConfig.allowImpersonation "true") }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-apiserver-auth-proxy + namespace: {{ .Release.Namespace }} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -16,9 +25,14 @@ kind: ClusterRoleBinding metadata: name: tailscale-auth-proxy subjects: +{{- if eq .Values.apiServerProxyConfig.mode "true" }} - kind: ServiceAccount name: operator namespace: {{ .Release.Namespace }} +{{- end }} +- kind: ServiceAccount + name: kube-apiserver-auth-proxy + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: tailscale-auth-proxy diff --git a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml index 1b9b97186b6ca..51d0a88c36671 100644 --- a/cmd/k8s-operator/deploy/chart/templates/deployment.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/deployment.yaml @@ -68,6 +68,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OPERATOR_LOGIN_SERVER + value: {{ .Values.loginServer }} + - name: OPERATOR_INGRESS_CLASS_NAME + value: {{ .Values.ingressClass.name }} - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE diff --git a/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml b/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml index 208d58ee10f08..54851955db67a 100644 --- a/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/ingressclass.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: - name: tailscale # class name currently can not be changed + name: {{ .Values.ingressClass.name }} annotations: {} # we do not support default IngressClass annotation https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class spec: controller: tailscale.com/ts-ingress # controller name currently can not be changed diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 00d8318acdce4..5eb920a6f41c4 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -16,6 +16,9 @@ kind: ClusterRole metadata: name: tailscale-operator rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events", "services", "services/status"] verbs: ["create","delete","deletecollection","get","list","patch","update","watch"] diff --git a/cmd/k8s-operator/deploy/chart/values.yaml b/cmd/k8s-operator/deploy/chart/values.yaml index 24e94c0493d75..048a1d95ca2fa 100644 --- a/cmd/k8s-operator/deploy/chart/values.yaml +++ b/cmd/k8s-operator/deploy/chart/values.yaml @@ -9,6 +9,9 @@ oauth: {} # clientId: "" # clientSecret: "" +# URL of the control plane to be used by all resources managed by the operator. +loginServer: "" + # Secret volume. # If set it defines the volume the oauth secrets will be mounted from. # The volume needs to contain two files named `client_id` and `client_secret`. @@ -74,6 +77,10 @@ operatorConfig: # In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here ingressClass: + # Allows for customization of the ingress class name used by the operator to identify ingresses to reconcile. This does + # not allow multiple operator instances to manage different ingresses, but provides an onboarding route for users that + # may have previously set up ingress classes named "tailscale" prior to using the operator. + name: "tailscale" enabled: true # proxyConfig contains configuraton that will be applied to any ingress/egress @@ -85,6 +92,13 @@ ingressClass: # If you need more configuration options, take a look at ProxyClass: # https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource proxyConfig: + # Configure the proxy image to use instead of the default tailscale/tailscale:latest. + # Applying a ProxyClass with `spec.statefulSet.pod.tailscaleContainer.image` + # set will override any defaults here. + # + # Note that ProxyGroups of type "kube-apiserver" use a different default image, + # tailscale/k8s-proxy:latest, and it is currently only possible to override + # that image via the same ProxyClass field. image: # Repository defaults to DockerHub, but images are also synced to ghcr.io/tailscale/tailscale. repository: tailscale/tailscale @@ -108,6 +122,15 @@ proxyConfig: # Kubernetes API server. # https://tailscale.com/kb/1437/kubernetes-operator-api-server-proxy apiServerProxyConfig: + # Set to "true" to create the ClusterRole permissions required for the API + # server proxy's auth mode. In auth mode, the API server proxy impersonates + # groups and users based on tailnet ACL grants. Required for ProxyGroups of + # type "kube-apiserver" running in auth mode. + allowImpersonation: "false" # "true", "false" + + # If true or noauth, the operator will run an in-process API server proxy. + # You can deploy a ProxyGroup of type "kube-apiserver" to run a high + # availability set of API server proxies instead. mode: "false" # "true", "false", "noauth" imagePullSecrets: [] diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml index 268d978c15f37..bffad47f97191 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_dnsconfigs.yaml @@ -101,6 +101,13 @@ spec: tag: description: Tag defaults to unstable. type: string + service: + description: Service configuration. + type: object + properties: + clusterIP: + description: ClusterIP sets the static IP of the service used by the nameserver. + type: string status: description: |- Status describes the status of the DNSConfig. This is set @@ -172,7 +179,7 @@ spec: ip: description: |- IP is the ClusterIP of the Service fronting the deployed ts.net nameserver. - Currently you must manually update your cluster DNS config to add + Currently, you must manually update your cluster DNS config to add this address as a stub nameserver for ts.net for cluster workloads to be able to resolve MagicDNS names associated with egress or Ingress proxies. diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml index 1541234755029..c5dc9c3e96a83 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml @@ -1379,12 +1379,21 @@ spec: type: string image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -1655,7 +1664,9 @@ spec: PodSecurityContext, the value specified in SecurityContext takes precedence. type: string tailscaleInitContainer: - description: Configuration for the proxy init container that enables forwarding. + description: |- + Configuration for the proxy init container that enables forwarding. + Not valid to apply to ProxyGroups of type "kube-apiserver". type: object properties: debug: @@ -1709,12 +1720,21 @@ spec: type: string image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -2203,6 +2223,51 @@ spec: won't make it *more* imbalanced. It's a required field. type: string + staticEndpoints: + description: |- + Configuration for 'static endpoints' on proxies in order to facilitate + direct connections from other devices on the tailnet. + See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. + type: object + required: + - nodePort + properties: + nodePort: + description: The configuration for static endpoints using NodePort Services. + type: object + required: + - ports + properties: + ports: + description: |- + The port ranges from which the operator will select NodePorts for the Services. + You must ensure that firewall rules allow UDP ingress traffic for these ports + to the node's external IPs. + The ports must be in the range of service node ports for the cluster (default `30000-32767`). + See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. + type: array + minItems: 1 + items: + type: object + required: + - port + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be used. This field cannot be defined if the port field is not defined. + The endPort must be either unset, or equal or greater than port. + type: integer + port: + description: port represents a port selected to be used. This is a required field. + type: integer + selector: + description: |- + A selector which will be used to select the node's that will have their `ExternalIP`'s advertised + by the ProxyGroup as Static Endpoints. + type: object + additionalProperties: + type: string tailscale: description: |- TailscaleConfig contains options to configure the tailscale-specific diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml index 4b9149e23e55b..98ca1c378ab8d 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_proxygroups.yaml @@ -20,6 +20,10 @@ spec: jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason name: Status type: string + - description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver. + jsonPath: .status.url + name: URL + type: string - description: ProxyGroup type. jsonPath: .spec.type name: Type @@ -32,15 +36,22 @@ spec: openAPIV3Schema: description: |- ProxyGroup defines a set of Tailscale devices that will act as proxies. - Currently only egress ProxyGroups are supported. + Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver + proxies. In addition to running a highly available set of proxies, ingress + and egress ProxyGroups also allow for serving many annotated Services from a + single set of proxies to minimise resource consumption. + + For ingress and egress, use the tailscale.com/proxy-group annotation on a + Service to specify that the proxy should be implemented by a ProxyGroup + instead of a single dedicated proxy. - Use the tailscale.com/proxy-group annotation on a Service to specify that - the egress proxy should be implemented by a ProxyGroup instead of a single - dedicated proxy. In addition to running a highly available set of proxies, - ProxyGroup also allows for serving many annotated Services from a single - set of proxies to minimise resource consumption. + More info: + * https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + * https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress - More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + For kube-apiserver, the ProxyGroup is a standalone resource. Use the + spec.kubeAPIServer field to configure options specific to the kube-apiserver + ProxyGroup type. type: object required: - spec @@ -77,6 +88,30 @@ spec: must not start with a dash and must be between 1 and 62 characters long. type: string pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + kubeAPIServer: + description: |- + KubeAPIServer contains configuration specific to the kube-apiserver + ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + type: object + properties: + hostname: + description: |- + Hostname is the hostname with which to expose the Kubernetes API server + proxies. Must be a valid DNS label no longer than 63 characters. If not + specified, the name of the ProxyGroup is used as the hostname. Must be + unique across the whole tailnet. + type: string + pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ + mode: + description: |- + Mode to run the API server proxy in. Supported modes are auth and noauth. + In auth mode, requests from the tailnet proxied over to the Kubernetes + API server are additionally impersonated using the sender's tailnet identity. + If not specified, defaults to auth mode. + type: string + enum: + - auth + - noauth proxyClass: description: |- ProxyClass is the name of the ProxyClass custom resource that contains @@ -106,12 +141,13 @@ spec: pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ type: description: |- - Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. Type is immutable once a ProxyGroup is created. type: string enum: - egress - ingress + - kube-apiserver x-kubernetes-validations: - rule: self == oldSelf message: ProxyGroup type is immutable @@ -124,7 +160,20 @@ spec: conditions: description: |- List of status conditions to indicate the status of the ProxyGroup - resources. Known condition types are `ProxyGroupReady`. + resources. Known condition types include `ProxyGroupReady` and + `ProxyGroupAvailable`. + + * `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and + all expected conditions are true. + * `ProxyGroupAvailable` indicates that at least one proxy is ready to + serve traffic. + + For ProxyGroups of type kube-apiserver, there are two additional conditions: + + * `KubeAPIServerProxyConfigured` indicates that at least one API server + proxy is configured and ready to serve traffic. + * `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is + valid. type: array items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -196,6 +245,11 @@ spec: If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the node. type: string + staticEndpoints: + description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. + type: array + items: + type: string tailnetIPs: description: |- TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) @@ -206,6 +260,11 @@ spec: x-kubernetes-list-map-keys: - hostname x-kubernetes-list-type: map + url: + description: |- + URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if + any. Only applies to ProxyGroups of type kube-apiserver. + type: string served: true storage: true subresources: diff --git a/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml b/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml index ddbdda32e476e..5818fa69fff7d 100644 --- a/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml +++ b/cmd/k8s-operator/deploy/manifests/authproxy-rbac.yaml @@ -1,6 +1,12 @@ # Copyright (c) Tailscale Inc & AUTHORS # SPDX-License-Identifier: BSD-3-Clause +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-apiserver-auth-proxy + namespace: tailscale +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -18,6 +24,9 @@ subjects: - kind: ServiceAccount name: operator namespace: tailscale +- kind: ServiceAccount + name: kube-apiserver-auth-proxy + namespace: tailscale roleRef: kind: ClusterRole name: tailscale-auth-proxy diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 1d910cf92c6c6..175f2a7fbe9ba 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -389,6 +389,13 @@ spec: description: Tag defaults to unstable. type: string type: object + service: + description: Service configuration. + properties: + clusterIP: + description: ClusterIP sets the static IP of the service used by the nameserver. + type: string + type: object type: object required: - nameserver @@ -462,7 +469,7 @@ spec: ip: description: |- IP is the ClusterIP of the Service fronting the deployed ts.net nameserver. - Currently you must manually update your cluster DNS config to add + Currently, you must manually update your cluster DNS config to add this address as a stub nameserver for ts.net for cluster workloads to be able to resolve MagicDNS names associated with egress or Ingress proxies. @@ -1852,12 +1859,21 @@ spec: type: array image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -2129,7 +2145,9 @@ spec: type: object type: object tailscaleInitContainer: - description: Configuration for the proxy init container that enables forwarding. + description: |- + Configuration for the proxy init container that enables forwarding. + Not valid to apply to ProxyGroups of type "kube-apiserver". properties: debug: description: |- @@ -2182,12 +2200,21 @@ spec: type: array image: description: |- - Container image name. By default images are pulled from - docker.io/tailscale/tailscale, but the official images are also - available at ghcr.io/tailscale/tailscale. Specifying image name here - will override any proxy image values specified via the Kubernetes - operator's Helm chart values or PROXY_IMAGE env var in the operator - Deployment. + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image type: string imagePullPolicy: @@ -2679,6 +2706,51 @@ spec: type: array type: object type: object + staticEndpoints: + description: |- + Configuration for 'static endpoints' on proxies in order to facilitate + direct connections from other devices on the tailnet. + See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. + properties: + nodePort: + description: The configuration for static endpoints using NodePort Services. + properties: + ports: + description: |- + The port ranges from which the operator will select NodePorts for the Services. + You must ensure that firewall rules allow UDP ingress traffic for these ports + to the node's external IPs. + The ports must be in the range of service node ports for the cluster (default `30000-32767`). + See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. + items: + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be used. This field cannot be defined if the port field is not defined. + The endPort must be either unset, or equal or greater than port. + type: integer + port: + description: port represents a port selected to be used. This is a required field. + type: integer + required: + - port + type: object + minItems: 1 + type: array + selector: + additionalProperties: + type: string + description: |- + A selector which will be used to select the node's that will have their `ExternalIP`'s advertised + by the ProxyGroup as Static Endpoints. + type: object + required: + - ports + type: object + required: + - nodePort + type: object tailscale: description: |- TailscaleConfig contains options to configure the tailscale-specific @@ -2808,6 +2880,10 @@ spec: jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason name: Status type: string + - description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver. + jsonPath: .status.url + name: URL + type: string - description: ProxyGroup type. jsonPath: .spec.type name: Type @@ -2820,15 +2896,22 @@ spec: openAPIV3Schema: description: |- ProxyGroup defines a set of Tailscale devices that will act as proxies. - Currently only egress ProxyGroups are supported. + Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver + proxies. In addition to running a highly available set of proxies, ingress + and egress ProxyGroups also allow for serving many annotated Services from a + single set of proxies to minimise resource consumption. - Use the tailscale.com/proxy-group annotation on a Service to specify that - the egress proxy should be implemented by a ProxyGroup instead of a single - dedicated proxy. In addition to running a highly available set of proxies, - ProxyGroup also allows for serving many annotated Services from a single - set of proxies to minimise resource consumption. + For ingress and egress, use the tailscale.com/proxy-group annotation on a + Service to specify that the proxy should be implemented by a ProxyGroup + instead of a single dedicated proxy. - More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + More info: + * https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + * https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress + + For kube-apiserver, the ProxyGroup is a standalone resource. Use the + spec.kubeAPIServer field to configure options specific to the kube-apiserver + ProxyGroup type. properties: apiVersion: description: |- @@ -2859,6 +2942,30 @@ spec: must not start with a dash and must be between 1 and 62 characters long. pattern: ^[a-z0-9][a-z0-9-]{0,61}$ type: string + kubeAPIServer: + description: |- + KubeAPIServer contains configuration specific to the kube-apiserver + ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + properties: + hostname: + description: |- + Hostname is the hostname with which to expose the Kubernetes API server + proxies. Must be a valid DNS label no longer than 63 characters. If not + specified, the name of the ProxyGroup is used as the hostname. Must be + unique across the whole tailnet. + pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ + type: string + mode: + description: |- + Mode to run the API server proxy in. Supported modes are auth and noauth. + In auth mode, requests from the tailnet proxied over to the Kubernetes + API server are additionally impersonated using the sender's tailnet identity. + If not specified, defaults to auth mode. + enum: + - auth + - noauth + type: string + type: object proxyClass: description: |- ProxyClass is the name of the ProxyClass custom resource that contains @@ -2888,11 +2995,12 @@ spec: type: array type: description: |- - Type of the ProxyGroup proxies. Supported types are egress and ingress. + Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. Type is immutable once a ProxyGroup is created. enum: - egress - ingress + - kube-apiserver type: string x-kubernetes-validations: - message: ProxyGroup type is immutable @@ -2908,7 +3016,20 @@ spec: conditions: description: |- List of status conditions to indicate the status of the ProxyGroup - resources. Known condition types are `ProxyGroupReady`. + resources. Known condition types include `ProxyGroupReady` and + `ProxyGroupAvailable`. + + * `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and + all expected conditions are true. + * `ProxyGroupAvailable` indicates that at least one proxy is ready to + serve traffic. + + For ProxyGroups of type kube-apiserver, there are two additional conditions: + + * `KubeAPIServerProxyConfigured` indicates that at least one API server + proxy is configured and ready to serve traffic. + * `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is + valid. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: @@ -2976,6 +3097,11 @@ spec: If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the node. type: string + staticEndpoints: + description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. + items: + type: string + type: array tailnetIPs: description: |- TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) @@ -2990,6 +3116,11 @@ spec: x-kubernetes-list-map-keys: - hostname x-kubernetes-list-type: map + url: + description: |- + URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if + any. Only applies to ProxyGroups of type kube-apiserver. + type: string type: object required: - spec @@ -4791,6 +4922,14 @@ kind: ClusterRole metadata: name: tailscale-operator rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -5066,6 +5205,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: OPERATOR_LOGIN_SERVER + value: null + - name: OPERATOR_INGRESS_CLASS_NAME + value: tailscale - name: CLIENT_ID_FILE value: /oauth/client_id - name: CLIENT_SECRET_FILE diff --git a/cmd/k8s-operator/egress-eps_test.go b/cmd/k8s-operator/egress-eps_test.go index bd81071cb5e4f..bd80112aeb8a2 100644 --- a/cmd/k8s-operator/egress-eps_test.go +++ b/cmd/k8s-operator/egress-eps_test.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" + "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/util/mak" ) @@ -200,7 +201,7 @@ func podAndSecretForProxyGroup(pg string) (*corev1.Pod, *corev1.Secret) { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-0", pg), Namespace: "operator-ns", - Labels: pgSecretLabels(pg, "state"), + Labels: pgSecretLabels(pg, kubetypes.LabelSecretTypeState), }, } return p, s diff --git a/cmd/k8s-operator/egress-services-readiness.go b/cmd/k8s-operator/egress-services-readiness.go index 5e95a52790395..ecf99b63cda44 100644 --- a/cmd/k8s-operator/egress-services-readiness.go +++ b/cmd/k8s-operator/egress-services-readiness.go @@ -102,7 +102,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re msg = err.Error() return res, err } - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { l.Infof("ProxyGroup for Service is not ready, waiting...") reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady st = metav1.ConditionFalse diff --git a/cmd/k8s-operator/egress-services-readiness_test.go b/cmd/k8s-operator/egress-services-readiness_test.go index ce947329ddfb8..f80759aef927b 100644 --- a/cmd/k8s-operator/egress-services-readiness_test.go +++ b/cmd/k8s-operator/egress-services-readiness_test.go @@ -137,7 +137,7 @@ func setReady(svc *corev1.Service, cl tstime.Clock, l *zap.SugaredLogger, replic } func setPGReady(pg *tsapi.ProxyGroup, cl tstime.Clock, l *zap.SugaredLogger) { - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, "foo", "foo", pg.Generation, cl, l) } func setEndpointForReplica(pg *tsapi.ProxyGroup, ordinal int32, eps *discoveryv1.EndpointSlice) { diff --git a/cmd/k8s-operator/egress-services.go b/cmd/k8s-operator/egress-services.go index 7103205ac2c3f..ca6562071eba7 100644 --- a/cmd/k8s-operator/egress-services.go +++ b/cmd/k8s-operator/egress-services.go @@ -531,7 +531,7 @@ func (esr *egressSvcsReconciler) validateClusterResources(ctx context.Context, s tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) return false, nil } - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { tsoperator.SetServiceCondition(svc, tsapi.EgressSvcValid, metav1.ConditionUnknown, reasonProxyGroupNotReady, reasonProxyGroupNotReady, esr.clock, l) tsoperator.RemoveServiceCondition(svc, tsapi.EgressSvcConfigured) } diff --git a/cmd/k8s-operator/ingress-for-pg.go b/cmd/k8s-operator/ingress-for-pg.go index 5f9c549407717..3afeb528f7f8f 100644 --- a/cmd/k8s-operator/ingress-for-pg.go +++ b/cmd/k8s-operator/ingress-for-pg.go @@ -68,14 +68,15 @@ var gaugePGIngressResources = clientmetric.NewGauge(kubetypes.MetricIngressPGRes type HAIngressReconciler struct { client.Client - recorder record.EventRecorder - logger *zap.SugaredLogger - tsClient tsClient - tsnetServer tsnetServer - tsNamespace string - lc localClient - defaultTags []string - operatorID string // stableID of the operator's Tailscale device + recorder record.EventRecorder + logger *zap.SugaredLogger + tsClient tsClient + tsnetServer tsnetServer + tsNamespace string + lc localClient + defaultTags []string + operatorID string // stableID of the operator's Tailscale device + ingressClassName string mu sync.Mutex // protects following // managedIngresses is a set of all ingress resources that we're currently @@ -162,7 +163,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin return false, fmt.Errorf("error getting Tailscale Service %q: %w", hostname, err) } - if err := validateIngressClass(ctx, r.Client); err != nil { + if err := validateIngressClass(ctx, r.Client, r.ingressClassName); err != nil { logger.Infof("error validating tailscale IngressClass: %v.", err) return false, nil } @@ -182,7 +183,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) } - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { logger.Infof("ProxyGroup is not (yet) ready") return false, nil } @@ -238,7 +239,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin // This checks and ensures that Tailscale Service's owner references are updated // for this Ingress and errors if that is not possible (i.e. because it // appears that the Tailscale Service has been created by a non-operator actor). - updatedAnnotations, err := r.ownerAnnotations(existingTSSvc) + updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc) if err != nil { const instr = "To proceed, you can either manually delete the existing Tailscale Service or choose a different MagicDNS name at `.spec.tls.hosts[0] in the Ingress definition" msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) @@ -247,12 +248,12 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin return false, nil } // 3. Ensure that TLS Secret and RBAC exists - tcd, err := r.tailnetCertDomain(ctx) + tcd, err := tailnetCertDomain(ctx, r.lc) if err != nil { return false, fmt.Errorf("error determining DNS name base: %w", err) } dnsName := hostname + "." + tcd - if err := r.ensureCertResources(ctx, pgName, dnsName, ing); err != nil { + if err := r.ensureCertResources(ctx, pg, dnsName, ing); err != nil { return false, fmt.Errorf("error ensuring cert resources: %w", err) } @@ -357,7 +358,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin } // 6. Update Ingress status if ProxyGroup Pods are ready. - count, err := r.numberPodsAdvertising(ctx, pg.Name, serviceName) + count, err := numberPodsAdvertising(ctx, r.Client, r.tsNamespace, pg.Name, serviceName) if err != nil { return false, fmt.Errorf("failed to check if any Pods are configured: %w", err) } @@ -369,7 +370,7 @@ func (r *HAIngressReconciler) maybeProvision(ctx context.Context, hostname strin ing.Status.LoadBalancer.Ingress = nil default: var ports []networkingv1.IngressPortStatus - hasCerts, err := r.hasCerts(ctx, serviceName) + hasCerts, err := hasCerts(ctx, r.Client, r.lc, r.tsNamespace, serviceName) if err != nil { return false, fmt.Errorf("error checking TLS credentials provisioned for Ingress: %w", err) } @@ -480,7 +481,7 @@ func (r *HAIngressReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG delete(cfg.Services, tsSvcName) serveConfigChanged = true } - if err := r.cleanupCertResources(ctx, proxyGroupName, tsSvcName); err != nil { + if err := cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, proxyGroupName, tsSvcName); err != nil { return false, fmt.Errorf("failed to clean up cert resources: %w", err) } } @@ -556,7 +557,7 @@ func (r *HAIngressReconciler) maybeCleanup(ctx context.Context, hostname string, } // 3. Clean up any cluster resources - if err := r.cleanupCertResources(ctx, pg, serviceName); err != nil { + if err := cleanupCertResources(ctx, r.Client, r.lc, r.tsNamespace, pg, serviceName); err != nil { return false, fmt.Errorf("failed to clean up cert resources: %w", err) } @@ -633,8 +634,8 @@ type localClient interface { } // tailnetCertDomain returns the base domain (TCD) of the current tailnet. -func (r *HAIngressReconciler) tailnetCertDomain(ctx context.Context) (string, error) { - st, err := r.lc.StatusWithoutPeers(ctx) +func tailnetCertDomain(ctx context.Context, lc localClient) (string, error) { + st, err := lc.StatusWithoutPeers(ctx) if err != nil { return "", fmt.Errorf("error getting tailscale status: %w", err) } @@ -645,7 +646,7 @@ func (r *HAIngressReconciler) tailnetCertDomain(ctx context.Context) (string, er func (r *HAIngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool { isTSIngress := ing != nil && ing.Spec.IngressClassName != nil && - *ing.Spec.IngressClassName == tailscaleIngressClassName + *ing.Spec.IngressClassName == r.ingressClassName pgAnnot := ing.Annotations[AnnotationProxyGroup] return isTSIngress && pgAnnot != "" } @@ -660,18 +661,13 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki var errs []error // Validate tags if present - if tstr, ok := ing.Annotations[AnnotationTags]; ok { - tags := strings.Split(tstr, ",") - for _, tag := range tags { - tag = strings.TrimSpace(tag) - if err := tailcfg.CheckTag(tag); err != nil { - errs = append(errs, fmt.Errorf("tailscale.com/tags annotation contains invalid tag %q: %w", tag, err)) - } - } + violations := tagViolations(ing) + if len(violations) > 0 { + errs = append(errs, fmt.Errorf("Ingress contains invalid tags: %v", strings.Join(violations, ","))) } // Validate TLS configuration - if ing.Spec.TLS != nil && len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) { + if len(ing.Spec.TLS) > 0 && (len(ing.Spec.TLS) > 1 || len(ing.Spec.TLS[0].Hosts) > 1) { errs = append(errs, fmt.Errorf("Ingress contains invalid TLS block %v: only a single TLS entry with a single host is allowed", ing.Spec.TLS)) } @@ -688,7 +684,7 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki } // Validate ProxyGroup readiness - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { errs = append(errs, fmt.Errorf("ProxyGroup %q is not ready", pg.Name)) } @@ -699,8 +695,8 @@ func (r *HAIngressReconciler) validateIngress(ctx context.Context, ing *networki return errors.Join(errs...) } for _, i := range ingList.Items { - if r.shouldExpose(&i) && hostnameForIngress(&i) == hostname && i.Name != ing.Name { - errs = append(errs, fmt.Errorf("found duplicate Ingress %q for hostname %q - multiple Ingresses for the same hostname in the same cluster are not allowed", i.Name, hostname)) + if r.shouldExpose(&i) && hostnameForIngress(&i) == hostname && i.UID != ing.UID { + errs = append(errs, fmt.Errorf("found duplicate Ingress %q for hostname %q - multiple Ingresses for the same hostname in the same cluster are not allowed", client.ObjectKeyFromObject(&i), hostname)) } } return errors.Join(errs...) @@ -765,7 +761,7 @@ const ( func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Context, pgName string, serviceName tailcfg.ServiceName, mode serviceAdvertisementMode, logger *zap.SugaredLogger) (err error) { // Get all config Secrets for this ProxyGroup. secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil { + if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig))); err != nil { return fmt.Errorf("failed to list config Secrets: %w", err) } @@ -777,7 +773,7 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // The only exception is Ingresses with an HTTP endpoint enabled - if an // Ingress has an HTTP endpoint enabled, it will be advertised even if the // TLS cert is not yet provisioned. - hasCert, err := a.hasCerts(ctx, serviceName) + hasCert, err := hasCerts(ctx, a.Client, a.lc, a.tsNamespace, serviceName) if err != nil { return fmt.Errorf("error checking TLS credentials provisioned for service %q: %w", serviceName, err) } @@ -826,10 +822,10 @@ func (a *HAIngressReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con return nil } -func (a *HAIngressReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) { +func numberPodsAdvertising(ctx context.Context, cl client.Client, tsNamespace, pgName string, serviceName tailcfg.ServiceName) (int, error) { // Get all state Secrets for this ProxyGroup. secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil { + if err := cl.List(ctx, secrets, client.InNamespace(tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeState))); err != nil { return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err) } @@ -863,7 +859,14 @@ type ownerAnnotationValue struct { // Kubernetes operator instance. type OwnerRef struct { // OperatorID is the stable ID of the operator's Tailscale device. - OperatorID string `json:"operatorID,omitempty"` + OperatorID string `json:"operatorID,omitempty"` + Resource *Resource `json:"resource,omitempty"` // optional, used to identify the ProxyGroup that owns this Tailscale Service. +} + +type Resource struct { + Kind string `json:"kind,omitempty"` // "ProxyGroup" + Name string `json:"name,omitempty"` // Name of the ProxyGroup that owns this Tailscale Service. Informational only. + UID string `json:"uid,omitempty"` // UID of the ProxyGroup that owns this Tailscale Service. } // ownerAnnotations returns the updated annotations required to ensure this @@ -871,9 +874,9 @@ type OwnerRef struct { // nil, but does not contain an owner reference we return an error as this likely means // that the Service was created by somthing other than a Tailscale // Kubernetes operator. -func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) { +func ownerAnnotations(operatorID string, svc *tailscale.VIPService) (map[string]string, error) { ref := OwnerRef{ - OperatorID: r.operatorID, + OperatorID: operatorID, } if svc == nil { c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} @@ -895,6 +898,9 @@ func (r *HAIngressReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[s if slices.Contains(o.OwnerRefs, ref) { // up to date return svc.Annotations, nil } + if o.OwnerRefs[0].Resource != nil { + return nil, fmt.Errorf("Tailscale Service %s is owned by another resource: %#v; cannot be reused for an Ingress", svc.Name, o.OwnerRefs[0].Resource) + } o.OwnerRefs = append(o.OwnerRefs, ref) json, err := json.Marshal(o) if err != nil { @@ -936,37 +942,50 @@ func ownersAreSetAndEqual(a, b *tailscale.VIPService) bool { // (domain) is a valid Kubernetes resource name. // https://github.com/tailscale/tailscale/blob/8b1e7f646ee4730ad06c9b70c13e7861b964949b/util/dnsname/dnsname.go#L99 // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names -func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pgName, domain string, ing *networkingv1.Ingress) error { - secret := certSecret(pgName, r.tsNamespace, domain, ing) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, nil); err != nil { +func (r *HAIngressReconciler) ensureCertResources(ctx context.Context, pg *tsapi.ProxyGroup, domain string, ing *networkingv1.Ingress) error { + secret := certSecret(pg.Name, r.tsNamespace, domain, ing) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, secret, func(s *corev1.Secret) { + // Labels might have changed if the Ingress has been updated to use a + // different ProxyGroup. + s.Labels = secret.Labels + }); err != nil { return fmt.Errorf("failed to create or update Secret %s: %w", secret.Name, err) } - role := certSecretRole(pgName, r.tsNamespace, domain) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, nil); err != nil { + role := certSecretRole(pg.Name, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { + // Labels might have changed if the Ingress has been updated to use a + // different ProxyGroup. + r.Labels = role.Labels + }); err != nil { return fmt.Errorf("failed to create or update Role %s: %w", role.Name, err) } - rb := certSecretRoleBinding(pgName, r.tsNamespace, domain) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rb, nil); err != nil { - return fmt.Errorf("failed to create or update RoleBinding %s: %w", rb.Name, err) + rolebinding := certSecretRoleBinding(pg, r.tsNamespace, domain) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, rolebinding, func(rb *rbacv1.RoleBinding) { + // Labels and subjects might have changed if the Ingress has been updated to use a + // different ProxyGroup. + rb.Labels = rolebinding.Labels + rb.Subjects = rolebinding.Subjects + }); err != nil { + return fmt.Errorf("failed to create or update RoleBinding %s: %w", rolebinding.Name, err) } return nil } // cleanupCertResources ensures that the TLS Secret and associated RBAC // resources that allow proxies to read/write to the Secret are deleted. -func (r *HAIngressReconciler) cleanupCertResources(ctx context.Context, pgName string, name tailcfg.ServiceName) error { - domainName, err := r.dnsNameForService(ctx, tailcfg.ServiceName(name)) +func cleanupCertResources(ctx context.Context, cl client.Client, lc localClient, tsNamespace, pgName string, serviceName tailcfg.ServiceName) error { + domainName, err := dnsNameForService(ctx, lc, serviceName) if err != nil { - return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", name, err) + return fmt.Errorf("error getting DNS name for Tailscale Service %s: %w", serviceName, err) } labels := certResourceLabels(pgName, domainName) - if err := r.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + if err := cl.DeleteAllOf(ctx, &rbacv1.RoleBinding{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil { return fmt.Errorf("error deleting RoleBinding for domain name %s: %w", domainName, err) } - if err := r.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + if err := cl.DeleteAllOf(ctx, &rbacv1.Role{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil { return fmt.Errorf("error deleting Role for domain name %s: %w", domainName, err) } - if err := r.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + if err := cl.DeleteAllOf(ctx, &corev1.Secret{}, client.InNamespace(tsNamespace), client.MatchingLabels(labels)); err != nil { return fmt.Errorf("error deleting Secret for domain name %s: %w", domainName, err) } return nil @@ -1009,17 +1028,17 @@ func certSecretRole(pgName, namespace, domain string) *rbacv1.Role { // certSecretRoleBinding creates a RoleBinding for Role that will allow proxies // to manage the TLS Secret for the given domain. Domain must be a valid // Kubernetes resource name. -func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding { +func certSecretRoleBinding(pg *tsapi.ProxyGroup, namespace, domain string) *rbacv1.RoleBinding { return &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: domain, Namespace: namespace, - Labels: certResourceLabels(pgName, domain), + Labels: certResourceLabels(pg.Name, domain), }, Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: pgName, + Name: pgServiceAccountName(pg), Namespace: namespace, }, }, @@ -1032,14 +1051,17 @@ func certSecretRoleBinding(pgName, namespace, domain string) *rbacv1.RoleBinding // certSecret creates a Secret that will store the TLS certificate and private // key for the given domain. Domain must be a valid Kubernetes resource name. -func certSecret(pgName, namespace, domain string, ing *networkingv1.Ingress) *corev1.Secret { +func certSecret(pgName, namespace, domain string, parent client.Object) *corev1.Secret { labels := certResourceLabels(pgName, domain) - labels[kubetypes.LabelSecretType] = "certs" + labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeCerts // Labels that let us identify the Ingress resource lets us reconcile // the Ingress when the TLS Secret is updated (for example, when TLS // certs have been provisioned). - labels[LabelParentName] = ing.Name - labels[LabelParentNamespace] = ing.Namespace + labels[LabelParentType] = strings.ToLower(parent.GetObjectKind().GroupVersionKind().Kind) + labels[LabelParentName] = parent.GetName() + if ns := parent.GetNamespace(); ns != "" { + labels[LabelParentNamespace] = ns + } return &corev1.Secret{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", @@ -1067,9 +1089,9 @@ func certResourceLabels(pgName, domain string) map[string]string { } // dnsNameForService returns the DNS name for the given Tailscale Service's name. -func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) { +func dnsNameForService(ctx context.Context, lc localClient, svc tailcfg.ServiceName) (string, error) { s := svc.WithoutPrefix() - tcd, err := r.tailnetCertDomain(ctx) + tcd, err := tailnetCertDomain(ctx, lc) if err != nil { return "", fmt.Errorf("error determining DNS name base: %w", err) } @@ -1077,14 +1099,14 @@ func (r *HAIngressReconciler) dnsNameForService(ctx context.Context, svc tailcfg } // hasCerts checks if the TLS Secret for the given service has non-zero cert and key data. -func (r *HAIngressReconciler) hasCerts(ctx context.Context, svc tailcfg.ServiceName) (bool, error) { - domain, err := r.dnsNameForService(ctx, svc) +func hasCerts(ctx context.Context, cl client.Client, lc localClient, ns string, svc tailcfg.ServiceName) (bool, error) { + domain, err := dnsNameForService(ctx, lc, svc) if err != nil { return false, fmt.Errorf("failed to get DNS name for service: %w", err) } secret := &corev1.Secret{} - err = r.Get(ctx, client.ObjectKey{ - Namespace: r.tsNamespace, + err = cl.Get(ctx, client.ObjectKey{ + Namespace: ns, Name: domain, }, secret) if err != nil { @@ -1113,3 +1135,22 @@ func isErrorTailscaleServiceNotFound(err error) bool { ok := errors.As(err, &errResp) return ok && errResp.Status == http.StatusNotFound } + +func tagViolations(obj client.Object) []string { + var violations []string + if obj == nil { + return nil + } + tags, ok := obj.GetAnnotations()[AnnotationTags] + if !ok { + return nil + } + + for _, tag := range strings.Split(tags, ",") { + tag = strings.TrimSpace(tag) + if err := tailcfg.CheckTag(tag); err != nil { + violations = append(violations, fmt.Sprintf("invalid tag %q: %v", tag, err)) + } + } + return violations +} diff --git a/cmd/k8s-operator/ingress-for-pg_test.go b/cmd/k8s-operator/ingress-for-pg_test.go index f155963030b4f..77e5ecb37a677 100644 --- a/cmd/k8s-operator/ingress-for-pg_test.go +++ b/cmd/k8s-operator/ingress-for-pg_test.go @@ -12,8 +12,10 @@ import ( "maps" "reflect" "slices" + "strings" "testing" + "github.com/google/go-cmp/cmp" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -69,12 +71,17 @@ func TestIngressPGReconciler(t *testing.T) { expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:my-svc", false) verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) - verifyTailscaledConfig(t, fc, []string{"svc:my-svc"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc"}) // Verify that Role and RoleBinding have been created for the first Ingress. // Do not verify the cert Secret as that was already verified implicitly above. + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg", + }, + } expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-svc.ts.net")) - expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-svc.ts.net")) + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-svc.ts.net")) mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) { ing.Annotations["tailscale.com/tags"] = "tag:custom,tag:test" @@ -132,16 +139,16 @@ func TestIngressPGReconciler(t *testing.T) { verifyServeConfig(t, fc, "svc:my-other-svc", false) verifyTailscaleService(t, ft, "svc:my-other-svc", []string{"tcp:443"}) - // Verify that Role and RoleBinding have been created for the first Ingress. + // Verify that Role and RoleBinding have been created for the second Ingress. // Do not verify the cert Secret as that was already verified implicitly above. expectEqual(t, fc, certSecretRole("test-pg", "operator-ns", "my-other-svc.ts.net")) - expectEqual(t, fc, certSecretRoleBinding("test-pg", "operator-ns", "my-other-svc.ts.net")) + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-other-svc.ts.net")) // Verify first Ingress is still working verifyServeConfig(t, fc, "svc:my-svc", false) verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) - verifyTailscaledConfig(t, fc, []string{"svc:my-svc", "svc:my-other-svc"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc", "svc:my-other-svc"}) // Delete second Ingress if err := fc.Delete(context.Background(), ing2); err != nil { @@ -172,11 +179,25 @@ func TestIngressPGReconciler(t *testing.T) { t.Error("second Ingress service config was not cleaned up") } - verifyTailscaledConfig(t, fc, []string{"svc:my-svc"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc"}) expectMissing[corev1.Secret](t, fc, "operator-ns", "my-other-svc.ts.net") expectMissing[rbacv1.Role](t, fc, "operator-ns", "my-other-svc.ts.net") expectMissing[rbacv1.RoleBinding](t, fc, "operator-ns", "my-other-svc.ts.net") + // Test Ingress ProxyGroup change + createPGResources(t, fc, "test-pg-second") + mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) { + ing.Annotations["tailscale.com/proxy-group"] = "test-pg-second" + }) + expectReconciled(t, ingPGR, "default", "test-ingress") + expectEqual(t, fc, certSecretRole("test-pg-second", "operator-ns", "my-svc.ts.net")) + pg = &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pg-second", + }, + } + expectEqual(t, fc, certSecretRoleBinding(pg, "operator-ns", "my-svc.ts.net")) + // Delete the first Ingress and verify cleanup if err := fc.Delete(context.Background(), ing); err != nil { t.Fatalf("deleting Ingress: %v", err) @@ -187,7 +208,7 @@ func TestIngressPGReconciler(t *testing.T) { // Verify the ConfigMap was cleaned up cm = &corev1.ConfigMap{} if err := fc.Get(context.Background(), types.NamespacedName{ - Name: "test-pg-ingress-config", + Name: "test-pg-second-ingress-config", Namespace: "operator-ns", }, cm); err != nil { t.Fatalf("getting ConfigMap: %v", err) @@ -201,7 +222,7 @@ func TestIngressPGReconciler(t *testing.T) { if len(cfg.Services) > 0 { t.Error("serve config not cleaned up") } - verifyTailscaledConfig(t, fc, nil) + verifyTailscaledConfig(t, fc, "test-pg-second", nil) // Add verification that cert resources were cleaned up expectMissing[corev1.Secret](t, fc, "operator-ns", "my-svc.ts.net") @@ -245,7 +266,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) { expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:my-svc", false) verifyTailscaleService(t, ft, "svc:my-svc", []string{"tcp:443"}) - verifyTailscaledConfig(t, fc, []string{"svc:my-svc"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:my-svc"}) // Update the Ingress hostname and make sure the original Tailscale Service is deleted. mustUpdate(t, fc, "default", "test-ingress", func(ing *networkingv1.Ingress) { @@ -256,7 +277,7 @@ func TestIngressPGReconciler_UpdateIngressHostname(t *testing.T) { expectReconciled(t, ingPGR, "default", "test-ingress") verifyServeConfig(t, fc, "svc:updated-svc", false) verifyTailscaleService(t, ft, "svc:updated-svc", []string{"tcp:443"}) - verifyTailscaledConfig(t, fc, []string{"svc:updated-svc"}) + verifyTailscaledConfig(t, fc, "test-pg", []string{"svc:updated-svc"}) _, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName("svc:my-svc")) if err == nil { @@ -272,6 +293,7 @@ func TestValidateIngress(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test-ingress", Namespace: "default", + UID: types.UID("1234-UID"), Annotations: map[string]string{ AnnotationProxyGroup: "test-pg", }, @@ -295,7 +317,7 @@ func TestValidateIngress(t *testing.T) { Status: tsapi.ProxyGroupStatus{ Conditions: []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, @@ -339,7 +361,7 @@ func TestValidateIngress(t *testing.T) { }, }, pg: readyProxyGroup, - wantErr: "tailscale.com/tags annotation contains invalid tag \"tag:invalid!\": tag names can only contain numbers, letters, or dashes", + wantErr: "Ingress contains invalid tags: invalid tag \"tag:invalid!\": tag names can only contain numbers, letters, or dashes", }, { name: "multiple_TLS_entries", @@ -389,7 +411,7 @@ func TestValidateIngress(t *testing.T) { Status: tsapi.ProxyGroupStatus{ Conditions: []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionFalse, ObservedGeneration: 1, }, @@ -417,7 +439,7 @@ func TestValidateIngress(t *testing.T) { }, }, }}, - wantErr: `found duplicate Ingress "existing-ingress" for hostname "test" - multiple Ingresses for the same hostname in the same cluster are not allowed`, + wantErr: `found duplicate Ingress "default/existing-ingress" for hostname "test" - multiple Ingresses for the same hostname in the same cluster are not allowed`, }, } @@ -428,7 +450,12 @@ func TestValidateIngress(t *testing.T) { WithObjects(tt.ing). WithLists(&networkingv1.IngressList{Items: tt.existingIngs}). Build() + r := &HAIngressReconciler{Client: fc} + if tt.ing.Spec.IngressClassName != nil { + r.ingressClassName = *tt.ing.Spec.IngressClassName + } + err := r.validateIngress(context.Background(), tt.ing, tt.pg) if (err == nil && tt.wantErr != "") || (err != nil && err.Error() != tt.wantErr) { t.Errorf("validateIngress() error = %v, wantErr %v", err, tt.wantErr) @@ -498,7 +525,7 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test-pg-0", Namespace: "operator-ns", - Labels: pgSecretLabels("test-pg", "state"), + Labels: pgSecretLabels("test-pg", kubetypes.LabelSecretTypeState), }, Data: map[string][]byte{ "_current-profile": []byte("profile-foo"), @@ -549,6 +576,172 @@ func TestIngressPGReconciler_HTTPEndpoint(t *testing.T) { } } +func TestIngressPGReconciler_MultiCluster(t *testing.T) { + ingPGR, fc, ft := setupIngressTest(t) + ingPGR.operatorID = "operator-1" + + // Create initial Ingress + ing := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + }, + }, + Spec: networkingv1.IngressSpec{ + IngressClassName: ptr.To("tailscale"), + TLS: []networkingv1.IngressTLS{ + {Hosts: []string{"my-svc"}}, + }, + }, + } + mustCreate(t, fc, ing) + + // Simulate existing Tailscale Service from another cluster + existingVIPSvc := &tailscale.VIPService{ + Name: "svc:my-svc", + Annotations: map[string]string{ + ownerAnnotation: `{"ownerrefs":[{"operatorID":"operator-2"}]}`, + }, + } + ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{ + "svc:my-svc": existingVIPSvc, + } + + // Verify reconciliation adds our operator reference + expectReconciled(t, ingPGR, "default", "test-ingress") + + tsSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc") + if err != nil { + t.Fatalf("getting Tailscale Service: %v", err) + } + if tsSvc == nil { + t.Fatal("Tailscale Service not found") + } + + o, err := parseOwnerAnnotation(tsSvc) + if err != nil { + t.Fatalf("parsing owner annotation: %v", err) + } + + wantOwnerRefs := []OwnerRef{ + {OperatorID: "operator-2"}, + {OperatorID: "operator-1"}, + } + if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) { + t.Errorf("incorrect owner refs\ngot: %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs) + } + + // Delete the Ingress and verify Tailscale Service still exists with one owner ref + if err := fc.Delete(context.Background(), ing); err != nil { + t.Fatalf("deleting Ingress: %v", err) + } + expectRequeue(t, ingPGR, "default", "test-ingress") + + tsSvc, err = ft.GetVIPService(context.Background(), "svc:my-svc") + if err != nil { + t.Fatalf("getting Tailscale Service after deletion: %v", err) + } + if tsSvc == nil { + t.Fatal("Tailscale Service was incorrectly deleted") + } + + o, err = parseOwnerAnnotation(tsSvc) + if err != nil { + t.Fatalf("parsing owner annotation: %v", err) + } + + wantOwnerRefs = []OwnerRef{ + {OperatorID: "operator-2"}, + } + if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) { + t.Errorf("incorrect owner refs after deletion\ngot: %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs) + } +} + +func TestOwnerAnnotations(t *testing.T) { + singleSelfOwner := map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id"}]}`, + } + + for name, tc := range map[string]struct { + svc *tailscale.VIPService + wantAnnotations map[string]string + wantErr string + }{ + "no_svc": { + svc: nil, + wantAnnotations: singleSelfOwner, + }, + "empty_svc": { + svc: &tailscale.VIPService{}, + wantErr: "likely a resource created by something other than the Tailscale Kubernetes operator", + }, + "already_owner": { + svc: &tailscale.VIPService{ + Annotations: singleSelfOwner, + }, + wantAnnotations: singleSelfOwner, + }, + "add_owner": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"}]}`, + }, + }, + wantAnnotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"operator-2"},{"operatorID":"self-id"}]}`, + }, + }, + "owned_by_proxygroup": { + svc: &tailscale.VIPService{ + Annotations: map[string]string{ + ownerAnnotation: `{"ownerRefs":[{"operatorID":"self-id","resource":{"kind":"ProxyGroup","name":"test-pg","uid":"1234-UID"}}]}`, + }, + }, + wantErr: "owned by another resource", + }, + } { + t.Run(name, func(t *testing.T) { + got, err := ownerAnnotations("self-id", tc.svc) + if tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("ownerAnnotations() error = %v, wantErr %v", err, tc.wantErr) + } + if diff := cmp.Diff(tc.wantAnnotations, got); diff != "" { + t.Errorf("ownerAnnotations() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: domain, + Namespace: "operator-ns", + Labels: map[string]string{ + kubetypes.LabelManaged: "true", + labelProxyGroup: pgName, + labelDomain: domain, + kubetypes.LabelSecretType: kubetypes.LabelSecretTypeCerts, + }, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSCertKey: []byte("fake-cert"), + corev1.TLSPrivateKeyKey: []byte("fake-key"), + }, + } + + _, err := createOrUpdate(ctx, c, "operator-ns", secret, func(s *corev1.Secret) { + s.Data = secret.Data + }) + return err +} + func verifyTailscaleService(t *testing.T, ft *fakeTSClient, serviceName string, wantPorts []string) { t.Helper() tsSvc, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(serviceName)) @@ -617,7 +810,7 @@ func verifyServeConfig(t *testing.T, fc client.Client, serviceName string, wantH } } -func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []string) { +func verifyTailscaledConfig(t *testing.T, fc client.Client, pgName string, expectedServices []string) { t.Helper() var expected string if expectedServices != nil && len(expectedServices) > 0 { @@ -629,66 +822,57 @@ func verifyTailscaledConfig(t *testing.T, fc client.Client, expectedServices []s } expectEqual(t, fc, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: pgConfigSecretName("test-pg", 0), + Name: pgConfigSecretName(pgName, 0), Namespace: "operator-ns", - Labels: pgSecretLabels("test-pg", "config"), + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig), }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)), + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(fmt.Sprintf(`{"Version":""%s}`, expected)), }, }) } -func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) { - tsIngressClass := &networkingv1.IngressClass{ - ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, - Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}, - } - +func createPGResources(t *testing.T, fc client.Client, pgName string) { + t.Helper() // Pre-create the ProxyGroup pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-pg", + Name: pgName, Generation: 1, }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, }, } + mustCreate(t, fc, pg) // Pre-create the ConfigMap for the ProxyGroup pgConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-pg-ingress-config", + Name: fmt.Sprintf("%s-ingress-config", pgName), Namespace: "operator-ns", }, BinaryData: map[string][]byte{ "serve-config.json": []byte(`{"Services":{}}`), }, } + mustCreate(t, fc, pgConfigMap) // Pre-create a config Secret for the ProxyGroup pgCfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: pgConfigSecretName("test-pg", 0), + Name: pgConfigSecretName(pgName, 0), Namespace: "operator-ns", - Labels: pgSecretLabels("test-pg", "config"), + Labels: pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig), }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): []byte("{}"), + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte("{}"), }, } - - fc := fake.NewClientBuilder(). - WithScheme(tsapi.GlobalScheme). - WithObjects(pg, pgCfgSecret, pgConfigMap, tsIngressClass). - WithStatusSubresource(pg). - Build() - - // Set ProxyGroup status to ready + mustCreate(t, fc, pgCfgSecret) pg.Status.Conditions = []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, @@ -696,6 +880,22 @@ func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeT if err := fc.Status().Update(context.Background(), pg); err != nil { t.Fatal(err) } +} + +func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeTSClient) { + tsIngressClass := &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, + Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}, + } + + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(tsIngressClass). + WithStatusSubresource(&tsapi.ProxyGroup{}). + Build() + + createPGResources(t, fc, "test-pg") + fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} ft := &fakeTSClient{} @@ -713,126 +913,16 @@ func setupIngressTest(t *testing.T) (*HAIngressReconciler, client.Client, *fakeT } ingPGR := &HAIngressReconciler{ - Client: fc, - tsClient: ft, - defaultTags: []string{"tag:k8s"}, - tsNamespace: "operator-ns", - tsnetServer: fakeTsnetServer, - logger: zl.Sugar(), - recorder: record.NewFakeRecorder(10), - lc: lc, + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + tsNamespace: "operator-ns", + tsnetServer: fakeTsnetServer, + logger: zl.Sugar(), + recorder: record.NewFakeRecorder(10), + lc: lc, + ingressClassName: tsIngressClass.Name, } return ingPGR, fc, ft } - -func TestIngressPGReconciler_MultiCluster(t *testing.T) { - ingPGR, fc, ft := setupIngressTest(t) - ingPGR.operatorID = "operator-1" - - // Create initial Ingress - ing := &networkingv1.Ingress{ - TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress", - Namespace: "default", - UID: types.UID("1234-UID"), - Annotations: map[string]string{ - "tailscale.com/proxy-group": "test-pg", - }, - }, - Spec: networkingv1.IngressSpec{ - IngressClassName: ptr.To("tailscale"), - TLS: []networkingv1.IngressTLS{ - {Hosts: []string{"my-svc"}}, - }, - }, - } - mustCreate(t, fc, ing) - - // Simulate existing Tailscale Service from another cluster - existingVIPSvc := &tailscale.VIPService{ - Name: "svc:my-svc", - Annotations: map[string]string{ - ownerAnnotation: `{"ownerrefs":[{"operatorID":"operator-2"}]}`, - }, - } - ft.vipServices = map[tailcfg.ServiceName]*tailscale.VIPService{ - "svc:my-svc": existingVIPSvc, - } - - // Verify reconciliation adds our operator reference - expectReconciled(t, ingPGR, "default", "test-ingress") - - tsSvc, err := ft.GetVIPService(context.Background(), "svc:my-svc") - if err != nil { - t.Fatalf("getting Tailscale Service: %v", err) - } - if tsSvc == nil { - t.Fatal("Tailscale Service not found") - } - - o, err := parseOwnerAnnotation(tsSvc) - if err != nil { - t.Fatalf("parsing owner annotation: %v", err) - } - - wantOwnerRefs := []OwnerRef{ - {OperatorID: "operator-2"}, - {OperatorID: "operator-1"}, - } - if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) { - t.Errorf("incorrect owner refs\ngot: %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs) - } - - // Delete the Ingress and verify Tailscale Service still exists with one owner ref - if err := fc.Delete(context.Background(), ing); err != nil { - t.Fatalf("deleting Ingress: %v", err) - } - expectRequeue(t, ingPGR, "default", "test-ingress") - - tsSvc, err = ft.GetVIPService(context.Background(), "svc:my-svc") - if err != nil { - t.Fatalf("getting Tailscale Service after deletion: %v", err) - } - if tsSvc == nil { - t.Fatal("Tailscale Service was incorrectly deleted") - } - - o, err = parseOwnerAnnotation(tsSvc) - if err != nil { - t.Fatalf("parsing owner annotation: %v", err) - } - - wantOwnerRefs = []OwnerRef{ - {OperatorID: "operator-2"}, - } - if !reflect.DeepEqual(o.OwnerRefs, wantOwnerRefs) { - t.Errorf("incorrect owner refs after deletion\ngot: %+v\nwant: %+v", o.OwnerRefs, wantOwnerRefs) - } -} - -func populateTLSSecret(ctx context.Context, c client.Client, pgName, domain string) error { - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: domain, - Namespace: "operator-ns", - Labels: map[string]string{ - kubetypes.LabelManaged: "true", - labelProxyGroup: pgName, - labelDomain: domain, - kubetypes.LabelSecretType: "certs", - }, - }, - Type: corev1.SecretTypeTLS, - Data: map[string][]byte{ - corev1.TLSCertKey: []byte("fake-cert"), - corev1.TLSPrivateKeyKey: []byte("fake-key"), - }, - } - - _, err := createOrUpdate(ctx, c, "operator-ns", secret, func(s *corev1.Secret) { - s.Data = secret.Data - }) - return err -} diff --git a/cmd/k8s-operator/ingress.go b/cmd/k8s-operator/ingress.go index 6c50e10b2ba94..d66cf9116f14a 100644 --- a/cmd/k8s-operator/ingress.go +++ b/cmd/k8s-operator/ingress.go @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/ipn" "tailscale.com/kube/kubetypes" "tailscale.com/types/opt" @@ -31,9 +32,9 @@ import ( ) const ( - tailscaleIngressClassName = "tailscale" // ingressClass.metadata.name for tailscale IngressClass resource tailscaleIngressControllerName = "tailscale.com/ts-ingress" // ingressClass.spec.controllerName for tailscale IngressClass resource ingressClassDefaultAnnotation = "ingressclass.kubernetes.io/is-default-class" // we do not support this https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class + indexIngressProxyClass = ".metadata.annotations.ingress-proxy-class" ) type IngressReconciler struct { @@ -50,6 +51,7 @@ type IngressReconciler struct { managedIngresses set.Slice[types.UID] defaultProxyClass string + ingressClassName string } var ( @@ -130,7 +132,7 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare // This function adds a finalizer to ing, ensuring that we can handle orderly // deprovisioning later. func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.SugaredLogger, ing *networkingv1.Ingress) error { - if err := validateIngressClass(ctx, a.Client); err != nil { + if err := validateIngressClass(ctx, a.Client, a.ingressClassName); err != nil { logger.Warnf("error validating tailscale IngressClass: %v. In future this might be a terminal error.", err) } if !slices.Contains(ing.Finalizers, FinalizerName) { @@ -218,6 +220,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga ChildResourceLabels: crl, ProxyClassName: proxyClass, proxyType: proxyTypeIngressResource, + LoginServer: a.ssr.loginServer, } if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" { @@ -263,17 +266,17 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga func (a *IngressReconciler) shouldExpose(ing *networkingv1.Ingress) bool { return ing != nil && ing.Spec.IngressClassName != nil && - *ing.Spec.IngressClassName == tailscaleIngressClassName && + *ing.Spec.IngressClassName == a.ingressClassName && ing.Annotations[AnnotationProxyGroup] == "" } // validateIngressClass attempts to validate that 'tailscale' IngressClass // included in Tailscale installation manifests exists and has not been modified // to attempt to enable features that we do not support. -func validateIngressClass(ctx context.Context, cl client.Client) error { +func validateIngressClass(ctx context.Context, cl client.Client, ingressClassName string) error { ic := &networkingv1.IngressClass{ ObjectMeta: metav1.ObjectMeta{ - Name: tailscaleIngressClassName, + Name: ingressClassName, }, } if err := cl.Get(ctx, client.ObjectKeyFromObject(ic), ic); apierrors.IsNotFound(err) { diff --git a/cmd/k8s-operator/ingress_test.go b/cmd/k8s-operator/ingress_test.go index a975fec7a6329..fe4d90c785c47 100644 --- a/cmd/k8s-operator/ingress_test.go +++ b/cmd/k8s-operator/ingress_test.go @@ -36,7 +36,8 @@ func TestTailscaleIngress(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: ft, @@ -71,7 +72,7 @@ func TestTailscaleIngress(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 2. Ingress status gets updated with ingress proxy's MagicDNS name // once that becomes available. @@ -98,7 +99,7 @@ func TestTailscaleIngress(t *testing.T) { }) opts.shouldEnableForwardingClusterTrafficViaIngress = true expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 4. Resources get cleaned up when Ingress class is unset mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { @@ -120,7 +121,8 @@ func TestTailscaleIngressHostname(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: ft, @@ -162,7 +164,7 @@ func TestTailscaleIngressHostname(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 2. Ingress proxy with capability version >= 110 does not have an HTTPS endpoint set mustUpdate(t, fc, "operator-ns", opts.secretName, func(secret *corev1.Secret) { @@ -230,7 +232,8 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { Spec: tsapi.ProxyClassSpec{StatefulSet: &tsapi.StatefulSet{ Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"bar.io/foo": "some-val"}, - Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}, + }}, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -244,7 +247,8 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: ft, @@ -280,15 +284,15 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 2. Ingress is updated to specify a ProxyClass, ProxyClass is not yet // ready, so proxy resource configuration does not change. mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { - mak.Set(&ing.ObjectMeta.Labels, LabelProxyClass, "custom-metadata") + mak.Set(&ing.ObjectMeta.Labels, LabelAnnotationProxyClass, "custom-metadata") }) expectReconciled(t, ingR, "default", "test") - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 3. ProxyClass is set to Ready by proxy-class reconciler. Ingress get // reconciled and configuration from the ProxyClass is applied to the @@ -299,21 +303,22 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: pc.Generation, - }}} + }}, + } }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = pc.Name - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) // 4. tailscale.com/proxy-class label is removed from the Ingress, the // Ingress gets reconciled and the custom ProxyClass configuration is // removed from the proxy resources. mustUpdate(t, fc, "default", "test", func(ing *networkingv1.Ingress) { - delete(ing.ObjectMeta.Labels, LabelProxyClass) + delete(ing.ObjectMeta.Labels, LabelAnnotationProxyClass) }) expectReconciled(t, ingR, "default", "test") opts.proxyClass = "" - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) } func TestTailscaleIngressWithServiceMonitor(t *testing.T) { @@ -325,14 +330,15 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: 1, - }}}, + }}, + }, } crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}} // Create fake client with ProxyClass, IngressClass, Ingress with metrics ProxyClass, and Service ing := ingress() ing.Labels = map[string]string{ - LabelProxyClass: "metrics", + LabelAnnotationProxyClass: "metrics", } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -347,7 +353,8 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: ft, @@ -421,13 +428,121 @@ func TestTailscaleIngressWithServiceMonitor(t *testing.T) { // ServiceMonitor gets garbage collected when the Service is deleted - we cannot test that here. } +func TestIngressProxyClassAnnotation(t *testing.T) { + cl := tstest.NewClock(tstest.ClockOpts{}) + zl := zap.Must(zap.NewDevelopment()) + + pcLEStaging, pcLEStagingFalse, _ := proxyClassesForLEStagingTest() + + testCases := []struct { + name string + proxyClassAnnotation string + proxyClassLabel string + proxyClassDefault string + expectedProxyClass string + expectEvents []string + }{ + { + name: "via_label", + proxyClassLabel: pcLEStaging.Name, + expectedProxyClass: pcLEStaging.Name, + }, + { + name: "via_annotation", + proxyClassAnnotation: pcLEStaging.Name, + expectedProxyClass: pcLEStaging.Name, + }, + { + name: "via_default", + proxyClassDefault: pcLEStaging.Name, + expectedProxyClass: pcLEStaging.Name, + }, + { + name: "via_label_override_annotation", + proxyClassLabel: pcLEStaging.Name, + proxyClassAnnotation: pcLEStagingFalse.Name, + expectedProxyClass: pcLEStaging.Name, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + builder := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme) + + builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse). + WithStatusSubresource(pcLEStaging, pcLEStagingFalse) + + fc := builder.Build() + + if tt.proxyClassAnnotation != "" || tt.proxyClassLabel != "" || tt.proxyClassDefault != "" { + name := tt.proxyClassDefault + if name == "" { + name = tt.proxyClassLabel + if name == "" { + name = tt.proxyClassAnnotation + } + } + setProxyClassReady(t, fc, cl, name) + } + + mustCreate(t, fc, ingressClass()) + mustCreate(t, fc, service()) + ing := ingress() + if tt.proxyClassLabel != "" { + ing.Labels = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassLabel, + } + } + if tt.proxyClassAnnotation != "" { + ing.Annotations = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassAnnotation, + } + } + mustCreate(t, fc, ing) + + ingR := &IngressReconciler{ + Client: fc, + ingressClassName: "tailscale", + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: &fakeTSClient{}, + tsnetServer: &fakeTSNetServer{certDomains: []string{"test-host"}}, + defaultTags: []string{"tag:test"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale:test", + }, + logger: zl.Sugar(), + defaultProxyClass: tt.proxyClassDefault, + } + + expectReconciled(t, ingR, "default", "test") + + _, shortName := findGenName(t, fc, "default", "test", "ingress") + sts := &appsv1.StatefulSet{} + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + switch tt.expectedProxyClass { + case pcLEStaging.Name: + verifyEnvVar(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint) + case pcLEStagingFalse.Name: + verifyEnvVarNotPresent(t, sts, "TS_DEBUG_ACME_DIRECTORY_URL") + default: + t.Fatalf("unexpected expected ProxyClass %q", tt.expectedProxyClass) + } + }) + } +} + func TestIngressLetsEncryptStaging(t *testing.T) { cl := tstest.NewClock(tstest.ClockOpts{}) zl := zap.Must(zap.NewDevelopment()) pcLEStaging, pcLEStagingFalse, pcOther := proxyClassesForLEStagingTest() - testCases := testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther) + testCases := testCasesForLEStagingTests() for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { @@ -452,13 +567,14 @@ func TestIngressLetsEncryptStaging(t *testing.T) { ing := ingress() if tt.proxyClassPerResource != "" { ing.Labels = map[string]string{ - LabelProxyClass: tt.proxyClassPerResource, + LabelAnnotationProxyClass: tt.proxyClassPerResource, } } mustCreate(t, fc, ing) ingR := &IngressReconciler{ - Client: fc, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: &fakeTSClient{}, @@ -565,8 +681,9 @@ func TestEmptyPath(t *testing.T) { t.Fatal(err) } ingR := &IngressReconciler{ - recorder: fr, - Client: fc, + recorder: fr, + Client: fc, + ingressClassName: "tailscale", ssr: &tailscaleSTSReconciler{ Client: fc, tsClient: ft, @@ -608,7 +725,7 @@ func TestEmptyPath(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "ingress")) - expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeResourceReqs) expectEvents(t, fr, tt.expectedEvents) }) diff --git a/cmd/k8s-operator/nameserver.go b/cmd/k8s-operator/nameserver.go index ef0762a1234e6..983a28c918276 100644 --- a/cmd/k8s-operator/nameserver.go +++ b/cmd/k8s-operator/nameserver.go @@ -7,14 +7,13 @@ package main import ( "context" + _ "embed" + "errors" "fmt" "slices" "strings" "sync" - _ "embed" - - "github.com/pkg/errors" "go.uber.org/zap" xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" @@ -106,7 +105,7 @@ func (a *NameserverReconciler) Reconcile(ctx context.Context, req reconcile.Requ if !apiequality.Semantic.DeepEqual(oldCnStatus, &dnsCfg.Status) { // An error encountered here should get returned by the Reconcile function. if updateErr := a.Client.Status().Update(ctx, dnsCfg); updateErr != nil { - err = errors.Wrap(err, updateErr.Error()) + err = errors.Join(err, updateErr) } } return res, err @@ -183,6 +182,10 @@ func (a *NameserverReconciler) maybeProvision(ctx context.Context, tsDNSCfg *tsa if tsDNSCfg.Spec.Nameserver.Image != nil && tsDNSCfg.Spec.Nameserver.Image.Tag != "" { dCfg.imageTag = tsDNSCfg.Spec.Nameserver.Image.Tag } + if tsDNSCfg.Spec.Nameserver.Service != nil { + dCfg.clusterIP = tsDNSCfg.Spec.Nameserver.Service.ClusterIP + } + for _, deployable := range []deployable{saDeployable, deployDeployable, svcDeployable, cmDeployable} { if err := deployable.updateObj(ctx, dCfg, a.Client); err != nil { return fmt.Errorf("error reconciling %s: %w", deployable.kind, err) @@ -213,6 +216,7 @@ type deployConfig struct { labels map[string]string ownerRefs []metav1.OwnerReference namespace string + clusterIP string } var ( @@ -267,6 +271,7 @@ var ( svc.ObjectMeta.Labels = cfg.labels svc.ObjectMeta.OwnerReferences = cfg.ownerRefs svc.ObjectMeta.Namespace = cfg.namespace + svc.Spec.ClusterIP = cfg.clusterIP _, err := createOrUpdate[corev1.Service](ctx, kubeClient, cfg.namespace, svc, func(*corev1.Service) {}) return err }, diff --git a/cmd/k8s-operator/nameserver_test.go b/cmd/k8s-operator/nameserver_test.go index cec95b84ee719..55a998ac31979 100644 --- a/cmd/k8s-operator/nameserver_test.go +++ b/cmd/k8s-operator/nameserver_test.go @@ -26,7 +26,7 @@ import ( ) func TestNameserverReconciler(t *testing.T) { - dnsCfg := &tsapi.DNSConfig{ + dnsConfig := &tsapi.DNSConfig{ TypeMeta: metav1.TypeMeta{Kind: "DNSConfig", APIVersion: "tailscale.com/v1alpha1"}, ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -37,91 +37,130 @@ func TestNameserverReconciler(t *testing.T) { Repo: "test", Tag: "v0.0.1", }, + Service: &tsapi.NameserverService{ + ClusterIP: "5.4.3.2", + }, }, }, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). - WithObjects(dnsCfg). - WithStatusSubresource(dnsCfg). + WithObjects(dnsConfig). + WithStatusSubresource(dnsConfig). Build() - zl, err := zap.NewDevelopment() + + logger, err := zap.NewDevelopment() if err != nil { t.Fatal(err) } - cl := tstest.NewClock(tstest.ClockOpts{}) - nr := &NameserverReconciler{ + + clock := tstest.NewClock(tstest.ClockOpts{}) + reconciler := &NameserverReconciler{ Client: fc, - clock: cl, - logger: zl.Sugar(), - tsNamespace: "tailscale", + clock: clock, + logger: logger.Sugar(), + tsNamespace: tsNamespace, } - expectReconciled(t, nr, "", "test") - // Verify that nameserver Deployment has been created and has the expected fields. - wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: "tailscale"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}} - if err := yaml.Unmarshal(deployYaml, wantsDeploy); err != nil { - t.Fatalf("unmarshalling yaml: %v", err) - } - dnsCfgOwnerRef := metav1.NewControllerRef(dnsCfg, tsapi.SchemeGroupVersion.WithKind("DNSConfig")) - wantsDeploy.OwnerReferences = []metav1.OwnerReference{*dnsCfgOwnerRef} - wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1" - wantsDeploy.Namespace = "tailscale" - labels := nameserverResourceLabels("test", "tailscale") - wantsDeploy.ObjectMeta.Labels = labels - expectEqual(t, fc, wantsDeploy) - - // Verify that DNSConfig advertizes the nameserver's Service IP address, - // has the ready status condition and tailscale finalizer. - mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { - svc.Spec.ClusterIP = "1.2.3.4" + expectReconciled(t, reconciler, "", "test") + + ownerReference := metav1.NewControllerRef(dnsConfig, tsapi.SchemeGroupVersion.WithKind("DNSConfig")) + nameserverLabels := nameserverResourceLabels(dnsConfig.Name, tsNamespace) + + wantsDeploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.Identifier()}} + t.Run("deployment has expected fields", func(t *testing.T) { + if err = yaml.Unmarshal(deployYaml, wantsDeploy); err != nil { + t.Fatalf("unmarshalling yaml: %v", err) + } + wantsDeploy.OwnerReferences = []metav1.OwnerReference{*ownerReference} + wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.1" + wantsDeploy.Namespace = tsNamespace + wantsDeploy.ObjectMeta.Labels = nameserverLabels + expectEqual(t, fc, wantsDeploy) }) - expectReconciled(t, nr, "", "test") - dnsCfg.Status.Nameserver = &tsapi.NameserverStatus{ - IP: "1.2.3.4", - } - dnsCfg.Finalizers = []string{FinalizerName} - dnsCfg.Status.Conditions = append(dnsCfg.Status.Conditions, metav1.Condition{ - Type: string(tsapi.NameserverReady), - Status: metav1.ConditionTrue, - Reason: reasonNameserverCreated, - Message: reasonNameserverCreated, - LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + + wantsSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nameserver", Namespace: tsNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: corev1.SchemeGroupVersion.Identifier()}} + t.Run("service has expected fields", func(t *testing.T) { + if err = yaml.Unmarshal(svcYaml, wantsSvc); err != nil { + t.Fatalf("unmarshalling yaml: %v", err) + } + wantsSvc.Spec.ClusterIP = dnsConfig.Spec.Nameserver.Service.ClusterIP + wantsSvc.OwnerReferences = []metav1.OwnerReference{*ownerReference} + wantsSvc.Namespace = tsNamespace + wantsSvc.ObjectMeta.Labels = nameserverLabels + expectEqual(t, fc, wantsSvc) }) - expectEqual(t, fc, dnsCfg) - // // Verify that nameserver image gets updated to match DNSConfig spec. - mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { - dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2" + t.Run("dns config status is set", func(t *testing.T) { + // Verify that DNSConfig advertizes the nameserver's Service IP address, + // has the ready status condition and tailscale finalizer. + mustUpdate(t, fc, "tailscale", "nameserver", func(svc *corev1.Service) { + svc.Spec.ClusterIP = "1.2.3.4" + }) + expectReconciled(t, reconciler, "", "test") + + dnsConfig.Finalizers = []string{FinalizerName} + dnsConfig.Status.Nameserver = &tsapi.NameserverStatus{ + IP: "1.2.3.4", + } + dnsConfig.Status.Conditions = append(dnsConfig.Status.Conditions, metav1.Condition{ + Type: string(tsapi.NameserverReady), + Status: metav1.ConditionTrue, + Reason: reasonNameserverCreated, + Message: reasonNameserverCreated, + LastTransitionTime: metav1.Time{Time: clock.Now().Truncate(time.Second)}, + }) + + expectEqual(t, fc, dnsConfig) }) - expectReconciled(t, nr, "", "test") - wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2" - expectEqual(t, fc, wantsDeploy) - - // Verify that when another actor sets ConfigMap data, it does not get - // overwritten by nameserver reconciler. - dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}} - bs, err := json.Marshal(dnsRecords) - if err != nil { - t.Fatalf("error marshalling ConfigMap contents: %v", err) - } - mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) { - mak.Set(&cm.Data, "records.json", string(bs)) + + t.Run("nameserver image can be updated", func(t *testing.T) { + // Verify that nameserver image gets updated to match DNSConfig spec. + mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { + dnsCfg.Spec.Nameserver.Image.Tag = "v0.0.2" + }) + expectReconciled(t, reconciler, "", "test") + wantsDeploy.Spec.Template.Spec.Containers[0].Image = "test:v0.0.2" + expectEqual(t, fc, wantsDeploy) + }) + + t.Run("reconciler does not overwrite custom configuration", func(t *testing.T) { + // Verify that when another actor sets ConfigMap data, it does not get + // overwritten by nameserver reconciler. + dnsRecords := &operatorutils.Records{Version: "v1alpha1", IP4: map[string][]string{"foo.ts.net": {"1.2.3.4"}}} + bs, err := json.Marshal(dnsRecords) + if err != nil { + t.Fatalf("error marshalling ConfigMap contents: %v", err) + } + + mustUpdate(t, fc, "tailscale", "dnsrecords", func(cm *corev1.ConfigMap) { + mak.Set(&cm.Data, "records.json", string(bs)) + }) + + expectReconciled(t, reconciler, "", "test") + + wantCm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dnsrecords", + Namespace: "tailscale", + Labels: nameserverLabels, + OwnerReferences: []metav1.OwnerReference{*ownerReference}, + }, + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + Data: map[string]string{"records.json": string(bs)}, + } + + expectEqual(t, fc, wantCm) }) - expectReconciled(t, nr, "", "test") - wantCm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dnsrecords", - Namespace: "tailscale", Labels: labels, OwnerReferences: []metav1.OwnerReference{*dnsCfgOwnerRef}}, - TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, - Data: map[string]string{"records.json": string(bs)}, - } - expectEqual(t, fc, wantCm) - // Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset, - // the nameserver image defaults to tailscale/k8s-nameserver:unstable. - mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { - dnsCfg.Spec.Nameserver.Image = nil + t.Run("uses default nameserver image", func(t *testing.T) { + // Verify that if dnsconfig.spec.nameserver.image.{repo,tag} are unset, + // the nameserver image defaults to tailscale/k8s-nameserver:unstable. + mustUpdate(t, fc, "", "test", func(dnsCfg *tsapi.DNSConfig) { + dnsCfg.Spec.Nameserver.Image = nil + }) + expectReconciled(t, reconciler, "", "test") + wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable" + expectEqual(t, fc, wantsDeploy) }) - expectReconciled(t, nr, "", "test") - wantsDeploy.Spec.Template.Spec.Containers[0].Image = "tailscale/k8s-nameserver:unstable" - expectEqual(t, fc, wantsDeploy) } diff --git a/cmd/k8s-operator/nodeport-service-ports.go b/cmd/k8s-operator/nodeport-service-ports.go new file mode 100644 index 0000000000000..a9504e3e94f88 --- /dev/null +++ b/cmd/k8s-operator/nodeport-service-ports.go @@ -0,0 +1,203 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "context" + "fmt" + "math/rand/v2" + "regexp" + "sort" + "strconv" + "strings" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + k8soperator "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/kubetypes" +) + +const ( + tailscaledPortMax = 65535 + tailscaledPortMin = 1024 + testSvcName = "test-node-port-range" + + invalidSvcNodePort = 777777 +) + +// getServicesNodePortRange is a hacky function that attempts to determine Service NodePort range by +// creating a deliberately invalid Service with a NodePort that is too large and parsing the returned +// validation error. Returns nil if unable to determine port range. +// https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport +func getServicesNodePortRange(ctx context.Context, c client.Client, tsNamespace string, logger *zap.SugaredLogger) *tsapi.PortRange { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: testSvcName, + Namespace: tsNamespace, + Labels: map[string]string{ + kubetypes.LabelManaged: "true", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + { + Name: testSvcName, + Port: 8080, + TargetPort: intstr.FromInt32(8080), + Protocol: corev1.ProtocolUDP, + NodePort: invalidSvcNodePort, + }, + }, + }, + } + + // NOTE(ChaosInTheCRD): ideally this would be a server side dry-run but could not get it working + err := c.Create(ctx, svc) + if err == nil { + return nil + } + + if validPorts := getServicesNodePortRangeFromErr(err.Error()); validPorts != "" { + pr, err := parseServicesNodePortRange(validPorts) + if err != nil { + logger.Debugf("failed to parse NodePort range set for Kubernetes Cluster: %w", err) + return nil + } + + return pr + } + + return nil +} + +func getServicesNodePortRangeFromErr(err string) string { + reg := regexp.MustCompile(`\d{1,5}-\d{1,5}`) + matches := reg.FindAllString(err, -1) + if len(matches) != 1 { + return "" + } + + return matches[0] +} + +// parseServicesNodePortRange converts the `ValidPorts` string field in the Kubernetes PortAllocator error and converts it to +// PortRange +func parseServicesNodePortRange(p string) (*tsapi.PortRange, error) { + parts := strings.Split(p, "-") + s, err := strconv.ParseUint(parts[0], 10, 16) + if err != nil { + return nil, fmt.Errorf("failed to parse string as uint16: %w", err) + } + + var e uint64 + switch len(parts) { + case 1: + e = uint64(s) + case 2: + e, err = strconv.ParseUint(parts[1], 10, 16) + if err != nil { + return nil, fmt.Errorf("failed to parse string as uint16: %w", err) + } + default: + return nil, fmt.Errorf("failed to parse port range %q", p) + } + + portRange := &tsapi.PortRange{Port: uint16(s), EndPort: uint16(e)} + if !portRange.IsValid() { + return nil, fmt.Errorf("port range %q is not valid", portRange.String()) + } + + return portRange, nil +} + +// validateNodePortRanges checks that the port range specified is valid. It also ensures that the specified ranges +// lie within the NodePort Service port range specified for the Kubernetes API Server. +func validateNodePortRanges(ctx context.Context, c client.Client, kubeRange *tsapi.PortRange, pc *tsapi.ProxyClass) error { + if pc.Spec.StaticEndpoints == nil { + return nil + } + + portRanges := pc.Spec.StaticEndpoints.NodePort.Ports + + if kubeRange != nil { + for _, pr := range portRanges { + if !kubeRange.Contains(pr.Port) || (pr.EndPort != 0 && !kubeRange.Contains(pr.EndPort)) { + return fmt.Errorf("range %q is not within Cluster configured range %q", pr.String(), kubeRange.String()) + } + } + } + + for _, r := range portRanges { + if !r.IsValid() { + return fmt.Errorf("port range %q is invalid", r.String()) + } + } + + // TODO(ChaosInTheCRD): if a ProxyClass that made another invalid (due to port range clash) is deleted, + // the invalid ProxyClass doesn't get reconciled on, and therefore will not go valid. We should fix this. + proxyClassRanges, err := getPortsForProxyClasses(ctx, c) + if err != nil { + return fmt.Errorf("failed to get port ranges for ProxyClasses: %w", err) + } + + for _, r := range portRanges { + for pcName, pcr := range proxyClassRanges { + if pcName == pc.Name { + continue + } + if pcr.ClashesWith(r) { + return fmt.Errorf("port ranges for ProxyClass %q clash with existing ProxyClass %q", pc.Name, pcName) + } + } + } + + if len(portRanges) == 1 { + return nil + } + + sort.Slice(portRanges, func(i, j int) bool { + return portRanges[i].Port < portRanges[j].Port + }) + + for i := 1; i < len(portRanges); i++ { + prev := portRanges[i-1] + curr := portRanges[i] + if curr.Port <= prev.Port || curr.Port <= prev.EndPort { + return fmt.Errorf("overlapping ranges: %q and %q", prev.String(), curr.String()) + } + } + + return nil +} + +// getPortsForProxyClasses gets the port ranges for all the other existing ProxyClasses +func getPortsForProxyClasses(ctx context.Context, c client.Client) (map[string]tsapi.PortRanges, error) { + pcs := new(tsapi.ProxyClassList) + + err := c.List(ctx, pcs) + if err != nil { + return nil, fmt.Errorf("failed to list ProxyClasses: %w", err) + } + + portRanges := make(map[string]tsapi.PortRanges) + for _, i := range pcs.Items { + if !k8soperator.ProxyClassIsReady(&i) { + continue + } + if se := i.Spec.StaticEndpoints; se != nil && se.NodePort != nil { + portRanges[i.Name] = se.NodePort.Ports + } + } + + return portRanges, nil +} + +func getRandomPort() uint16 { + return uint16(rand.IntN(tailscaledPortMax-tailscaledPortMin+1) + tailscaledPortMin) +} diff --git a/cmd/k8s-operator/nodeport-services-ports_test.go b/cmd/k8s-operator/nodeport-services-ports_test.go new file mode 100644 index 0000000000000..9418bb8446bd8 --- /dev/null +++ b/cmd/k8s-operator/nodeport-services-ports_test.go @@ -0,0 +1,277 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package main + +import ( + "context" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/tstest" +) + +func TestGetServicesNodePortRangeFromErr(t *testing.T) { + tests := []struct { + name string + errStr string + want string + }{ + { + name: "valid_error_string", + errStr: "NodePort 777777 is not in the allowed range 30000-32767", + want: "30000-32767", + }, + { + name: "error_string_with_different_message", + errStr: "some other error without a port range", + want: "", + }, + { + name: "error_string_with_multiple_port_ranges", + errStr: "range 1000-2000 and another range 3000-4000", + want: "", + }, + { + name: "empty_error_string", + errStr: "", + want: "", + }, + { + name: "error_string_with_range_at_start", + errStr: "30000-32767 is the range", + want: "30000-32767", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getServicesNodePortRangeFromErr(tt.errStr); got != tt.want { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseServicesNodePortRange(t *testing.T) { + tests := []struct { + name string + p string + want *tsapi.PortRange + wantErr bool + }{ + { + name: "valid_range", + p: "30000-32767", + want: &tsapi.PortRange{Port: 30000, EndPort: 32767}, + wantErr: false, + }, + { + name: "single_port_range", + p: "30000", + want: &tsapi.PortRange{Port: 30000, EndPort: 30000}, + wantErr: false, + }, + { + name: "invalid_format_non_numeric_end", + p: "30000-abc", + want: nil, + wantErr: true, + }, + { + name: "invalid_format_non_numeric_start", + p: "abc-32767", + want: nil, + wantErr: true, + }, + { + name: "empty_string", + p: "", + want: nil, + wantErr: true, + }, + { + name: "too_many_parts", + p: "1-2-3", + want: nil, + wantErr: true, + }, + { + name: "port_too_large_start", + p: "65536-65537", + want: nil, + wantErr: true, + }, + { + name: "port_too_large_end", + p: "30000-65536", + want: nil, + wantErr: true, + }, + { + name: "inverted_range", + p: "32767-30000", + want: nil, + wantErr: true, // IsValid() will fail + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + portRange, err := parseServicesNodePortRange(tt.p) + if (err != nil) != tt.wantErr { + t.Errorf("error = %v, wantErr %v", err, tt.wantErr) + return + } + + if tt.wantErr { + return + } + + if portRange == nil { + t.Fatalf("got nil port range, expected %v", tt.want) + } + + if portRange.Port != tt.want.Port || portRange.EndPort != tt.want.EndPort { + t.Errorf("got = %v, want %v", portRange, tt.want) + } + }) + } +} + +func TestValidateNodePortRanges(t *testing.T) { + tests := []struct { + name string + portRanges []tsapi.PortRange + wantErr bool + }{ + { + name: "valid_ranges_with_unknown_kube_range", + portRanges: []tsapi.PortRange{ + {Port: 30003, EndPort: 30005}, + {Port: 30006, EndPort: 30007}, + }, + wantErr: false, + }, + { + name: "overlapping_ranges", + portRanges: []tsapi.PortRange{ + {Port: 30000, EndPort: 30010}, + {Port: 30005, EndPort: 30015}, + }, + wantErr: true, + }, + { + name: "adjacent_ranges_no_overlap", + portRanges: []tsapi.PortRange{ + {Port: 30010, EndPort: 30020}, + {Port: 30021, EndPort: 30022}, + }, + wantErr: false, + }, + { + name: "identical_ranges_are_overlapping", + portRanges: []tsapi.PortRange{ + {Port: 30005, EndPort: 30010}, + {Port: 30005, EndPort: 30010}, + }, + wantErr: true, + }, + { + name: "range_clashes_with_existing_proxyclass", + portRanges: []tsapi.PortRange{ + {Port: 31005, EndPort: 32070}, + }, + wantErr: true, + }, + } + + // as part of this test, we want to create an adjacent ProxyClass in order to ensure that if it clashes with the one created in this test + // that we get an error + cl := tstest.NewClock(tstest.ClockOpts{}) + opc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Annotations: defaultProxyClassAnnotations, + }, + StaticEndpoints: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 31000}, {Port: 32000}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + }, + Status: tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + Reason: reasonProxyClassValid, + Message: reasonProxyClassValid, + LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }}, + }, + } + + fc := fake.NewClientBuilder(). + WithObjects(opc). + WithStatusSubresource(opc). + WithScheme(tsapi.GlobalScheme). + Build() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Annotations: defaultProxyClassAnnotations, + }, + StaticEndpoints: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: tt.portRanges, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + }, + Status: tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + Reason: reasonProxyClassValid, + Message: reasonProxyClassValid, + LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }}, + }, + } + err := validateNodePortRanges(context.Background(), fc, &tsapi.PortRange{Port: 30000, EndPort: 32767}, pc) + if (err != nil) != tt.wantErr { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +func TestGetRandomPort(t *testing.T) { + for range 100 { + port := getRandomPort() + if port < tailscaledPortMin || port > tailscaledPortMax { + t.Errorf("generated port %d which is out of range [%d, %d]", port, tailscaledPortMin, tailscaledPortMax) + } + } +} diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index a08dd4da8c52f..76d2df51d47d2 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -26,7 +26,10 @@ import ( networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + klabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" @@ -39,7 +42,9 @@ import ( kzap "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/client/local" "tailscale.com/client/tailscale" "tailscale.com/hostinfo" @@ -51,6 +56,7 @@ import ( "tailscale.com/tsnet" "tailscale.com/tstime" "tailscale.com/types/logger" + "tailscale.com/util/set" "tailscale.com/version" ) @@ -72,11 +78,14 @@ func main() { tsNamespace = defaultEnv("OPERATOR_NAMESPACE", "") tslogging = defaultEnv("OPERATOR_LOGGING", "info") image = defaultEnv("PROXY_IMAGE", "tailscale/tailscale:latest") + k8sProxyImage = defaultEnv("K8S_PROXY_IMAGE", "tailscale/k8s-proxy:latest") priorityClassName = defaultEnv("PROXY_PRIORITY_CLASS_NAME", "") tags = defaultEnv("PROXY_TAGS", "tag:k8s") tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "") defaultProxyClass = defaultEnv("PROXY_DEFAULT_CLASS", "") isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false) + loginServer = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/") + ingressClassName = defaultEnv("OPERATOR_INGRESS_CLASS_NAME", "tailscale") ) var opts []kzap.Opts @@ -103,17 +112,27 @@ func main() { // The operator can run either as a plain operator or it can // additionally act as api-server proxy // https://tailscale.com/kb/1236/kubernetes-operator/?q=kubernetes#accessing-the-kubernetes-control-plane-using-an-api-server-proxy. - mode := apiproxy.ParseAPIProxyMode() - if mode == apiproxy.APIServerProxyModeDisabled { + mode := parseAPIProxyMode() + if mode == nil { hostinfo.SetApp(kubetypes.AppOperator) } else { - hostinfo.SetApp(kubetypes.AppAPIServerProxy) + hostinfo.SetApp(kubetypes.AppInProcessAPIServerProxy) } - s, tsc := initTSNet(zlog) + s, tsc := initTSNet(zlog, loginServer) defer s.Close() restConfig := config.GetConfigOrDie() - apiproxy.MaybeLaunchAPIServerProxy(zlog, restConfig, s, mode) + if mode != nil { + ap, err := apiproxy.NewAPIServerProxy(zlog, restConfig, s, *mode, true) + if err != nil { + zlog.Fatalf("error creating API server proxy: %v", err) + } + go func() { + if err := ap.Run(context.Background()); err != nil { + zlog.Fatalf("error running API server proxy: %v", err) + } + }() + } rOpts := reconcilerOpts{ log: zlog, tsServer: s, @@ -121,11 +140,14 @@ func main() { tailscaleNamespace: tsNamespace, restConfig: restConfig, proxyImage: image, + k8sProxyImage: k8sProxyImage, proxyPriorityClassName: priorityClassName, proxyActAsDefaultLoadBalancer: isDefaultLoadBalancer, proxyTags: tags, proxyFirewallMode: tsFirewallMode, defaultProxyClass: defaultProxyClass, + loginServer: loginServer, + ingressClassName: ingressClassName, } runReconcilers(rOpts) } @@ -133,7 +155,7 @@ func main() { // initTSNet initializes the tsnet.Server and logs in to Tailscale. It uses the // CLIENT_ID_FILE and CLIENT_SECRET_FILE environment variables to authenticate // with Tailscale. -func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) { +func initTSNet(zlog *zap.SugaredLogger, loginServer string) (*tsnet.Server, tsClient) { var ( clientIDPath = defaultEnv("CLIENT_ID_FILE", "") clientSecretPath = defaultEnv("CLIENT_SECRET_FILE", "") @@ -145,13 +167,14 @@ func initTSNet(zlog *zap.SugaredLogger) (*tsnet.Server, tsClient) { if clientIDPath == "" || clientSecretPath == "" { startlog.Fatalf("CLIENT_ID_FILE and CLIENT_SECRET_FILE must be set") } - tsc, err := newTSClient(context.Background(), clientIDPath, clientSecretPath) + tsc, err := newTSClient(context.Background(), clientIDPath, clientSecretPath, loginServer) if err != nil { startlog.Fatalf("error creating Tailscale client: %v", err) } s := &tsnet.Server{ - Hostname: hostname, - Logf: zlog.Named("tailscaled").Debugf, + Hostname: hostname, + Logf: zlog.Named("tailscaled").Debugf, + ControlURL: loginServer, } if p := os.Getenv("TS_PORT"); p != "" { port, err := strconv.ParseUint(p, 10, 16) @@ -228,6 +251,17 @@ waitOnline: return s, tsc } +// predicate function for filtering to ensure we *don't* reconcile on tailscale managed Kubernetes Services +func serviceManagedResourceFilterPredicate() predicate.Predicate { + return predicate.NewPredicateFuncs(func(object client.Object) bool { + if svc, ok := object.(*corev1.Service); !ok { + return false + } else { + return !isManagedResource(svc) + } + }) +} + // runReconcilers starts the controller-runtime manager and registers the // ServiceReconciler. It blocks forever. func runReconcilers(opts reconcilerOpts) { @@ -292,7 +326,9 @@ func runReconcilers(opts reconcilerOpts) { proxyImage: opts.proxyImage, proxyPriorityClassName: opts.proxyPriorityClassName, tsFirewallMode: opts.proxyFirewallMode, + loginServer: opts.tsServer.ControlURL, } + err = builder. ControllerManagedBy(mgr). Named("service-reconciler"). @@ -313,12 +349,16 @@ func runReconcilers(opts reconcilerOpts) { if err != nil { startlog.Fatalf("could not create service reconciler: %v", err) } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(corev1.Service), indexServiceProxyClass, indexProxyClass); err != nil { + startlog.Fatalf("failed setting up ProxyClass indexer for Services: %v", err) + } + ingressChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("ingress")) // If a ProxyClassChanges, enqueue all Ingresses labeled with that // ProxyClass's name. proxyClassFilterForIngress := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForIngress(mgr.GetClient(), startlog)) // Enque Ingress if a managed Service or backend Service associated with a tailscale Ingress changes. - svcHandlerForIngress := handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngress(mgr.GetClient(), startlog)) + svcHandlerForIngress := handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngress(mgr.GetClient(), startlog, opts.ingressClassName)) err = builder. ControllerManagedBy(mgr). For(&networkingv1.Ingress{}). @@ -333,10 +373,15 @@ func runReconcilers(opts reconcilerOpts) { Client: mgr.GetClient(), logger: opts.log.Named("ingress-reconciler"), defaultProxyClass: opts.defaultProxyClass, + ingressClassName: opts.ingressClassName, }) if err != nil { startlog.Fatalf("could not create ingress reconciler: %v", err) } + if err := mgr.GetFieldIndexer().IndexField(context.Background(), new(networkingv1.Ingress), indexIngressProxyClass, indexProxyClass); err != nil { + startlog.Fatalf("failed setting up ProxyClass indexer for Ingresses: %v", err) + } + lc, err := opts.tsServer.LocalClient() if err != nil { startlog.Fatalf("could not get local client: %v", err) @@ -350,19 +395,20 @@ func runReconcilers(opts reconcilerOpts) { ControllerManagedBy(mgr). For(&networkingv1.Ingress{}). Named("ingress-pg-reconciler"). - Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog))). + Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(serviceHandlerForIngressPG(mgr.GetClient(), startlog, opts.ingressClassName))). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(HAIngressesFromSecret(mgr.GetClient(), startlog))). Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter). Complete(&HAIngressReconciler{ - recorder: eventRecorder, - tsClient: opts.tsClient, - tsnetServer: opts.tsServer, - defaultTags: strings.Split(opts.proxyTags, ","), - Client: mgr.GetClient(), - logger: opts.log.Named("ingress-pg-reconciler"), - lc: lc, - operatorID: id, - tsNamespace: opts.tailscaleNamespace, + recorder: eventRecorder, + tsClient: opts.tsClient, + tsnetServer: opts.tsServer, + defaultTags: strings.Split(opts.proxyTags, ","), + Client: mgr.GetClient(), + logger: opts.log.Named("ingress-pg-reconciler"), + lc: lc, + operatorID: id, + tsNamespace: opts.tailscaleNamespace, + ingressClassName: opts.ingressClassName, }) if err != nil { startlog.Fatalf("could not create ingress-pg-reconciler: %v", err) @@ -374,7 +420,7 @@ func runReconcilers(opts reconcilerOpts) { ingressSvcFromEpsFilter := handler.EnqueueRequestsFromMapFunc(ingressSvcFromEps(mgr.GetClient(), opts.log.Named("service-pg-reconciler"))) err = builder. ControllerManagedBy(mgr). - For(&corev1.Service{}). + For(&corev1.Service{}, builder.WithPredicates(serviceManagedResourceFilterPredicate())). Named("service-pg-reconciler"). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(HAServicesFromSecret(mgr.GetClient(), startlog))). Watches(&tsapi.ProxyGroup{}, ingressProxyGroupFilter). @@ -382,7 +428,6 @@ func runReconcilers(opts reconcilerOpts) { Complete(&HAServiceReconciler{ recorder: eventRecorder, tsClient: opts.tsClient, - tsnetServer: opts.tsServer, defaultTags: strings.Split(opts.proxyTags, ","), Client: mgr.GetClient(), logger: opts.log.Named("service-pg-reconciler"), @@ -519,16 +564,19 @@ func runReconcilers(opts reconcilerOpts) { // ProxyClass reconciler gets triggered on ServiceMonitor CRD changes to ensure that any ProxyClasses, that // define that a ServiceMonitor should be created, were set to invalid because the CRD did not exist get // reconciled if the CRD is applied at a later point. + kPortRange := getServicesNodePortRange(context.Background(), mgr.GetClient(), opts.tailscaleNamespace, startlog) serviceMonitorFilter := handler.EnqueueRequestsFromMapFunc(proxyClassesWithServiceMonitor(mgr.GetClient(), opts.log)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyClass{}). Named("proxyclass-reconciler"). Watches(&apiextensionsv1.CustomResourceDefinition{}, serviceMonitorFilter). Complete(&ProxyClassReconciler{ - Client: mgr.GetClient(), - recorder: eventRecorder, - logger: opts.log.Named("proxyclass-reconciler"), - clock: tstime.DefaultClock{}, + Client: mgr.GetClient(), + nodePortRange: kPortRange, + recorder: eventRecorder, + tsNamespace: opts.tailscaleNamespace, + logger: opts.log.Named("proxyclass-reconciler"), + clock: tstime.DefaultClock{}, }) if err != nil { startlog.Fatal("could not create proxyclass reconciler: %v", err) @@ -579,24 +627,55 @@ func runReconcilers(opts reconcilerOpts) { l: opts.log.Named("recorder-reconciler"), clock: tstime.DefaultClock{}, tsClient: opts.tsClient, + loginServer: opts.loginServer, }) if err != nil { startlog.Fatalf("could not create Recorder reconciler: %v", err) } + // kube-apiserver's Tailscale Service reconciler. + err = builder. + ControllerManagedBy(mgr). + For(&tsapi.ProxyGroup{}, builder.WithPredicates( + predicate.NewPredicateFuncs(func(obj client.Object) bool { + pg, ok := obj.(*tsapi.ProxyGroup) + return ok && pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer + }), + )). + Named("kube-apiserver-ts-service-reconciler"). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(kubeAPIServerPGsFromSecret(mgr.GetClient(), startlog))). + Complete(&KubeAPIServerTSServiceReconciler{ + Client: mgr.GetClient(), + recorder: eventRecorder, + logger: opts.log.Named("kube-apiserver-ts-service-reconciler"), + tsClient: opts.tsClient, + tsNamespace: opts.tailscaleNamespace, + lc: lc, + defaultTags: strings.Split(opts.proxyTags, ","), + operatorID: id, + clock: tstime.DefaultClock{}, + }) + if err != nil { + startlog.Fatalf("could not create Kubernetes API server Tailscale Service reconciler: %v", err) + } + // ProxyGroup reconciler. ownedByProxyGroupFilter := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &tsapi.ProxyGroup{}) proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog)) + nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog)) + saFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(serviceAccountHandlerForProxyGroup(mgr.GetClient(), startlog)) err = builder.ControllerManagedBy(mgr). For(&tsapi.ProxyGroup{}). Named("proxygroup-reconciler"). + Watches(&corev1.Service{}, ownedByProxyGroupFilter). Watches(&appsv1.StatefulSet{}, ownedByProxyGroupFilter). Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter). - Watches(&corev1.ServiceAccount{}, ownedByProxyGroupFilter). + Watches(&corev1.ServiceAccount{}, saFilterForProxyGroup). Watches(&corev1.Secret{}, ownedByProxyGroupFilter). Watches(&rbacv1.Role{}, ownedByProxyGroupFilter). Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter). Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup). + Watches(&corev1.Node{}, nodeFilterForProxyGroup). Complete(&ProxyGroupReconciler{ recorder: eventRecorder, Client: mgr.GetClient(), @@ -605,10 +684,12 @@ func runReconcilers(opts reconcilerOpts) { tsClient: opts.tsClient, tsNamespace: opts.tailscaleNamespace, - proxyImage: opts.proxyImage, + tsProxyImage: opts.proxyImage, + k8sProxyImage: opts.k8sProxyImage, defaultTags: strings.Split(opts.proxyTags, ","), tsFirewallMode: opts.proxyFirewallMode, defaultProxyClass: opts.defaultProxyClass, + loginServer: opts.tsServer.ControlURL, }) if err != nil { startlog.Fatalf("could not create ProxyGroup reconciler: %v", err) @@ -627,6 +708,7 @@ type reconcilerOpts struct { tailscaleNamespace string // namespace in which operator resources will be deployed restConfig *rest.Config // config for connecting to the kube API server proxyImage string // : + k8sProxyImage string // : // proxyPriorityClassName isPriorityClass to be set for proxy Pods. This // is a legacy mechanism for cluster resource configuration options - // going forward use ProxyClass. @@ -658,6 +740,11 @@ type reconcilerOpts struct { // class for proxies that do not have a ProxyClass set. // this is defined by an operator env variable. defaultProxyClass string + // loginServer is the coordination server URL that should be used by managed resources. + loginServer string + // ingressClassName is the name of the ingress class used by reconcilers of Ingress resources. This defaults + // to "tailscale" but can be customised. + ingressClassName string } // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each @@ -777,6 +864,16 @@ func managedResourceHandlerForType(typ string) handler.MapFunc { } } +// indexProxyClass is used to select ProxyClass-backed objects which are +// locally indexed in the cache for efficient listing without requiring labels. +func indexProxyClass(o client.Object) []string { + if !hasProxyClassAnnotation(o) { + return nil + } + + return []string{o.GetAnnotations()[LabelAnnotationProxyClass]} +} + // proxyClassHandlerForSvc returns a handler that, for a given ProxyClass, // returns a list of reconcile requests for all Services labeled with // tailscale.com/proxy-class: . @@ -784,16 +881,37 @@ func proxyClassHandlerForSvc(cl client.Client, logger *zap.SugaredLogger) handle return func(ctx context.Context, o client.Object) []reconcile.Request { svcList := new(corev1.ServiceList) labels := map[string]string{ - LabelProxyClass: o.GetName(), + LabelAnnotationProxyClass: o.GetName(), } + if err := cl.List(ctx, svcList, client.MatchingLabels(labels)); err != nil { logger.Debugf("error listing Services for ProxyClass: %v", err) return nil } + reqs := make([]reconcile.Request, 0) + seenSvcs := make(set.Set[string]) for _, svc := range svcList.Items { reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)}) + seenSvcs.Add(fmt.Sprintf("%s/%s", svc.Namespace, svc.Name)) } + + svcAnnotationList := new(corev1.ServiceList) + if err := cl.List(ctx, svcAnnotationList, client.MatchingFields{indexServiceProxyClass: o.GetName()}); err != nil { + logger.Debugf("error listing Services for ProxyClass: %v", err) + return nil + } + + for _, svc := range svcAnnotationList.Items { + nsname := fmt.Sprintf("%s/%s", svc.Namespace, svc.Name) + if seenSvcs.Contains(nsname) { + continue + } + + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&svc)}) + seenSvcs.Add(nsname) + } + return reqs } } @@ -805,16 +923,36 @@ func proxyClassHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) ha return func(ctx context.Context, o client.Object) []reconcile.Request { ingList := new(networkingv1.IngressList) labels := map[string]string{ - LabelProxyClass: o.GetName(), + LabelAnnotationProxyClass: o.GetName(), } if err := cl.List(ctx, ingList, client.MatchingLabels(labels)); err != nil { logger.Debugf("error listing Ingresses for ProxyClass: %v", err) return nil } + reqs := make([]reconcile.Request, 0) + seenIngs := make(set.Set[string]) for _, ing := range ingList.Items { reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) + seenIngs.Add(fmt.Sprintf("%s/%s", ing.Namespace, ing.Name)) + } + + ingAnnotationList := new(networkingv1.IngressList) + if err := cl.List(ctx, ingAnnotationList, client.MatchingFields{indexIngressProxyClass: o.GetName()}); err != nil { + logger.Debugf("error listing Ingreses for ProxyClass: %v", err) + return nil + } + + for _, ing := range ingAnnotationList.Items { + nsname := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name) + if seenIngs.Contains(nsname) { + continue + } + + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&ing)}) + seenIngs.Add(nsname) } + return reqs } } @@ -840,9 +978,67 @@ func proxyClassHandlerForConnector(cl client.Client, logger *zap.SugaredLogger) } } +// nodeHandlerForProxyGroup returns a handler that, for a given Node, returns a +// list of reconcile requests for ProxyGroups that should be reconciled for the +// Node event. ProxyGroups need to be reconciled for Node events if they are +// configured to expose tailscaled static endpoints to tailnet using NodePort +// Services. +func nodeHandlerForProxyGroup(cl client.Client, defaultProxyClass string, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + pgList := new(tsapi.ProxyGroupList) + if err := cl.List(ctx, pgList); err != nil { + logger.Debugf("error listing ProxyGroups for ProxyClass: %v", err) + return nil + } + + reqs := make([]reconcile.Request, 0) + for _, pg := range pgList.Items { + if pg.Spec.ProxyClass == "" && defaultProxyClass == "" { + continue + } + + pc := defaultProxyClass + if pc == "" { + pc = pg.Spec.ProxyClass + } + + proxyClass := &tsapi.ProxyClass{} + if err := cl.Get(ctx, types.NamespacedName{Name: pc}, proxyClass); err != nil { + logger.Debugf("error getting ProxyClass %q: %v", pg.Spec.ProxyClass, err) + return nil + } + + stat := proxyClass.Spec.StaticEndpoints + if stat == nil { + continue + } + + // If the selector is empty, all nodes match. + // TODO(ChaosInTheCRD): think about how this must be handled if we want to limit the number of nodes used + if len(stat.NodePort.Selector) == 0 { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + continue + } + + selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + MatchLabels: stat.NodePort.Selector, + }) + if err != nil { + logger.Debugf("error converting `spec.staticEndpoints.nodePort.selector` to Selector: %v", err) + return nil + } + + if selector.Matches(klabels.Set(o.GetLabels())) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + } + } + return reqs + } +} + // proxyClassHandlerForProxyGroup returns a handler that, for a given ProxyClass, -// returns a list of reconcile requests for all Connectors that have -// .spec.proxyClass set. +// returns a list of reconcile requests for all ProxyGroups that have +// .spec.proxyClass set to that ProxyClass. func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { pgList := new(tsapi.ProxyGroupList) @@ -861,13 +1057,44 @@ func proxyClassHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) } } +// serviceAccountHandlerForProxyGroup returns a handler that, for a given ServiceAccount, +// returns a list of reconcile requests for all ProxyGroups that use that ServiceAccount. +// For most ProxyGroups, this will be a dedicated ServiceAccount owned by a specific +// ProxyGroup. But for kube-apiserver ProxyGroups running in auth mode, they use a shared +// static ServiceAccount named "kube-apiserver-auth-proxy". +func serviceAccountHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + pgList := new(tsapi.ProxyGroupList) + if err := cl.List(ctx, pgList); err != nil { + logger.Debugf("error listing ProxyGroups for ServiceAccount: %v", err) + return nil + } + reqs := make([]reconcile.Request, 0) + saName := o.GetName() + for _, pg := range pgList.Items { + if saName == authAPIServerProxySAName && isAuthAPIServerProxy(&pg) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + } + expectedOwner := pgOwnerReference(&pg)[0] + saOwnerRefs := o.GetOwnerReferences() + for _, ref := range saOwnerRefs { + if apiequality.Semantic.DeepEqual(ref, expectedOwner) { + reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)}) + break + } + } + } + return reqs + } +} + // serviceHandlerForIngress returns a handler for Service events for ingress // reconciler that ensures that if the Service associated with an event is of // interest to the reconciler, the associated Ingress(es) gets be reconciled. // The Services of interest are backend Services for tailscale Ingress and // managed Services for an StatefulSet for a proxy configured for tailscale // Ingress -func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { +func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger, ingressClassName string) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { if isManagedByType(o, "ingress") { ingName := parentFromObjectLabels(o) @@ -880,7 +1107,7 @@ func serviceHandlerForIngress(cl client.Client, logger *zap.SugaredLogger) handl } reqs := make([]reconcile.Request, 0) for _, ing := range ingList.Items { - if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName { + if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != ingressClassName { return nil } if hasProxyGroupAnnotation(&ing) { @@ -1013,7 +1240,7 @@ func egressEpsFromPGStateSecrets(cl client.Client, ns string) handler.MapFunc { if parentType := o.GetLabels()[LabelParentType]; parentType != "proxygroup" { return nil } - if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != "state" { + if secretType := o.GetLabels()[kubetypes.LabelSecretType]; secretType != kubetypes.LabelSecretTypeState { return nil } pg, ok := o.GetLabels()[LabelParentName] @@ -1103,7 +1330,7 @@ func reconcileRequestsForPG(pg string, cl client.Client, ns string) []reconcile. func isTLSSecret(secret *corev1.Secret) bool { return secret.Type == corev1.SecretTypeTLS && secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" && - secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == "certs" && + secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeCerts && secret.ObjectMeta.Labels[labelDomain] != "" && secret.ObjectMeta.Labels[labelProxyGroup] != "" } @@ -1111,7 +1338,7 @@ func isTLSSecret(secret *corev1.Secret) bool { func isPGStateSecret(secret *corev1.Secret) bool { return secret.ObjectMeta.Labels[kubetypes.LabelManaged] == "true" && secret.ObjectMeta.Labels[LabelParentType] == "proxygroup" && - secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == "state" + secret.ObjectMeta.Labels[kubetypes.LabelSecretType] == kubetypes.LabelSecretTypeState } // HAIngressesFromSecret returns a handler that returns reconcile requests for @@ -1193,6 +1420,42 @@ func HAServicesFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.M } } +// kubeAPIServerPGsFromSecret finds ProxyGroups of type "kube-apiserver" that +// need to be reconciled after a ProxyGroup-owned Secret is updated. +func kubeAPIServerPGsFromSecret(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + secret, ok := o.(*corev1.Secret) + if !ok { + logger.Infof("[unexpected] Secret handler triggered for an object that is not a Secret") + return nil + } + if secret.ObjectMeta.Labels[kubetypes.LabelManaged] != "true" || + secret.ObjectMeta.Labels[LabelParentType] != "proxygroup" { + return nil + } + + var pg tsapi.ProxyGroup + if err := cl.Get(ctx, types.NamespacedName{Name: secret.ObjectMeta.Labels[LabelParentName]}, &pg); err != nil { + logger.Infof("error getting ProxyGroup %s: %v", secret.ObjectMeta.Labels[LabelParentName], err) + return nil + } + + if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer { + return nil + } + + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: secret.ObjectMeta.Labels[LabelParentNamespace], + Name: secret.ObjectMeta.Labels[LabelParentName], + }, + }, + } + + } +} + // egressSvcsFromEgressProxyGroup is an event handler for egress ProxyGroups. It returns reconcile requests for all // user-created ExternalName Services that should be exposed on this ProxyGroup. func egressSvcsFromEgressProxyGroup(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { @@ -1385,7 +1648,7 @@ func indexPGIngresses(o client.Object) []string { // serviceHandlerForIngressPG returns a handler for Service events that ensures that if the Service // associated with an event is a backend Service for a tailscale Ingress with ProxyGroup annotation, // the associated Ingress gets reconciled. -func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) handler.MapFunc { +func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger, ingressClassName string) handler.MapFunc { return func(ctx context.Context, o client.Object) []reconcile.Request { ingList := networkingv1.IngressList{} if err := cl.List(ctx, &ingList, client.InNamespace(o.GetNamespace())); err != nil { @@ -1394,7 +1657,7 @@ func serviceHandlerForIngressPG(cl client.Client, logger *zap.SugaredLogger) han } reqs := make([]reconcile.Request, 0) for _, ing := range ingList.Items { - if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != tailscaleIngressClassName { + if ing.Spec.IngressClassName == nil || *ing.Spec.IngressClassName != ingressClassName { continue } if !hasProxyGroupAnnotation(&ing) { @@ -1422,6 +1685,10 @@ func hasProxyGroupAnnotation(obj client.Object) bool { return obj.GetAnnotations()[AnnotationProxyGroup] != "" } +func hasProxyClassAnnotation(obj client.Object) bool { + return obj.GetAnnotations()[LabelAnnotationProxyClass] != "" +} + func id(ctx context.Context, lc *local.Client) (string, error) { st, err := lc.StatusWithoutPeers(ctx) if err != nil { diff --git a/cmd/k8s-operator/operator_test.go b/cmd/k8s-operator/operator_test.go index f4b0db01cf108..1f700f13a4fc0 100644 --- a/cmd/k8s-operator/operator_test.go +++ b/cmd/k8s-operator/operator_test.go @@ -7,6 +7,7 @@ package main import ( "context" + "encoding/json" "fmt" "testing" "time" @@ -20,8 +21,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/k8s-operator/apis/v1alpha1" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" "tailscale.com/net/dns/resolvconffile" @@ -130,7 +133,7 @@ func TestLoadBalancerClass(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) want.Annotations = nil want.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"} @@ -268,7 +271,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -291,7 +294,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) { expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Change the tailscale-target-fqdn annotation which should update the // StatefulSet @@ -380,7 +383,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -403,7 +406,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) { expectEqual(t, fc, want) expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Change the tailscale-target-ip annotation which should update the // StatefulSet @@ -631,7 +634,7 @@ func TestAnnotations(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -737,7 +740,7 @@ func TestAnnotationIntoLB(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, since it would have normally happened at @@ -781,7 +784,7 @@ func TestAnnotationIntoLB(t *testing.T) { expectReconciled(t, sr, "default", "test") // None of the proxy machinery should have changed... expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // ... but the service should have a LoadBalancer status. want = &corev1.Service{ @@ -867,7 +870,7 @@ func TestLBIntoAnnotation(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) // Normally the Tailscale proxy pod would come up here and write its info // into the secret. Simulate that, then verify reconcile again and verify @@ -927,7 +930,7 @@ func TestLBIntoAnnotation(t *testing.T) { expectReconciled(t, sr, "default", "test") expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want = &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -1007,7 +1010,7 @@ func TestCustomHostname(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, o)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) want := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -1118,7 +1121,183 @@ func TestCustomPriorityClassName(t *testing.T) { app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) +} + +func TestServiceProxyClassAnnotation(t *testing.T) { + cl := tstest.NewClock(tstest.ClockOpts{}) + zl := zap.Must(zap.NewDevelopment()) + + pcIfNotPresent := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "if-not-present", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &v1alpha1.Container{ + ImagePullPolicy: corev1.PullIfNotPresent, + }, + }, + }, + }, + } + + pcAlways := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "always", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{ + TailscaleContainer: &v1alpha1.Container{ + ImagePullPolicy: corev1.PullAlways, + }, + }, + }, + }, + } + + builder := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme) + builder = builder.WithObjects(pcIfNotPresent, pcAlways). + WithStatusSubresource(pcIfNotPresent, pcAlways) + fc := builder.Build() + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + // The apiserver is supposed to set the UID, but the fake client + // doesn't. So, set it explicitly because other code later depends + // on it being set. + UID: types.UID("1234-UID"), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.20.30.40", + Type: corev1.ServiceTypeLoadBalancer, + }, + } + + mustCreate(t, fc, svc) + + testCases := []struct { + name string + proxyClassAnnotation string + proxyClassLabel string + proxyClassDefault string + expectedProxyClass string + expectEvents []string + }{ + { + name: "via_label", + proxyClassLabel: pcIfNotPresent.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + { + name: "via_annotation", + proxyClassAnnotation: pcIfNotPresent.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + { + name: "via_default", + proxyClassDefault: pcIfNotPresent.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + { + name: "via_label_override_annotation", + proxyClassLabel: pcIfNotPresent.Name, + proxyClassAnnotation: pcAlways.Name, + expectedProxyClass: pcIfNotPresent.Name, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + ft := &fakeTSClient{} + + if tt.proxyClassAnnotation != "" || tt.proxyClassLabel != "" || tt.proxyClassDefault != "" { + name := tt.proxyClassDefault + if name == "" { + name = tt.proxyClassLabel + if name == "" { + name = tt.proxyClassAnnotation + } + } + setProxyClassReady(t, fc, cl, name) + } + + sr := &ServiceReconciler{ + Client: fc, + ssr: &tailscaleSTSReconciler{ + Client: fc, + tsClient: ft, + defaultTags: []string{"tag:k8s"}, + operatorNamespace: "operator-ns", + proxyImage: "tailscale/tailscale", + }, + defaultProxyClass: tt.proxyClassDefault, + logger: zl.Sugar(), + clock: cl, + isDefaultLoadBalancer: true, + } + + if tt.proxyClassLabel != "" { + svc.Labels = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassLabel, + } + } + if tt.proxyClassAnnotation != "" { + svc.Annotations = map[string]string{ + LabelAnnotationProxyClass: tt.proxyClassAnnotation, + } + } + + mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) { + s.Labels = svc.Labels + s.Annotations = svc.Annotations + }) + + expectReconciled(t, sr, "default", "test") + + list := &corev1.ServiceList{} + fc.List(context.Background(), list, client.InNamespace("default")) + + for _, i := range list.Items { + t.Logf("found service %s", i.Name) + } + + slist := &corev1.SecretList{} + fc.List(context.Background(), slist, client.InNamespace("operator-ns")) + for _, i := range slist.Items { + l, _ := json.Marshal(i.Labels) + t.Logf("found secret %q with labels %q ", i.Name, string(l)) + } + + _, shortName := findGenName(t, fc, "default", "test", "svc") + sts := &appsv1.StatefulSet{} + if err := fc.Get(context.Background(), client.ObjectKey{Namespace: "operator-ns", Name: shortName}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + switch tt.expectedProxyClass { + case pcIfNotPresent.Name: + for _, cont := range sts.Spec.Template.Spec.Containers { + if cont.Name == "tailscale" && cont.ImagePullPolicy != corev1.PullIfNotPresent { + t.Fatalf("ImagePullPolicy %q does not match ProxyClass %q with value %q", cont.ImagePullPolicy, pcIfNotPresent.Name, pcIfNotPresent.Spec.StatefulSet.Pod.TailscaleContainer.ImagePullPolicy) + } + } + case pcAlways.Name: + for _, cont := range sts.Spec.Template.Spec.Containers { + if cont.Name == "tailscale" && cont.ImagePullPolicy != corev1.PullAlways { + t.Fatalf("ImagePullPolicy %q does not match ProxyClass %q with value %q", cont.ImagePullPolicy, pcAlways.Name, pcAlways.Spec.StatefulSet.Pod.TailscaleContainer.ImagePullPolicy) + } + } + default: + t.Fatalf("unexpected expected ProxyClass %q", tt.expectedProxyClass) + } + }) + } } func TestProxyClassForService(t *testing.T) { @@ -1132,7 +1311,9 @@ func TestProxyClassForService(t *testing.T) { StatefulSet: &tsapi.StatefulSet{ Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"bar.io/foo": "some-val"}, - Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}}}, + Pod: &tsapi.Pod{Annotations: map[string]string{"foo.io/bar": "some-val"}}, + }, + }, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). @@ -1188,16 +1369,16 @@ func TestProxyClassForService(t *testing.T) { } expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. The Service gets updated with tailscale.com/proxy-class label // pointing at the 'custom-metadata' ProxyClass. The ProxyClass is not // yet ready, so no changes are actually applied to the proxy resources. mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) { - mak.Set(&svc.Labels, LabelProxyClass, "custom-metadata") + mak.Set(&svc.Labels, LabelAnnotationProxyClass, "custom-metadata") }) expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) expectEqual(t, fc, expectedSecret(t, fc, opts)) // 3. ProxyClass is set to Ready, the Service gets reconciled by the @@ -1209,22 +1390,23 @@ func TestProxyClassForService(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: pc.Generation, - }}} + }}, + } }) opts.proxyClass = pc.Name expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) expectEqual(t, fc, expectedSecret(t, fc, opts), removeAuthKeyIfExistsModifier(t)) // 4. tailscale.com/proxy-class label is removed from the Service, the // configuration from the ProxyClass is removed from the cluster // resources. mustUpdate(t, fc, "default", "test", func(svc *corev1.Service) { - delete(svc.Labels, LabelProxyClass) + delete(svc.Labels, LabelAnnotationProxyClass) }) opts.proxyClass = "" expectReconciled(t, sr, "default", "test") - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) } func TestDefaultLoadBalancer(t *testing.T) { @@ -1280,7 +1462,7 @@ func TestDefaultLoadBalancer(t *testing.T) { clusterTargetIP: "10.20.30.40", app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) } func TestProxyFirewallMode(t *testing.T) { @@ -1336,7 +1518,7 @@ func TestProxyFirewallMode(t *testing.T) { clusterTargetIP: "10.20.30.40", app: kubetypes.AppIngressProxy, } - expectEqual(t, fc, expectedSTS(t, fc, o), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, o), removeResourceReqs) } func Test_isMagicDNSName(t *testing.T) { @@ -1367,6 +1549,8 @@ func Test_isMagicDNSName(t *testing.T) { } func Test_serviceHandlerForIngress(t *testing.T) { + const tailscaleIngressClassName = "tailscale" + fc := fake.NewFakeClient() zl, err := zap.NewDevelopment() if err != nil { @@ -1396,7 +1580,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { } mustCreate(t, fc, svc1) wantReqs := []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-1", Name: "ing-1"}}} - gotReqs := serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), svc1) + gotReqs := serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), svc1) if diff := cmp.Diff(gotReqs, wantReqs); diff != "" { t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff) } @@ -1423,7 +1607,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { } mustCreate(t, fc, backendSvc) wantReqs = []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-2", Name: "ing-2"}}} - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), backendSvc) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), backendSvc) if diff := cmp.Diff(gotReqs, wantReqs); diff != "" { t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff) } @@ -1439,7 +1623,8 @@ func Test_serviceHandlerForIngress(t *testing.T) { IngressClassName: ptr.To(tailscaleIngressClassName), Rules: []networkingv1.IngressRule{{IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{ Paths: []networkingv1.HTTPIngressPath{ - {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}}}, + {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "backend"}}}, + }, }}}}, }, }) @@ -1451,7 +1636,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { } mustCreate(t, fc, backendSvc2) wantReqs = []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: "ns-3", Name: "ing-3"}}} - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), backendSvc2) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), backendSvc2) if diff := cmp.Diff(gotReqs, wantReqs); diff != "" { t.Fatalf("unexpected reconcile requests (-got +want):\n%s", diff) } @@ -1466,7 +1651,8 @@ func Test_serviceHandlerForIngress(t *testing.T) { Spec: networkingv1.IngressSpec{ Rules: []networkingv1.IngressRule{{IngressRuleValue: networkingv1.IngressRuleValue{HTTP: &networkingv1.HTTPIngressRuleValue{ Paths: []networkingv1.HTTPIngressPath{ - {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "non-ts-backend"}}}}, + {Backend: networkingv1.IngressBackend{Service: &networkingv1.IngressServiceBackend{Name: "non-ts-backend"}}}, + }, }}}}, }, }) @@ -1477,7 +1663,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { }, } mustCreate(t, fc, nonTSBackend) - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), nonTSBackend) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), nonTSBackend) if len(gotReqs) > 0 { t.Errorf("unexpected reconcile request for a Service that does not belong to a Tailscale Ingress: %#+v\n", gotReqs) } @@ -1491,7 +1677,7 @@ func Test_serviceHandlerForIngress(t *testing.T) { }, } mustCreate(t, fc, someSvc) - gotReqs = serviceHandlerForIngress(fc, zl.Sugar())(context.Background(), someSvc) + gotReqs = serviceHandlerForIngress(fc, zl.Sugar(), tailscaleIngressClassName)(context.Background(), someSvc) if len(gotReqs) > 0 { t.Errorf("unexpected reconcile request for a Service that does not belong to any Ingress: %#+v\n", gotReqs) } @@ -1565,6 +1751,7 @@ func Test_clusterDomainFromResolverConf(t *testing.T) { }) } } + func Test_authKeyRemoval(t *testing.T) { fc := fake.NewFakeClient() ft := &fakeTSClient{} @@ -1617,7 +1804,7 @@ func Test_authKeyRemoval(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. Apply update to the Secret that imitates the proxy setting device_id. s := expectedSecret(t, fc, opts) @@ -1691,7 +1878,7 @@ func Test_externalNameService(t *testing.T) { expectEqual(t, fc, expectedSecret(t, fc, opts)) expectEqual(t, fc, expectedHeadlessService(shortName, "svc")) - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) // 2. Change the ExternalName and verify that changes get propagated. mustUpdate(t, sr, "default", "test", func(s *corev1.Service) { @@ -1699,7 +1886,7 @@ func Test_externalNameService(t *testing.T) { }) expectReconciled(t, sr, "default", "test") opts.clusterTargetDNS = "bar.com" - expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation, removeResourceReqs) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs) } func Test_metricsResourceCreation(t *testing.T) { @@ -1711,14 +1898,15 @@ func Test_metricsResourceCreation(t *testing.T) { Status: metav1.ConditionTrue, Type: string(tsapi.ProxyClassReady), ObservedGeneration: 1, - }}}, + }}, + }, } svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", UID: types.UID("1234-UID"), - Labels: map[string]string{LabelProxyClass: "metrics"}, + Labels: map[string]string{LabelAnnotationProxyClass: "metrics"}, }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", @@ -1804,7 +1992,7 @@ func Test_metricsResourceCreation(t *testing.T) { func TestIgnorePGService(t *testing.T) { // NOTE: creating proxygroup stuff just to be sure that it's all ignored - _, _, fc, _ := setupServiceTest(t) + _, _, fc, _, _ := setupServiceTest(t) ft := &fakeTSClient{} zl, err := zap.NewDevelopment() diff --git a/cmd/k8s-operator/proxyclass.go b/cmd/k8s-operator/proxyclass.go index 5ec9897d0a8b7..2d51b351d3907 100644 --- a/cmd/k8s-operator/proxyclass.go +++ b/cmd/k8s-operator/proxyclass.go @@ -44,22 +44,24 @@ const ( type ProxyClassReconciler struct { client.Client - recorder record.EventRecorder - logger *zap.SugaredLogger - clock tstime.Clock + recorder record.EventRecorder + logger *zap.SugaredLogger + clock tstime.Clock + tsNamespace string mu sync.Mutex // protects following // managedProxyClasses is a set of all ProxyClass resources that we're currently // managing. This is only used for metrics. managedProxyClasses set.Slice[types.UID] + // nodePortRange is the NodePort range set for the Kubernetes Cluster. This is used + // when validating port ranges configured by users for spec.StaticEndpoints + nodePortRange *tsapi.PortRange } -var ( - // gaugeProxyClassResources tracks the number of ProxyClass resources - // that we're currently managing. - gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources") -) +// gaugeProxyClassResources tracks the number of ProxyClass resources +// that we're currently managing. +var gaugeProxyClassResources = clientmetric.NewGauge("k8s_proxyclass_resources") func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { logger := pcr.logger.With("ProxyClass", req.Name) @@ -96,7 +98,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re pcr.mu.Unlock() oldPCStatus := pc.Status.DeepCopy() - if errs := pcr.validate(ctx, pc); errs != nil { + if errs := pcr.validate(ctx, pc, logger); errs != nil { msg := fmt.Sprintf(messageProxyClassInvalid, errs.ToAggregate().Error()) pcr.recorder.Event(pc, corev1.EventTypeWarning, reasonProxyClassInvalid, msg) tsoperator.SetProxyClassCondition(pc, tsapi.ProxyClassReady, metav1.ConditionFalse, reasonProxyClassInvalid, msg, pc.Generation, pcr.clock, logger) @@ -112,7 +114,7 @@ func (pcr *ProxyClassReconciler) Reconcile(ctx context.Context, req reconcile.Re return reconcile.Result{}, nil } -func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass) (violations field.ErrorList) { +func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) (violations field.ErrorList) { if sts := pc.Spec.StatefulSet; sts != nil { if len(sts.Labels) > 0 { if errs := metavalidation.ValidateLabels(sts.Labels.Parse(), field.NewPath(".spec.statefulSet.labels")); errs != nil { @@ -183,6 +185,17 @@ func (pcr *ProxyClassReconciler) validate(ctx context.Context, pc *tsapi.ProxyCl violations = append(violations, errs...) } } + + if stat := pc.Spec.StaticEndpoints; stat != nil { + if err := validateNodePortRanges(ctx, pcr.Client, pcr.nodePortRange, pc); err != nil { + var prs tsapi.PortRanges = stat.NodePort.Ports + violations = append(violations, field.TypeInvalid(field.NewPath("spec", "staticEndpoints", "nodePort", "ports"), prs.String(), err.Error())) + } + + if len(stat.NodePort.Selector) < 1 { + logger.Debug("no Selectors specified on `spec.staticEndpoints.nodePort.selectors` field") + } + } // We do not validate embedded fields (security context, resource // requirements etc) as we inherit upstream validation for those fields. // Invalid values would get rejected by upstream validations at apply diff --git a/cmd/k8s-operator/proxyclass_test.go b/cmd/k8s-operator/proxyclass_test.go index 48290eea782b5..ae0f63d99ea4d 100644 --- a/cmd/k8s-operator/proxyclass_test.go +++ b/cmd/k8s-operator/proxyclass_test.go @@ -131,9 +131,11 @@ func TestProxyClass(t *testing.T) { proxyClass.Spec.StatefulSet.Pod.TailscaleInitContainer.Image = pc.Spec.StatefulSet.Pod.TailscaleInitContainer.Image proxyClass.Spec.StatefulSet.Pod.TailscaleContainer.Env = []tsapi.Env{{Name: "TS_USERSPACE", Value: "true"}, {Name: "EXPERIMENTAL_TS_CONFIGFILE_PATH"}, {Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS"}} }) - expectedEvents := []string{"Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", + expectedEvents := []string{ + "Warning CustomTSEnvVar ProxyClass overrides the default value for TS_USERSPACE env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", "Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_TS_CONFIGFILE_PATH env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", - "Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future."} + "Warning CustomTSEnvVar ProxyClass overrides the default value for EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS env var for tailscale container. Running with custom values for Tailscale env vars is not recommended and might break in the future.", + } expectReconciled(t, pcr, "", "test") expectEvents(t, fr, expectedEvents) @@ -176,6 +178,110 @@ func TestProxyClass(t *testing.T) { expectEqual(t, fc, pc) } +func TestValidateProxyClassStaticEndpoints(t *testing.T) { + for name, tc := range map[string]struct { + staticEndpointConfig *tsapi.StaticEndpointsConfig + valid bool + }{ + "no_static_endpoints": { + staticEndpointConfig: nil, + valid: true, + }, + "valid_specific_ports": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: true, + }, + "valid_port_ranges": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3000, EndPort: 3002}, + {Port: 3005, EndPort: 3007}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: true, + }, + "overlapping_port_ranges": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 1000, EndPort: 2000}, + {Port: 1500, EndPort: 1800}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: false, + }, + "clashing_port_and_range": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3005}, + {Port: 3001, EndPort: 3010}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: false, + }, + "malformed_port_range": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001, EndPort: 3000}, + }, + Selector: map[string]string{"kubernetes.io/hostname": "foobar"}, + }, + }, + valid: false, + }, + "empty_selector": { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{{Port: 3000}}, + Selector: map[string]string{}, + }, + }, + valid: true, + }, + } { + t.Run(name, func(t *testing.T) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + Build() + zl, _ := zap.NewDevelopment() + pcr := &ProxyClassReconciler{ + logger: zl.Sugar(), + Client: fc, + } + + pc := &tsapi.ProxyClass{ + Spec: tsapi.ProxyClassSpec{ + StaticEndpoints: tc.staticEndpointConfig, + }, + } + + logger := pcr.logger.With("ProxyClass", pc) + err := pcr.validate(context.Background(), pc, logger) + valid := err == nil + if valid != tc.valid { + t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err) + } + }) + } +} + func TestValidateProxyClass(t *testing.T) { for name, tc := range map[string]struct { pc *tsapi.ProxyClass @@ -219,8 +325,12 @@ func TestValidateProxyClass(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - pcr := &ProxyClassReconciler{} - err := pcr.validate(context.Background(), tc.pc) + zl, _ := zap.NewDevelopment() + pcr := &ProxyClassReconciler{ + logger: zl.Sugar(), + } + logger := pcr.logger.With("ProxyClass", tc.pc) + err := pcr.validate(context.Background(), tc.pc, logger) valid := err == nil if valid != tc.valid { t.Errorf("expected valid=%v, got valid=%v, err=%v", tc.valid, valid, err) diff --git a/cmd/k8s-operator/proxygroup.go b/cmd/k8s-operator/proxygroup.go index f263829d73963..debeb5c6b3442 100644 --- a/cmd/k8s-operator/proxygroup.go +++ b/cmd/k8s-operator/proxygroup.go @@ -7,15 +7,17 @@ package main import ( "context" - "crypto/sha256" "encoding/json" + "errors" "fmt" "net/http" + "net/netip" "slices" + "sort" "strings" "sync" - "github.com/pkg/errors" + dockerref "github.com/distribution/reference" "go.uber.org/zap" xslices "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" @@ -25,17 +27,21 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "tailscale.com/client/tailscale" "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" + "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" "tailscale.com/tstime" + "tailscale.com/types/opt" "tailscale.com/types/ptr" "tailscale.com/util/clientmetric" "tailscale.com/util/mak" @@ -45,16 +51,30 @@ import ( const ( reasonProxyGroupCreationFailed = "ProxyGroupCreationFailed" reasonProxyGroupReady = "ProxyGroupReady" + reasonProxyGroupAvailable = "ProxyGroupAvailable" reasonProxyGroupCreating = "ProxyGroupCreating" reasonProxyGroupInvalid = "ProxyGroupInvalid" // Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c - optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" + optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" + staticEndpointsMaxAddrs = 2 + + // The minimum tailcfg.CapabilityVersion that deployed clients are expected + // to support to be compatible with the current ProxyGroup controller. + // If the controller needs to depend on newer client behaviour, it should + // maintain backwards compatible logic for older capability versions for 3 + // stable releases, as per documentation on supported version drift: + // https://tailscale.com/kb/1236/kubernetes-operator#supported-versions + // + // tailcfg.CurrentCapabilityVersion was 106 when the ProxyGroup controller was + // first introduced. + pgMinCapabilityVersion = 106 ) var ( - gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount) - gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount) + gaugeEgressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupEgressCount) + gaugeIngressProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupIngressCount) + gaugeAPIServerProxyGroupResources = clientmetric.NewGauge(kubetypes.MetricProxyGroupAPIServerCount) ) // ProxyGroupReconciler ensures cluster resources for a ProxyGroup definition. @@ -67,14 +87,17 @@ type ProxyGroupReconciler struct { // User-specified defaults from the helm installation. tsNamespace string - proxyImage string + tsProxyImage string + k8sProxyImage string defaultTags []string tsFirewallMode string defaultProxyClass string + loginServer string - mu sync.Mutex // protects following - egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge - ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge + mu sync.Mutex // protects following + egressProxyGroups set.Slice[types.UID] // for egress proxygroups gauge + ingressProxyGroups set.Slice[types.UID] // for ingress proxygroups gauge + apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge } func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger { @@ -103,6 +126,10 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } if done, err := r.maybeCleanup(ctx, pg); err != nil { + if strings.Contains(err.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error, retrying: %s", err) + return reconcile.Result{}, nil + } return reconcile.Result{}, err } else if !done { logger.Debugf("ProxyGroup resource cleanup not yet finished, will retry...") @@ -117,17 +144,15 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ } oldPGStatus := pg.Status.DeepCopy() - setStatusReady := func(pg *tsapi.ProxyGroup, status metav1.ConditionStatus, reason, message string) (reconcile.Result, error) { - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger) - if !apiequality.Semantic.DeepEqual(oldPGStatus, &pg.Status) { - // An error encountered here should get returned by the Reconcile function. - if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { - err = errors.Wrap(err, updateErr.Error()) - } - } - return reconcile.Result{}, err - } + staticEndpoints, nrr, err := r.reconcilePG(ctx, pg, logger) + return reconcile.Result{}, errors.Join(err, r.maybeUpdateStatus(ctx, logger, pg, oldPGStatus, nrr, staticEndpoints)) +} +// reconcilePG handles all reconciliation of a ProxyGroup that is not marked +// for deletion. It is separated out from Reconcile to make a clear separation +// between reconciling the ProxyGroup, and posting the status of its created +// resources onto the ProxyGroup status field. +func (r *ProxyGroupReconciler) reconcilePG(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (map[string][]netip.AddrPort, *notReadyReason, error) { if !slices.Contains(pg.Finalizers, FinalizerName) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, @@ -135,18 +160,11 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ // operation is underway. logger.Infof("ensuring ProxyGroup is set up") pg.Finalizers = append(pg.Finalizers, FinalizerName) - if err = r.Update(ctx, pg); err != nil { - err = fmt.Errorf("error adding finalizer: %w", err) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, reasonProxyGroupCreationFailed) + if err := r.Update(ctx, pg); err != nil { + return r.notReadyErrf(pg, logger, "error adding finalizer: %w", err) } } - if err = r.validate(pg); err != nil { - message := fmt.Sprintf("ProxyGroup is invalid: %s", err) - r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupInvalid, message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupInvalid, message) - } - proxyClassName := r.defaultProxyClass if pg.Spec.ProxyClass != "" { proxyClassName = pg.Spec.ProxyClass @@ -157,60 +175,33 @@ func (r *ProxyGroupReconciler) Reconcile(ctx context.Context, req reconcile.Requ proxyClass = new(tsapi.ProxyClass) err := r.Get(ctx, types.NamespacedName{Name: proxyClassName}, proxyClass) if apierrors.IsNotFound(err) { - err = nil - message := fmt.Sprintf("the ProxyGroup's ProxyClass %s does not (yet) exist", proxyClassName) - logger.Info(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) + msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q does not (yet) exist", proxyClassName) + logger.Info(msg) + return notReady(reasonProxyGroupCreating, msg) } if err != nil { - err = fmt.Errorf("error getting ProxyGroup's ProxyClass %s: %s", proxyClassName, err) - r.recorder.Eventf(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error()) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreationFailed, err.Error()) + return r.notReadyErrf(pg, logger, "error getting ProxyGroup's ProxyClass %q: %w", proxyClassName, err) } - validateProxyClassForPG(logger, pg, proxyClass) if !tsoperator.ProxyClassIsReady(proxyClass) { - message := fmt.Sprintf("the ProxyGroup's ProxyClass %s is not yet in a ready state, waiting...", proxyClassName) - logger.Info(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) - } - } - - if err = r.maybeProvision(ctx, pg, proxyClass); err != nil { - reason := reasonProxyGroupCreationFailed - msg := fmt.Sprintf("error provisioning ProxyGroup resources: %s", err) - if strings.Contains(err.Error(), optimisticLockErrorMsg) { - reason = reasonProxyGroupCreating - msg = fmt.Sprintf("optimistic lock error, retrying: %s", err) - err = nil + msg := fmt.Sprintf("the ProxyGroup's ProxyClass %q is not yet in a ready state, waiting...", proxyClassName) logger.Info(msg) - } else { - r.recorder.Eventf(pg, corev1.EventTypeWarning, reason, msg) + return notReady(reasonProxyGroupCreating, msg) } - return setStatusReady(pg, metav1.ConditionFalse, reason, msg) } - desiredReplicas := int(pgReplicas(pg)) - if len(pg.Status.Devices) < desiredReplicas { - message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(pg.Status.Devices), desiredReplicas) - logger.Debug(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) + if err := r.validate(ctx, pg, proxyClass, logger); err != nil { + return notReady(reasonProxyGroupInvalid, fmt.Sprintf("invalid ProxyGroup spec: %v", err)) } - if len(pg.Status.Devices) > desiredReplicas { - message := fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(pg.Status.Devices)-desiredReplicas) - logger.Debug(message) - return setStatusReady(pg, metav1.ConditionFalse, reasonProxyGroupCreating, message) + staticEndpoints, nrr, err := r.maybeProvision(ctx, pg, proxyClass) + if err != nil { + return nil, nrr, err } - logger.Info("ProxyGroup resources synced") - return setStatusReady(pg, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady) + return staticEndpoints, nrr, nil } -// validateProxyClassForPG applies custom validation logic for ProxyClass applied to ProxyGroup. -func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) { - if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { - return - } +func (r *ProxyGroupReconciler) validate(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, logger *zap.SugaredLogger) error { // Our custom logic for ensuring minimum downtime ProxyGroup update rollouts relies on the local health check // beig accessible on the replica Pod IP:9002. This address can also be modified by users, via // TS_LOCAL_ADDR_PORT env var. @@ -222,25 +213,107 @@ func validateProxyClassForPG(logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, pc // shouldn't need to set their own). // // TODO(irbekrm): maybe disallow configuring this env var in future (in Tailscale 1.84 or later). - if hasLocalAddrPortSet(pc) { + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress && hasLocalAddrPortSet(pc) { msg := fmt.Sprintf("ProxyClass %s applied to an egress ProxyGroup has TS_LOCAL_ADDR_PORT env var set to a custom value."+ "This will disable the ProxyGroup graceful failover mechanism, so you might experience downtime when ProxyGroup pods are restarted."+ "In future we will remove the ability to set custom TS_LOCAL_ADDR_PORT for egress ProxyGroups."+ "Please raise an issue if you expect that this will cause issues for your workflow.", pc.Name) logger.Warn(msg) } + + // image is the value of pc.Spec.StatefulSet.Pod.TailscaleContainer.Image or "" + // imagePath is a slash-delimited path ending with the image name, e.g. + // "tailscale/tailscale" or maybe "k8s-proxy" if hosted at example.com/k8s-proxy. + var image, imagePath string + if pc != nil && + pc.Spec.StatefulSet != nil && + pc.Spec.StatefulSet.Pod != nil && + pc.Spec.StatefulSet.Pod.TailscaleContainer != nil && + pc.Spec.StatefulSet.Pod.TailscaleContainer.Image != "" { + image, err := dockerref.ParseNormalizedNamed(pc.Spec.StatefulSet.Pod.TailscaleContainer.Image) + if err != nil { + // Shouldn't be possible as the ProxyClass won't be marked ready + // without successfully parsing the image. + return fmt.Errorf("error parsing %q as a container image reference: %w", pc.Spec.StatefulSet.Pod.TailscaleContainer.Image, err) + } + imagePath = dockerref.Path(image) + } + + var errs []error + if isAuthAPIServerProxy(pg) { + // Validate that the static ServiceAccount already exists. + sa := &corev1.ServiceAccount{} + if err := r.Get(ctx, types.NamespacedName{Namespace: r.tsNamespace, Name: authAPIServerProxySAName}, sa); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("error validating that ServiceAccount %q exists: %w", authAPIServerProxySAName, err) + } + + errs = append(errs, fmt.Errorf("the ServiceAccount %q used for the API server proxy in auth mode does not exist but "+ + "should have been created during operator installation; use apiServerProxyConfig.allowImpersonation=true "+ + "in the helm chart, or authproxy-rbac.yaml from the static manifests", authAPIServerProxySAName)) + } + } else { + // Validate that the ServiceAccount we create won't overwrite the static one. + // TODO(tomhjp): This doesn't cover other controllers that could create a + // ServiceAccount. Perhaps should have some guards to ensure that an update + // would never change the ownership of a resource we expect to already be owned. + if pgServiceAccountName(pg) == authAPIServerProxySAName { + errs = append(errs, fmt.Errorf("the name of the ProxyGroup %q conflicts with the static ServiceAccount used for the API server proxy in auth mode", pg.Name)) + } + } + + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + if strings.HasSuffix(imagePath, "tailscale") { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "k8s-proxy", pg.Spec.Type)) + } + + if pc != nil && pc.Spec.StatefulSet != nil && pc.Spec.StatefulSet.Pod != nil && pc.Spec.StatefulSet.Pod.TailscaleInitContainer != nil { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies Tailscale init container config, but ProxyGroups of type %q do not use init containers", pc.Name, pg.Spec.Type)) + } + } else { + if strings.HasSuffix(imagePath, "k8s-proxy") { + errs = append(errs, fmt.Errorf("the configured ProxyClass %q specifies to use image %q but expected a %q image for ProxyGroup of type %q", pc.Name, image, "tailscale", pg.Spec.Type)) + } + } + + return errors.Join(errs...) } -func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) error { +func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (map[string][]netip.AddrPort, *notReadyReason, error) { logger := r.logger(pg.Name) r.mu.Lock() r.ensureAddedToGaugeForProxyGroup(pg) r.mu.Unlock() - cfgHash, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass) + svcToNodePorts := make(map[string]uint16) + var tailscaledPort *uint16 + if proxyClass != nil && proxyClass.Spec.StaticEndpoints != nil { + var err error + svcToNodePorts, tailscaledPort, err = r.ensureNodePortServiceCreated(ctx, pg, proxyClass) + if err != nil { + var allocatePortErr *allocatePortsErr + if errors.As(err, &allocatePortErr) { + reason := reasonProxyGroupCreationFailed + msg := fmt.Sprintf("error provisioning NodePort Services for static endpoints: %v", err) + r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg) + return notReady(reason, msg) + } + return r.notReadyErrf(pg, logger, "error provisioning NodePort Services for static endpoints: %w", err) + } + } + + staticEndpoints, err := r.ensureConfigSecretsCreated(ctx, pg, proxyClass, svcToNodePorts) if err != nil { - return fmt.Errorf("error provisioning config Secrets: %w", err) + var selectorErr *FindStaticEndpointErr + if errors.As(err, &selectorErr) { + reason := reasonProxyGroupCreationFailed + msg := fmt.Sprintf("error provisioning config Secrets: %v", err) + r.recorder.Event(pg, corev1.EventTypeWarning, reason, msg) + return notReady(reason, msg) + } + return r.notReadyErrf(pg, logger, "error provisioning config Secrets: %w", err) } + // State secrets are precreated so we can use the ProxyGroup CR as their owner ref. stateSecrets := pgStateSecrets(pg, r.tsNamespace) for _, sec := range stateSecrets { @@ -249,17 +322,24 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro s.ObjectMeta.Annotations = sec.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = sec.ObjectMeta.OwnerReferences }); err != nil { - return fmt.Errorf("error provisioning state Secrets: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning state Secrets: %w", err) } } - sa := pgServiceAccount(pg, r.tsNamespace) - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) { - s.ObjectMeta.Labels = sa.ObjectMeta.Labels - s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations - s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences - }); err != nil { - return fmt.Errorf("error provisioning ServiceAccount: %w", err) + + // auth mode kube-apiserver ProxyGroups use a statically created + // ServiceAccount to keep ClusterRole creation permissions limited to the + // helm chart installer. + if !isAuthAPIServerProxy(pg) { + sa := pgServiceAccount(pg, r.tsNamespace) + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, sa, func(s *corev1.ServiceAccount) { + s.ObjectMeta.Labels = sa.ObjectMeta.Labels + s.ObjectMeta.Annotations = sa.ObjectMeta.Annotations + s.ObjectMeta.OwnerReferences = sa.ObjectMeta.OwnerReferences + }); err != nil { + return r.notReadyErrf(pg, logger, "error provisioning ServiceAccount: %w", err) + } } + role := pgRole(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) { r.ObjectMeta.Labels = role.ObjectMeta.Labels @@ -267,8 +347,9 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.ObjectMeta.OwnerReferences = role.ObjectMeta.OwnerReferences r.Rules = role.Rules }); err != nil { - return fmt.Errorf("error provisioning Role: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning Role: %w", err) } + roleBinding := pgRoleBinding(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, roleBinding, func(r *rbacv1.RoleBinding) { r.ObjectMeta.Labels = roleBinding.ObjectMeta.Labels @@ -277,8 +358,9 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro r.RoleRef = roleBinding.RoleRef r.Subjects = roleBinding.Subjects }); err != nil { - return fmt.Errorf("error provisioning RoleBinding: %w", err) + return r.notReadyErrf(pg, logger, "error provisioning RoleBinding: %w", err) } + if pg.Spec.Type == tsapi.ProxyGroupTypeEgress { cm, hp := pgEgressCM(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { @@ -286,61 +368,42 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences mak.Set(&existing.BinaryData, egressservices.KeyHEPPings, hp) }); err != nil { - return fmt.Errorf("error provisioning egress ConfigMap %q: %w", cm.Name, err) + return r.notReadyErrf(pg, logger, "error provisioning egress ConfigMap %q: %w", cm.Name, err) } } + if pg.Spec.Type == tsapi.ProxyGroupTypeIngress { cm := pgIngressCM(pg, r.tsNamespace) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, cm, func(existing *corev1.ConfigMap) { existing.ObjectMeta.Labels = cm.ObjectMeta.Labels existing.ObjectMeta.OwnerReferences = cm.ObjectMeta.OwnerReferences }); err != nil { - return fmt.Errorf("error provisioning ingress ConfigMap %q: %w", cm.Name, err) + return r.notReadyErrf(pg, logger, "error provisioning ingress ConfigMap %q: %w", cm.Name, err) } } - ss, err := pgStatefulSet(pg, r.tsNamespace, r.proxyImage, r.tsFirewallMode, proxyClass) + + defaultImage := r.tsProxyImage + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + defaultImage = r.k8sProxyImage + } + ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass) if err != nil { - return fmt.Errorf("error generating StatefulSet spec: %w", err) + return r.notReadyErrf(pg, logger, "error generating StatefulSet spec: %w", err) } cfg := &tailscaleSTSConfig{ proxyType: string(pg.Spec.Type), } ss = applyProxyClassToStatefulSet(proxyClass, ss, cfg, logger) - capver, err := r.capVerForPG(ctx, pg, logger) - if err != nil { - return fmt.Errorf("error getting device info: %w", err) - } - - updateSS := func(s *appsv1.StatefulSet) { - - // This is a temporary workaround to ensure that egress ProxyGroup proxies with capver older than 110 - // are restarted when tailscaled configfile contents have changed. - // This workaround ensures that: - // 1. The hash mechanism is used to trigger pod restarts for proxies below capver 110. - // 2. Proxies above capver are not unnecessarily restarted when the configfile contents change. - // 3. If the hash has alreay been set, but the capver is above 110, the old hash is preserved to avoid - // unnecessary pod restarts that could result in an update loop where capver cannot be determined for a - // restarting Pod and the hash is re-added again. - // Note that this workaround is only applied to egress ProxyGroups, because ingress ProxyGroup was added after capver 110. - // Note also that the hash annotation is only set on updates, not creation, because if the StatefulSet is - // being created, there is no need for a restart. - // TODO(irbekrm): remove this in 1.84. - hash := cfgHash - if capver >= 110 { - hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash] - } - s.Spec = ss.Spec - if hash != "" && pg.Spec.Type == tsapi.ProxyGroupTypeEgress { - mak.Set(&s.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, hash) - } + if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { + s.Spec = ss.Spec s.ObjectMeta.Labels = ss.ObjectMeta.Labels s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations s.ObjectMeta.OwnerReferences = ss.ObjectMeta.OwnerReferences + }); err != nil { + return r.notReadyErrf(pg, logger, "error provisioning StatefulSet: %w", err) } - if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, updateSS); err != nil { - return fmt.Errorf("error provisioning StatefulSet: %w", err) - } + mo := &metricsOpts{ tsNamespace: r.tsNamespace, proxyStsName: pg.Name, @@ -348,26 +411,207 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, pg *tsapi.Pro proxyType: "proxygroup", } if err := reconcileMetricsResources(ctx, logger, mo, proxyClass, r.Client); err != nil { - return fmt.Errorf("error reconciling metrics resources: %w", err) + return r.notReadyErrf(pg, logger, "error reconciling metrics resources: %w", err) } - if err := r.cleanupDanglingResources(ctx, pg); err != nil { - return fmt.Errorf("error cleaning up dangling resources: %w", err) + if err := r.cleanupDanglingResources(ctx, pg, proxyClass); err != nil { + return r.notReadyErrf(pg, logger, "error cleaning up dangling resources: %w", err) } - devices, err := r.getDeviceInfo(ctx, pg) + logger.Info("ProxyGroup resources synced") + + return staticEndpoints, nil, nil +} + +func (r *ProxyGroupReconciler) maybeUpdateStatus(ctx context.Context, logger *zap.SugaredLogger, pg *tsapi.ProxyGroup, oldPGStatus *tsapi.ProxyGroupStatus, nrr *notReadyReason, endpoints map[string][]netip.AddrPort) (err error) { + defer func() { + if !apiequality.Semantic.DeepEqual(*oldPGStatus, pg.Status) { + if updateErr := r.Client.Status().Update(ctx, pg); updateErr != nil { + if strings.Contains(updateErr.Error(), optimisticLockErrorMsg) { + logger.Infof("optimistic lock error updating status, retrying: %s", updateErr) + updateErr = nil + } + err = errors.Join(err, updateErr) + } + } + }() + + devices, err := r.getRunningProxies(ctx, pg, endpoints) if err != nil { - return fmt.Errorf("failed to get device info: %w", err) + return fmt.Errorf("failed to list running proxies: %w", err) } pg.Status.Devices = devices + desiredReplicas := int(pgReplicas(pg)) + + // Set ProxyGroupAvailable condition. + status := metav1.ConditionFalse + reason := reasonProxyGroupCreating + message := fmt.Sprintf("%d/%d ProxyGroup pods running", len(devices), desiredReplicas) + if len(devices) > 0 { + status = metav1.ConditionTrue + if len(devices) == desiredReplicas { + reason = reasonProxyGroupAvailable + } + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, status, reason, message, 0, r.clock, logger) + + // Set ProxyGroupReady condition. + tsSvcValid, tsSvcSet := tsoperator.KubeAPIServerProxyValid(pg) + status = metav1.ConditionFalse + reason = reasonProxyGroupCreating + switch { + case nrr != nil: + // If we failed earlier, that reason takes precedence. + reason = nrr.reason + message = nrr.message + case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && tsSvcSet && !tsSvcValid: + reason = reasonProxyGroupInvalid + message = "waiting for config in spec.kubeAPIServer to be marked valid" + case len(devices) < desiredReplicas: + case len(devices) > desiredReplicas: + message = fmt.Sprintf("waiting for %d ProxyGroup pods to shut down", len(devices)-desiredReplicas) + case pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer && !tsoperator.KubeAPIServerProxyConfigured(pg): + reason = reasonProxyGroupCreating + message = "waiting for proxies to start advertising the kube-apiserver proxy's hostname" + default: + status = metav1.ConditionTrue + reason = reasonProxyGroupReady + message = reasonProxyGroupReady + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, status, reason, message, pg.Generation, r.clock, logger) + return nil } +// getServicePortsForProxyGroups returns a map of ProxyGroup Service names to their NodePorts, +// and a set of all allocated NodePorts for quick occupancy checking. +func getServicePortsForProxyGroups(ctx context.Context, c client.Client, namespace string, portRanges tsapi.PortRanges) (map[string]uint16, set.Set[uint16], error) { + svcs := new(corev1.ServiceList) + matchingLabels := client.MatchingLabels(map[string]string{ + LabelParentType: "proxygroup", + }) + + err := c.List(ctx, svcs, matchingLabels, client.InNamespace(namespace)) + if err != nil { + return nil, nil, fmt.Errorf("failed to list ProxyGroup Services: %w", err) + } + + svcToNodePorts := map[string]uint16{} + usedPorts := set.Set[uint16]{} + for _, svc := range svcs.Items { + if len(svc.Spec.Ports) == 1 && svc.Spec.Ports[0].NodePort != 0 { + p := uint16(svc.Spec.Ports[0].NodePort) + if portRanges.Contains(p) { + svcToNodePorts[svc.Name] = p + usedPorts.Add(p) + } + } + } + + return svcToNodePorts, usedPorts, nil +} + +type allocatePortsErr struct { + msg string +} + +func (e *allocatePortsErr) Error() string { + return e.msg +} + +func (r *ProxyGroupReconciler) allocatePorts(ctx context.Context, pg *tsapi.ProxyGroup, proxyClassName string, portRanges tsapi.PortRanges) (map[string]uint16, error) { + replicaCount := int(pgReplicas(pg)) + svcToNodePorts, usedPorts, err := getServicePortsForProxyGroups(ctx, r.Client, r.tsNamespace, portRanges) + if err != nil { + return nil, &allocatePortsErr{msg: fmt.Sprintf("failed to find ports for existing ProxyGroup NodePort Services: %s", err.Error())} + } + + replicasAllocated := 0 + for i := range pgReplicas(pg) { + if _, ok := svcToNodePorts[pgNodePortServiceName(pg.Name, i)]; !ok { + svcToNodePorts[pgNodePortServiceName(pg.Name, i)] = 0 + } else { + replicasAllocated++ + } + } + + for replica, port := range svcToNodePorts { + if port == 0 { + for p := range portRanges.All() { + if !usedPorts.Contains(p) { + svcToNodePorts[replica] = p + usedPorts.Add(p) + replicasAllocated++ + break + } + } + } + } + + if replicasAllocated < replicaCount { + return nil, &allocatePortsErr{msg: fmt.Sprintf("not enough available ports to allocate all replicas (needed %d, got %d). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass %q must have bigger range allocated", replicaCount, usedPorts.Len(), proxyClassName)} + } + + return svcToNodePorts, nil +} + +func (r *ProxyGroupReconciler) ensureNodePortServiceCreated(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) (map[string]uint16, *uint16, error) { + // NOTE: (ChaosInTheCRD) we want the same TargetPort for every static endpoint NodePort Service for the ProxyGroup + tailscaledPort := getRandomPort() + svcs := []*corev1.Service{} + for i := range pgReplicas(pg) { + nodePortSvcName := pgNodePortServiceName(pg.Name, i) + + svc := &corev1.Service{} + err := r.Get(ctx, types.NamespacedName{Name: nodePortSvcName, Namespace: r.tsNamespace}, svc) + if err != nil && !apierrors.IsNotFound(err) { + return nil, nil, fmt.Errorf("error getting Kubernetes Service %q: %w", nodePortSvcName, err) + } + if apierrors.IsNotFound(err) { + svcs = append(svcs, pgNodePortService(pg, nodePortSvcName, r.tsNamespace)) + } else { + // NOTE: if we can we want to recover the random port used for tailscaled, + // as well as the NodePort previously used for that Service + if len(svc.Spec.Ports) == 1 { + if svc.Spec.Ports[0].Port != 0 { + tailscaledPort = uint16(svc.Spec.Ports[0].Port) + } + } + svcs = append(svcs, svc) + } + } + + svcToNodePorts, err := r.allocatePorts(ctx, pg, pc.Name, pc.Spec.StaticEndpoints.NodePort.Ports) + if err != nil { + return nil, nil, fmt.Errorf("failed to allocate NodePorts to ProxyGroup Services: %w", err) + } + + for _, svc := range svcs { + // NOTE: we know that every service is going to have 1 port here + svc.Spec.Ports[0].Port = int32(tailscaledPort) + svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(tailscaledPort)) + svc.Spec.Ports[0].NodePort = int32(svcToNodePorts[svc.Name]) + + _, err = createOrUpdate(ctx, r.Client, r.tsNamespace, svc, func(s *corev1.Service) { + s.ObjectMeta.Labels = svc.ObjectMeta.Labels + s.ObjectMeta.Annotations = svc.ObjectMeta.Annotations + s.ObjectMeta.OwnerReferences = svc.ObjectMeta.OwnerReferences + s.Spec.Selector = svc.Spec.Selector + s.Spec.Ports = svc.Spec.Ports + }) + if err != nil { + return nil, nil, fmt.Errorf("error creating/updating Kubernetes NodePort Service %q: %w", svc.Name, err) + } + } + + return svcToNodePorts, ptr.To(tailscaledPort), nil +} + // cleanupDanglingResources ensures we don't leak config secrets, state secrets, and // tailnet devices when the number of replicas specified is reduced. -func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup) error { +func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass) error { logger := r.logger(pg.Name) metadata, err := r.getNodeMetadata(ctx, pg) if err != nil { @@ -384,20 +628,40 @@ func (r *ProxyGroupReconciler) cleanupDanglingResources(ctx context.Context, pg if err := r.deleteTailnetDevice(ctx, m.tsID, logger); err != nil { return err } - if err := r.Delete(ctx, m.stateSecret); err != nil { - if !apierrors.IsNotFound(err) { - return fmt.Errorf("error deleting state Secret %s: %w", m.stateSecret.Name, err) - } + if err := r.Delete(ctx, m.stateSecret); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting state Secret %q: %w", m.stateSecret.Name, err) } configSecret := m.stateSecret.DeepCopy() configSecret.Name += "-config" - if err := r.Delete(ctx, configSecret); err != nil { + if err := r.Delete(ctx, configSecret); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting config Secret %q: %w", configSecret.Name, err) + } + // NOTE(ChaosInTheCRD): we shouldn't need to get the service first, checking for a not found error should be enough + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-nodeport", m.stateSecret.Name), + Namespace: m.stateSecret.Namespace, + }, + } + if err := r.Delete(ctx, svc); err != nil { if !apierrors.IsNotFound(err) { - return fmt.Errorf("error deleting config Secret %s: %w", configSecret.Name, err) + return fmt.Errorf("error deleting static endpoints Kubernetes Service %q: %w", svc.Name, err) } } } + // If the ProxyClass has its StaticEndpoints config removed, we want to remove all of the NodePort Services + if pc != nil && pc.Spec.StaticEndpoints == nil { + labels := map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentType: proxyTypeProxyGroup, + LabelParentName: pg.Name, + } + if err := r.DeleteAllOf(ctx, &corev1.Service{}, client.InNamespace(r.tsNamespace), client.MatchingLabels(labels)); err != nil { + return fmt.Errorf("error deleting Kubernetes Services for static endpoints: %w", err) + } + } + return nil } @@ -421,7 +685,8 @@ func (r *ProxyGroupReconciler) maybeCleanup(ctx context.Context, pg *tsapi.Proxy mo := &metricsOpts{ proxyLabels: pgLabels(pg.Name, nil), tsNamespace: r.tsNamespace, - proxyType: "proxygroup"} + proxyType: "proxygroup", + } if err := maybeCleanupMetricsResources(ctx, mo, r.Client); err != nil { return false, fmt.Errorf("error cleaning up metrics resources: %w", err) } @@ -449,15 +714,15 @@ func (r *ProxyGroupReconciler) deleteTailnetDevice(ctx context.Context, id tailc return nil } -func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass) (hash string, err error) { +func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, pg *tsapi.ProxyGroup, proxyClass *tsapi.ProxyClass, svcToNodePorts map[string]uint16) (endpoints map[string][]netip.AddrPort, err error) { logger := r.logger(pg.Name) - var configSHA256Sum string + endpoints = make(map[string][]netip.AddrPort, pgReplicas(pg)) // keyed by Service name. for i := range pgReplicas(pg) { cfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName(pg.Name, i), Namespace: r.tsNamespace, - Labels: pgSecretLabels(pg.Name, "config"), + Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig), OwnerReferences: pgOwnerReference(pg), }, } @@ -467,77 +732,272 @@ func (r *ProxyGroupReconciler) ensureConfigSecretsCreated(ctx context.Context, p logger.Debugf("Secret %s/%s already exists", cfgSecret.GetNamespace(), cfgSecret.GetName()) existingCfgSecret = cfgSecret.DeepCopy() } else if !apierrors.IsNotFound(err) { - return "", err + return nil, err } - var authKey string + var authKey *string if existingCfgSecret == nil { logger.Debugf("Creating authkey for new ProxyGroup proxy") tags := pg.Spec.Tags.Stringify() if len(tags) == 0 { tags = r.defaultTags } - authKey, err = newAuthKey(ctx, r.tsClient, tags) + key, err := newAuthKey(ctx, r.tsClient, tags) if err != nil { - return "", err + return nil, err } + authKey = &key } - configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, existingCfgSecret) - if err != nil { - return "", fmt.Errorf("error creating tailscaled config: %w", err) + if authKey == nil { + // Get state Secret to check if it's already authed. + stateSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgStateSecretName(pg.Name, i), + Namespace: r.tsNamespace, + }, + } + if err := r.Get(ctx, client.ObjectKeyFromObject(stateSecret), stateSecret); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + + if shouldRetainAuthKey(stateSecret) && existingCfgSecret != nil { + authKey, err = authKeyFromSecret(existingCfgSecret) + if err != nil { + return nil, fmt.Errorf("error retrieving auth key from existing config Secret: %w", err) + } + } } - for cap, cfg := range configs { - cfgJSON, err := json.Marshal(cfg) + nodePortSvcName := pgNodePortServiceName(pg.Name, i) + if len(svcToNodePorts) > 0 { + replicaName := fmt.Sprintf("%s-%d", pg.Name, i) + port, ok := svcToNodePorts[nodePortSvcName] + if !ok { + return nil, fmt.Errorf("could not find configured NodePort for ProxyGroup replica %q", replicaName) + } + + endpoints[nodePortSvcName], err = r.findStaticEndpoints(ctx, existingCfgSecret, proxyClass, port, logger) if err != nil { - return "", fmt.Errorf("error marshalling tailscaled config: %w", err) - } - mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON) - } - - // The config sha256 sum is a value for a hash annotation used to trigger - // pod restarts when tailscaled config changes. Any config changes apply - // to all replicas, so it is sufficient to only hash the config for the - // first replica. - // - // In future, we're aiming to eliminate restarts altogether and have - // pods dynamically reload their config when it changes. - if i == 0 { - sum := sha256.New() - for _, cfg := range configs { - // Zero out the auth key so it doesn't affect the sha256 hash when we - // remove it from the config after the pods have all authed. Otherwise - // all the pods will need to restart immediately after authing. - cfg.AuthKey = nil - b, err := json.Marshal(cfg) - if err != nil { - return "", err + return nil, fmt.Errorf("could not find static endpoints for replica %q: %w", replicaName, err) + } + } + + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + hostname := pgHostname(pg, i) + + if authKey == nil && existingCfgSecret != nil { + deviceAuthed := false + for _, d := range pg.Status.Devices { + if d.Hostname == hostname { + deviceAuthed = true + break + } } - if _, err := sum.Write(b); err != nil { - return "", err + if !deviceAuthed { + existingCfg := conf.ConfigV1Alpha1{} + if err := json.Unmarshal(existingCfgSecret.Data[kubetypes.KubeAPIServerConfigFile], &existingCfg); err != nil { + return nil, fmt.Errorf("error unmarshalling existing config: %w", err) + } + if existingCfg.AuthKey != nil { + authKey = existingCfg.AuthKey + } } } - configSHA256Sum = fmt.Sprintf("%x", sum.Sum(nil)) + mode := kubetypes.APIServerProxyModeAuth + if !isAuthAPIServerProxy(pg) { + mode = kubetypes.APIServerProxyModeNoAuth + } + cfg := conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &conf.ConfigV1Alpha1{ + AuthKey: authKey, + State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, i))), + App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer), + LogLevel: ptr.To(logger.Level().String()), + + // Reloadable fields. + Hostname: &hostname, + APIServerProxy: &conf.APIServerProxyConfig{ + Enabled: opt.NewBool(true), + Mode: &mode, + // The first replica is elected as the cert issuer, same + // as containerboot does for ingress-pg-reconciler. + IssueCerts: opt.NewBool(i == 0), + }, + LocalPort: ptr.To(uint16(9002)), + HealthCheckEnabled: opt.NewBool(true), + }, + } + + // Copy over config that the apiserver-proxy-service-reconciler sets. + if existingCfgSecret != nil { + if k8sProxyCfg, ok := cfgSecret.Data[kubetypes.KubeAPIServerConfigFile]; ok { + k8sCfg := &conf.ConfigV1Alpha1{} + if err := json.Unmarshal(k8sProxyCfg, k8sCfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal kube-apiserver config: %w", err) + } + + cfg.AdvertiseServices = k8sCfg.AdvertiseServices + if k8sCfg.APIServerProxy != nil { + cfg.APIServerProxy.ServiceName = k8sCfg.APIServerProxy.ServiceName + } + } + } + + if r.loginServer != "" { + cfg.ServerURL = &r.loginServer + } + + if proxyClass != nil && proxyClass.Spec.TailscaleConfig != nil { + cfg.AcceptRoutes = opt.NewBool(proxyClass.Spec.TailscaleConfig.AcceptRoutes) + } + + if proxyClass != nil && proxyClass.Spec.Metrics != nil { + cfg.MetricsEnabled = opt.NewBool(proxyClass.Spec.Metrics.Enable) + } + + if len(endpoints[nodePortSvcName]) > 0 { + cfg.StaticEndpoints = endpoints[nodePortSvcName] + } + + cfgB, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("error marshalling k8s-proxy config: %w", err) + } + mak.Set(&cfgSecret.Data, kubetypes.KubeAPIServerConfigFile, cfgB) + } else { + // AdvertiseServices config is set by ingress-pg-reconciler, so make sure we + // don't overwrite it if already set. + existingAdvertiseServices, err := extractAdvertiseServicesConfig(existingCfgSecret) + if err != nil { + return nil, err + } + + configs, err := pgTailscaledConfig(pg, proxyClass, i, authKey, endpoints[nodePortSvcName], existingAdvertiseServices, r.loginServer) + if err != nil { + return nil, fmt.Errorf("error creating tailscaled config: %w", err) + } + + for cap, cfg := range configs { + cfgJSON, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("error marshalling tailscaled config: %w", err) + } + mak.Set(&cfgSecret.Data, tsoperator.TailscaledConfigFileName(cap), cfgJSON) + } } if existingCfgSecret != nil { if !apiequality.Semantic.DeepEqual(existingCfgSecret, cfgSecret) { logger.Debugf("Updating the existing ProxyGroup config Secret %s", cfgSecret.Name) if err := r.Update(ctx, cfgSecret); err != nil { - return "", err + return nil, err } } } else { logger.Debugf("Creating a new config Secret %s for the ProxyGroup", cfgSecret.Name) if err := r.Create(ctx, cfgSecret); err != nil { - return "", err + return nil, err + } + } + } + + return endpoints, nil +} + +type FindStaticEndpointErr struct { + msg string +} + +func (e *FindStaticEndpointErr) Error() string { + return e.msg +} + +// findStaticEndpoints returns up to two `netip.AddrPort` entries, derived from the ExternalIPs of Nodes that +// match the `proxyClass`'s selector within the StaticEndpoints configuration. The port is set to the replica's NodePort Service Port. +func (r *ProxyGroupReconciler) findStaticEndpoints(ctx context.Context, existingCfgSecret *corev1.Secret, proxyClass *tsapi.ProxyClass, port uint16, logger *zap.SugaredLogger) ([]netip.AddrPort, error) { + var currAddrs []netip.AddrPort + if existingCfgSecret != nil { + oldConfB := existingCfgSecret.Data[tsoperator.TailscaledConfigFileName(106)] + if len(oldConfB) > 0 { + var oldConf ipn.ConfigVAlpha + if err := json.Unmarshal(oldConfB, &oldConf); err == nil { + currAddrs = oldConf.StaticEndpoints + } else { + logger.Debugf("failed to unmarshal tailscaled config from secret %q: %v", existingCfgSecret.Name, err) + } + } else { + logger.Debugf("failed to get tailscaled config from secret %q: empty data", existingCfgSecret.Name) + } + } + + nodes := new(corev1.NodeList) + selectors := client.MatchingLabels(proxyClass.Spec.StaticEndpoints.NodePort.Selector) + + err := r.List(ctx, nodes, selectors) + if err != nil { + return nil, fmt.Errorf("failed to list nodes: %w", err) + } + + if len(nodes.Items) == 0 { + return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass %q", proxyClass.Name)} + } + + endpoints := []netip.AddrPort{} + + // NOTE(ChaosInTheCRD): Setting a hard limit of two static endpoints. + newAddrs := []netip.AddrPort{} + for _, n := range nodes.Items { + for _, a := range n.Status.Addresses { + if a.Type == corev1.NodeExternalIP { + addr := getStaticEndpointAddress(&a, port) + if addr == nil { + logger.Debugf("failed to parse %q address on node %q: %q", corev1.NodeExternalIP, n.Name, a.Address) + continue + } + + // we want to add the currently used IPs first before + // adding new ones. + if currAddrs != nil && slices.Contains(currAddrs, *addr) { + endpoints = append(endpoints, *addr) + } else { + newAddrs = append(newAddrs, *addr) + } + } + + if len(endpoints) == 2 { + break } } } - return configSHA256Sum, nil + // if the 2 endpoints limit hasn't been reached, we + // can start adding newIPs. + if len(endpoints) < 2 { + for _, a := range newAddrs { + endpoints = append(endpoints, a) + if len(endpoints) == 2 { + break + } + } + } + + if len(endpoints) == 0 { + return nil, &FindStaticEndpointErr{msg: fmt.Sprintf("failed to find any `status.addresses` of type %q on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass %q", corev1.NodeExternalIP, proxyClass.Name)} + } + + return endpoints, nil +} + +func getStaticEndpointAddress(a *corev1.NodeAddress, port uint16) *netip.AddrPort { + addr, err := netip.ParseAddr(a.Address) + if err != nil { + return nil + } + + return ptr.To(netip.AddrPortFrom(addr, port)) } // ensureAddedToGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource is updated when the ProxyGroup @@ -548,9 +1008,12 @@ func (r *ProxyGroupReconciler) ensureAddedToGaugeForProxyGroup(pg *tsapi.ProxyGr r.egressProxyGroups.Add(pg.UID) case tsapi.ProxyGroupTypeIngress: r.ingressProxyGroups.Add(pg.UID) + case tsapi.ProxyGroupTypeKubernetesAPIServer: + r.apiServerProxyGroups.Add(pg.UID) } gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) + gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) } // ensureRemovedFromGaugeForProxyGroup ensures the gauge metric for the ProxyGroup resource type is updated when the @@ -561,87 +1024,69 @@ func (r *ProxyGroupReconciler) ensureRemovedFromGaugeForProxyGroup(pg *tsapi.Pro r.egressProxyGroups.Remove(pg.UID) case tsapi.ProxyGroupTypeIngress: r.ingressProxyGroups.Remove(pg.UID) + case tsapi.ProxyGroupTypeKubernetesAPIServer: + r.apiServerProxyGroups.Remove(pg.UID) } gaugeEgressProxyGroupResources.Set(int64(r.egressProxyGroups.Len())) gaugeIngressProxyGroupResources.Set(int64(r.ingressProxyGroups.Len())) + gaugeAPIServerProxyGroupResources.Set(int64(r.apiServerProxyGroups.Len())) } -func pgTailscaledConfig(pg *tsapi.ProxyGroup, class *tsapi.ProxyClass, idx int32, authKey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { +func pgTailscaledConfig(pg *tsapi.ProxyGroup, pc *tsapi.ProxyClass, idx int32, authKey *string, staticEndpoints []netip.AddrPort, oldAdvertiseServices []string, loginServer string) (tailscaledConfigs, error) { conf := &ipn.ConfigVAlpha{ - Version: "alpha0", - AcceptDNS: "false", - AcceptRoutes: "false", // AcceptRoutes defaults to true - Locked: "false", - Hostname: ptr.To(fmt.Sprintf("%s-%d", pg.Name, idx)), + Version: "alpha0", + AcceptDNS: "false", + AcceptRoutes: "false", // AcceptRoutes defaults to true + Locked: "false", + Hostname: ptr.To(pgHostname(pg, idx)), + AdvertiseServices: oldAdvertiseServices, + AuthKey: authKey, } - if pg.Spec.HostnamePrefix != "" { - conf.Hostname = ptr.To(fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, idx)) + if loginServer != "" { + conf.ServerURL = &loginServer } - if shouldAcceptRoutes(class) { + if shouldAcceptRoutes(pc) { conf.AcceptRoutes = "true" } - deviceAuthed := false - for _, d := range pg.Status.Devices { - if d.Hostname == *conf.Hostname { - deviceAuthed = true - break - } + if len(staticEndpoints) > 0 { + conf.StaticEndpoints = staticEndpoints } - if authKey != "" { - conf.AuthKey = &authKey - } else if !deviceAuthed { - key, err := authKeyFromSecret(oldSecret) - if err != nil { - return nil, fmt.Errorf("error retrieving auth key from Secret: %w", err) - } - conf.AuthKey = key - } - capVerConfigs := make(map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha) - - // AdvertiseServices config is set by ingress-pg-reconciler, so make sure we - // don't overwrite it here. - if err := copyAdvertiseServicesConfig(conf, oldSecret, 106); err != nil { - return nil, err - } - capVerConfigs[106] = *conf - return capVerConfigs, nil + return map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha{ + pgMinCapabilityVersion: *conf, + }, nil } -func copyAdvertiseServicesConfig(conf *ipn.ConfigVAlpha, oldSecret *corev1.Secret, capVer tailcfg.CapabilityVersion) error { - if oldSecret == nil { - return nil +func extractAdvertiseServicesConfig(cfgSecret *corev1.Secret) ([]string, error) { + if cfgSecret == nil { + return nil, nil } - oldConfB := oldSecret.Data[tsoperator.TailscaledConfigFileName(capVer)] - if len(oldConfB) == 0 { - return nil + cfg, err := latestConfigFromSecret(cfgSecret) + if err != nil { + return nil, err } - var oldConf ipn.ConfigVAlpha - if err := json.Unmarshal(oldConfB, &oldConf); err != nil { - return fmt.Errorf("error unmarshalling existing config: %w", err) + if cfg == nil { + return nil, nil } - conf.AdvertiseServices = oldConf.AdvertiseServices - - return nil -} -func (r *ProxyGroupReconciler) validate(_ *tsapi.ProxyGroup) error { - return nil + return cfg.AdvertiseServices, nil } // getNodeMetadata gets metadata for all the pods owned by this ProxyGroup by // querying their state Secrets. It may not return the same number of items as // specified in the ProxyGroup spec if e.g. it is getting scaled up or down, or // some pods have failed to write state. +// +// The returned metadata will contain an entry for each state Secret that exists. func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.ProxyGroup) (metadata []nodeMetadata, _ error) { - // List all state secrets owned by this ProxyGroup. + // List all state Secrets owned by this ProxyGroup. secrets := &corev1.SecretList{} - if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, "state"))); err != nil { + if err := r.List(ctx, secrets, client.InNamespace(r.tsNamespace), client.MatchingLabels(pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState))); err != nil { return nil, fmt.Errorf("failed to list state Secrets: %w", err) } for _, secret := range secrets.Items { @@ -650,22 +1095,22 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr return nil, fmt.Errorf("unexpected secret %s was labelled as owned by the ProxyGroup %s: %w", secret.Name, pg.Name, err) } + nm := nodeMetadata{ + ordinal: ordinal, + stateSecret: &secret, + } + prefs, ok, err := getDevicePrefs(&secret) if err != nil { return nil, err } - if !ok { - continue + if ok { + nm.tsID = prefs.Config.NodeID + nm.dnsName = prefs.Config.UserProfile.LoginName } - nm := nodeMetadata{ - ordinal: ordinal, - stateSecret: &secret, - tsID: prefs.Config.NodeID, - dnsName: prefs.Config.UserProfile.LoginName, - } pod := &corev1.Pod{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: secret.Name}, pod); err != nil && !apierrors.IsNotFound(err) { + if err := r.Get(ctx, client.ObjectKey{Namespace: r.tsNamespace, Name: fmt.Sprintf("%s-%d", pg.Name, ordinal)}, pod); err != nil && !apierrors.IsNotFound(err) { return nil, err } else if err == nil { nm.podUID = string(pod.UID) @@ -673,27 +1118,56 @@ func (r *ProxyGroupReconciler) getNodeMetadata(ctx context.Context, pg *tsapi.Pr metadata = append(metadata, nm) } + // Sort for predictable ordering and status. + sort.Slice(metadata, func(i, j int) bool { + return metadata[i].ordinal < metadata[j].ordinal + }) + return metadata, nil } -func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.ProxyGroup) (devices []tsapi.TailnetDevice, _ error) { +// getRunningProxies will return status for all proxy Pods whose state Secret +// has an up to date Pod UID and at least a hostname. +func (r *ProxyGroupReconciler) getRunningProxies(ctx context.Context, pg *tsapi.ProxyGroup, staticEndpoints map[string][]netip.AddrPort) (devices []tsapi.TailnetDevice, _ error) { metadata, err := r.getNodeMetadata(ctx, pg) if err != nil { return nil, err } - for _, m := range metadata { - device, ok, err := getDeviceInfo(ctx, r.tsClient, m.stateSecret) - if err != nil { - return nil, err + for i, m := range metadata { + if m.podUID == "" || !strings.EqualFold(string(m.stateSecret.Data[kubetypes.KeyPodUID]), m.podUID) { + // Current Pod has not yet written its UID to the state Secret, data may + // be stale. + continue } - if !ok { + + device := tsapi.TailnetDevice{} + if hostname, _, ok := strings.Cut(string(m.stateSecret.Data[kubetypes.KeyDeviceFQDN]), "."); ok { + device.Hostname = hostname + } else { continue } - devices = append(devices, tsapi.TailnetDevice{ - Hostname: device.Hostname, - TailnetIPs: device.TailnetIPs, - }) + + if ipsB := m.stateSecret.Data[kubetypes.KeyDeviceIPs]; len(ipsB) > 0 { + ips := []string{} + if err := json.Unmarshal(ipsB, &ips); err != nil { + return nil, fmt.Errorf("failed to extract device IPs from state Secret %q: %w", m.stateSecret.Name, err) + } + device.TailnetIPs = ips + } + + // TODO(tomhjp): This is our input to the proxy, but we should instead + // read this back from the proxy's state in some way to more accurately + // reflect its status. + if ep, ok := staticEndpoints[pgNodePortServiceName(pg.Name, int32(i))]; ok && len(ep) > 0 { + eps := make([]string, 0, len(ep)) + for _, e := range ep { + eps = append(eps, e.String()) + } + device.StaticEndpoints = eps + } + + devices = append(devices, device) } return devices, nil @@ -702,29 +1176,34 @@ func (r *ProxyGroupReconciler) getDeviceInfo(ctx context.Context, pg *tsapi.Prox type nodeMetadata struct { ordinal int stateSecret *corev1.Secret - // podUID is the UID of the current Pod or empty if the Pod does not exist. - podUID string - tsID tailcfg.StableNodeID - dnsName string + podUID string // or empty if the Pod no longer exists. + tsID tailcfg.StableNodeID + dnsName string } -// capVerForPG returns best effort capability version for the given ProxyGroup. It attempts to find it by looking at the -// Secret + Pod for the replica with ordinal 0. Returns -1 if it is not possible to determine the capability version -// (i.e there is no Pod yet). -func (r *ProxyGroupReconciler) capVerForPG(ctx context.Context, pg *tsapi.ProxyGroup, logger *zap.SugaredLogger) (tailcfg.CapabilityVersion, error) { - metas, err := r.getNodeMetadata(ctx, pg) - if err != nil { - return -1, fmt.Errorf("error getting node metadata: %w", err) - } - if len(metas) == 0 { - return -1, nil - } - dev, err := deviceInfo(metas[0].stateSecret, metas[0].podUID, logger) - if err != nil { - return -1, fmt.Errorf("error getting device info: %w", err) - } - if dev == nil { - return -1, nil +func notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason, error) { + return nil, ¬ReadyReason{ + reason: reason, + message: msg, + }, nil +} + +func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) { + err := fmt.Errorf(format, a...) + if strings.Contains(err.Error(), optimisticLockErrorMsg) { + msg := fmt.Sprintf("optimistic lock error, retrying: %s", err.Error()) + logger.Info(msg) + return notReady(reasonProxyGroupCreating, msg) } - return dev.capver, nil + + r.recorder.Event(pg, corev1.EventTypeWarning, reasonProxyGroupCreationFailed, err.Error()) + return nil, ¬ReadyReason{ + reason: reasonProxyGroupCreationFailed, + message: err.Error(), + }, err +} + +type notReadyReason struct { + reason string + message string } diff --git a/cmd/k8s-operator/proxygroup_specs.go b/cmd/k8s-operator/proxygroup_specs.go index 1d12c39e0241e..e185499f0e19d 100644 --- a/cmd/k8s-operator/proxygroup_specs.go +++ b/cmd/k8s-operator/proxygroup_specs.go @@ -9,11 +9,13 @@ import ( "fmt" "slices" "strconv" + "strings" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/yaml" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" @@ -23,12 +25,49 @@ import ( "tailscale.com/types/ptr" ) -// deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully. -const deletionGracePeriodSeconds int64 = 360 +const ( + // deletionGracePeriodSeconds is set to 6 minutes to ensure that the pre-stop hook of these proxies have enough chance to terminate gracefully. + deletionGracePeriodSeconds int64 = 360 + staticEndpointPortName = "static-endpoint-port" + // authAPIServerProxySAName is the ServiceAccount deployed by the helm chart + // if apiServerProxy.authEnabled is true. + authAPIServerProxySAName = "kube-apiserver-auth-proxy" +) + +func pgNodePortServiceName(proxyGroupName string, replica int32) string { + return fmt.Sprintf("%s-%d-nodeport", proxyGroupName, replica) +} + +func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + // NOTE(ChaosInTheCRD): we set the ports once we've iterated over every svc and found any old configuration we want to persist. + { + Name: staticEndpointPortName, + Protocol: corev1.ProtocolUDP, + }, + }, + Selector: map[string]string{ + appsv1.StatefulSetPodNameLabel: strings.TrimSuffix(name, "-nodeport"), + }, + }, + } +} // Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be // applied over the top after. -func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { +func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) { + if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer { + return kubeAPIServerStatefulSet(pg, namespace, image, port) + } ss := new(appsv1.StatefulSet) if err := yaml.Unmarshal(proxyYaml, &ss); err != nil { return nil, fmt.Errorf("failed to unmarshal proxy spec: %w", err) @@ -135,6 +174,7 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string Value: "$(POD_NAME)", }, { + // TODO(tomhjp): This is tsrecorder-specific and does nothing. Delete. Name: "TS_STATE", Value: "kube:$(POD_NAME)", }, @@ -144,6 +184,13 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string }, } + if port != nil { + envs = append(envs, corev1.EnvVar{ + Name: "PORT", + Value: strconv.Itoa(int(*port)), + }) + } + if tsFirewallMode != "" { envs = append(envs, corev1.EnvVar{ Name: "TS_DEBUG_FIREWALL_MODE", @@ -225,9 +272,109 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string // gracefully. ss.Spec.Template.DeletionGracePeriodSeconds = ptr.To(deletionGracePeriodSeconds) } + return ss, nil } +func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, port *uint16) (*appsv1.StatefulSet, error) { + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: pg.Name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + OwnerReferences: pgOwnerReference(pg), + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(pgReplicas(pg)), + Selector: &metav1.LabelSelector{ + MatchLabels: pgLabels(pg.Name, nil), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: pg.Name, + Namespace: namespace, + Labels: pgLabels(pg.Name, nil), + DeletionGracePeriodSeconds: ptr.To[int64](10), + }, + Spec: corev1.PodSpec{ + ServiceAccountName: pgServiceAccountName(pg), + Containers: []corev1.Container{ + { + Name: mainContainerName, + Image: image, + Env: func() []corev1.EnvVar { + envs := []corev1.EnvVar{ + { + // Used as default hostname and in Secret names. + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + { + // Used by kubeclient to post Events about the Pod's lifecycle. + Name: "POD_UID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.uid", + }, + }, + }, + { + // Used in an interpolated env var if metrics enabled. + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + // Included for completeness with POD_IP and easier backwards compatibility in future. + Name: "POD_IPS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, + }, + }, + { + Name: "TS_K8S_PROXY_CONFIG", + Value: "kube:" + types.NamespacedName{ + Namespace: namespace, + Name: "$(POD_NAME)-config", + }.String(), + }, + } + + if port != nil { + envs = append(envs, corev1.EnvVar{ + Name: "PORT", + Value: strconv.Itoa(int(*port)), + }) + } + + return envs + }(), + Ports: []corev1.ContainerPort{ + { + Name: "k8s-proxy", + ContainerPort: 443, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + }, + }, + } + + return sts, nil +} + func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAccount { return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ @@ -253,6 +400,7 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role { Resources: []string{"secrets"}, Verbs: []string{ "list", + "watch", // For k8s-proxy. }, }, { @@ -266,8 +414,8 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role { ResourceNames: func() (secrets []string) { for i := range pgReplicas(pg) { secrets = append(secrets, - pgConfigSecretName(pg.Name, i), // Config with auth key. - fmt.Sprintf("%s-%d", pg.Name, i), // State. + pgConfigSecretName(pg.Name, i), // Config with auth key. + pgPodName(pg.Name, i), // State. ) } return secrets @@ -297,7 +445,7 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding { Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: pg.Name, + Name: pgServiceAccountName(pg), Namespace: namespace, }, }, @@ -308,13 +456,34 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding { } } +// kube-apiserver proxies in auth mode use a static ServiceAccount. Everything +// else uses a per-ProxyGroup ServiceAccount. +func pgServiceAccountName(pg *tsapi.ProxyGroup) string { + if isAuthAPIServerProxy(pg) { + return authAPIServerProxySAName + } + + return pg.Name +} + +func isAuthAPIServerProxy(pg *tsapi.ProxyGroup) bool { + if pg.Spec.Type != tsapi.ProxyGroupTypeKubernetesAPIServer { + return false + } + + // The default is auth mode. + return pg.Spec.KubeAPIServer == nil || + pg.Spec.KubeAPIServer.Mode == nil || + *pg.Spec.KubeAPIServer.Mode == tsapi.APIServerProxyModeAuth +} + func pgStateSecrets(pg *tsapi.ProxyGroup, namespace string) (secrets []*corev1.Secret) { for i := range pgReplicas(pg) { secrets = append(secrets, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%d", pg.Name, i), + Name: pgStateSecretName(pg.Name, i), Namespace: namespace, - Labels: pgSecretLabels(pg.Name, "state"), + Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeState), OwnerReferences: pgOwnerReference(pg), }, }) @@ -379,10 +548,26 @@ func pgReplicas(pg *tsapi.ProxyGroup) int32 { return 2 } +func pgPodName(pgName string, i int32) string { + return fmt.Sprintf("%s-%d", pgName, i) +} + +func pgHostname(pg *tsapi.ProxyGroup, i int32) string { + if pg.Spec.HostnamePrefix != "" { + return fmt.Sprintf("%s-%d", pg.Spec.HostnamePrefix, i) + } + + return fmt.Sprintf("%s-%d", pg.Name, i) +} + func pgConfigSecretName(pgName string, i int32) string { return fmt.Sprintf("%s-%d-config", pgName, i) } +func pgStateSecretName(pgName string, i int32) string { + return fmt.Sprintf("%s-%d", pgName, i) +} + func pgEgressCMName(pg string) string { return fmt.Sprintf("%s-egress-config", pg) } diff --git a/cmd/k8s-operator/proxygroup_test.go b/cmd/k8s-operator/proxygroup_test.go index 159329eda2335..d763cf92276ec 100644 --- a/cmd/k8s-operator/proxygroup_test.go +++ b/cmd/k8s-operator/proxygroup_test.go @@ -6,9 +6,10 @@ package main import ( - "context" "encoding/json" "fmt" + "net/netip" + "slices" "testing" "time" @@ -18,30 +19,792 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "tailscale.com/client/tailscale" "tailscale.com/ipn" + kube "tailscale.com/k8s-operator" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/kube/k8s-proxy/conf" "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" "tailscale.com/tstest" + "tailscale.com/types/opt" "tailscale.com/types/ptr" - "tailscale.com/util/mak" ) -const testProxyImage = "tailscale/tailscale:test" +const ( + testProxyImage = "tailscale/tailscale:test" + initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196" +) + +var ( + defaultProxyClassAnnotations = map[string]string{ + "some-annotation": "from-the-proxy-class", + } + + defaultReplicas = ptr.To(int32(2)) + defaultStaticEndpointConfig = &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 30001}, {Port: 30002}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + } +) -var defaultProxyClassAnnotations = map[string]string{ - "some-annotation": "from-the-proxy-class", +func TestProxyGroupWithStaticEndpoints(t *testing.T) { + type testNodeAddr struct { + ip string + addrType corev1.NodeAddressType + } + + type testNode struct { + name string + addresses []testNodeAddr + labels map[string]string + } + + type reconcile struct { + staticEndpointConfig *tsapi.StaticEndpointsConfig + replicas *int32 + nodes []testNode + expectedIPs []netip.Addr + expectedEvents []string + expectedErr string + expectStatefulSet bool + } + + testCases := []struct { + name string + description string + reconciles []reconcile + }{ + { + // the reconciler should manage to create static endpoints when Nodes have IPv6 addresses. + name: "IPv6", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + {Port: 3009}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: ptr.To(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "2001:0db8::1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "2001:0db8::2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "2001:0db8::3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("2001:0db8::1"), netip.MustParseAddr("2001:0db8::2"), netip.MustParseAddr("2001:0db8::3")}, + expectedEvents: []string{}, + expectedErr: "", + expectStatefulSet: true, + }, + }, + }, + { + // declaring specific ports (with no `endPort`s) in the `spec.staticEndpoints.nodePort` should work. + name: "SpecificPorts", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + {Port: 3009}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: ptr.To(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("192.168.0.1"), netip.MustParseAddr("192.168.0.2"), netip.MustParseAddr("192.168.0.3")}, + expectedEvents: []string{}, + expectedErr: "", + expectStatefulSet: true, + }, + }, + }, + { + // if too narrow a range of `spec.staticEndpoints.nodePort.Ports` on the proxyClass should result in no StatefulSet being created. + name: "NotEnoughPorts", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: ptr.To(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning NodePort Services for static endpoints: failed to allocate NodePorts to ProxyGroup Services: not enough available ports to allocate all replicas (needed 4, got 3). Field 'spec.staticEndpoints.nodePort.ports' on ProxyClass \"default-pc\" must have bigger range allocated"}, + expectedErr: "", + expectStatefulSet: false, + }, + }, + }, + { + // when supplying a variety of ranges that are not clashing, the reconciler should manage to create a StatefulSet. + name: "NonClashingRanges", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3000, EndPort: 3002}, + {Port: 3003, EndPort: 3005}, + {Port: 3006}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: ptr.To(int32(3)), + nodes: []testNode{ + {name: "node1", addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}}, + {name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}}, + {name: "node3", addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"foo/bar": "baz"}}, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2"), netip.MustParseAddr("10.0.0.3")}, + expectedEvents: []string{}, + expectedErr: "", + expectStatefulSet: true, + }, + }, + }, + { + // when there isn't a node that matches the selector, the ProxyGroup enters a failed state as there are no valid Static Endpoints. + // while it does create an event on the resource, It does not return an error + name: "NoMatchingNodes", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3000, EndPort: 3005}, + }, + Selector: map[string]string{ + "zone": "us-west", + }, + }, + }, + replicas: defaultReplicas, + nodes: []testNode{ + {name: "node1", addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, labels: map[string]string{"zone": "eu-central"}}, + {name: "node2", addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}}, labels: map[string]string{"zone": "eu-central"}}, + }, + expectedIPs: []netip.Addr{}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning config Secrets: could not find static endpoints for replica \"test-0\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""}, + expectedErr: "", + expectStatefulSet: false, + }, + }, + }, + { + // when all the nodes have only have addresses of type InternalIP populated in their status, the ProxyGroup enters a failed state as there are no valid Static Endpoints. + // while it does create an event on the resource, It does not return an error + name: "AllInternalIPAddresses", + reconciles: []reconcile{ + { + staticEndpointConfig: &tsapi.StaticEndpointsConfig{ + NodePort: &tsapi.NodePortConfig{ + Ports: []tsapi.PortRange{ + {Port: 3001}, + {Port: 3005}, + {Port: 3007}, + {Port: 3009}, + }, + Selector: map[string]string{ + "foo/bar": "baz", + }, + }, + }, + replicas: ptr.To(int32(4)), + nodes: []testNode{ + { + name: "foobar", + addresses: []testNodeAddr{{ip: "192.168.0.1", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbaz", + addresses: []testNodeAddr{{ip: "192.168.0.2", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "foobarbazz", + addresses: []testNodeAddr{{ip: "192.168.0.3", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning config Secrets: could not find static endpoints for replica \"test-0\": failed to find any `status.addresses` of type \"ExternalIP\" on nodes using configured Selectors on `spec.staticEndpoints.nodePort.selectors` for ProxyClass \"default-pc\""}, + expectedErr: "", + expectStatefulSet: false, + }, + }, + }, + { + // When the node's (and some of their addresses) change between reconciles, the reconciler should first pick addresses that + // have been used previously (provided that they are still populated on a node that matches the selector) + name: "NodeIPChangesAndPersists", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.10", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectStatefulSet: true, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + }, + }, + }, + { + // given a new node being created with a new IP, and a node previously used for Static Endpoints being removed, the Static Endpoints should be updated + // correctly + name: "NodeIPChangesWithNewNode", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.3", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.3")}, + expectStatefulSet: true, + }, + }, + }, + { + // when all the node IPs change, they should all update + name: "AllNodeIPsChange", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.100", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.200", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.100"), netip.MustParseAddr("10.0.0.200")}, + expectStatefulSet: true, + }, + }, + }, + { + // if there are less ExternalIPs after changes to the nodes between reconciles, the reconciler should complete without issues + name: "LessExternalIPsAfterChange", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeInternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1")}, + expectStatefulSet: true, + }, + }, + }, + { + // if node address parsing fails (given an invalid address), the reconciler should continue without failure and find other + // valid addresses + name: "NodeAddressParsingFails", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "invalid-ip", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "invalid-ip", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + }, + }, + { + // if the node's become unlabeled, the ProxyGroup should enter a ProxyGroupInvalid state, but the reconciler should not fail + name: "NodesBecomeUnlabeled", + reconciles: []reconcile{ + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node1", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + { + name: "node2", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{"foo/bar": "baz"}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectStatefulSet: true, + }, + { + staticEndpointConfig: defaultStaticEndpointConfig, + replicas: defaultReplicas, + nodes: []testNode{ + { + name: "node3", + addresses: []testNodeAddr{{ip: "10.0.0.1", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{}, + }, + { + name: "node4", + addresses: []testNodeAddr{{ip: "10.0.0.2", addrType: corev1.NodeExternalIP}}, + labels: map[string]string{}, + }, + }, + expectedIPs: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("10.0.0.2")}, + expectedEvents: []string{"Warning ProxyGroupCreationFailed error provisioning config Secrets: could not find static endpoints for replica \"test-0\": failed to match nodes to configured Selectors on `spec.staticEndpoints.nodePort.selectors` field for ProxyClass \"default-pc\""}, + expectStatefulSet: true, + }, + }, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + tsClient := &fakeTSClient{} + zl, _ := zap.NewDevelopment() + fr := record.NewFakeRecorder(10) + cl := tstest.NewClock(tstest.ClockOpts{}) + + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Annotations: defaultProxyClassAnnotations, + }, + }, + Status: tsapi.ProxyClassStatus{ + Conditions: []metav1.Condition{{ + Type: string(tsapi.ProxyClassReady), + Status: metav1.ConditionTrue, + Reason: reasonProxyClassValid, + Message: reasonProxyClassValid, + LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, + }}, + }, + } + + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeEgress, + ProxyClass: pc.Name, + }, + } + + fc := fake.NewClientBuilder(). + WithObjects(pc, pg). + WithStatusSubresource(pc, pg). + WithScheme(tsapi.GlobalScheme). + Build() + + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + defaultTags: []string{"tag:test-tag"}, + tsFirewallMode: "auto", + defaultProxyClass: "default-pc", + + Client: fc, + tsClient: tsClient, + recorder: fr, + clock: cl, + } + + for i, r := range tt.reconciles { + createdNodes := []corev1.Node{} + t.Run(tt.name, func(t *testing.T) { + for _, n := range r.nodes { + no := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: n.name, + Labels: n.labels, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{}, + }, + } + for _, addr := range n.addresses { + no.Status.Addresses = append(no.Status.Addresses, corev1.NodeAddress{ + Type: addr.addrType, + Address: addr.ip, + }) + } + if err := fc.Create(t.Context(), no); err != nil { + t.Fatalf("failed to create node %q: %v", n.name, err) + } + createdNodes = append(createdNodes, *no) + t.Logf("created node %q with data", n.name) + } + + reconciler.l = zl.Sugar().With("TestName", tt.name).With("Reconcile", i) + pg.Spec.Replicas = r.replicas + pc.Spec.StaticEndpoints = r.staticEndpointConfig + + createOrUpdate(t.Context(), fc, "", pg, func(o *tsapi.ProxyGroup) { + o.Spec.Replicas = pg.Spec.Replicas + }) + + createOrUpdate(t.Context(), fc, "", pc, func(o *tsapi.ProxyClass) { + o.Spec.StaticEndpoints = pc.Spec.StaticEndpoints + }) + + if r.expectedErr != "" { + expectError(t, reconciler, "", pg.Name) + } else { + expectReconciled(t, reconciler, "", pg.Name) + } + expectEvents(t, fr, r.expectedEvents) + + sts := &appsv1.StatefulSet{} + err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts) + if r.expectStatefulSet { + if err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + for j := range 2 { + sec := &corev1.Secret{} + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: fmt.Sprintf("%s-%d-config", pg.Name, j)}, sec); err != nil { + t.Fatalf("failed to get state Secret for replica %d: %v", j, err) + } + + config := &ipn.ConfigVAlpha{} + foundConfig := false + for _, d := range sec.Data { + if err := json.Unmarshal(d, config); err == nil { + foundConfig = true + break + } + } + if !foundConfig { + t.Fatalf("could not unmarshal config from secret data for replica %d", j) + } + + if len(config.StaticEndpoints) > staticEndpointsMaxAddrs { + t.Fatalf("expected %d StaticEndpoints in config Secret, but got %d for replica %d. Found Static Endpoints: %v", staticEndpointsMaxAddrs, len(config.StaticEndpoints), j, config.StaticEndpoints) + } + + for _, e := range config.StaticEndpoints { + if !slices.Contains(r.expectedIPs, e.Addr()) { + t.Fatalf("found unexpected static endpoint IP %q for replica %d. Expected one of %v", e.Addr().String(), j, r.expectedIPs) + } + if c := r.staticEndpointConfig; c != nil && c.NodePort.Ports != nil { + var ports tsapi.PortRanges = c.NodePort.Ports + found := false + for port := range ports.All() { + if port == e.Port() { + found = true + break + } + } + + if !found { + t.Fatalf("found unexpected static endpoint port %d for replica %d. Expected one of %v .", e.Port(), j, ports.All()) + } + } else { + if e.Port() != 3001 && e.Port() != 3002 { + t.Fatalf("found unexpected static endpoint port %d for replica %d. Expected 3001 or 3002.", e.Port(), j) + } + } + } + } + + pgroup := &tsapi.ProxyGroup{} + err = fc.Get(t.Context(), client.ObjectKey{Name: pg.Name}, pgroup) + if err != nil { + t.Fatalf("failed to get ProxyGroup %q: %v", pg.Name, err) + } + + t.Logf("getting proxygroup after reconcile") + for _, d := range pgroup.Status.Devices { + t.Logf("found device %q", d.Hostname) + for _, e := range d.StaticEndpoints { + t.Logf("found static endpoint %q", e) + } + } + } else { + if err == nil { + t.Fatal("expected error when getting Statefulset") + } + } + }) + + // node cleanup between reconciles + // we created a new set of nodes for each + for _, n := range createdNodes { + err := fc.Delete(t.Context(), &n) + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("failed to delete node: %v", err) + } + } + } + + t.Run("delete_and_cleanup", func(t *testing.T) { + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + defaultTags: []string{"tag:test-tag"}, + tsFirewallMode: "auto", + defaultProxyClass: "default-pc", + + Client: fc, + tsClient: tsClient, + recorder: fr, + l: zl.Sugar().With("TestName", tt.name).With("Reconcile", "cleanup"), + clock: cl, + } + + if err := fc.Delete(t.Context(), pg); err != nil { + t.Fatalf("error deleting ProxyGroup: %v", err) + } + + expectReconciled(t, reconciler, "", pg.Name) + expectMissing[tsapi.ProxyGroup](t, fc, "", pg.Name) + + if err := fc.Delete(t.Context(), pc); err != nil { + t.Fatalf("error deleting ProxyClass: %v", err) + } + expectMissing[tsapi.ProxyClass](t, fc, "", pc.Name) + }) + }) + } } func TestProxyGroup(t *testing.T) { - const initialCfgHash = "6632726be70cf224049580deb4d317bba065915b5fd415461d60ed621c91b196" - pc := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{ Name: "default-pc", @@ -56,6 +819,7 @@ func TestProxyGroup(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test", Finalizers: []string{"tailscale.com/finalizer"}, + Generation: 1, }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeEgress, @@ -73,7 +837,7 @@ func TestProxyGroup(t *testing.T) { cl := tstest.NewClock(tstest.ClockOpts{}) reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test-tag"}, tsFirewallMode: "auto", defaultProxyClass: "default-pc", @@ -96,9 +860,13 @@ func TestProxyGroup(t *testing.T) { t.Run("proxyclass_not_ready", func(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass default-pc is not yet in a ready state, waiting...", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 1, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, false, "", pc) + expectProxyGroupResources(t, fc, pg, false, pc) + if kube.ProxyGroupAvailable(pg) { + t.Fatal("expected ProxyGroup to not be available") + } }) t.Run("observe_ProxyGroupCreating_status_reason", func(t *testing.T) { @@ -111,19 +879,26 @@ func TestProxyGroup(t *testing.T) { LastTransitionTime: metav1.Time{Time: cl.Now().Truncate(time.Second)}, }}, } - if err := fc.Status().Update(context.Background(), pc); err != nil { + if err := fc.Status().Update(t.Context(), pc); err != nil { t.Fatal(err) } - + pg.ObjectMeta.Generation = 2 + mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + p.ObjectMeta.Generation = pg.ObjectMeta.Generation + }) expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 2, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, "", pc) + expectProxyGroupResources(t, fc, pg, true, pc) + if kube.ProxyGroupAvailable(pg) { + t.Fatal("expected ProxyGroup to not be available") + } if expected := 1; reconciler.egressProxyGroups.Len() != expected { t.Fatalf("expected %d egress ProxyGroups, got %d", expected, reconciler.egressProxyGroups.Len()) } - expectProxyGroupResources(t, fc, pg, true, "", pc) + expectProxyGroupResources(t, fc, pg, true, pc) keyReq := tailscale.KeyCapabilities{ Devices: tailscale.KeyDeviceCapabilities{ Create: tailscale.KeyDeviceCreateCapabilities{ @@ -141,6 +916,10 @@ func TestProxyGroup(t *testing.T) { t.Run("simulate_successful_device_auth", func(t *testing.T) { addNodeIDToStateSecrets(t, fc, pg) + pg.ObjectMeta.Generation = 3 + mustUpdate(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + p.ObjectMeta.Generation = pg.ObjectMeta.Generation + }) expectReconciled(t, reconciler, "", pg.Name) pg.Status.Devices = []tsapi.TailnetDevice{ @@ -153,9 +932,13 @@ func TestProxyGroup(t *testing.T) { TailnetIPs: []string{"1.2.3.4", "::1"}, }, } - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) + expectProxyGroupResources(t, fc, pg, true, pc) + if !kube.ProxyGroupAvailable(pg) { + t.Fatal("expected ProxyGroup to be available") + } }) t.Run("scale_up_to_3", func(t *testing.T) { @@ -164,19 +947,21 @@ func TestProxyGroup(t *testing.T) { p.Spec = pg.Spec }) expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 3, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupCreating, "2/3 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) + expectProxyGroupResources(t, fc, pg, true, pc) addNodeIDToStateSecrets(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) - tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 0, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, reasonProxyGroupReady, 3, cl, zl.Sugar()) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "3/3 ProxyGroup pods running", 0, cl, zl.Sugar()) pg.Status.Devices = append(pg.Status.Devices, tsapi.TailnetDevice{ Hostname: "hostname-nodeid-2", TailnetIPs: []string{"1.2.3.4", "::1"}, }) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) + expectProxyGroupResources(t, fc, pg, true, pc) }) t.Run("scale_down_to_1", func(t *testing.T) { @@ -188,22 +973,9 @@ func TestProxyGroup(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) pg.Status.Devices = pg.Status.Devices[:1] // truncate to only the first device. + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "1/1 ProxyGroup pods running", 0, cl, zl.Sugar()) expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, initialCfgHash, pc) - }) - - t.Run("trigger_config_change_and_observe_new_config_hash", func(t *testing.T) { - pc.Spec.TailscaleConfig = &tsapi.TailscaleConfig{ - AcceptRoutes: true, - } - mustUpdate(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { - p.Spec = pc.Spec - }) - - expectReconciled(t, reconciler, "", pg.Name) - - expectEqual(t, fc, pg) - expectProxyGroupResources(t, fc, pg, true, "518a86e9fae64f270f8e0ec2a2ea6ca06c10f725035d3d6caca132cd61e42a74", pc) + expectProxyGroupResources(t, fc, pg, true, pc) }) t.Run("enable_metrics", func(t *testing.T) { @@ -228,7 +1000,7 @@ func TestProxyGroup(t *testing.T) { }) t.Run("delete_and_cleanup", func(t *testing.T) { - if err := fc.Delete(context.Background(), pg); err != nil { + if err := fc.Delete(t.Context(), pg); err != nil { t.Fatal(err) } @@ -257,10 +1029,12 @@ func TestProxyGroupTypes(t *testing.T) { }, Spec: tsapi.ProxyClassSpec{}, } + // Passing ProxyGroup as status subresource is a way to get around fake + // client's limitations for updating resource statuses. fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). WithObjects(pc). - WithStatusSubresource(pc). + WithStatusSubresource(pc, &tsapi.ProxyGroup{}). Build() mustUpdateStatus(t, fc, "", pc.Name, func(p *tsapi.ProxyClass) { p.Status.Conditions = []metav1.Condition{{ @@ -272,12 +1046,12 @@ func TestProxyGroupTypes(t *testing.T) { zl, _ := zap.NewDevelopment() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - proxyImage: testProxyImage, - Client: fc, - l: zl.Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + l: zl.Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), } t.Run("egress_type", func(t *testing.T) { @@ -294,10 +1068,10 @@ func TestProxyGroupTypes(t *testing.T) { mustCreate(t, fc, pg) expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 0, 1) + verifyProxyGroupCounts(t, reconciler, 0, 1, 0) sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupEgress) @@ -307,7 +1081,7 @@ func TestProxyGroupTypes(t *testing.T) { // Verify that egress configuration has been set up. cm := &corev1.ConfigMap{} cmName := fmt.Sprintf("%s-egress-config", pg.Name) - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: cmName}, cm); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: cmName}, cm); err != nil { t.Fatalf("failed to get ConfigMap: %v", err) } @@ -383,7 +1157,7 @@ func TestProxyGroupTypes(t *testing.T) { expectReconciled(t, reconciler, "", pg.Name) sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } @@ -403,15 +1177,15 @@ func TestProxyGroupTypes(t *testing.T) { Replicas: ptr.To[int32](0), }, } - if err := fc.Create(context.Background(), pg); err != nil { + if err := fc.Create(t.Context(), pg); err != nil { t.Fatal(err) } expectReconciled(t, reconciler, "", pg.Name) - verifyProxyGroupCounts(t, reconciler, 1, 2) + verifyProxyGroupCounts(t, reconciler, 1, 2, 0) sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress) @@ -445,19 +1219,217 @@ func TestProxyGroupTypes(t *testing.T) { t.Errorf("unexpected volume mounts (-want +got):\n%s", diff) } }) + + t.Run("kubernetes_api_server_type", func(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-k8s-apiserver", + UID: "test-k8s-apiserver-uid", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + Replicas: ptr.To[int32](2), + KubeAPIServer: &tsapi.KubeAPIServerConfig{ + Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), + }, + }, + } + if err := fc.Create(t.Context(), pg); err != nil { + t.Fatal(err) + } + + expectReconciled(t, reconciler, "", pg.Name) + verifyProxyGroupCounts(t, reconciler, 1, 2, 1) + + sts := &appsv1.StatefulSet{} + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + t.Fatalf("failed to get StatefulSet: %v", err) + } + + // Verify the StatefulSet configuration for KubernetesAPIServer type. + if sts.Spec.Template.Spec.Containers[0].Name != mainContainerName { + t.Errorf("unexpected container name %s, want %s", sts.Spec.Template.Spec.Containers[0].Name, mainContainerName) + } + if sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort != 443 { + t.Errorf("unexpected container port %d, want 443", sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort) + } + if sts.Spec.Template.Spec.Containers[0].Ports[0].Name != "k8s-proxy" { + t.Errorf("unexpected port name %s, want k8s-proxy", sts.Spec.Template.Spec.Containers[0].Ports[0].Name) + } + }) +} + +func TestKubeAPIServerStatusConditionFlow(t *testing.T) { + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-k8s-apiserver", + UID: "test-k8s-apiserver-uid", + Generation: 1, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + Replicas: ptr.To[int32](1), + KubeAPIServer: &tsapi.KubeAPIServerConfig{ + Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), + }, + }, + } + stateSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgStateSecretName(pg.Name, 0), + Namespace: tsNamespace, + }, + } + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithObjects(pg, stateSecret). + WithStatusSubresource(pg). + Build() + r := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + l: zap.Must(zap.NewDevelopment()).Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), + } + + expectReconciled(t, r, "", pg.Name) + pg.ObjectMeta.Finalizers = append(pg.ObjectMeta.Finalizers, FinalizerName) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "", 0, r.clock, r.l) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Set kube-apiserver valid. + mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.l) + }) + expectReconciled(t, r, "", pg.Name) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid, metav1.ConditionTrue, reasonKubeAPIServerProxyValid, "", 1, r.clock, r.l) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Set available. + addNodeIDToStateSecrets(t, fc, pg) + expectReconciled(t, r, "", pg.Name) + pg.Status.Devices = []tsapi.TailnetDevice{ + { + Hostname: "hostname-nodeid-0", + TailnetIPs: []string{"1.2.3.4", "::1"}, + }, + } + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "", 0, r.clock, r.l) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "", 1, r.clock, r.l) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) + + // Set kube-apiserver configured. + mustUpdateStatus(t, fc, "", pg.Name, func(p *tsapi.ProxyGroup) { + tsoperator.SetProxyGroupCondition(p, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.l) + }) + expectReconciled(t, r, "", pg.Name) + tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.l) + tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionTrue, reasonProxyGroupReady, "", 1, r.clock, r.l) + expectEqual(t, fc, pg, omitPGStatusConditionMessages) +} + +func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) { + fc := fake.NewClientBuilder(). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.ProxyGroup{}). + Build() + + reconciler := &ProxyGroupReconciler{ + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + l: zap.Must(zap.NewDevelopment()).Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), + } + + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-k8s-apiserver", + UID: "test-k8s-apiserver-uid", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tsapi.ProxyGroupTypeKubernetesAPIServer, + Replicas: ptr.To[int32](1), + KubeAPIServer: &tsapi.KubeAPIServerConfig{ + Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), // Avoid needing to pre-create the static ServiceAccount. + }, + }, + } + if err := fc.Create(t.Context(), pg); err != nil { + t.Fatal(err) + } + expectReconciled(t, reconciler, "", pg.Name) + + cfg := conf.VersionedConfig{ + Version: "v1alpha1", + ConfigV1Alpha1: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("secret-authkey"), + State: ptr.To(fmt.Sprintf("kube:%s", pgPodName(pg.Name, 0))), + App: ptr.To(kubetypes.AppProxyGroupKubeAPIServer), + LogLevel: ptr.To("debug"), + + Hostname: ptr.To("test-k8s-apiserver-0"), + APIServerProxy: &conf.APIServerProxyConfig{ + Enabled: opt.NewBool(true), + Mode: ptr.To(kubetypes.APIServerProxyModeNoAuth), + IssueCerts: opt.NewBool(true), + }, + LocalPort: ptr.To(uint16(9002)), + HealthCheckEnabled: opt.NewBool(true), + }, + } + cfgB, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("failed to marshal config: %v", err) + } + + cfgSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgConfigSecretName(pg.Name, 0), + Namespace: tsNamespace, + Labels: pgSecretLabels(pg.Name, kubetypes.LabelSecretTypeConfig), + OwnerReferences: pgOwnerReference(pg), + }, + Data: map[string][]byte{ + kubetypes.KubeAPIServerConfigFile: cfgB, + }, + } + expectEqual(t, fc, cfgSecret) + + // Now simulate the kube-apiserver services reconciler updating config, + // then check the proxygroup reconciler doesn't overwrite it. + cfg.APIServerProxy.ServiceName = ptr.To(tailcfg.ServiceName("svc:some-svc-name")) + cfg.AdvertiseServices = []string{"svc:should-not-be-overwritten"} + cfgB, err = json.Marshal(cfg) + if err != nil { + t.Fatalf("failed to marshal config: %v", err) + } + mustUpdate(t, fc, tsNamespace, cfgSecret.Name, func(s *corev1.Secret) { + s.Data[kubetypes.KubeAPIServerConfigFile] = cfgB + }) + expectReconciled(t, reconciler, "", pg.Name) + + cfgSecret.Data[kubetypes.KubeAPIServerConfigFile] = cfgB + expectEqual(t, fc, cfgSecret) } func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.ProxyGroup{}). Build() reconciler := &ProxyGroupReconciler{ - tsNamespace: tsNamespace, - proxyImage: testProxyImage, - Client: fc, - l: zap.Must(zap.NewDevelopment()).Sugar(), - tsClient: &fakeTSClient{}, - clock: tstest.NewClock(tstest.ClockOpts{}), + tsNamespace: tsNamespace, + tsProxyImage: testProxyImage, + Client: fc, + l: zap.Must(zap.NewDevelopment()).Sugar(), + tsClient: &fakeTSClient{}, + clock: tstest.NewClock(tstest.ClockOpts{}), } existingServices := []string{"svc1", "svc2"} @@ -476,7 +1448,7 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { Namespace: tsNamespace, }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): existingConfigBytes, + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): existingConfigBytes, }, }) @@ -513,11 +1485,175 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) { ResourceVersion: "2", }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): expectedConfigBytes, + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): expectedConfigBytes, }, }) } +func TestValidateProxyGroup(t *testing.T) { + type testCase struct { + typ tsapi.ProxyGroupType + pgName string + image string + noauth bool + initContainer bool + staticSAExists bool + expectedErrs int + } + + for name, tc := range map[string]testCase{ + "default_ingress": { + typ: tsapi.ProxyGroupTypeIngress, + }, + "default_kube": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + }, + "default_kube_noauth": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + noauth: true, + // Does not require the static ServiceAccount to exist. + }, + "kube_static_sa_missing": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: false, + expectedErrs: 1, + }, + "kube_noauth_would_overwrite_static_sa": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + noauth: true, + pgName: authAPIServerProxySAName, + expectedErrs: 1, + }, + "ingress_would_overwrite_static_sa": { + typ: tsapi.ProxyGroupTypeIngress, + staticSAExists: true, + pgName: authAPIServerProxySAName, + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_1": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale/tailscale", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_2": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_3": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "example.com/tailscale/tailscale:latest", + expectedErrs: 1, + }, + "tailscale_image_for_kube_pg_4": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + image: "tailscale/tailscale", + expectedErrs: 1, + }, + "k8s_proxy_image_for_ingress_pg": { + typ: tsapi.ProxyGroupTypeIngress, + image: "example.com/k8s-proxy", + expectedErrs: 1, + }, + "init_container_for_kube_pg": { + typ: tsapi.ProxyGroupTypeKubernetesAPIServer, + staticSAExists: true, + initContainer: true, + expectedErrs: 1, + }, + "init_container_for_ingress_pg": { + typ: tsapi.ProxyGroupTypeIngress, + initContainer: true, + }, + "init_container_for_egress_pg": { + typ: tsapi.ProxyGroupTypeEgress, + initContainer: true, + }, + } { + t.Run(name, func(t *testing.T) { + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-pc", + }, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Pod: &tsapi.Pod{}, + }, + }, + } + if tc.image != "" { + pc.Spec.StatefulSet.Pod.TailscaleContainer = &tsapi.Container{ + Image: tc.image, + } + } + if tc.initContainer { + pc.Spec.StatefulSet.Pod.TailscaleInitContainer = &tsapi.Container{} + } + pgName := "some-pg" + if tc.pgName != "" { + pgName = tc.pgName + } + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: pgName, + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tc.typ, + }, + } + if tc.noauth { + pg.Spec.KubeAPIServer = &tsapi.KubeAPIServerConfig{ + Mode: ptr.To(tsapi.APIServerProxyModeNoAuth), + } + } + + var objs []client.Object + if tc.staticSAExists { + objs = append(objs, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: authAPIServerProxySAName, + Namespace: tsNamespace, + }, + }) + } + r := ProxyGroupReconciler{ + tsNamespace: tsNamespace, + Client: fake.NewClientBuilder(). + WithObjects(objs...). + Build(), + } + + logger, _ := zap.NewDevelopment() + err := r.validate(t.Context(), pg, pc, logger.Sugar()) + if tc.expectedErrs == 0 { + if err != nil { + t.Fatalf("expected no errors, got: %v", err) + } + // Test finished. + return + } + + if err == nil { + t.Fatalf("expected %d errors, got none", tc.expectedErrs) + } + + type unwrapper interface { + Unwrap() []error + } + errs := err.(unwrapper) + if len(errs.Unwrap()) != tc.expectedErrs { + t.Fatalf("expected %d errors, got %d: %v", tc.expectedErrs, len(errs.Unwrap()), err) + } + }) + } +} + func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsapi.ProxyClass) { pcLEStaging := &tsapi.ProxyClass{ ObjectMeta: metav1.ObjectMeta{ @@ -553,7 +1689,7 @@ func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsap func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name string) *tsapi.ProxyClass { t.Helper() pc := &tsapi.ProxyClass{} - if err := fc.Get(context.Background(), client.ObjectKey{Name: name}, pc); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Name: name}, pc); err != nil { t.Fatal(err) } pc.Status = tsapi.ProxyClassStatus{ @@ -566,13 +1702,13 @@ func setProxyClassReady(t *testing.T, fc client.Client, cl *tstest.Clock, name s ObservedGeneration: pc.Generation, }}, } - if err := fc.Status().Update(context.Background(), pc); err != nil { + if err := fc.Status().Update(t.Context(), pc); err != nil { t.Fatal(err) } return pc } -func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress int) { +func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, wantEgress, wantAPIServer int) { t.Helper() if r.ingressProxyGroups.Len() != wantIngress { t.Errorf("expected %d ingress proxy groups, got %d", wantIngress, r.ingressProxyGroups.Len()) @@ -580,6 +1716,9 @@ func verifyProxyGroupCounts(t *testing.T, r *ProxyGroupReconciler, wantIngress, if r.egressProxyGroups.Len() != wantEgress { t.Errorf("expected %d egress proxy groups, got %d", wantEgress, r.egressProxyGroups.Len()) } + if r.apiServerProxyGroups.Len() != wantAPIServer { + t.Errorf("expected %d kube-apiserver proxy groups, got %d", wantAPIServer, r.apiServerProxyGroups.Len()) + } } func verifyEnvVar(t *testing.T, sts *appsv1.StatefulSet, name, expectedValue string) { @@ -605,20 +1744,17 @@ func verifyEnvVarNotPresent(t *testing.T, sts *appsv1.StatefulSet, name string) } } -func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, cfgHash string, proxyClass *tsapi.ProxyClass) { +func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, proxyClass *tsapi.ProxyClass) { t.Helper() role := pgRole(pg, tsNamespace) roleBinding := pgRoleBinding(pg, tsNamespace) serviceAccount := pgServiceAccount(pg, tsNamespace) - statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", proxyClass) + statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass) if err != nil { t.Fatal(err) } statefulSet.Annotations = defaultProxyClassAnnotations - if cfgHash != "" { - mak.Set(&statefulSet.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, cfgHash) - } if shouldExist { expectEqual(t, fc, role) @@ -648,7 +1784,7 @@ func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) { t.Helper() secrets := &corev1.SecretList{} - if err := fc.List(context.Background(), secrets); err != nil { + if err := fc.List(t.Context(), secrets); err != nil { t.Fatal(err) } @@ -663,6 +1799,7 @@ func expectSecrets(t *testing.T, fc client.WithWatch, expected []string) { } func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup) { + t.Helper() const key = "profile-abc" for i := range pgReplicas(pg) { bytes, err := json.Marshal(map[string]any{ @@ -674,10 +1811,27 @@ func addNodeIDToStateSecrets(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyG t.Fatal(err) } - mustUpdate(t, fc, tsNamespace, fmt.Sprintf("test-%d", i), func(s *corev1.Secret) { + podUID := fmt.Sprintf("pod-uid-%d", i) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", pg.Name, i), + Namespace: "tailscale", + UID: types.UID(podUID), + }, + } + if _, err := createOrUpdate(t.Context(), fc, "tailscale", pod, nil); err != nil { + t.Fatalf("failed to create or update Pod %s: %v", pod.Name, err) + } + mustUpdate(t, fc, tsNamespace, pgStateSecretName(pg.Name, i), func(s *corev1.Secret) { s.Data = map[string][]byte{ - currentProfileKey: []byte(key), - key: bytes, + currentProfileKey: []byte(key), + key: bytes, + kubetypes.KeyDeviceIPs: []byte(`["1.2.3.4", "::1"]`), + kubetypes.KeyDeviceFQDN: []byte(fmt.Sprintf("hostname-nodeid-%d.tails-scales.ts.net", i)), + // TODO(tomhjp): We have two different mechanisms to retrieve device IDs. + // Consolidate on this one. + kubetypes.KeyDeviceID: []byte(fmt.Sprintf("nodeid-%d", i)), + kubetypes.KeyPodUID: []byte(podUID), } }) } @@ -693,7 +1847,7 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { pgType tsapi.ProxyGroupType } pcLEStaging, pcLEStagingFalse, pcOther := proxyClassesForLEStagingTest() - sharedTestCases := testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther) + sharedTestCases := testCasesForLEStagingTests() var tests []proxyGroupLETestCase for _, tt := range sharedTestCases { tests = append(tests, proxyGroupLETestCase{ @@ -715,9 +1869,20 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { builder := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme) + pg := &tsapi.ProxyGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: tsapi.ProxyGroupSpec{ + Type: tt.pgType, + Replicas: ptr.To[int32](1), + ProxyClass: tt.proxyClassPerResource, + }, + } + // Pre-populate the fake client with ProxyClasses. - builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse, pcOther). - WithStatusSubresource(pcLEStaging, pcLEStagingFalse, pcOther) + builder = builder.WithObjects(pcLEStaging, pcLEStagingFalse, pcOther, pg). + WithStatusSubresource(pcLEStaging, pcLEStagingFalse, pcOther, pg) fc := builder.Build() @@ -730,22 +1895,9 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { setProxyClassReady(t, fc, cl, name) } - // Create ProxyGroup - pg := &tsapi.ProxyGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - Spec: tsapi.ProxyGroupSpec{ - Type: tt.pgType, - Replicas: ptr.To[int32](1), - ProxyClass: tt.proxyClassPerResource, - }, - } - mustCreate(t, fc, pg) - reconciler := &ProxyGroupReconciler{ tsNamespace: tsNamespace, - proxyImage: testProxyImage, + tsProxyImage: testProxyImage, defaultTags: []string{"tag:test"}, defaultProxyClass: tt.defaultProxyClass, Client: fc, @@ -759,7 +1911,7 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) { // Verify that the StatefulSet created for ProxyGrup has // the expected setting for the staging endpoint. sts := &appsv1.StatefulSet{} - if err := fc.Get(context.Background(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { + if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil { t.Fatalf("failed to get StatefulSet: %v", err) } @@ -783,7 +1935,7 @@ type leStagingTestCase struct { // Shared test cases for LE staging endpoint configuration for ProxyGroup and // non-HA Ingress. -func testCasesForLEStagingTests(pcLEStaging, pcLEStagingFalse, pcOther *tsapi.ProxyClass) []leStagingTestCase { +func testCasesForLEStagingTests() []leStagingTestCase { return []leStagingTestCase{ { name: "with_staging_proxyclass", diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 70b25f2d28784..df12554e0feca 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -7,7 +7,6 @@ package main import ( "context" - "crypto/sha256" _ "embed" "encoding/json" "errors" @@ -28,6 +27,7 @@ import ( "k8s.io/apiserver/pkg/storage/names" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" + "tailscale.com/client/tailscale" "tailscale.com/ipn" tsoperator "tailscale.com/k8s-operator" @@ -51,7 +51,7 @@ const ( // LabelProxyClass can be set by users on tailscale Ingresses and Services that define cluster ingress or // cluster egress, to specify that configuration in this ProxyClass should be applied to resources created for // the Ingress or Service. - LabelProxyClass = "tailscale.com/proxy-class" + LabelAnnotationProxyClass = "tailscale.com/proxy-class" FinalizerName = "tailscale.com/finalizer" @@ -91,8 +91,6 @@ const ( podAnnotationLastSetClusterDNSName = "tailscale.com/operator-last-set-cluster-dns-name" podAnnotationLastSetTailnetTargetIP = "tailscale.com/operator-last-set-ts-tailnet-target-ip" podAnnotationLastSetTailnetTargetFQDN = "tailscale.com/operator-last-set-ts-tailnet-target-fqdn" - // podAnnotationLastSetConfigFileHash is sha256 hash of the current tailscaled configuration contents. - podAnnotationLastSetConfigFileHash = "tailscale.com/operator-last-set-config-file-hash" proxyTypeEgress = "egress_service" proxyTypeIngressService = "ingress_service" @@ -104,13 +102,15 @@ const ( defaultLocalAddrPort = 9002 // metrics and health check port letsEncryptStagingEndpoint = "https://acme-staging-v02.api.letsencrypt.org/directory" + + mainContainerName = "tailscale" ) var ( // tailscaleManagedLabels are label keys that tailscale operator sets on StatefulSets and Pods. tailscaleManagedLabels = []string{kubetypes.LabelManaged, LabelParentType, LabelParentName, LabelParentNamespace, "app"} // tailscaleManagedAnnotations are annotation keys that tailscale operator sets on StatefulSets and Pods. - tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN, podAnnotationLastSetConfigFileHash} + tailscaleManagedAnnotations = []string{podAnnotationLastSetClusterIP, podAnnotationLastSetTailnetTargetIP, podAnnotationLastSetTailnetTargetFQDN} ) type tailscaleSTSConfig struct { @@ -141,6 +141,9 @@ type tailscaleSTSConfig struct { ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one) + + // LoginServer denotes the URL of the control plane that should be used by the proxy. + LoginServer string } type connector struct { @@ -165,6 +168,7 @@ type tailscaleSTSReconciler struct { proxyImage string proxyPriorityClassName string tsFirewallMode string + loginServer string } func (sts tailscaleSTSReconciler) validate() error { @@ -201,11 +205,11 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga } sts.ProxyClass = proxyClass - secretName, tsConfigHash, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc) + secretName, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc) if err != nil { return nil, fmt.Errorf("failed to create or get API key secret: %w", err) } - _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName, tsConfigHash) + _, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName) if err != nil { return nil, fmt.Errorf("failed to reconcile statefulset: %w", err) } @@ -335,7 +339,7 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec }) } -func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName, hash string, configs tailscaledConfigs, _ error) { +func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName string, configs tailscaledConfigs, _ error) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ // Hardcode a -0 suffix so that in future, if we support @@ -351,7 +355,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName()) orig = secret.DeepCopy() } else if !apierrors.IsNotFound(err) { - return "", "", nil, err + return "", nil, err } var authKey string @@ -361,13 +365,13 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * // ACME account key. sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels) if err != nil { - return "", "", nil, err + return "", nil, err } if sts != nil { // StatefulSet exists, so we have already created the secret. // If the secret is missing, they should delete the StatefulSet. logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName()) - return "", "", nil, nil + return "", nil, nil } // Create API Key secret which is going to be used by the statefulset // to authenticate with Tailscale. @@ -378,25 +382,20 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * } authKey, err = newAuthKey(ctx, a.tsClient, tags) if err != nil { - return "", "", nil, err + return "", nil, err } } configs, err := tailscaledConfig(stsC, authKey, orig) if err != nil { - return "", "", nil, fmt.Errorf("error creating tailscaled config: %w", err) + return "", nil, fmt.Errorf("error creating tailscaled config: %w", err) } - hash, err = tailscaledConfigHash(configs) - if err != nil { - return "", "", nil, fmt.Errorf("error calculating hash of tailscaled configs: %w", err) - } - latest := tailcfg.CapabilityVersion(-1) var latestConfig ipn.ConfigVAlpha for key, val := range configs { fn := tsoperator.TailscaledConfigFileName(key) b, err := json.Marshal(val) if err != nil { - return "", "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err) + return "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err) } mak.Set(&secret.StringData, fn, string(b)) if key > latest { @@ -408,7 +407,7 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * if stsC.ServeConfig != nil { j, err := json.Marshal(stsC.ServeConfig) if err != nil { - return "", "", nil, err + return "", nil, err } mak.Set(&secret.StringData, "serve-config", string(j)) } @@ -416,15 +415,15 @@ func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger * if orig != nil { logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig)) if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil { - return "", "", nil, err + return "", nil, err } } else { logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig)) if err := a.Create(ctx, secret); err != nil { - return "", "", nil, err + return "", nil, err } } - return secret.Name, hash, configs, nil + return secret.Name, configs, nil } // sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted @@ -535,7 +534,7 @@ var proxyYaml []byte //go:embed deploy/manifests/userspace-proxy.yaml var userspaceProxyYaml []byte -func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret, tsConfigHash string) (*appsv1.StatefulSet, error) { +func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret string) (*appsv1.StatefulSet, error) { ss := new(appsv1.StatefulSet) if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil { @@ -662,11 +661,6 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S }) } - dev, err := a.DeviceInfo(ctx, sts.ChildResourceLabels, logger) - if err != nil { - return nil, fmt.Errorf("failed to get device info: %w", err) - } - app, err := appInfoForProxy(sts) if err != nil { // No need to error out if now or in future we end up in a @@ -685,25 +679,7 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S ss = applyProxyClassToStatefulSet(sts.ProxyClass, ss, sts, logger) } updateSS := func(s *appsv1.StatefulSet) { - // This is a temporary workaround to ensure that proxies with capver older than 110 - // are restarted when tailscaled configfile contents have changed. - // This workaround ensures that: - // 1. The hash mechanism is used to trigger pod restarts for proxies below capver 110. - // 2. Proxies above capver are not unnecessarily restarted when the configfile contents change. - // 3. If the hash has alreay been set, but the capver is above 110, the old hash is preserved to avoid - // unnecessary pod restarts that could result in an update loop where capver cannot be determined for a - // restarting Pod and the hash is re-added again. - // Note that the hash annotation is only set on updates not creation, because if the StatefulSet is - // being created, there is no need for a restart. - // TODO(irbekrm): remove this in 1.84. - hash := tsConfigHash - if dev == nil || dev.capver >= 110 { - hash = s.Spec.Template.GetAnnotations()[podAnnotationLastSetConfigFileHash] - } s.Spec = ss.Spec - if hash != "" { - mak.Set(&s.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash, hash) - } s.ObjectMeta.Labels = ss.Labels s.ObjectMeta.Annotations = ss.Annotations } @@ -785,14 +761,21 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, enableEndpoints(ss, metricsEnabled, debugEnabled) } } - if pc.Spec.UseLetsEncryptStagingEnvironment && (stsCfg.proxyType == proxyTypeIngressResource || stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress)) { - for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { - ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ - Name: "TS_DEBUG_ACME_DIRECTORY_URL", - Value: letsEncryptStagingEndpoint, - }) - break + + if stsCfg != nil { + usesLetsEncrypt := stsCfg.proxyType == proxyTypeIngressResource || + stsCfg.proxyType == string(tsapi.ProxyGroupTypeIngress) || + stsCfg.proxyType == string(tsapi.ProxyGroupTypeKubernetesAPIServer) + + if pc.Spec.UseLetsEncryptStagingEnvironment && usesLetsEncrypt { + for i, c := range ss.Spec.Template.Spec.Containers { + if isMainContainer(&c) { + ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ + Name: "TS_DEBUG_ACME_DIRECTORY_URL", + Value: letsEncryptStagingEndpoint, + }) + break + } } } } @@ -855,7 +838,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, return base } for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { + if isMainContainer(&c) { ss.Spec.Template.Spec.Containers[i] = updateContainer(wantsPod.TailscaleContainer, ss.Spec.Template.Spec.Containers[i]) break } @@ -873,7 +856,7 @@ func applyProxyClassToStatefulSet(pc *tsapi.ProxyClass, ss *appsv1.StatefulSet, func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == "tailscale" { + if isMainContainer(&c) { if debug { ss.Spec.Template.Spec.Containers[i].Env = append(ss.Spec.Template.Spec.Containers[i].Env, // Serve tailscaled's debug metrics on on @@ -928,17 +911,12 @@ func enableEndpoints(ss *appsv1.StatefulSet, metrics, debug bool) { } } -func readAuthKey(secret *corev1.Secret, key string) (*string, error) { - origConf := &ipn.ConfigVAlpha{} - if err := json.Unmarshal([]byte(secret.Data[key]), origConf); err != nil { - return nil, fmt.Errorf("error unmarshaling previous tailscaled config in %q: %w", key, err) - } - return origConf.AuthKey, nil +func isMainContainer(c *corev1.Container) bool { + return c.Name == mainContainerName } // tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy -// state and auth key and returns tailscaled config files for currently supported proxy versions and a hash of that -// configuration. +// state and auth key and returns tailscaled config files for currently supported proxy versions. func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) { conf := &ipn.ConfigVAlpha{ Version: "alpha0", @@ -950,6 +928,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co AppConnector: &ipn.AppConnectorPrefs{Advertise: false}, } + if stsC.LoginServer != "" { + conf.ServerURL = &stsC.LoginServer + } + if stsC.Connector != nil { routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode) if err != nil { @@ -983,7 +965,10 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co return capVerConfigs, nil } -func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { +// latestConfigFromSecret returns the ipn.ConfigVAlpha with the highest capver +// as found in the Secret's key names, e.g. "cap-107.hujson" has capver 107. +// If no config is found, it returns nil. +func latestConfigFromSecret(s *corev1.Secret) (*ipn.ConfigVAlpha, error) { latest := tailcfg.CapabilityVersion(-1) latestStr := "" for k, data := range s.Data { @@ -1000,12 +985,31 @@ func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { latest = v } } + + var conf *ipn.ConfigVAlpha + if latestStr != "" { + conf = &ipn.ConfigVAlpha{} + if err := json.Unmarshal([]byte(s.Data[latestStr]), conf); err != nil { + return nil, fmt.Errorf("error unmarshaling tailscaled config from Secret %q in field %q: %w", s.Name, latestStr, err) + } + } + + return conf, nil +} + +func authKeyFromSecret(s *corev1.Secret) (key *string, err error) { + conf, err := latestConfigFromSecret(s) + if err != nil { + return nil, err + } + // Allow for configs that don't contain an auth key. Perhaps // users have some mechanisms to delete them. Auth key is // normally not needed after the initial login. - if latestStr != "" { - return readAuthKey(s, latestStr) + if conf != nil { + key = conf.AuthKey } + return key, nil } @@ -1031,27 +1035,6 @@ type ptrObject[T any] interface { type tailscaledConfigs map[tailcfg.CapabilityVersion]ipn.ConfigVAlpha -// hashBytes produces a hash for the provided tailscaled config that is the same across -// different invocations of this code. We do not use the -// tailscale.com/deephash.Hash here because that produces a different hash for -// the same value in different tailscale builds. The hash we are producing here -// is used to determine if the container running the Connector Tailscale node -// needs to be restarted. The container does not need restarting when the only -// thing that changed is operator version (the hash is also exposed to users via -// an annotation and might be confusing if it changes without the config having -// changed). -func tailscaledConfigHash(c tailscaledConfigs) (string, error) { - b, err := json.Marshal(c) - if err != nil { - return "", fmt.Errorf("error marshalling tailscaled configs: %w", err) - } - h := sha256.New() - if _, err = h.Write(b); err != nil { - return "", fmt.Errorf("error calculating hash: %w", err) - } - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - // createOrMaybeUpdate adds obj to the k8s cluster, unless the object already exists, // in which case update is called to make changes to it. If update is nil or returns // an error, the object is returned unmodified. @@ -1166,6 +1149,22 @@ func nameForService(svc *corev1.Service) string { return svc.Namespace + "-" + svc.Name } +// proxyClassForObject returns the proxy class for the given object. If the +// object does not have a proxy class label, it returns the default proxy class +func proxyClassForObject(o client.Object, proxyDefaultClass string) string { + proxyClass, exists := o.GetLabels()[LabelAnnotationProxyClass] + if exists { + return proxyClass + } + + proxyClass, exists = o.GetAnnotations()[LabelAnnotationProxyClass] + if exists { + return proxyClass + } + + return proxyDefaultClass +} + func isValidFirewallMode(m string) bool { return m == "auto" || m == "nftables" || m == "iptables" } diff --git a/cmd/k8s-operator/sts_test.go b/cmd/k8s-operator/sts_test.go index 35c512c8cd05b..afa791ccc7904 100644 --- a/cmd/k8s-operator/sts_test.go +++ b/cmd/k8s-operator/sts_test.go @@ -61,6 +61,7 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { // Setup proxyClassAllOpts := &tsapi.ProxyClass{ Spec: tsapi.ProxyClassSpec{ + UseLetsEncryptStagingEnvironment: true, StatefulSet: &tsapi.StatefulSet{ Labels: tsapi.Labels{"foo": "bar"}, Annotations: map[string]string{"foo.io/bar": "foo"}, @@ -292,6 +293,10 @@ func Test_applyProxyClassToStatefulSet(t *testing.T) { if diff := cmp.Diff(gotSS, wantSS); diff != "" { t.Errorf("Unexpected result applying ProxyClass with metrics enabled to a StatefulSet (-got +want):\n%s", diff) } + + // 8. A Kubernetes API proxy with letsencrypt staging enabled + gotSS = applyProxyClassToStatefulSet(proxyClassAllOpts, nonUserspaceProxySS.DeepCopy(), &tailscaleSTSConfig{proxyType: string(tsapi.ProxyGroupTypeKubernetesAPIServer)}, zl.Sugar()) + verifyEnvVar(t, gotSS, "TS_DEBUG_ACME_DIRECTORY_URL", letsEncryptStagingEndpoint) } func Test_mergeStatefulSetLabelsOrAnnots(t *testing.T) { diff --git a/cmd/k8s-operator/svc-for-pg.go b/cmd/k8s-operator/svc-for-pg.go index 779f2714e72f5..62cc36bd4a82b 100644 --- a/cmd/k8s-operator/svc-for-pg.go +++ b/cmd/k8s-operator/svc-for-pg.go @@ -41,7 +41,7 @@ import ( ) const ( - finalizerName = "tailscale.com/service-pg-finalizer" + svcPGFinalizerName = "tailscale.com/service-pg-finalizer" reasonIngressSvcInvalid = "IngressSvcInvalid" reasonIngressSvcValid = "IngressSvcValid" @@ -60,7 +60,6 @@ type HAServiceReconciler struct { recorder record.EventRecorder logger *zap.SugaredLogger tsClient tsClient - tsnetServer tsnetServer tsNamespace string lc localClient defaultTags []string @@ -164,27 +163,24 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin } return false, fmt.Errorf("getting ProxyGroup %q: %w", pgName, err) } - if !tsoperator.ProxyGroupIsReady(pg) { + if !tsoperator.ProxyGroupAvailable(pg) { logger.Infof("ProxyGroup is not (yet) ready") return false, nil } - // Validate Service configuration - if violations := validateService(svc); len(violations) > 0 { - msg := fmt.Sprintf("unable to provision proxy resources: invalid Service: %s", strings.Join(violations, ", ")) - r.recorder.Event(svc, corev1.EventTypeWarning, "INVALIDSERVICE", msg) - r.logger.Error(msg) - tsoperator.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, msg, r.clock, logger) + if err := r.validateService(ctx, svc, pg); err != nil { + r.recorder.Event(svc, corev1.EventTypeWarning, reasonIngressSvcInvalid, err.Error()) + tsoperator.SetServiceCondition(svc, tsapi.IngressSvcValid, metav1.ConditionFalse, reasonIngressSvcInvalid, err.Error(), r.clock, logger) return false, nil } - if !slices.Contains(svc.Finalizers, finalizerName) { + if !slices.Contains(svc.Finalizers, svcPGFinalizerName) { // This log line is printed exactly once during initial provisioning, // because once the finalizer is in place this block gets skipped. So, // this is a nice place to tell the operator that the high level, // multi-reconcile operation is underway. logger.Infof("exposing Service over tailscale") - svc.Finalizers = append(svc.Finalizers, finalizerName) + svc.Finalizers = append(svc.Finalizers, svcPGFinalizerName) if err := r.Update(ctx, svc); err != nil { return false, fmt.Errorf("failed to add finalizer: %w", err) } @@ -224,7 +220,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin // This checks and ensures that Tailscale Service's owner references are updated // for this Service and errors if that is not possible (i.e. because it // appears that the Tailscale Service has been created by a non-operator actor). - updatedAnnotations, err := r.ownerAnnotations(existingTSSvc) + updatedAnnotations, err := ownerAnnotations(r.operatorID, existingTSSvc) if err != nil { instr := fmt.Sprintf("To proceed, you can either manually delete the existing Tailscale Service or choose a different hostname with the '%s' annotaion", AnnotationHostname) msg := fmt.Sprintf("error ensuring ownership of Tailscale Service %s: %v. %s", hostname, err, instr) @@ -382,7 +378,7 @@ func (r *HAServiceReconciler) maybeProvision(ctx context.Context, hostname strin // corresponding to this Service. func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, svc *corev1.Service, logger *zap.SugaredLogger) (svcChanged bool, err error) { logger.Debugf("Ensuring any resources for Service are cleaned up") - ix := slices.Index(svc.Finalizers, finalizerName) + ix := slices.Index(svc.Finalizers, svcPGFinalizerName) if ix < 0 { logger.Debugf("no finalizer, nothing to do") return false, nil @@ -398,7 +394,7 @@ func (r *HAServiceReconciler) maybeCleanup(ctx context.Context, hostname string, serviceName := tailcfg.ServiceName("svc:" + hostname) // 1. Clean up the Tailscale Service. - svcChanged, err = r.cleanupTailscaleService(ctx, serviceName, logger) + svcChanged, err = cleanupTailscaleService(ctx, r.tsClient, serviceName, r.operatorID, logger) if err != nil { return false, fmt.Errorf("error deleting Tailscale Service: %w", err) } @@ -459,7 +455,7 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG return false, fmt.Errorf("failed to update tailscaled config services: %w", err) } - svcsChanged, err = r.cleanupTailscaleService(ctx, tailcfg.ServiceName(tsSvcName), logger) + svcsChanged, err = cleanupTailscaleService(ctx, r.tsClient, tailcfg.ServiceName(tsSvcName), r.operatorID, logger) if err != nil { return false, fmt.Errorf("deleting Tailscale Service %q: %w", tsSvcName, err) } @@ -489,12 +485,12 @@ func (r *HAServiceReconciler) maybeCleanupProxyGroup(ctx context.Context, proxyG func (r *HAServiceReconciler) deleteFinalizer(ctx context.Context, svc *corev1.Service, logger *zap.SugaredLogger) error { svc.Finalizers = slices.DeleteFunc(svc.Finalizers, func(f string) bool { - return f == finalizerName + return f == svcPGFinalizerName }) - logger.Debugf("ensure %q finalizer is removed", finalizerName) + logger.Debugf("ensure %q finalizer is removed", svcPGFinalizerName) if err := r.Update(ctx, svc); err != nil { - return fmt.Errorf("failed to remove finalizer %q: %w", finalizerName, err) + return fmt.Errorf("failed to remove finalizer %q: %w", svcPGFinalizerName, err) } r.mu.Lock() defer r.mu.Unlock() @@ -532,8 +528,8 @@ func (r *HAServiceReconciler) tailnetCertDomain(ctx context.Context) (string, er // If a Tailscale Service is found, but contains other owner references, only removes this operator's owner reference. // If a Tailscale Service by the given name is not found or does not contain this operator's owner reference, do nothing. // It returns true if an existing Tailscale Service was updated to remove owner reference, as well as any error that occurred. -func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name tailcfg.ServiceName, logger *zap.SugaredLogger) (updated bool, err error) { - svc, err := r.tsClient.GetVIPService(ctx, name) +func cleanupTailscaleService(ctx context.Context, tsClient tsClient, name tailcfg.ServiceName, operatorID string, logger *zap.SugaredLogger) (updated bool, err error) { + svc, err := tsClient.GetVIPService(ctx, name) if isErrorFeatureFlagNotEnabled(err) { msg := fmt.Sprintf("Unable to proceed with cleanup: %s.", msgFeatureFlagNotEnabled) logger.Warn(msg) @@ -566,14 +562,14 @@ func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name // cluster before deleting the Ingress. Perhaps the comparison could be // 'if or.OperatorID == r.operatorID || or.ingressUID == r.ingressUID'. ix := slices.IndexFunc(o.OwnerRefs, func(or OwnerRef) bool { - return or.OperatorID == r.operatorID + return or.OperatorID == operatorID }) if ix == -1 { return false, nil } if len(o.OwnerRefs) == 1 { logger.Infof("Deleting Tailscale Service %q", name) - return false, r.tsClient.DeleteVIPService(ctx, name) + return false, tsClient.DeleteVIPService(ctx, name) } o.OwnerRefs = slices.Delete(o.OwnerRefs, ix, ix+1) logger.Infof("Updating Tailscale Service %q", name) @@ -582,7 +578,7 @@ func (r *HAServiceReconciler) cleanupTailscaleService(ctx context.Context, name return false, fmt.Errorf("error marshalling updated Tailscale Service owner reference: %w", err) } svc.Annotations[ownerAnnotation] = string(json) - return true, r.tsClient.CreateOrUpdateVIPService(ctx, svc) + return true, tsClient.CreateOrUpdateVIPService(ctx, svc) } func (a *HAServiceReconciler) backendRoutesSetup(ctx context.Context, serviceName, replicaName, pgName string, wantsCfg *ingressservices.Config, logger *zap.SugaredLogger) (bool, error) { @@ -657,7 +653,7 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con // Get all config Secrets for this ProxyGroup. // Get all Pods secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "config"))); err != nil { + if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeConfig))); err != nil { return fmt.Errorf("failed to list config Secrets: %w", err) } @@ -724,7 +720,7 @@ func (a *HAServiceReconciler) maybeUpdateAdvertiseServicesConfig(ctx context.Con func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName string, serviceName tailcfg.ServiceName) (int, error) { // Get all state Secrets for this ProxyGroup. secrets := &corev1.SecretList{} - if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, "state"))); err != nil { + if err := a.List(ctx, secrets, client.InNamespace(a.tsNamespace), client.MatchingLabels(pgSecretLabels(pgName, kubetypes.LabelSecretTypeState))); err != nil { return 0, fmt.Errorf("failed to list ProxyGroup %q state Secrets: %w", pgName, err) } @@ -745,49 +741,6 @@ func (a *HAServiceReconciler) numberPodsAdvertising(ctx context.Context, pgName return count, nil } -// ownerAnnotations returns the updated annotations required to ensure this -// instance of the operator is included as an owner. If the Tailscale Service is not -// nil, but does not contain an owner we return an error as this likely means -// that the Tailscale Service was created by something other than a Tailscale -// Kubernetes operator. -func (r *HAServiceReconciler) ownerAnnotations(svc *tailscale.VIPService) (map[string]string, error) { - ref := OwnerRef{ - OperatorID: r.operatorID, - } - if svc == nil { - c := ownerAnnotationValue{OwnerRefs: []OwnerRef{ref}} - json, err := json.Marshal(c) - if err != nil { - return nil, fmt.Errorf("[unexpected] unable to marshal Tailscale Service owner annotation contents: %w, please report this", err) - } - return map[string]string{ - ownerAnnotation: string(json), - }, nil - } - o, err := parseOwnerAnnotation(svc) - if err != nil { - return nil, err - } - if o == nil || len(o.OwnerRefs) == 0 { - return nil, fmt.Errorf("Tailscale Service %s exists, but does not contain owner annotation with owner references; not proceeding as this is likely a resource created by something other than the Tailscale Kubernetes operator", svc.Name) - } - if slices.Contains(o.OwnerRefs, ref) { // up to date - return svc.Annotations, nil - } - o.OwnerRefs = append(o.OwnerRefs, ref) - json, err := json.Marshal(o) - if err != nil { - return nil, fmt.Errorf("error marshalling updated owner references: %w", err) - } - - newAnnots := make(map[string]string, len(svc.Annotations)+1) - for k, v := range svc.Annotations { - newAnnots[k] = v - } - newAnnots[ownerAnnotation] = string(json) - return newAnnots, nil -} - // dnsNameForService returns the DNS name for the given Tailscale Service name. func (r *HAServiceReconciler) dnsNameForService(ctx context.Context, svc tailcfg.ServiceName) (string, error) { s := svc.WithoutPrefix() @@ -857,3 +810,26 @@ func (r *HAServiceReconciler) checkEndpointsReady(ctx context.Context, svc *core logger.Debugf("could not find any ready Endpoints in EndpointSlice") return false, nil } + +func (r *HAServiceReconciler) validateService(ctx context.Context, svc *corev1.Service, pg *tsapi.ProxyGroup) error { + var errs []error + if pg.Spec.Type != tsapi.ProxyGroupTypeIngress { + errs = append(errs, fmt.Errorf("ProxyGroup %q is of type %q but must be of type %q", + pg.Name, pg.Spec.Type, tsapi.ProxyGroupTypeIngress)) + } + if violations := validateService(svc); len(violations) > 0 { + errs = append(errs, fmt.Errorf("invalid Service: %s", strings.Join(violations, ", "))) + } + svcList := &corev1.ServiceList{} + if err := r.List(ctx, svcList); err != nil { + errs = append(errs, fmt.Errorf("[unexpected] error listing Services: %w", err)) + return errors.Join(errs...) + } + svcName := nameForService(svc) + for _, s := range svcList.Items { + if r.shouldExpose(&s) && nameForService(&s) == svcName && s.UID != svc.UID { + errs = append(errs, fmt.Errorf("found duplicate Service %q for hostname %q - multiple HA Services for the same hostname in the same cluster are not allowed", client.ObjectKeyFromObject(&s), svcName)) + } + } + return errors.Join(errs...) +} diff --git a/cmd/k8s-operator/svc-for-pg_test.go b/cmd/k8s-operator/svc-for-pg_test.go index 4bb633cb8ff66..baaa07727df06 100644 --- a/cmd/k8s-operator/svc-for-pg_test.go +++ b/cmd/k8s-operator/svc-for-pg_test.go @@ -12,6 +12,7 @@ import ( "math/rand/v2" "net/netip" "testing" + "time" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -25,6 +26,7 @@ import ( tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/ingressservices" + "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/types/ptr" "tailscale.com/util/mak" @@ -33,7 +35,7 @@ import ( ) func TestServicePGReconciler(t *testing.T) { - svcPGR, stateSecret, fc, ft := setupServiceTest(t) + svcPGR, stateSecret, fc, ft, _ := setupServiceTest(t) svcs := []*corev1.Service{} config := []string{} for i := range 4 { @@ -45,7 +47,7 @@ func TestServicePGReconciler(t *testing.T) { config = append(config, fmt.Sprintf("svc:default-%s", svc.Name)) verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"}) - verifyTailscaledConfig(t, fc, config) + verifyTailscaledConfig(t, fc, "test-pg", config) } for i, svc := range svcs { @@ -74,12 +76,12 @@ func TestServicePGReconciler(t *testing.T) { } config = removeEl(config, fmt.Sprintf("svc:default-%s", svc.Name)) - verifyTailscaledConfig(t, fc, config) + verifyTailscaledConfig(t, fc, "test-pg", config) } } func TestServicePGReconciler_UpdateHostname(t *testing.T) { - svcPGR, stateSecret, fc, ft := setupServiceTest(t) + svcPGR, stateSecret, fc, ft, _ := setupServiceTest(t) cip := "4.1.6.7" svc, _ := setupTestService(t, "test-service", "", cip, fc, stateSecret) @@ -87,7 +89,7 @@ func TestServicePGReconciler_UpdateHostname(t *testing.T) { expectReconciled(t, svcPGR, "default", svc.Name) verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"}) - verifyTailscaledConfig(t, fc, []string{fmt.Sprintf("svc:default-%s", svc.Name)}) + verifyTailscaledConfig(t, fc, "test-pg", []string{fmt.Sprintf("svc:default-%s", svc.Name)}) hostname := "foobarbaz" mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) { @@ -99,7 +101,7 @@ func TestServicePGReconciler_UpdateHostname(t *testing.T) { expectReconciled(t, svcPGR, "default", svc.Name) verifyTailscaleService(t, ft, fmt.Sprintf("svc:%s", hostname), []string{"do-not-validate"}) - verifyTailscaledConfig(t, fc, []string{fmt.Sprintf("svc:%s", hostname)}) + verifyTailscaledConfig(t, fc, "test-pg", []string{fmt.Sprintf("svc:%s", hostname)}) _, err := ft.GetVIPService(context.Background(), tailcfg.ServiceName(fmt.Sprintf("svc:default-%s", svc.Name))) if err == nil { @@ -110,7 +112,7 @@ func TestServicePGReconciler_UpdateHostname(t *testing.T) { } } -func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, client.Client, *fakeTSClient) { +func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, client.Client, *fakeTSClient, *tstest.Clock) { // Pre-create the ProxyGroup pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ @@ -138,10 +140,10 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName("test-pg", 0), Namespace: "operator-ns", - Labels: pgSecretLabels("test-pg", "config"), + Labels: pgSecretLabels("test-pg", kubetypes.LabelSecretTypeConfig), }, Data: map[string][]byte{ - tsoperator.TailscaledConfigFileName(106): []byte(`{"Version":""}`), + tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(`{"Version":""}`), }, } @@ -178,7 +180,7 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien // Set ProxyGroup status to ready pg.Status.Conditions = []metav1.Condition{ { - Type: string(tsapi.ProxyGroupReady), + Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, @@ -186,7 +188,6 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien if err := fc.Status().Update(context.Background(), pg); err != nil { t.Fatal(err) } - fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}} ft := &fakeTSClient{} zl, err := zap.NewDevelopment() @@ -209,20 +210,79 @@ func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, clien clock: cl, defaultTags: []string{"tag:k8s"}, tsNamespace: "operator-ns", - tsnetServer: fakeTsnetServer, logger: zl.Sugar(), recorder: record.NewFakeRecorder(10), lc: lc, } - return svcPGR, pgStateSecret, fc, ft + return svcPGR, pgStateSecret, fc, ft, cl +} + +func TestValidateService(t *testing.T) { + // Test that no more than one Kubernetes Service in a cluster refers to the same Tailscale Service. + pgr, _, lc, _, cl := setupServiceTest(t) + svc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app", + Namespace: "ns-1", + UID: types.UID("1234-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + "tailscale.com/hostname": "my-app", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.4", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: ptr.To("tailscale"), + }, + } + svc2 := &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-app2", + Namespace: "ns-2", + UID: types.UID("1235-UID"), + Annotations: map[string]string{ + "tailscale.com/proxy-group": "test-pg", + "tailscale.com/hostname": "my-app", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "1.2.3.5", + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerClass: ptr.To("tailscale"), + }, + } + wantSvc := &corev1.Service{ + ObjectMeta: svc.ObjectMeta, + TypeMeta: svc.TypeMeta, + Spec: svc.Spec, + Status: corev1.ServiceStatus{ + Conditions: []metav1.Condition{ + { + Type: string(tsapi.IngressSvcValid), + Status: metav1.ConditionFalse, + Reason: reasonIngressSvcInvalid, + LastTransitionTime: metav1.NewTime(cl.Now().Truncate(time.Second)), + Message: `found duplicate Service "ns-2/my-app2" for hostname "my-app" - multiple HA Services for the same hostname in the same cluster are not allowed`, + }, + }, + }, + } + + mustCreate(t, lc, svc) + mustCreate(t, lc, svc2) + expectReconciled(t, pgr, svc.Namespace, svc.Name) + expectEqual(t, lc, wantSvc) } func TestServicePGReconciler_MultiCluster(t *testing.T) { var ft *fakeTSClient var lc localClient for i := 0; i <= 10; i++ { - pgr, stateSecret, fc, fti := setupServiceTest(t) + pgr, stateSecret, fc, fti, _ := setupServiceTest(t) if i == 0 { ft = fti lc = pgr.lc @@ -239,18 +299,18 @@ func TestServicePGReconciler_MultiCluster(t *testing.T) { t.Fatalf("getting Tailscale Service: %v", err) } - if len(tsSvcs) != 1 { - t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs)) + if len(tsSvcs.VIPServices) != 1 { + t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs.VIPServices)) } - for name := range tsSvcs { - t.Logf("found Tailscale Service with name %q", name.String()) + for _, svc := range tsSvcs.VIPServices { + t.Logf("found Tailscale Service with name %q", svc.Name) } } } func TestIgnoreRegularService(t *testing.T) { - pgr, _, fc, ft := setupServiceTest(t) + pgr, _, fc, ft, _ := setupServiceTest(t) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -273,11 +333,11 @@ func TestIgnoreRegularService(t *testing.T) { mustCreate(t, fc, svc) expectReconciled(t, pgr, "default", "test") - verifyTailscaledConfig(t, fc, nil) + verifyTailscaledConfig(t, fc, "test-pg", nil) tsSvcs, err := ft.ListVIPServices(context.Background()) if err == nil { - if len(tsSvcs) > 0 { + if len(tsSvcs.VIPServices) > 0 { t.Fatal("unexpected Tailscale Services found") } } diff --git a/cmd/k8s-operator/svc.go b/cmd/k8s-operator/svc.go index d6a6f440feda9..52c8bec7ff32a 100644 --- a/cmd/k8s-operator/svc.go +++ b/cmd/k8s-operator/svc.go @@ -23,6 +23,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/kubetypes" @@ -41,6 +42,8 @@ const ( reasonProxyInvalid = "ProxyInvalid" reasonProxyFailed = "ProxyFailed" reasonProxyPending = "ProxyPending" + + indexServiceProxyClass = ".metadata.annotations.service-proxy-class" ) type ServiceReconciler struct { @@ -268,6 +271,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga Tags: tags, ChildResourceLabels: crl, ProxyClassName: proxyClass, + LoginServer: a.ssr.loginServer, } sts.proxyType = proxyTypeEgress if a.shouldExpose(svc) { @@ -392,6 +396,7 @@ func validateService(svc *corev1.Service) []string { violations = append(violations, fmt.Sprintf("invalid Tailscale hostname %q, use %q annotation to override: %s", svcName, AnnotationHostname, err)) } } + violations = append(violations, tagViolations(svc)...) return violations } @@ -437,16 +442,6 @@ func tailnetTargetAnnotation(svc *corev1.Service) string { return svc.Annotations[annotationTailnetTargetIPOld] } -// proxyClassForObject returns the proxy class for the given object. If the -// object does not have a proxy class label, it returns the default proxy class -func proxyClassForObject(o client.Object, proxyDefaultClass string) string { - proxyClass, exists := o.GetLabels()[LabelProxyClass] - if !exists { - proxyClass = proxyDefaultClass - } - return proxyClass -} - func proxyClassIsReady(ctx context.Context, name string, cl client.Client) (bool, error) { proxyClass := new(tsapi.ProxyClass) if err := cl.Get(ctx, types.NamespacedName{Name: name}, proxyClass); err != nil { diff --git a/cmd/k8s-operator/testutils_test.go b/cmd/k8s-operator/testutils_test.go index 619aecc56816e..6ae32d6fbac13 100644 --- a/cmd/k8s-operator/testutils_test.go +++ b/cmd/k8s-operator/testutils_test.go @@ -62,7 +62,6 @@ type configOpts struct { subnetRoutes string isExitNode bool isAppConnector bool - confFileHash string serveConfig *ipn.ServeConfig shouldEnableForwardingClusterTrafficViaIngress bool proxyClass string // configuration from the named ProxyClass should be applied to proxy resources @@ -120,9 +119,6 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef ReadOnly: true, MountPath: "/etc/tsconfig", }} - if opts.confFileHash != "" { - mak.Set(&annots, "tailscale.com/operator-last-set-config-file-hash", opts.confFileHash) - } if opts.firewallMode != "" { tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{ Name: "TS_DEBUG_FIREWALL_MODE", @@ -358,10 +354,6 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps }, }, } - ss.Spec.Template.Annotations = map[string]string{} - if opts.confFileHash != "" { - ss.Spec.Template.Annotations["tailscale.com/operator-last-set-config-file-hash"] = opts.confFileHash - } // If opts.proxyClass is set, retrieve the ProxyClass and apply // configuration from that to the StatefulSet. if opts.proxyClass != "" { @@ -842,17 +834,6 @@ func (c *fakeTSClient) Deleted() []string { return c.deleted } -// removeHashAnnotation can be used to remove declarative tailscaled config hash -// annotation from proxy StatefulSets to make the tests more maintainable (so -// that we don't have to change the annotation in each test case after any -// change to the configfile contents). -func removeHashAnnotation(sts *appsv1.StatefulSet) { - delete(sts.Spec.Template.Annotations, podAnnotationLastSetConfigFileHash) - if len(sts.Spec.Template.Annotations) == 0 { - sts.Spec.Template.Annotations = nil - } -} - func removeResourceReqs(sts *appsv1.StatefulSet) { if sts != nil { sts.Spec.Template.Spec.Resources = nil @@ -910,13 +891,17 @@ func (c *fakeTSClient) GetVIPService(ctx context.Context, name tailcfg.ServiceNa return svc, nil } -func (c *fakeTSClient) ListVIPServices(ctx context.Context) (map[tailcfg.ServiceName]*tailscale.VIPService, error) { +func (c *fakeTSClient) ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error) { c.Lock() defer c.Unlock() if c.vipServices == nil { return nil, &tailscale.ErrResponse{Status: http.StatusNotFound} } - return c.vipServices, nil + result := &tailscale.VIPServiceList{} + for _, svc := range c.vipServices { + result.VIPServices = append(result.VIPServices, *svc) + } + return result, nil } func (c *fakeTSClient) CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error { diff --git a/cmd/k8s-operator/tsclient.go b/cmd/k8s-operator/tsclient.go index f49f84af96ed4..50620c26ddf27 100644 --- a/cmd/k8s-operator/tsclient.go +++ b/cmd/k8s-operator/tsclient.go @@ -12,6 +12,7 @@ import ( "golang.org/x/oauth2/clientcredentials" "tailscale.com/internal/client/tailscale" + "tailscale.com/ipn" "tailscale.com/tailcfg" ) @@ -19,10 +20,9 @@ import ( // call should be performed on the default tailnet for the provided credentials. const ( defaultTailnet = "-" - defaultBaseURL = "https://api.tailscale.com" ) -func newTSClient(ctx context.Context, clientIDPath, clientSecretPath string) (tsClient, error) { +func newTSClient(ctx context.Context, clientIDPath, clientSecretPath, loginServer string) (tsClient, error) { clientID, err := os.ReadFile(clientIDPath) if err != nil { return nil, fmt.Errorf("error reading client ID %q: %w", clientIDPath, err) @@ -31,14 +31,22 @@ func newTSClient(ctx context.Context, clientIDPath, clientSecretPath string) (ts if err != nil { return nil, fmt.Errorf("reading client secret %q: %w", clientSecretPath, err) } + const tokenURLPath = "/api/v2/oauth/token" + tokenURL := fmt.Sprintf("%s%s", ipn.DefaultControlURL, tokenURLPath) + if loginServer != "" { + tokenURL = fmt.Sprintf("%s%s", loginServer, tokenURLPath) + } credentials := clientcredentials.Config{ ClientID: string(clientID), ClientSecret: string(clientSecret), - TokenURL: "https://login.tailscale.com/api/v2/oauth/token", + TokenURL: tokenURL, } c := tailscale.NewClient(defaultTailnet, nil) c.UserAgent = "tailscale-k8s-operator" c.HTTPClient = credentials.Client(ctx) + if loginServer != "" { + c.BaseURL = loginServer + } return c, nil } @@ -48,6 +56,8 @@ type tsClient interface { DeleteDevice(ctx context.Context, nodeStableID string) error // GetVIPService is a method for getting a Tailscale Service. VIPService is the original name for Tailscale Service. GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*tailscale.VIPService, error) + // ListVIPServices is a method for listing all Tailscale Services. VIPService is the original name for Tailscale Service. + ListVIPServices(ctx context.Context) (*tailscale.VIPServiceList, error) // CreateOrUpdateVIPService is a method for creating or updating a Tailscale Service. CreateOrUpdateVIPService(ctx context.Context, svc *tailscale.VIPService) error // DeleteVIPService is a method for deleting a Tailscale Service. diff --git a/cmd/k8s-operator/tsrecorder.go b/cmd/k8s-operator/tsrecorder.go index 081543cd384db..ec95ecf40dab5 100644 --- a/cmd/k8s-operator/tsrecorder.go +++ b/cmd/k8s-operator/tsrecorder.go @@ -59,6 +59,7 @@ type RecorderReconciler struct { clock tstime.Clock tsNamespace string tsClient tsClient + loginServer string mu sync.Mutex // protects following recorders set.Slice[types.UID] // for recorders gauge @@ -202,7 +203,7 @@ func (r *RecorderReconciler) maybeProvision(ctx context.Context, tsr *tsapi.Reco }); err != nil { return fmt.Errorf("error creating RoleBinding: %w", err) } - ss := tsrStatefulSet(tsr, r.tsNamespace) + ss := tsrStatefulSet(tsr, r.tsNamespace, r.loginServer) if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, ss, func(s *appsv1.StatefulSet) { s.ObjectMeta.Labels = ss.ObjectMeta.Labels s.ObjectMeta.Annotations = ss.ObjectMeta.Annotations @@ -446,18 +447,15 @@ func (r *RecorderReconciler) getDeviceInfo(ctx context.Context, tsrName string) return tsapi.RecorderTailnetDevice{}, false, err } - return getDeviceInfo(ctx, r.tsClient, secret) -} - -func getDeviceInfo(ctx context.Context, tsClient tsClient, secret *corev1.Secret) (d tsapi.RecorderTailnetDevice, ok bool, err error) { prefs, ok, err := getDevicePrefs(secret) if !ok || err != nil { return tsapi.RecorderTailnetDevice{}, false, err } // TODO(tomhjp): The profile info doesn't include addresses, which is why we - // need the API. Should we instead update the profile to include addresses? - device, err := tsClient.Device(ctx, string(prefs.Config.NodeID), nil) + // need the API. Should maybe update tsrecorder to write IPs to the state + // Secret like containerboot does. + device, err := r.tsClient.Device(ctx, string(prefs.Config.NodeID), nil) if err != nil { return tsapi.RecorderTailnetDevice{}, false, fmt.Errorf("failed to get device info from API: %w", err) } diff --git a/cmd/k8s-operator/tsrecorder_specs.go b/cmd/k8s-operator/tsrecorder_specs.go index 7c6e80aed56fd..f5eedc2a1d1da 100644 --- a/cmd/k8s-operator/tsrecorder_specs.go +++ b/cmd/k8s-operator/tsrecorder_specs.go @@ -17,7 +17,7 @@ import ( "tailscale.com/version" ) -func tsrStatefulSet(tsr *tsapi.Recorder, namespace string) *appsv1.StatefulSet { +func tsrStatefulSet(tsr *tsapi.Recorder, namespace string, loginServer string) *appsv1.StatefulSet { return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: tsr.Name, @@ -59,7 +59,7 @@ func tsrStatefulSet(tsr *tsapi.Recorder, namespace string) *appsv1.StatefulSet { ImagePullPolicy: tsr.Spec.StatefulSet.Pod.Container.ImagePullPolicy, Resources: tsr.Spec.StatefulSet.Pod.Container.Resources, SecurityContext: tsr.Spec.StatefulSet.Pod.Container.SecurityContext, - Env: env(tsr), + Env: env(tsr, loginServer), EnvFrom: func() []corev1.EnvFromSource { if tsr.Spec.Storage.S3 == nil || tsr.Spec.Storage.S3.Credentials.Secret.Name == "" { return nil @@ -201,7 +201,7 @@ func tsrStateSecret(tsr *tsapi.Recorder, namespace string) *corev1.Secret { } } -func env(tsr *tsapi.Recorder) []corev1.EnvVar { +func env(tsr *tsapi.Recorder, loginServer string) []corev1.EnvVar { envs := []corev1.EnvVar{ { Name: "TS_AUTHKEY", @@ -239,6 +239,10 @@ func env(tsr *tsapi.Recorder) []corev1.EnvVar { Name: "TSRECORDER_HOSTNAME", Value: "$(POD_NAME)", }, + { + Name: "TSRECORDER_LOGIN_SERVER", + Value: loginServer, + }, } for _, env := range tsr.Spec.StatefulSet.Pod.Container.Env { diff --git a/cmd/k8s-operator/tsrecorder_specs_test.go b/cmd/k8s-operator/tsrecorder_specs_test.go index 94a8a816c69f5..49332d09b6a08 100644 --- a/cmd/k8s-operator/tsrecorder_specs_test.go +++ b/cmd/k8s-operator/tsrecorder_specs_test.go @@ -90,7 +90,7 @@ func TestRecorderSpecs(t *testing.T) { }, } - ss := tsrStatefulSet(tsr, tsNamespace) + ss := tsrStatefulSet(tsr, tsNamespace, tsLoginServer) // StatefulSet-level. if diff := cmp.Diff(ss.Annotations, tsr.Spec.StatefulSet.Annotations); diff != "" { @@ -124,7 +124,7 @@ func TestRecorderSpecs(t *testing.T) { } // Container-level. - if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, env(tsr)); diff != "" { + if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Env, env(tsr, tsLoginServer)); diff != "" { t.Errorf("(-got +want):\n%s", diff) } if diff := cmp.Diff(ss.Spec.Template.Spec.Containers[0].Image, tsr.Spec.StatefulSet.Pod.Container.Image); diff != "" { diff --git a/cmd/k8s-operator/tsrecorder_test.go b/cmd/k8s-operator/tsrecorder_test.go index e6d56ef2f04c6..990bd68193e8b 100644 --- a/cmd/k8s-operator/tsrecorder_test.go +++ b/cmd/k8s-operator/tsrecorder_test.go @@ -25,7 +25,10 @@ import ( "tailscale.com/tstest" ) -const tsNamespace = "tailscale" +const ( + tsNamespace = "tailscale" + tsLoginServer = "example.tailscale.com" +) func TestRecorder(t *testing.T) { tsr := &tsapi.Recorder{ @@ -51,6 +54,7 @@ func TestRecorder(t *testing.T) { recorder: fr, l: zl.Sugar(), clock: cl, + loginServer: tsLoginServer, } t.Run("invalid_spec_gives_an_error_condition", func(t *testing.T) { @@ -234,7 +238,7 @@ func expectRecorderResources(t *testing.T, fc client.WithWatch, tsr *tsapi.Recor role := tsrRole(tsr, tsNamespace) roleBinding := tsrRoleBinding(tsr, tsNamespace) serviceAccount := tsrServiceAccount(tsr, tsNamespace) - statefulSet := tsrStatefulSet(tsr, tsNamespace) + statefulSet := tsrStatefulSet(tsr, tsNamespace, tsLoginServer) if shouldExist { expectEqual(t, fc, auth) diff --git a/cmd/k8s-proxy/internal/config/config.go b/cmd/k8s-proxy/internal/config/config.go new file mode 100644 index 0000000000000..4013047e76f0c --- /dev/null +++ b/cmd/k8s-proxy/internal/config/config.go @@ -0,0 +1,264 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package config provides watchers for the various supported ways to load a +// config file for k8s-proxy; currently file or Kubernetes Secret. +package config + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/types/ptr" + "tailscale.com/util/testenv" +) + +type configLoader struct { + logger *zap.SugaredLogger + client clientcorev1.CoreV1Interface + + cfgChan chan<- *conf.Config + previous []byte + + once sync.Once // For use in tests. To close cfgIgnored. + cfgIgnored chan struct{} // For use in tests. +} + +func NewConfigLoader(logger *zap.SugaredLogger, client clientcorev1.CoreV1Interface, cfgChan chan<- *conf.Config) *configLoader { + return &configLoader{ + logger: logger, + client: client, + cfgChan: cfgChan, + } +} + +func (l *configLoader) WatchConfig(ctx context.Context, path string) error { + secretNamespacedName, isKubeSecret := strings.CutPrefix(path, "kube:") + if isKubeSecret { + secretNamespace, secretName, ok := strings.Cut(secretNamespacedName, string(types.Separator)) + if !ok { + return fmt.Errorf("invalid Kubernetes Secret reference %q, expected format /", path) + } + if err := l.watchConfigSecretChanges(ctx, secretNamespace, secretName); err != nil && !errors.Is(err, context.Canceled) { + return fmt.Errorf("error watching config Secret %q: %w", secretNamespacedName, err) + } + + return nil + } + + if err := l.watchConfigFileChanges(ctx, path); err != nil && !errors.Is(err, context.Canceled) { + return fmt.Errorf("error watching config file %q: %w", path, err) + } + + return nil +} + +func (l *configLoader) reloadConfig(ctx context.Context, raw []byte) error { + if bytes.Equal(raw, l.previous) { + if l.cfgIgnored != nil && testenv.InTest() { + l.once.Do(func() { + close(l.cfgIgnored) + }) + } + return nil + } + + cfg, err := conf.Load(raw) + if err != nil { + return fmt.Errorf("error loading config: %w", err) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case l.cfgChan <- &cfg: + } + + l.previous = raw + return nil +} + +func (l *configLoader) watchConfigFileChanges(ctx context.Context, path string) error { + var ( + tickChan <-chan time.Time + eventChan <-chan fsnotify.Event + errChan <-chan error + ) + + if w, err := fsnotify.NewWatcher(); err != nil { + // Creating a new fsnotify watcher would fail for example if inotify was not able to create a new file descriptor. + // See https://github.com/tailscale/tailscale/issues/15081 + l.logger.Infof("Failed to create fsnotify watcher on config file %q; watching for changes on 5s timer: %v", path, err) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + tickChan = ticker.C + } else { + dir := filepath.Dir(path) + file := filepath.Base(path) + l.logger.Infof("Watching directory %q for changes to config file %q", dir, file) + defer w.Close() + if err := w.Add(dir); err != nil { + return fmt.Errorf("failed to add fsnotify watch: %w", err) + } + eventChan = w.Events + errChan = w.Errors + } + + // Read the initial config file, but after the watcher is already set up to + // avoid an unlucky race condition if the config file is edited in between. + b, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading config file %q: %w", path, err) + } + if err := l.reloadConfig(ctx, b); err != nil { + return fmt.Errorf("error loading initial config file %q: %w", path, err) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err, ok := <-errChan: + if !ok { + // Watcher was closed. + return nil + } + return fmt.Errorf("watcher error: %w", err) + case <-tickChan: + case ev, ok := <-eventChan: + if !ok { + // Watcher was closed. + return nil + } + if ev.Name != path || ev.Op&fsnotify.Write == 0 { + // Ignore irrelevant events. + continue + } + } + b, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading config file: %w", err) + } + // Writers such as os.WriteFile may truncate the file before writing + // new contents, so it's possible to read an empty file if we read before + // the write has completed. + if len(b) == 0 { + continue + } + if err := l.reloadConfig(ctx, b); err != nil { + return fmt.Errorf("error reloading config file %q: %v", path, err) + } + } +} + +func (l *configLoader) watchConfigSecretChanges(ctx context.Context, secretNamespace, secretName string) error { + secrets := l.client.Secrets(secretNamespace) + w, err := secrets.Watch(ctx, metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + // Re-watch regularly to avoid relying on long-lived connections. + // See https://github.com/kubernetes-client/javascript/issues/596#issuecomment-786419380 + TimeoutSeconds: ptr.To(int64(600)), + FieldSelector: fmt.Sprintf("metadata.name=%s", secretName), + Watch: true, + }) + if err != nil { + return fmt.Errorf("failed to watch config Secret %q: %w", secretName, err) + } + defer func() { + // May not be the original watcher by the time we exit. + if w != nil { + w.Stop() + } + }() + + // Get the initial config Secret now we've got the watcher set up. + secret, err := secrets.Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get config Secret %q: %w", secretName, err) + } + + if err := l.configFromSecret(ctx, secret); err != nil { + return fmt.Errorf("error loading initial config: %w", err) + } + + l.logger.Infof("Watching config Secret %q for changes", secretName) + for { + var secret *corev1.Secret + select { + case <-ctx.Done(): + return ctx.Err() + case ev, ok := <-w.ResultChan(): + if !ok { + w.Stop() + w, err = secrets.Watch(ctx, metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + TimeoutSeconds: ptr.To(int64(600)), + FieldSelector: fmt.Sprintf("metadata.name=%s", secretName), + Watch: true, + }) + if err != nil { + return fmt.Errorf("failed to re-watch config Secret %q: %w", secretName, err) + } + continue + } + + switch ev.Type { + case watch.Added, watch.Modified: + // New config available to load. + var ok bool + secret, ok = ev.Object.(*corev1.Secret) + if !ok { + return fmt.Errorf("unexpected object type %T in watch event for config Secret %q", ev.Object, secretName) + } + if secret == nil || secret.Data == nil { + continue + } + if err := l.configFromSecret(ctx, secret); err != nil { + return fmt.Errorf("error reloading config Secret %q: %v", secret.Name, err) + } + case watch.Error: + return fmt.Errorf("error watching config Secret %q: %v", secretName, ev.Object) + default: + // Ignore, no action required. + continue + } + } + } +} + +func (l *configLoader) configFromSecret(ctx context.Context, s *corev1.Secret) error { + b := s.Data[kubetypes.KubeAPIServerConfigFile] + if len(b) == 0 { + return fmt.Errorf("config Secret %q does not contain expected config in key %q", s.Name, kubetypes.KubeAPIServerConfigFile) + } + + if err := l.reloadConfig(ctx, b); err != nil { + return err + } + + return nil +} diff --git a/cmd/k8s-proxy/internal/config/config_test.go b/cmd/k8s-proxy/internal/config/config_test.go new file mode 100644 index 0000000000000..1603dbe1f398f --- /dev/null +++ b/cmd/k8s-proxy/internal/config/config_test.go @@ -0,0 +1,245 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package config + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + "tailscale.com/types/ptr" +) + +func TestWatchConfig(t *testing.T) { + type phase struct { + config string + cancel bool + expectedConf *conf.ConfigV1Alpha1 + expectedErr string + } + + // Same set of behaviour tests for each config source. + for _, env := range []string{"file", "kube"} { + t.Run(env, func(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + initialConfig string + phases []phase + }{ + { + name: "no_config", + phases: []phase{{ + expectedErr: "error loading initial config", + }}, + }, + { + name: "valid_config", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{{ + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("abc123"), + }, + }}, + }, + { + name: "can_cancel", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{ + { + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("abc123"), + }, + }, + { + cancel: true, + }, + }, + }, + { + name: "can_reload", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{ + { + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("abc123"), + }, + }, + { + config: `{"version": "v1alpha1", "authKey": "def456"}`, + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("def456"), + }, + }, + }, + }, + { + name: "ignores_events_with_no_changes", + initialConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + phases: []phase{ + { + expectedConf: &conf.ConfigV1Alpha1{ + AuthKey: ptr.To("abc123"), + }, + }, + { + config: `{"version": "v1alpha1", "authKey": "abc123"}`, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + cl := fake.NewClientset() + + var cfgPath string + var writeFile func(*testing.T, string) + if env == "file" { + cfgPath = filepath.Join(root, kubetypes.KubeAPIServerConfigFile) + writeFile = func(t *testing.T, content string) { + if err := os.WriteFile(cfgPath, []byte(content), 0o644); err != nil { + t.Fatalf("error writing config file %q: %v", cfgPath, err) + } + } + } else { + cfgPath = "kube:default/config-secret" + writeFile = func(t *testing.T, content string) { + s := secretFrom(content) + mustCreateOrUpdate(t, cl, s) + } + } + configChan := make(chan *conf.Config) + l := NewConfigLoader(zap.Must(zap.NewDevelopment()).Sugar(), cl.CoreV1(), configChan) + l.cfgIgnored = make(chan struct{}) + errs := make(chan error) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + writeFile(t, tc.initialConfig) + go func() { + errs <- l.WatchConfig(ctx, cfgPath) + }() + + for i, p := range tc.phases { + if p.config != "" { + writeFile(t, p.config) + } + if p.cancel { + cancel() + } + + select { + case cfg := <-configChan: + if diff := cmp.Diff(*p.expectedConf, cfg.Parsed); diff != "" { + t.Errorf("unexpected config (-want +got):\n%s", diff) + } + case err := <-errs: + if p.cancel { + if err != nil { + t.Fatalf("unexpected error after cancel: %v", err) + } + } else if p.expectedErr == "" { + t.Fatalf("unexpected error: %v", err) + } else if !strings.Contains(err.Error(), p.expectedErr) { + t.Fatalf("expected error to contain %q, got %q", p.expectedErr, err.Error()) + } + case <-l.cfgIgnored: + if p.expectedConf != nil { + t.Fatalf("expected config to be reloaded, but got ignored signal") + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for expected event in phase: %d", i) + } + } + }) + } + }) + } +} + +func TestWatchConfigSecret_Rewatches(t *testing.T) { + cl := fake.NewClientset() + var watchCount int + var watcher *watch.RaceFreeFakeWatcher + expected := []string{ + `{"version": "v1alpha1", "authKey": "abc123"}`, + `{"version": "v1alpha1", "authKey": "def456"}`, + `{"version": "v1alpha1", "authKey": "ghi789"}`, + } + cl.PrependWatchReactor("secrets", func(action ktesting.Action) (handled bool, ret watch.Interface, err error) { + watcher = watch.NewRaceFreeFake() + watcher.Add(secretFrom(expected[watchCount])) + if action.GetVerb() == "watch" && action.GetResource().Resource == "secrets" { + watchCount++ + } + return true, watcher, nil + }) + + configChan := make(chan *conf.Config) + l := NewConfigLoader(zap.Must(zap.NewDevelopment()).Sugar(), cl.CoreV1(), configChan) + + mustCreateOrUpdate(t, cl, secretFrom(expected[0])) + + errs := make(chan error) + go func() { + errs <- l.watchConfigSecretChanges(t.Context(), "default", "config-secret") + }() + + for i := range 2 { + select { + case cfg := <-configChan: + if exp := expected[i]; cfg.Parsed.AuthKey == nil || !strings.Contains(exp, *cfg.Parsed.AuthKey) { + t.Fatalf("expected config to have authKey %q, got: %v", exp, cfg.Parsed.AuthKey) + } + if i == 0 { + watcher.Stop() + } + case err := <-errs: + t.Fatalf("unexpected error: %v", err) + case <-l.cfgIgnored: + t.Fatalf("expected config to be reloaded, but got ignored signal") + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for expected event") + } + } + + if watchCount != 2 { + t.Fatalf("expected 2 watch API calls, got %d", watchCount) + } +} + +func secretFrom(content string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "config-secret", + }, + Data: map[string][]byte{ + kubetypes.KubeAPIServerConfigFile: []byte(content), + }, + } +} + +func mustCreateOrUpdate(t *testing.T, cl *fake.Clientset, s *corev1.Secret) { + t.Helper() + if _, err := cl.CoreV1().Secrets("default").Create(t.Context(), s, metav1.CreateOptions{}); err != nil { + if _, updateErr := cl.CoreV1().Secrets("default").Update(t.Context(), s, metav1.UpdateOptions{}); updateErr != nil { + t.Fatalf("error writing config Secret %q: %v", s.Name, updateErr) + } + } +} diff --git a/cmd/k8s-proxy/k8s-proxy.go b/cmd/k8s-proxy/k8s-proxy.go new file mode 100644 index 0000000000000..7a77072140568 --- /dev/null +++ b/cmd/k8s-proxy/k8s-proxy.go @@ -0,0 +1,474 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// k8s-proxy proxies between tailnet and Kubernetes cluster traffic. +// Currently, it only supports proxying tailnet clients to the Kubernetes API +// server. +package main + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "reflect" + "strconv" + "strings" + "syscall" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "golang.org/x/sync/errgroup" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/utils/strings/slices" + "tailscale.com/client/local" + "tailscale.com/cmd/k8s-proxy/internal/config" + "tailscale.com/hostinfo" + "tailscale.com/ipn" + "tailscale.com/ipn/store" + apiproxy "tailscale.com/k8s-operator/api-proxy" + "tailscale.com/kube/certs" + healthz "tailscale.com/kube/health" + "tailscale.com/kube/k8s-proxy/conf" + "tailscale.com/kube/kubetypes" + klc "tailscale.com/kube/localclient" + "tailscale.com/kube/metrics" + "tailscale.com/kube/services" + "tailscale.com/kube/state" + "tailscale.com/tailcfg" + "tailscale.com/tsnet" +) + +func main() { + encoderCfg := zap.NewProductionEncoderConfig() + encoderCfg.EncodeTime = zapcore.RFC3339TimeEncoder + logger := zap.Must(zap.Config{ + Level: zap.NewAtomicLevelAt(zap.DebugLevel), + Encoding: "json", + OutputPaths: []string{"stderr"}, + ErrorOutputPaths: []string{"stderr"}, + EncoderConfig: encoderCfg, + }.Build()).Sugar() + defer logger.Sync() + + if err := run(logger); err != nil { + logger.Fatal(err.Error()) + } +} + +func run(logger *zap.SugaredLogger) error { + var ( + configPath = os.Getenv("TS_K8S_PROXY_CONFIG") + podUID = os.Getenv("POD_UID") + podIP = os.Getenv("POD_IP") + ) + if configPath == "" { + return errors.New("TS_K8S_PROXY_CONFIG unset") + } + + // serveCtx to live for the lifetime of the process, only gets cancelled + // once the Tailscale Service has been drained + serveCtx, serveCancel := context.WithCancel(context.Background()) + defer serveCancel() + + // ctx to cancel to start the shutdown process. + ctx, cancel := context.WithCancel(serveCtx) + defer cancel() + + sigsChan := make(chan os.Signal, 1) + signal.Notify(sigsChan, syscall.SIGINT, syscall.SIGTERM) + go func() { + select { + case <-ctx.Done(): + case s := <-sigsChan: + logger.Infof("Received shutdown signal %s, exiting", s) + cancel() + } + }() + + var group *errgroup.Group + group, ctx = errgroup.WithContext(ctx) + + restConfig, err := getRestConfig(logger) + if err != nil { + return fmt.Errorf("error getting rest config: %w", err) + } + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("error creating Kubernetes clientset: %w", err) + } + + // Load and watch config. + cfgChan := make(chan *conf.Config) + cfgLoader := config.NewConfigLoader(logger, clientset.CoreV1(), cfgChan) + group.Go(func() error { + return cfgLoader.WatchConfig(ctx, configPath) + }) + + // Get initial config. + var cfg *conf.Config + select { + case <-ctx.Done(): + return group.Wait() + case cfg = <-cfgChan: + } + + if cfg.Parsed.LogLevel != nil { + level, err := zapcore.ParseLevel(*cfg.Parsed.LogLevel) + if err != nil { + return fmt.Errorf("error parsing log level %q: %w", *cfg.Parsed.LogLevel, err) + } + logger = logger.WithOptions(zap.IncreaseLevel(level)) + } + + // TODO:(ChaosInTheCRD) This is a temporary workaround until we can set static endpoints using prefs + if se := cfg.Parsed.StaticEndpoints; len(se) > 0 { + logger.Debugf("setting static endpoints '%v' via TS_DEBUG_PRETENDPOINT environment variable", cfg.Parsed.StaticEndpoints) + ses := make([]string, len(se)) + for i, e := range se { + ses[i] = e.String() + } + + err := os.Setenv("TS_DEBUG_PRETENDPOINT", strings.Join(ses, ",")) + if err != nil { + return err + } + } + + if cfg.Parsed.App != nil { + hostinfo.SetApp(*cfg.Parsed.App) + } + + // TODO(tomhjp): Pass this setting directly into the store instead of using + // environment variables. + if cfg.Parsed.APIServerProxy != nil && cfg.Parsed.APIServerProxy.IssueCerts.EqualBool(true) { + os.Setenv("TS_CERT_SHARE_MODE", "rw") + } else { + os.Setenv("TS_CERT_SHARE_MODE", "ro") + } + + st, err := getStateStore(cfg.Parsed.State, logger) + if err != nil { + return err + } + + // If Pod UID unset, assume we're running outside of a cluster/not managed + // by the operator, so no need to set additional state keys. + if podUID != "" { + if err := state.SetInitialKeys(st, podUID); err != nil { + return fmt.Errorf("error setting initial state: %w", err) + } + } + + var authKey string + if cfg.Parsed.AuthKey != nil { + authKey = *cfg.Parsed.AuthKey + } + + ts := &tsnet.Server{ + Logf: logger.Named("tsnet").Debugf, + UserLogf: logger.Named("tsnet").Infof, + Store: st, + AuthKey: authKey, + } + + if cfg.Parsed.ServerURL != nil { + ts.ControlURL = *cfg.Parsed.ServerURL + } + + if cfg.Parsed.Hostname != nil { + ts.Hostname = *cfg.Parsed.Hostname + } + + // Make sure we crash loop if Up doesn't complete in reasonable time. + upCtx, upCancel := context.WithTimeout(ctx, time.Minute) + defer upCancel() + if _, err := ts.Up(upCtx); err != nil { + return fmt.Errorf("error starting tailscale server: %w", err) + } + defer ts.Close() + lc, err := ts.LocalClient() + if err != nil { + return fmt.Errorf("error getting local client: %w", err) + } + + // Setup for updating state keys. + if podUID != "" { + group.Go(func() error { + return state.KeepKeysUpdated(ctx, st, klc.New(lc)) + }) + } + + if cfg.Parsed.HealthCheckEnabled.EqualBool(true) || cfg.Parsed.MetricsEnabled.EqualBool(true) { + addr := podIP + if addr == "" { + addr = cfg.GetLocalAddr() + } + + addrPort := getLocalAddrPort(addr, cfg.GetLocalPort()) + mux := http.NewServeMux() + localSrv := &http.Server{Addr: addrPort, Handler: mux} + + if cfg.Parsed.MetricsEnabled.EqualBool(true) { + logger.Infof("Running metrics endpoint at %s/metrics", addrPort) + metrics.RegisterMetricsHandlers(mux, lc, "") + } + + if cfg.Parsed.HealthCheckEnabled.EqualBool(true) { + ipV4, _ := ts.TailscaleIPs() + hz := healthz.RegisterHealthHandlers(mux, ipV4.String(), logger.Infof) + group.Go(func() error { + err := hz.MonitorHealth(ctx, lc) + if err == nil || errors.Is(err, context.Canceled) { + return nil + } + return err + }) + } + + group.Go(func() error { + errChan := make(chan error) + go func() { + if err := localSrv.ListenAndServe(); err != nil { + errChan <- err + } + close(errChan) + }() + + select { + case <-ctx.Done(): + sCtx, scancel := context.WithTimeout(serveCtx, 10*time.Second) + defer scancel() + return localSrv.Shutdown(sCtx) + case err := <-errChan: + return err + } + }) + } + + if v, ok := cfg.Parsed.AcceptRoutes.Get(); ok { + _, err = lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + RouteAllSet: true, + Prefs: ipn.Prefs{RouteAll: v}, + }) + if err != nil { + return fmt.Errorf("error editing prefs: %w", err) + } + } + + // TODO(tomhjp): There seems to be a bug that on restart the device does + // not get reassigned it's already working Service IPs unless we clear and + // reset the serve config. + if err := lc.SetServeConfig(ctx, &ipn.ServeConfig{}); err != nil { + return fmt.Errorf("error clearing existing ServeConfig: %w", err) + } + + var cm *certs.CertManager + if shouldIssueCerts(cfg) { + logger.Infof("Will issue TLS certs for Tailscale Service") + cm = certs.NewCertManager(klc.New(lc), logger.Infof) + } + if err := setServeConfig(ctx, lc, cm, apiServerProxyService(cfg)); err != nil { + return err + } + + if cfg.Parsed.AdvertiseServices != nil { + if _, err := lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: cfg.Parsed.AdvertiseServices, + }, + }); err != nil { + return fmt.Errorf("error setting prefs AdvertiseServices: %w", err) + } + } + + // Setup for the API server proxy. + mode := kubetypes.APIServerProxyModeAuth + if cfg.Parsed.APIServerProxy != nil && cfg.Parsed.APIServerProxy.Mode != nil { + mode = *cfg.Parsed.APIServerProxy.Mode + } + ap, err := apiproxy.NewAPIServerProxy(logger.Named("apiserver-proxy"), restConfig, ts, mode, false) + if err != nil { + return fmt.Errorf("error creating api server proxy: %w", err) + } + + group.Go(func() error { + if err := ap.Run(serveCtx); err != nil { + return fmt.Errorf("error running API server proxy: %w", err) + } + + return nil + }) + + for { + select { + case <-ctx.Done(): + // Context cancelled, exit. + logger.Info("Context cancelled, exiting") + shutdownCtx, shutdownCancel := context.WithTimeout(serveCtx, 20*time.Second) + unadvertiseErr := services.EnsureServicesNotAdvertised(shutdownCtx, lc, logger.Infof) + shutdownCancel() + serveCancel() + return errors.Join(unadvertiseErr, group.Wait()) + case cfg = <-cfgChan: + // Handle config reload. + // TODO(tomhjp): Make auth mode reloadable. + var prefs ipn.MaskedPrefs + cfgLogger := logger + currentPrefs, err := lc.GetPrefs(ctx) + if err != nil { + return fmt.Errorf("error getting current prefs: %w", err) + } + if !slices.Equal(currentPrefs.AdvertiseServices, cfg.Parsed.AdvertiseServices) { + cfgLogger = cfgLogger.With("AdvertiseServices", fmt.Sprintf("%v -> %v", currentPrefs.AdvertiseServices, cfg.Parsed.AdvertiseServices)) + prefs.AdvertiseServicesSet = true + prefs.Prefs.AdvertiseServices = cfg.Parsed.AdvertiseServices + } + if cfg.Parsed.Hostname != nil && *cfg.Parsed.Hostname != currentPrefs.Hostname { + cfgLogger = cfgLogger.With("Hostname", fmt.Sprintf("%s -> %s", currentPrefs.Hostname, *cfg.Parsed.Hostname)) + prefs.HostnameSet = true + prefs.Hostname = *cfg.Parsed.Hostname + } + if v, ok := cfg.Parsed.AcceptRoutes.Get(); ok && v != currentPrefs.RouteAll { + cfgLogger = cfgLogger.With("AcceptRoutes", fmt.Sprintf("%v -> %v", currentPrefs.RouteAll, v)) + prefs.RouteAllSet = true + prefs.Prefs.RouteAll = v + } + if !prefs.IsEmpty() { + if _, err := lc.EditPrefs(ctx, &prefs); err != nil { + return fmt.Errorf("error editing prefs: %w", err) + } + } + if err := setServeConfig(ctx, lc, cm, apiServerProxyService(cfg)); err != nil { + return fmt.Errorf("error setting serve config: %w", err) + } + + cfgLogger.Infof("Config reloaded") + } + } +} + +func getLocalAddrPort(addr string, port uint16) string { + return net.JoinHostPort(addr, strconv.FormatUint(uint64(port), 10)) +} + +func getStateStore(path *string, logger *zap.SugaredLogger) (ipn.StateStore, error) { + p := "mem:" + if path != nil { + p = *path + } else { + logger.Warn("No state Secret provided; using in-memory store, which will lose state on restart") + } + st, err := store.New(logger.Errorf, p) + if err != nil { + return nil, fmt.Errorf("error creating state store: %w", err) + } + + return st, nil +} + +func getRestConfig(logger *zap.SugaredLogger) (*rest.Config, error) { + restConfig, err := rest.InClusterConfig() + switch err { + case nil: + return restConfig, nil + case rest.ErrNotInCluster: + logger.Info("Not running in-cluster, falling back to kubeconfig") + default: + return nil, fmt.Errorf("error getting in-cluster config: %w", err) + } + + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, nil) + restConfig, err = clientConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("error loading kubeconfig: %w", err) + } + + return restConfig, nil +} + +func apiServerProxyService(cfg *conf.Config) tailcfg.ServiceName { + if cfg.Parsed.APIServerProxy != nil && + cfg.Parsed.APIServerProxy.Enabled.EqualBool(true) && + cfg.Parsed.APIServerProxy.ServiceName != nil && + *cfg.Parsed.APIServerProxy.ServiceName != "" { + return tailcfg.ServiceName(*cfg.Parsed.APIServerProxy.ServiceName) + } + + return "" +} + +func shouldIssueCerts(cfg *conf.Config) bool { + return cfg.Parsed.APIServerProxy != nil && + cfg.Parsed.APIServerProxy.IssueCerts.EqualBool(true) +} + +// setServeConfig sets up serve config such that it's serving for the passed in +// Tailscale Service, and does nothing if it's already up to date. +func setServeConfig(ctx context.Context, lc *local.Client, cm *certs.CertManager, name tailcfg.ServiceName) error { + existingServeConfig, err := lc.GetServeConfig(ctx) + if err != nil { + return fmt.Errorf("error getting existing serve config: %w", err) + } + + // Ensure serve config is cleared if no Tailscale Service. + if name == "" { + if reflect.DeepEqual(*existingServeConfig, ipn.ServeConfig{}) { + // Already up to date. + return nil + } + + if cm != nil { + cm.EnsureCertLoops(ctx, &ipn.ServeConfig{}) + } + return lc.SetServeConfig(ctx, &ipn.ServeConfig{}) + } + + status, err := lc.StatusWithoutPeers(ctx) + if err != nil { + return fmt.Errorf("error getting local client status: %w", err) + } + serviceHostPort := ipn.HostPort(fmt.Sprintf("%s.%s:443", name.WithoutPrefix(), status.CurrentTailnet.MagicDNSSuffix)) + + serveConfig := ipn.ServeConfig{ + // Configure for the Service hostname. + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + name: { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: { + HTTPS: true, + }, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + serviceHostPort: { + Handlers: map[string]*ipn.HTTPHandler{ + "/": { + Proxy: "http://localhost:80", + }, + }, + }, + }, + }, + }, + } + + if reflect.DeepEqual(*existingServeConfig, serveConfig) { + // Already up to date. + return nil + } + + if cm != nil { + cm.EnsureCertLoops(ctx, &serveConfig) + } + return lc.SetServeConfig(ctx, &serveConfig) +} diff --git a/cmd/natc/ippool/consensusippool.go b/cmd/natc/ippool/consensusippool.go new file mode 100644 index 0000000000000..3bc21bd0357dd --- /dev/null +++ b/cmd/natc/ippool/consensusippool.go @@ -0,0 +1,435 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ippool + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net/netip" + "time" + + "github.com/hashicorp/raft" + "go4.org/netipx" + "tailscale.com/syncs" + "tailscale.com/tailcfg" + "tailscale.com/tsconsensus" + "tailscale.com/tsnet" + "tailscale.com/util/mak" +) + +// ConsensusIPPool implements an [IPPool] that is distributed among members of a cluster for high availability. +// Writes are directed to a leader among the cluster and are slower than reads, reads are performed locally +// using information replicated from the leader. +// The cluster maintains consistency, reads can be stale and writes can be unavailable if sufficient cluster +// peers are unavailable. +type ConsensusIPPool struct { + IPSet *netipx.IPSet + perPeerMap *syncs.Map[tailcfg.NodeID, *consensusPerPeerState] + consensus commandExecutor + unusedAddressLifetime time.Duration +} + +func NewConsensusIPPool(ipSet *netipx.IPSet) *ConsensusIPPool { + return &ConsensusIPPool{ + unusedAddressLifetime: 48 * time.Hour, // TODO (fran) is this appropriate? should it be configurable? + IPSet: ipSet, + perPeerMap: &syncs.Map[tailcfg.NodeID, *consensusPerPeerState]{}, + } +} + +// IPForDomain looks up or creates an IP address allocation for the tailcfg.NodeID and domain pair. +// If no address association is found, one is allocated from the range of free addresses for this tailcfg.NodeID. +// If no more address are available, an error is returned. +func (ipp *ConsensusIPPool) IPForDomain(nid tailcfg.NodeID, domain string) (netip.Addr, error) { + now := time.Now() + // Check local state; local state may be stale. If we have an IP for this domain, and we are not + // close to the expiry time for the domain, it's safe to return what we have. + ps, psFound := ipp.perPeerMap.Load(nid) + if psFound { + if addr, addrFound := ps.domainToAddr[domain]; addrFound { + if ww, wwFound := ps.addrToDomain.Load(addr); wwFound { + if !isCloseToExpiry(ww.LastUsed, now, ipp.unusedAddressLifetime) { + ipp.fireAndForgetMarkLastUsed(nid, addr, ww, now) + return addr, nil + } + } + } + } + + // go via consensus + args := checkoutAddrArgs{ + NodeID: nid, + Domain: domain, + ReuseDeadline: now.Add(-1 * ipp.unusedAddressLifetime), + UpdatedAt: now, + } + bs, err := json.Marshal(args) + if err != nil { + return netip.Addr{}, err + } + c := tsconsensus.Command{ + Name: "checkoutAddr", + Args: bs, + } + result, err := ipp.consensus.ExecuteCommand(c) + if err != nil { + log.Printf("IPForDomain: raft error executing command: %v", err) + return netip.Addr{}, err + } + if result.Err != nil { + log.Printf("IPForDomain: error returned from state machine: %v", err) + return netip.Addr{}, result.Err + } + var addr netip.Addr + err = json.Unmarshal(result.Result, &addr) + return addr, err +} + +// DomainForIP looks up the domain associated with a tailcfg.NodeID and netip.Addr pair. +// If there is no association, the result is empty and ok is false. +func (ipp *ConsensusIPPool) DomainForIP(from tailcfg.NodeID, addr netip.Addr, updatedAt time.Time) (string, bool) { + // Look in local state, to save a consensus round trip; local state may be stale. + // + // The only time we expect ordering of commands to matter to clients is on first + // connection to a domain. In that case it may be that although we don't find the + // domain in our local state, it is in fact in the state of the state machine (ie + // the client did a DNS lookup, and we responded with an IP and _should_ know that + // domain when the TCP connection for that IP arrives.) + // + // So it's ok to return local state, unless local state doesn't recognize the domain, + // in which case we should check the consensus state machine to know for sure. + var domain string + ww, ok := ipp.domainLookup(from, addr) + if ok { + domain = ww.Domain + } else { + d, err := ipp.readDomainForIP(from, addr) + if err != nil { + log.Printf("error reading domain from consensus: %v", err) + return "", false + } + domain = d + } + if domain == "" { + log.Printf("did not find domain for node: %v, addr: %s", from, addr) + return "", false + } + ipp.fireAndForgetMarkLastUsed(from, addr, ww, updatedAt) + return domain, true +} + +func (ipp *ConsensusIPPool) fireAndForgetMarkLastUsed(from tailcfg.NodeID, addr netip.Addr, ww whereWhen, updatedAt time.Time) { + window := 5 * time.Minute + if updatedAt.Sub(ww.LastUsed).Abs() < window { + return + } + go func() { + err := ipp.markLastUsed(from, addr, ww.Domain, updatedAt) + if err != nil { + log.Printf("error marking last used: %v", err) + } + }() +} + +func (ipp *ConsensusIPPool) domainLookup(from tailcfg.NodeID, addr netip.Addr) (whereWhen, bool) { + ps, ok := ipp.perPeerMap.Load(from) + if !ok { + log.Printf("domainLookup: peer state absent for: %d", from) + return whereWhen{}, false + } + ww, ok := ps.addrToDomain.Load(addr) + if !ok { + log.Printf("domainLookup: peer state doesn't recognize addr: %s", addr) + return whereWhen{}, false + } + return ww, true +} + +// StartConsensus is part of the IPPool interface. It starts the raft background routines that handle consensus. +func (ipp *ConsensusIPPool) StartConsensus(ctx context.Context, ts *tsnet.Server, clusterTag string, clusterStateDir string) error { + cfg := tsconsensus.DefaultConfig() + cfg.ServeDebugMonitor = true + cfg.StateDirPath = clusterStateDir + cns, err := tsconsensus.Start(ctx, ts, ipp, clusterTag, cfg) + if err != nil { + return err + } + ipp.consensus = cns + return nil +} + +type whereWhen struct { + Domain string + LastUsed time.Time +} + +type consensusPerPeerState struct { + domainToAddr map[string]netip.Addr + addrToDomain *syncs.Map[netip.Addr, whereWhen] +} + +// StopConsensus is part of the IPPool interface. It stops the raft background routines that handle consensus. +func (ipp *ConsensusIPPool) StopConsensus(ctx context.Context) error { + return (ipp.consensus).(*tsconsensus.Consensus).Stop(ctx) +} + +// unusedIPV4 finds the next unused or expired IP address in the pool. +// IP addresses in the pool should be reused if they haven't been used for some period of time. +// reuseDeadline is the time before which addresses are considered to be expired. +// So if addresses are being reused after they haven't been used for 24 hours say, reuseDeadline +// would be 24 hours ago. +func (ps *consensusPerPeerState) unusedIPV4(ipset *netipx.IPSet, reuseDeadline time.Time) (netip.Addr, bool, string, error) { + // If we want to have a random IP choice behavior we could make that work with the state machine by doing something like + // passing the randomly chosen IP into the state machine call (so replaying logs would still be deterministic). + for _, r := range ipset.Ranges() { + ip := r.From() + toIP := r.To() + if !ip.IsValid() || !toIP.IsValid() { + continue + } + for toIP.Compare(ip) != -1 { + ww, ok := ps.addrToDomain.Load(ip) + if !ok { + return ip, false, "", nil + } + if ww.LastUsed.Before(reuseDeadline) { + return ip, true, ww.Domain, nil + } + ip = ip.Next() + } + } + return netip.Addr{}, false, "", errors.New("ip pool exhausted") +} + +// isCloseToExpiry returns true if the lastUsed and now times are more than +// half the lifetime apart +func isCloseToExpiry(lastUsed, now time.Time, lifetime time.Duration) bool { + return now.Sub(lastUsed).Abs() > (lifetime / 2) +} + +type readDomainForIPArgs struct { + NodeID tailcfg.NodeID + Addr netip.Addr +} + +// executeReadDomainForIP parses a readDomainForIP log entry and applies it. +func (ipp *ConsensusIPPool) executeReadDomainForIP(bs []byte) tsconsensus.CommandResult { + var args readDomainForIPArgs + err := json.Unmarshal(bs, &args) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + return ipp.applyReadDomainForIP(args.NodeID, args.Addr) +} + +func (ipp *ConsensusIPPool) applyReadDomainForIP(from tailcfg.NodeID, addr netip.Addr) tsconsensus.CommandResult { + domain := func() string { + ps, ok := ipp.perPeerMap.Load(from) + if !ok { + return "" + } + ww, ok := ps.addrToDomain.Load(addr) + if !ok { + return "" + } + return ww.Domain + }() + resultBs, err := json.Marshal(domain) + return tsconsensus.CommandResult{Result: resultBs, Err: err} +} + +// readDomainForIP executes a readDomainForIP command on the leader with raft. +func (ipp *ConsensusIPPool) readDomainForIP(nid tailcfg.NodeID, addr netip.Addr) (string, error) { + args := readDomainForIPArgs{ + NodeID: nid, + Addr: addr, + } + bs, err := json.Marshal(args) + if err != nil { + return "", err + } + c := tsconsensus.Command{ + Name: "readDomainForIP", + Args: bs, + } + result, err := ipp.consensus.ExecuteCommand(c) + if err != nil { + log.Printf("readDomainForIP: raft error executing command: %v", err) + return "", err + } + if result.Err != nil { + log.Printf("readDomainForIP: error returned from state machine: %v", err) + return "", result.Err + } + var domain string + err = json.Unmarshal(result.Result, &domain) + return domain, err +} + +type markLastUsedArgs struct { + NodeID tailcfg.NodeID + Addr netip.Addr + Domain string + UpdatedAt time.Time +} + +// executeMarkLastUsed parses a markLastUsed log entry and applies it. +func (ipp *ConsensusIPPool) executeMarkLastUsed(bs []byte) tsconsensus.CommandResult { + var args markLastUsedArgs + err := json.Unmarshal(bs, &args) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + err = ipp.applyMarkLastUsed(args.NodeID, args.Addr, args.Domain, args.UpdatedAt) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + return tsconsensus.CommandResult{} +} + +// applyMarkLastUsed applies the arguments from the log entry to the state. It updates an entry in the AddrToDomain +// map with a new LastUsed timestamp. +// applyMarkLastUsed is not safe for concurrent access. It's only called from raft which will +// not call it concurrently. +func (ipp *ConsensusIPPool) applyMarkLastUsed(from tailcfg.NodeID, addr netip.Addr, domain string, updatedAt time.Time) error { + ps, ok := ipp.perPeerMap.Load(from) + if !ok { + // There's nothing to mark. But this is unexpected, because we mark last used after we do things with peer state. + log.Printf("applyMarkLastUsed: could not find peer state, nodeID: %s", from) + return nil + } + ww, ok := ps.addrToDomain.Load(addr) + if !ok { + // The peer state didn't have an entry for the IP address (possibly it expired), so there's nothing to mark. + return nil + } + if ww.Domain != domain { + // The IP address expired and was reused for a new domain. Don't mark. + return nil + } + if ww.LastUsed.After(updatedAt) { + // This has been marked more recently. Don't mark. + return nil + } + ww.LastUsed = updatedAt + ps.addrToDomain.Store(addr, ww) + return nil +} + +// markLastUsed executes a markLastUsed command on the leader with raft. +func (ipp *ConsensusIPPool) markLastUsed(nid tailcfg.NodeID, addr netip.Addr, domain string, lastUsed time.Time) error { + args := markLastUsedArgs{ + NodeID: nid, + Addr: addr, + Domain: domain, + UpdatedAt: lastUsed, + } + bs, err := json.Marshal(args) + if err != nil { + return err + } + c := tsconsensus.Command{ + Name: "markLastUsed", + Args: bs, + } + result, err := ipp.consensus.ExecuteCommand(c) + if err != nil { + log.Printf("markLastUsed: raft error executing command: %v", err) + return err + } + if result.Err != nil { + log.Printf("markLastUsed: error returned from state machine: %v", err) + return result.Err + } + return nil +} + +type checkoutAddrArgs struct { + NodeID tailcfg.NodeID + Domain string + ReuseDeadline time.Time + UpdatedAt time.Time +} + +// executeCheckoutAddr parses a checkoutAddr raft log entry and applies it. +func (ipp *ConsensusIPPool) executeCheckoutAddr(bs []byte) tsconsensus.CommandResult { + var args checkoutAddrArgs + err := json.Unmarshal(bs, &args) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + addr, err := ipp.applyCheckoutAddr(args.NodeID, args.Domain, args.ReuseDeadline, args.UpdatedAt) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + resultBs, err := json.Marshal(addr) + if err != nil { + return tsconsensus.CommandResult{Err: err} + } + return tsconsensus.CommandResult{Result: resultBs} +} + +// applyCheckoutAddr finds the IP address for a nid+domain +// Each nid can use all of the addresses in the pool. +// updatedAt is the current time, the time at which we are wanting to get a new IP address. +// reuseDeadline is the time before which addresses are considered to be expired. +// So if addresses are being reused after they haven't been used for 24 hours say updatedAt would be now +// and reuseDeadline would be 24 hours ago. +// It is not safe for concurrent access (it's only called from raft, which will not call concurrently +// so that's fine). +func (ipp *ConsensusIPPool) applyCheckoutAddr(nid tailcfg.NodeID, domain string, reuseDeadline, updatedAt time.Time) (netip.Addr, error) { + ps, ok := ipp.perPeerMap.Load(nid) + if !ok { + ps = &consensusPerPeerState{ + addrToDomain: &syncs.Map[netip.Addr, whereWhen]{}, + } + ipp.perPeerMap.Store(nid, ps) + } + if existing, ok := ps.domainToAddr[domain]; ok { + ww, ok := ps.addrToDomain.Load(existing) + if ok { + ww.LastUsed = updatedAt + ps.addrToDomain.Store(existing, ww) + return existing, nil + } + log.Printf("applyCheckoutAddr: data out of sync, allocating new IP") + } + addr, wasInUse, previousDomain, err := ps.unusedIPV4(ipp.IPSet, reuseDeadline) + if err != nil { + return netip.Addr{}, err + } + mak.Set(&ps.domainToAddr, domain, addr) + if wasInUse { + delete(ps.domainToAddr, previousDomain) + } + ps.addrToDomain.Store(addr, whereWhen{Domain: domain, LastUsed: updatedAt}) + return addr, nil +} + +// Apply is part of the raft.FSM interface. It takes an incoming log entry and applies it to the state. +func (ipp *ConsensusIPPool) Apply(l *raft.Log) any { + var c tsconsensus.Command + if err := json.Unmarshal(l.Data, &c); err != nil { + panic(fmt.Sprintf("failed to unmarshal command: %s", err.Error())) + } + switch c.Name { + case "checkoutAddr": + return ipp.executeCheckoutAddr(c.Args) + case "markLastUsed": + return ipp.executeMarkLastUsed(c.Args) + case "readDomainForIP": + return ipp.executeReadDomainForIP(c.Args) + default: + panic(fmt.Sprintf("unrecognized command: %s", c.Name)) + } +} + +// commandExecutor is an interface covering the routing parts of consensus +// used to allow a fake in the tests +type commandExecutor interface { + ExecuteCommand(tsconsensus.Command) (tsconsensus.CommandResult, error) +} diff --git a/cmd/natc/ippool/consensusippool_test.go b/cmd/natc/ippool/consensusippool_test.go new file mode 100644 index 0000000000000..242cdffaf26d3 --- /dev/null +++ b/cmd/natc/ippool/consensusippool_test.go @@ -0,0 +1,383 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ippool + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/netip" + "testing" + "time" + + "github.com/hashicorp/raft" + "go4.org/netipx" + "tailscale.com/tailcfg" + "tailscale.com/tsconsensus" + "tailscale.com/util/must" +) + +func makeSetFromPrefix(pfx netip.Prefix) *netipx.IPSet { + var ipsb netipx.IPSetBuilder + ipsb.AddPrefix(pfx) + return must.Get(ipsb.IPSet()) +} + +type FakeConsensus struct { + ipp *ConsensusIPPool +} + +func (c *FakeConsensus) ExecuteCommand(cmd tsconsensus.Command) (tsconsensus.CommandResult, error) { + b, err := json.Marshal(cmd) + if err != nil { + return tsconsensus.CommandResult{}, err + } + result := c.ipp.Apply(&raft.Log{Data: b}) + return result.(tsconsensus.CommandResult), nil +} + +func makePool(pfx netip.Prefix) *ConsensusIPPool { + ipp := NewConsensusIPPool(makeSetFromPrefix(pfx)) + ipp.consensus = &FakeConsensus{ipp: ipp} + return ipp +} + +func TestConsensusIPForDomain(t *testing.T) { + pfx := netip.MustParsePrefix("100.64.0.0/16") + ipp := makePool(pfx) + from := tailcfg.NodeID(1) + + a, err := ipp.IPForDomain(from, "example.com") + if err != nil { + t.Fatal(err) + } + if !pfx.Contains(a) { + t.Fatalf("expected %v to be in the prefix %v", a, pfx) + } + + b, err := ipp.IPForDomain(from, "a.example.com") + if err != nil { + t.Fatal(err) + } + if !pfx.Contains(b) { + t.Fatalf("expected %v to be in the prefix %v", b, pfx) + } + if b == a { + t.Fatalf("same address issued twice %v, %v", a, b) + } + + c, err := ipp.IPForDomain(from, "example.com") + if err != nil { + t.Fatal(err) + } + if c != a { + t.Fatalf("expected %v to be remembered as the addr for example.com, but got %v", a, c) + } +} + +func TestConsensusPoolExhaustion(t *testing.T) { + ipp := makePool(netip.MustParsePrefix("100.64.0.0/31")) + from := tailcfg.NodeID(1) + + subdomains := []string{"a", "b", "c"} + for i, sd := range subdomains { + _, err := ipp.IPForDomain(from, fmt.Sprintf("%s.example.com", sd)) + if i < 2 && err != nil { + t.Fatal(err) + } + expected := "ip pool exhausted" + if i == 2 && err.Error() != expected { + t.Fatalf("expected error to be '%s', got '%s'", expected, err.Error()) + } + } +} + +func TestConsensusPoolExpiry(t *testing.T) { + ipp := makePool(netip.MustParsePrefix("100.64.0.0/31")) + firstIP := netip.MustParseAddr("100.64.0.0") + secondIP := netip.MustParseAddr("100.64.0.1") + timeOfUse := time.Now() + beforeTimeOfUse := timeOfUse.Add(-1 * time.Hour) + afterTimeOfUse := timeOfUse.Add(1 * time.Hour) + from := tailcfg.NodeID(1) + + // the pool is unused, we get an address, and it's marked as being used at timeOfUse + aAddr, err := ipp.applyCheckoutAddr(from, "a.example.com", time.Time{}, timeOfUse) + if err != nil { + t.Fatal(err) + } + if aAddr.Compare(firstIP) != 0 { + t.Fatalf("expected %s, got %s", firstIP, aAddr) + } + ww, ok := ipp.domainLookup(from, firstIP) + if !ok { + t.Fatal("expected wherewhen to be found") + } + if ww.Domain != "a.example.com" { + t.Fatalf("expected aAddr to look up to a.example.com, got: %s", ww.Domain) + } + + // the time before which we will reuse addresses is prior to timeOfUse, so no reuse + bAddr, err := ipp.applyCheckoutAddr(from, "b.example.com", beforeTimeOfUse, timeOfUse) + if err != nil { + t.Fatal(err) + } + if bAddr.Compare(secondIP) != 0 { + t.Fatalf("expected %s, got %s", secondIP, bAddr) + } + + // the time before which we will reuse addresses is after timeOfUse, so reuse addresses that were marked as used at timeOfUse. + cAddr, err := ipp.applyCheckoutAddr(from, "c.example.com", afterTimeOfUse, timeOfUse) + if err != nil { + t.Fatal(err) + } + if cAddr.Compare(firstIP) != 0 { + t.Fatalf("expected %s, got %s", firstIP, cAddr) + } + ww, ok = ipp.domainLookup(from, firstIP) + if !ok { + t.Fatal("expected wherewhen to be found") + } + if ww.Domain != "c.example.com" { + t.Fatalf("expected firstIP to look up to c.example.com, got: %s", ww.Domain) + } + + // the addr remains associated with c.example.com + cAddrAgain, err := ipp.applyCheckoutAddr(from, "c.example.com", afterTimeOfUse, timeOfUse) + if err != nil { + t.Fatal(err) + } + if cAddrAgain.Compare(cAddr) != 0 { + t.Fatalf("expected cAddrAgain to be cAddr, but they are different. cAddrAgain=%s cAddr=%s", cAddrAgain, cAddr) + } + ww, ok = ipp.domainLookup(from, firstIP) + if !ok { + t.Fatal("expected wherewhen to be found") + } + if ww.Domain != "c.example.com" { + t.Fatalf("expected firstIP to look up to c.example.com, got: %s", ww.Domain) + } +} + +func TestConsensusPoolApplyMarkLastUsed(t *testing.T) { + ipp := makePool(netip.MustParsePrefix("100.64.0.0/31")) + firstIP := netip.MustParseAddr("100.64.0.0") + time1 := time.Now() + time2 := time1.Add(1 * time.Hour) + from := tailcfg.NodeID(1) + domain := "example.com" + + aAddr, err := ipp.applyCheckoutAddr(from, domain, time.Time{}, time1) + if err != nil { + t.Fatal(err) + } + if aAddr.Compare(firstIP) != 0 { + t.Fatalf("expected %s, got %s", firstIP, aAddr) + } + // example.com LastUsed is now time1 + ww, ok := ipp.domainLookup(from, firstIP) + if !ok { + t.Fatal("expected wherewhen to be found") + } + if ww.LastUsed != time1 { + t.Fatalf("expected %s, got %s", time1, ww.LastUsed) + } + if ww.Domain != domain { + t.Fatalf("expected %s, got %s", domain, ww.Domain) + } + + err = ipp.applyMarkLastUsed(from, firstIP, domain, time2) + if err != nil { + t.Fatal(err) + } + + // example.com LastUsed is now time2 + ww, ok = ipp.domainLookup(from, firstIP) + if !ok { + t.Fatal("expected wherewhen to be found") + } + if ww.LastUsed != time2 { + t.Fatalf("expected %s, got %s", time2, ww.LastUsed) + } + if ww.Domain != domain { + t.Fatalf("expected %s, got %s", domain, ww.Domain) + } +} + +func TestConsensusDomainForIP(t *testing.T) { + ipp := makePool(netip.MustParsePrefix("100.64.0.0/16")) + from := tailcfg.NodeID(1) + domain := "example.com" + now := time.Now() + + d, ok := ipp.DomainForIP(from, netip.MustParseAddr("100.64.0.1"), now) + if d != "" { + t.Fatalf("expected an empty string if the addr is not found but got %s", d) + } + if ok { + t.Fatalf("expected domain to not be found for IP, as it has never been looked up") + } + a, err := ipp.IPForDomain(from, domain) + if err != nil { + t.Fatal(err) + } + d2, ok := ipp.DomainForIP(from, a, now) + if d2 != domain { + t.Fatalf("expected %s but got %s", domain, d2) + } + if !ok { + t.Fatalf("expected domain to be found for IP that was handed out for it") + } +} + +func TestConsensusReadDomainForIP(t *testing.T) { + ipp := makePool(netip.MustParsePrefix("100.64.0.0/16")) + from := tailcfg.NodeID(1) + domain := "example.com" + + d, err := ipp.readDomainForIP(from, netip.MustParseAddr("100.64.0.1")) + if err != nil { + t.Fatal(err) + } + if d != "" { + t.Fatalf("expected an empty string if the addr is not found but got %s", d) + } + a, err := ipp.IPForDomain(from, domain) + if err != nil { + t.Fatal(err) + } + d2, err := ipp.readDomainForIP(from, a) + if err != nil { + t.Fatal(err) + } + if d2 != domain { + t.Fatalf("expected %s but got %s", domain, d2) + } +} + +func TestConsensusSnapshot(t *testing.T) { + pfx := netip.MustParsePrefix("100.64.0.0/16") + ipp := makePool(pfx) + domain := "example.com" + expectedAddr := netip.MustParseAddr("100.64.0.0") + expectedFrom := expectedAddr + expectedTo := netip.MustParseAddr("100.64.255.255") + from := tailcfg.NodeID(1) + + // pool allocates first addr for from + if _, err := ipp.IPForDomain(from, domain); err != nil { + t.Fatal(err) + } + // take a snapshot + fsmSnap, err := ipp.Snapshot() + if err != nil { + t.Fatal(err) + } + snap := fsmSnap.(fsmSnapshot) + + // verify snapshot state matches the state we know ipp will have + // ipset matches ipp.IPSet + if len(snap.IPSet.Ranges) != 1 { + t.Fatalf("expected 1, got %d", len(snap.IPSet.Ranges)) + } + if snap.IPSet.Ranges[0].From != expectedFrom { + t.Fatalf("want %s, got %s", expectedFrom, snap.IPSet.Ranges[0].From) + } + if snap.IPSet.Ranges[0].To != expectedTo { + t.Fatalf("want %s, got %s", expectedTo, snap.IPSet.Ranges[0].To) + } + + // perPeerMap has one entry, for from + if len(snap.PerPeerMap) != 1 { + t.Fatalf("expected 1, got %d", len(snap.PerPeerMap)) + } + ps := snap.PerPeerMap[from] + + // the one peer state has allocated one address, the first in the prefix + if len(ps.DomainToAddr) != 1 { + t.Fatalf("expected 1, got %d", len(ps.DomainToAddr)) + } + addr := ps.DomainToAddr[domain] + if addr != expectedAddr { + t.Fatalf("want %s, got %s", expectedAddr.String(), addr.String()) + } + if len(ps.AddrToDomain) != 1 { + t.Fatalf("expected 1, got %d", len(ps.AddrToDomain)) + } + ww := ps.AddrToDomain[addr] + if ww.Domain != domain { + t.Fatalf("want %s, got %s", domain, ww.Domain) + } +} + +func TestConsensusRestore(t *testing.T) { + pfx := netip.MustParsePrefix("100.64.0.0/16") + ipp := makePool(pfx) + domain := "example.com" + expectedAddr := netip.MustParseAddr("100.64.0.0") + from := tailcfg.NodeID(1) + + if _, err := ipp.IPForDomain(from, domain); err != nil { + t.Fatal(err) + } + // take the snapshot after only 1 addr allocated + fsmSnap, err := ipp.Snapshot() + if err != nil { + t.Fatal(err) + } + snap := fsmSnap.(fsmSnapshot) + + if _, err := ipp.IPForDomain(from, "b.example.com"); err != nil { + t.Fatal(err) + } + if _, err := ipp.IPForDomain(from, "c.example.com"); err != nil { + t.Fatal(err) + } + if _, err := ipp.IPForDomain(from, "d.example.com"); err != nil { + t.Fatal(err) + } + // ipp now has 4 entries in domainToAddr + ps, _ := ipp.perPeerMap.Load(from) + if len(ps.domainToAddr) != 4 { + t.Fatalf("want 4, got %d", len(ps.domainToAddr)) + } + + // restore the snapshot + bs, err := json.Marshal(snap) + if err != nil { + t.Fatal(err) + } + err = ipp.Restore(io.NopCloser(bytes.NewBuffer(bs))) + if err != nil { + t.Fatal(err) + } + + // everything should be as it was when the snapshot was taken + if ipp.perPeerMap.Len() != 1 { + t.Fatalf("want 1, got %d", ipp.perPeerMap.Len()) + } + psAfter, _ := ipp.perPeerMap.Load(from) + if len(psAfter.domainToAddr) != 1 { + t.Fatalf("want 1, got %d", len(psAfter.domainToAddr)) + } + if psAfter.domainToAddr[domain] != expectedAddr { + t.Fatalf("want %s, got %s", expectedAddr, psAfter.domainToAddr[domain]) + } + ww, _ := psAfter.addrToDomain.Load(expectedAddr) + if ww.Domain != domain { + t.Fatalf("want %s, got %s", domain, ww.Domain) + } +} + +func TestConsensusIsCloseToExpiry(t *testing.T) { + a := time.Now() + b := a.Add(5 * time.Second) + if !isCloseToExpiry(a, b, 8*time.Second) { + t.Fatal("times are not within half the lifetime, expected true") + } + if isCloseToExpiry(a, b, 12*time.Second) { + t.Fatal("times are within half the lifetime, expected false") + } +} diff --git a/cmd/natc/ippool/consensusippoolserialize.go b/cmd/natc/ippool/consensusippoolserialize.go new file mode 100644 index 0000000000000..97dc02f2c7d7c --- /dev/null +++ b/cmd/natc/ippool/consensusippoolserialize.go @@ -0,0 +1,164 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ippool + +import ( + "encoding/json" + "io" + "log" + "maps" + "net/netip" + + "github.com/hashicorp/raft" + "go4.org/netipx" + "tailscale.com/syncs" + "tailscale.com/tailcfg" +) + +// Snapshot and Restore enable the raft lib to do log compaction. +// https://pkg.go.dev/github.com/hashicorp/raft#FSM + +// Snapshot is part of the raft.FSM interface. +// According to the docs it: +// - should return quickly +// - will not be called concurrently with Apply +// - the snapshot returned will have Persist called on it concurrently with Apply +// (so it should not contain pointers to the original data that's being mutated) +func (ipp *ConsensusIPPool) Snapshot() (raft.FSMSnapshot, error) { + // everything is safe for concurrent reads and this is not called concurrently with Apply which is + // the only thing that writes, so we do not need to lock + return ipp.getPersistable(), nil +} + +type persistableIPSet struct { + Ranges []persistableIPRange +} + +func getPersistableIPSet(i *netipx.IPSet) persistableIPSet { + rs := []persistableIPRange{} + for _, r := range i.Ranges() { + rs = append(rs, getPersistableIPRange(r)) + } + return persistableIPSet{Ranges: rs} +} + +func (mips *persistableIPSet) toIPSet() (*netipx.IPSet, error) { + b := netipx.IPSetBuilder{} + for _, r := range mips.Ranges { + b.AddRange(r.toIPRange()) + } + return b.IPSet() +} + +type persistableIPRange struct { + From netip.Addr + To netip.Addr +} + +func getPersistableIPRange(r netipx.IPRange) persistableIPRange { + return persistableIPRange{ + From: r.From(), + To: r.To(), + } +} + +func (mipr *persistableIPRange) toIPRange() netipx.IPRange { + return netipx.IPRangeFrom(mipr.From, mipr.To) +} + +// Restore is part of the raft.FSM interface. +// According to the docs it: +// - will not be called concurrently with any other command +// - the FSM must discard all previous state before restoring +func (ipp *ConsensusIPPool) Restore(rc io.ReadCloser) error { + var snap fsmSnapshot + if err := json.NewDecoder(rc).Decode(&snap); err != nil { + return err + } + ipset, ppm, err := snap.getData() + if err != nil { + return err + } + ipp.IPSet = ipset + ipp.perPeerMap = ppm + return nil +} + +type fsmSnapshot struct { + IPSet persistableIPSet + PerPeerMap map[tailcfg.NodeID]persistablePPS +} + +// Persist is part of the raft.FSMSnapshot interface +// According to the docs Persist may be called concurrently with Apply +func (f fsmSnapshot) Persist(sink raft.SnapshotSink) error { + if err := json.NewEncoder(sink).Encode(f); err != nil { + log.Printf("Error encoding snapshot as JSON: %v", err) + return sink.Cancel() + } + return sink.Close() +} + +// Release is part of the raft.FSMSnapshot interface +func (f fsmSnapshot) Release() {} + +// getPersistable returns an object that: +// - contains all the data in ConsensusIPPool +// - doesn't share any pointers with it +// - can be marshalled to JSON +// +// part of the raft snapshotting, getPersistable will be called during Snapshot +// and the results used during persist (concurrently with Apply) +func (ipp *ConsensusIPPool) getPersistable() fsmSnapshot { + ppm := map[tailcfg.NodeID]persistablePPS{} + for k, v := range ipp.perPeerMap.All() { + ppm[k] = v.getPersistable() + } + return fsmSnapshot{ + IPSet: getPersistableIPSet(ipp.IPSet), + PerPeerMap: ppm, + } +} + +func (f fsmSnapshot) getData() (*netipx.IPSet, *syncs.Map[tailcfg.NodeID, *consensusPerPeerState], error) { + ppm := syncs.Map[tailcfg.NodeID, *consensusPerPeerState]{} + for k, v := range f.PerPeerMap { + ppm.Store(k, v.toPerPeerState()) + } + ipset, err := f.IPSet.toIPSet() + if err != nil { + return nil, nil, err + } + return ipset, &ppm, nil +} + +// getPersistable returns an object that: +// - contains all the data in consensusPerPeerState +// - doesn't share any pointers with it +// - can be marshalled to JSON +// +// part of the raft snapshotting, getPersistable will be called during Snapshot +// and the results used during persist (concurrently with Apply) +func (ps *consensusPerPeerState) getPersistable() persistablePPS { + return persistablePPS{ + AddrToDomain: maps.Collect(ps.addrToDomain.All()), + DomainToAddr: maps.Clone(ps.domainToAddr), + } +} + +type persistablePPS struct { + DomainToAddr map[string]netip.Addr + AddrToDomain map[netip.Addr]whereWhen +} + +func (p persistablePPS) toPerPeerState() *consensusPerPeerState { + atd := &syncs.Map[netip.Addr, whereWhen]{} + for k, v := range p.AddrToDomain { + atd.Store(k, v) + } + return &consensusPerPeerState{ + domainToAddr: p.DomainToAddr, + addrToDomain: atd, + } +} diff --git a/cmd/natc/ippool/ippool.go b/cmd/natc/ippool/ippool.go index 3a46a6e7ad186..5a2dcbec911e0 100644 --- a/cmd/natc/ippool/ippool.go +++ b/cmd/natc/ippool/ippool.go @@ -10,6 +10,7 @@ import ( "math/big" "net/netip" "sync" + "time" "github.com/gaissmai/bart" "go4.org/netipx" @@ -21,12 +22,26 @@ import ( var ErrNoIPsAvailable = errors.New("no IPs available") -type IPPool struct { +// IPPool allocates IPv4 addresses from a pool to DNS domains, on a per tailcfg.NodeID basis. +// For each tailcfg.NodeID, IPv4 addresses are associated with at most one DNS domain. +// Addresses may be reused across other tailcfg.NodeID's for the same or other domains. +type IPPool interface { + // DomainForIP looks up the domain associated with a tailcfg.NodeID and netip.Addr pair. + // If there is no association, the result is empty and ok is false. + DomainForIP(tailcfg.NodeID, netip.Addr, time.Time) (string, bool) + + // IPForDomain looks up or creates an IP address allocation for the tailcfg.NodeID and domain pair. + // If no address association is found, one is allocated from the range of free addresses for this tailcfg.NodeID. + // If no more address are available, an error is returned. + IPForDomain(tailcfg.NodeID, string) (netip.Addr, error) +} + +type SingleMachineIPPool struct { perPeerMap syncs.Map[tailcfg.NodeID, *perPeerState] IPSet *netipx.IPSet } -func (ipp *IPPool) DomainForIP(from tailcfg.NodeID, addr netip.Addr) (string, bool) { +func (ipp *SingleMachineIPPool) DomainForIP(from tailcfg.NodeID, addr netip.Addr, _ time.Time) (string, bool) { ps, ok := ipp.perPeerMap.Load(from) if !ok { log.Printf("handleTCPFlow: no perPeerState for %v", from) @@ -40,7 +55,7 @@ func (ipp *IPPool) DomainForIP(from tailcfg.NodeID, addr netip.Addr) (string, bo return domain, ok } -func (ipp *IPPool) IPForDomain(from tailcfg.NodeID, domain string) (netip.Addr, error) { +func (ipp *SingleMachineIPPool) IPForDomain(from tailcfg.NodeID, domain string) (netip.Addr, error) { npps := &perPeerState{ ipset: ipp.IPSet, } diff --git a/cmd/natc/ippool/ippool_test.go b/cmd/natc/ippool/ippool_test.go index 2919d7757af8c..8d474f86a97ed 100644 --- a/cmd/natc/ippool/ippool_test.go +++ b/cmd/natc/ippool/ippool_test.go @@ -8,6 +8,7 @@ import ( "fmt" "net/netip" "testing" + "time" "go4.org/netipx" "tailscale.com/tailcfg" @@ -19,7 +20,7 @@ func TestIPPoolExhaustion(t *testing.T) { var ipsb netipx.IPSetBuilder ipsb.AddPrefix(smallPrefix) addrPool := must.Get(ipsb.IPSet()) - pool := IPPool{IPSet: addrPool} + pool := SingleMachineIPPool{IPSet: addrPool} assignedIPs := make(map[netip.Addr]string) @@ -68,7 +69,7 @@ func TestIPPool(t *testing.T) { var ipsb netipx.IPSetBuilder ipsb.AddPrefix(netip.MustParsePrefix("100.64.1.0/24")) addrPool := must.Get(ipsb.IPSet()) - pool := IPPool{ + pool := SingleMachineIPPool{ IPSet: addrPool, } from := tailcfg.NodeID(12345) @@ -89,7 +90,7 @@ func TestIPPool(t *testing.T) { t.Errorf("IPv4 address %s not in range %s", addr, addrPool) } - domain, ok := pool.DomainForIP(from, addr) + domain, ok := pool.DomainForIP(from, addr, time.Now()) if !ok { t.Errorf("domainForIP(%s) not found", addr) } else if domain != "example.com" { diff --git a/cmd/natc/natc.go b/cmd/natc/natc.go index b327f55bdc3ea..fdbce3da189b2 100644 --- a/cmd/natc/natc.go +++ b/cmd/natc/natc.go @@ -18,6 +18,7 @@ import ( "net/http" "net/netip" "os" + "path/filepath" "strings" "time" @@ -53,10 +54,14 @@ func main() { hostname = fs.String("hostname", "", "Hostname to register the service under") siteID = fs.Uint("site-id", 1, "an integer site ID to use for the ULA prefix which allows for multiple proxies to act in a HA configuration") v4PfxStr = fs.String("v4-pfx", "100.64.1.0/24", "comma-separated list of IPv4 prefixes to advertise") + dnsServers = fs.String("dns-servers", "", "comma separated list of upstream DNS to use, including host and port (use system if empty)") verboseTSNet = fs.Bool("verbose-tsnet", false, "enable verbose logging in tsnet") printULA = fs.Bool("print-ula", false, "print the ULA prefix and exit") ignoreDstPfxStr = fs.String("ignore-destinations", "", "comma-separated list of prefixes to ignore") wgPort = fs.Uint("wg-port", 0, "udp port for wireguard and peer to peer traffic") + clusterTag = fs.String("cluster-tag", "", "optionally run in a consensus cluster with other nodes with this tag") + server = fs.String("login-server", ipn.DefaultControlURL, "the base URL of control server") + stateDir = fs.String("state-dir", "", "path to directory in which to store app state") ) ff.Parse(fs, os.Args[1:], ff.WithEnvVarPrefix("TS_NATC")) @@ -74,7 +79,7 @@ func main() { } var ignoreDstTable *bart.Table[bool] - for _, s := range strings.Split(*ignoreDstPfxStr, ",") { + for s := range strings.SplitSeq(*ignoreDstPfxStr, ",") { s := strings.TrimSpace(s) if s == "" { continue @@ -93,7 +98,9 @@ func main() { } ts := &tsnet.Server{ Hostname: *hostname, + Dir: *stateDir, } + ts.ControlURL = *server if *wgPort != 0 { if *wgPort >= 1<<16 { log.Fatalf("wg-port must be in the range [0, 65535]") @@ -148,19 +155,68 @@ func main() { routes, dnsAddr, addrPool := calculateAddresses(prefixes) v6ULA := ula(uint16(*siteID)) + + var ipp ippool.IPPool + if *clusterTag != "" { + cipp := ippool.NewConsensusIPPool(addrPool) + clusterStateDir, err := getClusterStatePath(*stateDir) + if err != nil { + log.Fatalf("Creating cluster state dir failed: %v", err) + } + err = cipp.StartConsensus(ctx, ts, *clusterTag, clusterStateDir) + if err != nil { + log.Fatalf("StartConsensus: %v", err) + } + defer func() { + err := cipp.StopConsensus(ctx) + if err != nil { + log.Printf("Error stopping consensus: %v", err) + } + }() + ipp = cipp + } else { + ipp = &ippool.SingleMachineIPPool{IPSet: addrPool} + } + c := &connector{ ts: ts, whois: lc, v6ULA: v6ULA, ignoreDsts: ignoreDstTable, - ipPool: &ippool.IPPool{IPSet: addrPool}, + ipPool: ipp, routes: routes, dnsAddr: dnsAddr, - resolver: net.DefaultResolver, + resolver: getResolver(*dnsServers), } c.run(ctx, lc) } +// getResolver parses serverFlag and returns either the default resolver, or a +// resolver that uses the provided comma-separated DNS server AddrPort's, or +// panics. +func getResolver(serverFlag string) lookupNetIPer { + if serverFlag == "" { + return net.DefaultResolver + } + var addrs []string + for s := range strings.SplitSeq(serverFlag, ",") { + s = strings.TrimSpace(s) + addr, err := netip.ParseAddrPort(s) + if err != nil { + log.Fatalf("dns server provided: %q does not parse: %v", s, err) + } + addrs = append(addrs, addr.String()) + } + return &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network string, address string) (net.Conn, error) { + var dialer net.Dialer + // TODO(raggi): perhaps something other than random? + return dialer.DialContext(ctx, network, addrs[rand.N(len(addrs))]) + }, + } +} + func calculateAddresses(prefixes []netip.Prefix) (*netipx.IPSet, netip.Addr, *netipx.IPSet) { var ipsb netipx.IPSetBuilder for _, p := range prefixes { @@ -209,7 +265,7 @@ type connector struct { ignoreDsts *bart.Table[bool] // ipPool contains the per-peer IPv4 address assignments. - ipPool *ippool.IPPool + ipPool ippool.IPPool // resolver is used to lookup IP addresses for DNS queries. resolver lookupNetIPer @@ -453,7 +509,7 @@ func (c *connector) handleTCPFlow(src, dst netip.AddrPort) (handler func(net.Con if dstAddr.Is6() { dstAddr = v4ForV6(dstAddr) } - domain, ok := c.ipPool.DomainForIP(who.Node.ID, dstAddr) + domain, ok := c.ipPool.DomainForIP(who.Node.ID, dstAddr, time.Now()) if !ok { return nil, false } @@ -547,3 +603,28 @@ func proxyTCPConn(c net.Conn, dest string, ctor *connector) { p.Start() } + +func getClusterStatePath(stateDirFlag string) (string, error) { + var dirPath string + if stateDirFlag != "" { + dirPath = stateDirFlag + } else { + confDir, err := os.UserConfigDir() + if err != nil { + return "", err + } + dirPath = filepath.Join(confDir, "nat-connector-state") + } + dirPath = filepath.Join(dirPath, "cluster") + + if err := os.MkdirAll(dirPath, 0700); err != nil { + return "", err + } + if fi, err := os.Stat(dirPath); err != nil { + return "", err + } else if !fi.IsDir() { + return "", fmt.Errorf("%v is not a directory", dirPath) + } + + return dirPath, nil +} diff --git a/cmd/natc/natc_test.go b/cmd/natc/natc_test.go index 0320db8a4ea59..c0a66deb8a4da 100644 --- a/cmd/natc/natc_test.go +++ b/cmd/natc/natc_test.go @@ -9,6 +9,7 @@ import ( "io" "net" "net/netip" + "sync" "testing" "time" @@ -270,7 +271,7 @@ func TestDNSResponse(t *testing.T) { ignoreDsts: &bart.Table[bool]{}, routes: routes, v6ULA: v6ULA, - ipPool: &ippool.IPPool{IPSet: addrPool}, + ipPool: &ippool.SingleMachineIPPool{IPSet: addrPool}, dnsAddr: dnsAddr, } c.ignoreDsts.Insert(netip.MustParsePrefix("8.8.4.4/32"), true) @@ -480,3 +481,198 @@ func TestV6V4(t *testing.T) { } } } + +// echoServer is a simple server that just echos back data set to it. +type echoServer struct { + listener net.Listener + addr string + wg sync.WaitGroup + done chan struct{} +} + +// newEchoServer creates a new test DNS server on the specified network and address +func newEchoServer(t *testing.T, network, addr string) *echoServer { + listener, err := net.Listen(network, addr) + if err != nil { + t.Fatalf("Failed to create test DNS server: %v", err) + } + + server := &echoServer{ + listener: listener, + addr: listener.Addr().String(), + done: make(chan struct{}), + } + + server.wg.Add(1) + go server.serve() + + return server +} + +func (s *echoServer) serve() { + defer s.wg.Done() + + for { + select { + case <-s.done: + return + default: + conn, err := s.listener.Accept() + if err != nil { + select { + case <-s.done: + return + default: + continue + } + } + go s.handleConnection(conn) + } + } +} + +func (s *echoServer) handleConnection(conn net.Conn) { + defer conn.Close() + // Simple response - just echo back some data to confirm connectivity + buf := make([]byte, 1024) + n, err := conn.Read(buf) + if err != nil { + return + } + conn.Write(buf[:n]) +} + +func (s *echoServer) close() { + close(s.done) + s.listener.Close() + s.wg.Wait() +} + +func TestGetResolver(t *testing.T) { + tests := []struct { + name string + network string + addr string + }{ + { + name: "ipv4_loopback", + network: "tcp4", + addr: "127.0.0.1:0", + }, + { + name: "ipv6_loopback", + network: "tcp6", + addr: "[::1]:0", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := newEchoServer(t, tc.network, tc.addr) + defer server.close() + serverAddr := server.addr + resolver := getResolver(serverAddr) + if resolver == nil { + t.Fatal("getResolver returned nil") + } + + netResolver, ok := resolver.(*net.Resolver) + if !ok { + t.Fatal("getResolver did not return a *net.Resolver") + } + if netResolver.Dial == nil { + t.Fatal("resolver.Dial is nil") + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := netResolver.Dial(ctx, "tcp", "dummy.address:53") + if err != nil { + t.Fatalf("Failed to dial test DNS server: %v", err) + } + defer conn.Close() + + testData := []byte("test") + _, err = conn.Write(testData) + if err != nil { + t.Fatalf("Failed to write to connection: %v", err) + } + + response := make([]byte, len(testData)) + _, err = conn.Read(response) + if err != nil { + t.Fatalf("Failed to read from connection: %v", err) + } + + if string(response) != string(testData) { + t.Fatalf("Expected echo response %q, got %q", testData, response) + } + }) + } +} + +func TestGetResolverMultipleServers(t *testing.T) { + server1 := newEchoServer(t, "tcp4", "127.0.0.1:0") + defer server1.close() + server2 := newEchoServer(t, "tcp4", "127.0.0.1:0") + defer server2.close() + serverFlag := server1.addr + ", " + server2.addr + + resolver := getResolver(serverFlag) + netResolver, ok := resolver.(*net.Resolver) + if !ok { + t.Fatal("getResolver did not return a *net.Resolver") + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + servers := map[string]bool{ + server1.addr: false, + server2.addr: false, + } + + // Try up to 1000 times to hit all servers, this should be very quick, and + // if this fails randomness has regressed beyond reason. + for range 1000 { + conn, err := netResolver.Dial(ctx, "tcp", "dummy.address:53") + if err != nil { + t.Fatalf("Failed to dial test DNS server: %v", err) + } + + remoteAddr := conn.RemoteAddr().String() + + conn.Close() + + servers[remoteAddr] = true + + var allDone = true + for _, done := range servers { + if !done { + allDone = false + break + } + } + if allDone { + break + } + } + + var allDone = true + for _, done := range servers { + if !done { + allDone = false + break + } + } + if !allDone { + t.Errorf("after 1000 queries, not all servers were hit, significant lack of randomness: %#v", servers) + } +} + +func TestGetResolverEmpty(t *testing.T) { + resolver := getResolver("") + if resolver != net.DefaultResolver { + t.Fatal(`getResolver("") should return net.DefaultResolver`) + } +} diff --git a/cmd/proxy-test-server/proxy-test-server.go b/cmd/proxy-test-server/proxy-test-server.go new file mode 100644 index 0000000000000..9f8c94a384ea5 --- /dev/null +++ b/cmd/proxy-test-server/proxy-test-server.go @@ -0,0 +1,81 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// The proxy-test-server command is a simple HTTP proxy server for testing +// Tailscale's client proxy functionality. +package main + +import ( + "crypto/tls" + "flag" + "fmt" + "log" + "net" + "net/http" + "os" + "strings" + + "golang.org/x/crypto/acme/autocert" + "tailscale.com/net/connectproxy" + "tailscale.com/tempfork/acme" +) + +var ( + listen = flag.String("listen", ":8080", "Address to listen on for HTTPS proxy requests") + hostname = flag.String("hostname", "localhost", "Hostname for the proxy server") + tailscaleOnly = flag.Bool("tailscale-only", true, "Restrict proxy to Tailscale targets only") + extraAllowedHosts = flag.String("allow-hosts", "", "Comma-separated list of allowed target hosts to additionally allow if --tailscale-only is true") +) + +func main() { + flag.Parse() + + am := &autocert.Manager{ + HostPolicy: autocert.HostWhitelist(*hostname), + Prompt: autocert.AcceptTOS, + Cache: autocert.DirCache(os.ExpandEnv("$HOME/.cache/autocert/proxy-test-server")), + } + var allowTarget func(hostPort string) error + if *tailscaleOnly { + allowTarget = func(hostPort string) error { + host, port, err := net.SplitHostPort(hostPort) + if err != nil { + return fmt.Errorf("invalid target %q: %v", hostPort, err) + } + if port != "443" { + return fmt.Errorf("target %q must use port 443", hostPort) + } + for allowed := range strings.SplitSeq(*extraAllowedHosts, ",") { + if host == allowed { + return nil // explicitly allowed target + } + } + if !strings.HasSuffix(host, ".tailscale.com") { + return fmt.Errorf("target %q is not a Tailscale host", hostPort) + } + return nil // valid Tailscale target + } + } + + go func() { + if err := http.ListenAndServe(":http", am.HTTPHandler(nil)); err != nil { + log.Fatalf("autocert HTTP server failed: %v", err) + } + }() + hs := &http.Server{ + Addr: *listen, + Handler: &connectproxy.Handler{ + Check: allowTarget, + Logf: log.Printf, + }, + TLSConfig: &tls.Config{ + GetCertificate: am.GetCertificate, + NextProtos: []string{ + "http/1.1", // enable HTTP/2 + acme.ALPNProto, // enable tls-alpn ACME challenges + }, + }, + } + log.Printf("Starting proxy-test-server on %s (hostname: %q)\n", *listen, *hostname) + log.Fatal(hs.ListenAndServeTLS("", "")) // cert and key are provided by autocert +} diff --git a/cmd/proxy-to-grafana/proxy-to-grafana_test.go b/cmd/proxy-to-grafana/proxy-to-grafana_test.go index 083c4bc494ad6..4831d54364943 100644 --- a/cmd/proxy-to-grafana/proxy-to-grafana_test.go +++ b/cmd/proxy-to-grafana/proxy-to-grafana_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package main import ( diff --git a/cmd/stund/depaware.txt b/cmd/stund/depaware.txt index da768039431fe..81544b7505dc7 100644 --- a/cmd/stund/depaware.txt +++ b/cmd/stund/depaware.txt @@ -76,7 +76,7 @@ tailscale.com/cmd/stund dependencies: (generated by github.com/tailscale/depawar L 💣 tailscale.com/util/dirwalk from tailscale.com/metrics tailscale.com/util/dnsname from tailscale.com/tailcfg tailscale.com/util/lineiter from tailscale.com/version/distro - tailscale.com/util/mak from tailscale.com/syncs + tailscale.com/util/mak from tailscale.com/syncs+ tailscale.com/util/nocasemaps from tailscale.com/types/ipproto tailscale.com/util/rands from tailscale.com/tsweb tailscale.com/util/slicesx from tailscale.com/tailcfg diff --git a/cmd/tailscale/cli/advertise.go b/cmd/tailscale/cli/advertise.go deleted file mode 100644 index 83d1a35aa8a14..0000000000000 --- a/cmd/tailscale/cli/advertise.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package cli - -import ( - "context" - "flag" - "fmt" - "strings" - - "github.com/peterbourgon/ff/v3/ffcli" - "tailscale.com/envknob" - "tailscale.com/ipn" - "tailscale.com/tailcfg" -) - -var advertiseArgs struct { - services string // comma-separated list of services to advertise -} - -// TODO(naman): This flag may move to set.go or serve_v2.go after the WIPCode -// envknob is not needed. -func advertiseCmd() *ffcli.Command { - if !envknob.UseWIPCode() { - return nil - } - return &ffcli.Command{ - Name: "advertise", - ShortUsage: "tailscale advertise --services=", - ShortHelp: "Advertise this node as a destination for a service", - Exec: runAdvertise, - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("advertise") - fs.StringVar(&advertiseArgs.services, "services", "", "comma-separated services to advertise; each must start with \"svc:\" (e.g. \"svc:idp,svc:nas,svc:database\")") - return fs - })(), - } -} - -func runAdvertise(ctx context.Context, args []string) error { - if len(args) > 0 { - return flag.ErrHelp - } - - services, err := parseServiceNames(advertiseArgs.services) - if err != nil { - return err - } - - _, err = localClient.EditPrefs(ctx, &ipn.MaskedPrefs{ - AdvertiseServicesSet: true, - Prefs: ipn.Prefs{ - AdvertiseServices: services, - }, - }) - return err -} - -// parseServiceNames takes a comma-separated list of service names -// (eg. "svc:hello,svc:webserver,svc:catphotos"), splits them into -// a list and validates each service name. If valid, it returns -// the service names in a slice of strings. -func parseServiceNames(servicesArg string) ([]string, error) { - var services []string - if servicesArg != "" { - services = strings.Split(servicesArg, ",") - for _, svc := range services { - err := tailcfg.ServiceName(svc).Validate() - if err != nil { - return nil, fmt.Errorf("service %q: %s", svc, err) - } - } - } - return services, nil -} diff --git a/cmd/tailscale/cli/cli.go b/cmd/tailscale/cli/cli.go index d7e8e5ca22dce..bdfc7af423bf4 100644 --- a/cmd/tailscale/cli/cli.go +++ b/cmd/tailscale/cli/cli.go @@ -260,7 +260,6 @@ change in the future. debugCmd(), driveCmd, idTokenCmd, - advertiseCmd(), configureHostCmd(), ), FlagSet: rootfs, diff --git a/cmd/tailscale/cli/cli_test.go b/cmd/tailscale/cli/cli_test.go index 9aa3693fd92c5..2e1bec8c9bcb0 100644 --- a/cmd/tailscale/cli/cli_test.go +++ b/cmd/tailscale/cli/cli_test.go @@ -964,13 +964,16 @@ func TestPrefFlagMapping(t *testing.T) { // flag for this. continue case "AdvertiseServices": - // Handled by the tailscale advertise subcommand, we don't want a + // Handled by the tailscale serve subcommand, we don't want a // CLI flag for this. continue case "InternalExitNodePrior": // Used internally by LocalBackend as part of exit node usage toggling. // No CLI flag for this. continue + case "AutoExitNode": + // Handled by tailscale {set,up} --exit-node=auto:any. + continue } t.Errorf("unexpected new ipn.Pref field %q is not handled by up.go (see addPrefFlagMapping and checkForAccidentalSettingReverts)", prefName) } @@ -1334,6 +1337,27 @@ func TestUpdatePrefs(t *testing.T) { } }, }, + { + name: "auto_exit_node", + flags: []string{"--exit-node=auto:any"}, + curPrefs: &ipn.Prefs{ + ControlURL: ipn.DefaultControlURL, + CorpDNS: true, // enabled by [ipn.NewPrefs] by default + NetfilterMode: preftype.NetfilterOn, // enabled by [ipn.NewPrefs] by default + }, + wantJustEditMP: &ipn.MaskedPrefs{ + WantRunningSet: true, // enabled by default for tailscale up + AutoExitNodeSet: true, + ExitNodeIDSet: true, // we want ExitNodeID cleared + ExitNodeIPSet: true, // same for ExitNodeIP + }, + env: upCheckEnv{backendState: "Running"}, + checkUpdatePrefsMutations: func(t *testing.T, newPrefs *ipn.Prefs) { + if newPrefs.AutoExitNode != ipn.AnyExitNode { + t.Errorf("AutoExitNode: got %q; want %q", newPrefs.AutoExitNode, ipn.AnyExitNode) + } + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/tailscale/cli/configure-jetkvm.go b/cmd/tailscale/cli/configure-jetkvm.go new file mode 100644 index 0000000000000..a8e0a7cb542ef --- /dev/null +++ b/cmd/tailscale/cli/configure-jetkvm.go @@ -0,0 +1,81 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build linux && !android && arm + +package cli + +import ( + "bytes" + "context" + "errors" + "flag" + "os" + "runtime" + "strings" + + "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/version/distro" +) + +func init() { + maybeJetKVMConfigureCmd = jetKVMConfigureCmd +} + +func jetKVMConfigureCmd() *ffcli.Command { + if runtime.GOOS != "linux" || distro.Get() != distro.JetKVM { + return nil + } + return &ffcli.Command{ + Name: "jetkvm", + Exec: runConfigureJetKVM, + ShortUsage: "tailscale configure jetkvm", + ShortHelp: "Configure JetKVM to run tailscaled at boot", + LongHelp: strings.TrimSpace(` +This command configures the JetKVM host to run tailscaled at boot. +`), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("jetkvm") + return fs + })(), + } +} + +func runConfigureJetKVM(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("unknown arguments") + } + if runtime.GOOS != "linux" || distro.Get() != distro.JetKVM { + return errors.New("only implemented on JetKVM") + } + err := os.WriteFile("/etc/init.d/S22tailscale", bytes.TrimLeft([]byte(` +#!/bin/sh +# /etc/init.d/S22tailscale +# Start/stop tailscaled + +case "$1" in + start) + /userdata/tailscale/tailscaled > /dev/null 2>&1 & + ;; + stop) + killall tailscaled + ;; + *) + echo "Usage: $0 {start|stop}" + exit 1 + ;; +esac +`), "\n"), 0755) + if err != nil { + return err + } + + if err := os.Symlink("/userdata/tailscale/tailscale", "/bin/tailscale"); err != nil { + if !os.IsExist(err) { + return err + } + } + + printf("Done. Now restart your JetKVM.\n") + return nil +} diff --git a/cmd/tailscale/cli/configure-kube.go b/cmd/tailscale/cli/configure-kube.go index 6bc4e202efd4e..e74e8877996fe 100644 --- a/cmd/tailscale/cli/configure-kube.go +++ b/cmd/tailscale/cli/configure-kube.go @@ -9,17 +9,29 @@ import ( "errors" "flag" "fmt" + "net/netip" + "net/url" "os" "path/filepath" "slices" "strings" + "time" "github.com/peterbourgon/ff/v3/ffcli" "k8s.io/client-go/util/homedir" "sigs.k8s.io/yaml" + "tailscale.com/ipn" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" + "tailscale.com/types/netmap" + "tailscale.com/util/dnsname" "tailscale.com/version" ) +var configureKubeconfigArgs struct { + http bool // Use HTTP instead of HTTPS (default) for the auth proxy. +} + func configureKubeconfigCmd() *ffcli.Command { return &ffcli.Command{ Name: "kubeconfig", @@ -34,6 +46,7 @@ See: https://tailscale.com/s/k8s-auth-proxy `), FlagSet: (func() *flag.FlagSet { fs := newFlagSet("kubeconfig") + fs.BoolVar(&configureKubeconfigArgs.http, "http", false, "Use HTTP instead of HTTPS to connect to the auth proxy. Ignored if you include a scheme in the hostname argument.") return fs })(), Exec: runConfigureKubeconfig, @@ -70,10 +83,13 @@ func kubeconfigPath() (string, error) { } func runConfigureKubeconfig(ctx context.Context, args []string) error { - if len(args) != 1 { - return errors.New("unknown arguments") + if len(args) != 1 || args[0] == "" { + return flag.ErrHelp + } + hostOrFQDNOrIP, http, err := getInputs(args[0], configureKubeconfigArgs.http) + if err != nil { + return fmt.Errorf("error parsing inputs: %w", err) } - hostOrFQDN := args[0] st, err := localClient.Status(ctx) if err != nil { @@ -82,22 +98,45 @@ func runConfigureKubeconfig(ctx context.Context, args []string) error { if st.BackendState != "Running" { return errors.New("Tailscale is not running") } - targetFQDN, ok := nodeDNSNameFromArg(st, hostOrFQDN) - if !ok { - return fmt.Errorf("no peer found with hostname %q", hostOrFQDN) + nm, err := getNetMap(ctx) + if err != nil { + return err + } + + targetFQDN, err := nodeOrServiceDNSNameFromArg(st, nm, hostOrFQDNOrIP) + if err != nil { + return err } targetFQDN = strings.TrimSuffix(targetFQDN, ".") var kubeconfig string if kubeconfig, err = kubeconfigPath(); err != nil { return err } - if err = setKubeconfigForPeer(targetFQDN, kubeconfig); err != nil { + scheme := "https://" + if http { + scheme = "http://" + } + if err = setKubeconfigForPeer(scheme, targetFQDN, kubeconfig); err != nil { return err } - printf("kubeconfig configured for %q\n", hostOrFQDN) + printf("kubeconfig configured for %q at URL %q\n", targetFQDN, scheme+targetFQDN) return nil } +func getInputs(arg string, httpArg bool) (string, bool, error) { + u, err := url.Parse(arg) + if err != nil { + return "", false, err + } + + switch u.Scheme { + case "http", "https": + return u.Host, u.Scheme == "http", nil + default: + return arg, httpArg, nil + } +} + // appendOrSetNamed finds a map with a "name" key matching name in dst, and // replaces it with val. If no such map is found, val is appended to dst. func appendOrSetNamed(dst []any, name string, val map[string]any) []any { @@ -116,7 +155,7 @@ func appendOrSetNamed(dst []any, name string, val map[string]any) []any { var errInvalidKubeconfig = errors.New("invalid kubeconfig") -func updateKubeconfig(cfgYaml []byte, fqdn string) ([]byte, error) { +func updateKubeconfig(cfgYaml []byte, scheme, fqdn string) ([]byte, error) { var cfg map[string]any if len(cfgYaml) > 0 { if err := yaml.Unmarshal(cfgYaml, &cfg); err != nil { @@ -139,7 +178,7 @@ func updateKubeconfig(cfgYaml []byte, fqdn string) ([]byte, error) { cfg["clusters"] = appendOrSetNamed(clusters, fqdn, map[string]any{ "name": fqdn, "cluster": map[string]string{ - "server": "https://" + fqdn, + "server": scheme + fqdn, }, }) @@ -172,7 +211,7 @@ func updateKubeconfig(cfgYaml []byte, fqdn string) ([]byte, error) { return yaml.Marshal(cfg) } -func setKubeconfigForPeer(fqdn, filePath string) error { +func setKubeconfigForPeer(scheme, fqdn, filePath string) error { dir := filepath.Dir(filePath) if _, err := os.Stat(dir); err != nil { if !os.IsNotExist(err) { @@ -191,9 +230,97 @@ func setKubeconfigForPeer(fqdn, filePath string) error { if err != nil && !os.IsNotExist(err) { return fmt.Errorf("reading kubeconfig: %w", err) } - b, err = updateKubeconfig(b, fqdn) + b, err = updateKubeconfig(b, scheme, fqdn) if err != nil { return err } return os.WriteFile(filePath, b, 0600) } + +// nodeOrServiceDNSNameFromArg returns the PeerStatus.DNSName value from a peer +// in st that matches the input arg which can be a base name, full DNS name, or +// an IP. If none is found, it looks for a Tailscale Service +func nodeOrServiceDNSNameFromArg(st *ipnstate.Status, nm *netmap.NetworkMap, arg string) (string, error) { + // First check for a node DNS name. + if dnsName, ok := nodeDNSNameFromArg(st, arg); ok { + return dnsName, nil + } + + // If not found, check for a Tailscale Service DNS name. + rec, ok := serviceDNSRecordFromNetMap(nm, st.CurrentTailnet.MagicDNSSuffix, arg) + if !ok { + return "", fmt.Errorf("no peer found for %q", arg) + } + + // Validate we can see a peer advertising the Tailscale Service. + ip, err := netip.ParseAddr(rec.Value) + if err != nil { + return "", fmt.Errorf("error parsing ExtraRecord IP address %q: %w", rec.Value, err) + } + ipPrefix := netip.PrefixFrom(ip, ip.BitLen()) + for _, ps := range st.Peer { + for _, allowedIP := range ps.AllowedIPs.All() { + if allowedIP == ipPrefix { + return rec.Name, nil + } + } + } + + return "", fmt.Errorf("%q is in MagicDNS, but is not currently reachable on any known peer", arg) +} + +func getNetMap(ctx context.Context) (*netmap.NetworkMap, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + watcher, err := localClient.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) + if err != nil { + return nil, err + } + defer watcher.Close() + + n, err := watcher.Next() + if err != nil { + return nil, err + } + + return n.NetMap, nil +} + +func serviceDNSRecordFromNetMap(nm *netmap.NetworkMap, tcd, arg string) (rec tailcfg.DNSRecord, ok bool) { + argIP, _ := netip.ParseAddr(arg) + argFQDN, err := dnsname.ToFQDN(arg) + argFQDNValid := err == nil + if !argIP.IsValid() && !argFQDNValid { + return rec, false + } + + for _, rec := range nm.DNS.ExtraRecords { + if argIP.IsValid() { + recIP, _ := netip.ParseAddr(rec.Value) + if recIP == argIP { + return rec, true + } + continue + } + + if !argFQDNValid { + continue + } + + recFirstLabel := dnsname.FirstLabel(rec.Name) + if strings.EqualFold(arg, recFirstLabel) { + return rec, true + } + + recFQDN, err := dnsname.ToFQDN(rec.Name) + if err != nil { + continue + } + if strings.EqualFold(argFQDN.WithTrailingDot(), recFQDN.WithTrailingDot()) { + return rec, true + } + } + + return tailcfg.DNSRecord{}, false +} diff --git a/cmd/tailscale/cli/configure-kube_test.go b/cmd/tailscale/cli/configure-kube_test.go index d71a9b627e7f0..0c8b6b2b6cc0e 100644 --- a/cmd/tailscale/cli/configure-kube_test.go +++ b/cmd/tailscale/cli/configure-kube_test.go @@ -6,6 +6,7 @@ package cli import ( "bytes" + "fmt" "strings" "testing" @@ -16,6 +17,7 @@ func TestKubeconfig(t *testing.T) { const fqdn = "foo.tail-scale.ts.net" tests := []struct { name string + http bool in string want string wantErr error @@ -48,6 +50,27 @@ contexts: current-context: foo.tail-scale.ts.net kind: Config users: +- name: tailscale-auth + user: + token: unused`, + }, + { + name: "empty_http", + http: true, + in: "", + want: `apiVersion: v1 +clusters: +- cluster: + server: http://foo.tail-scale.ts.net + name: foo.tail-scale.ts.net +contexts: +- context: + cluster: foo.tail-scale.ts.net + user: tailscale-auth + name: foo.tail-scale.ts.net +current-context: foo.tail-scale.ts.net +kind: Config +users: - name: tailscale-auth user: token: unused`, @@ -202,7 +225,11 @@ users: } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := updateKubeconfig([]byte(tt.in), fqdn) + scheme := "https://" + if tt.http { + scheme = "http://" + } + got, err := updateKubeconfig([]byte(tt.in), scheme, fqdn) if err != nil { if err != tt.wantErr { t.Fatalf("updateKubeconfig() error = %v, wantErr %v", err, tt.wantErr) @@ -219,3 +246,30 @@ users: }) } } + +func TestGetInputs(t *testing.T) { + for _, arg := range []string{ + "foo.tail-scale.ts.net", + "foo", + "127.0.0.1", + } { + for _, prefix := range []string{"", "https://", "http://"} { + for _, httpFlag := range []bool{false, true} { + expectedHost := arg + expectedHTTP := (httpFlag && !strings.HasPrefix(prefix, "https://")) || strings.HasPrefix(prefix, "http://") + t.Run(fmt.Sprintf("%s%s_http=%v", prefix, arg, httpFlag), func(t *testing.T) { + host, http, err := getInputs(prefix+arg, httpFlag) + if err != nil { + t.Fatal(err) + } + if host != expectedHost { + t.Errorf("host = %v, want %v", host, expectedHost) + } + if http != expectedHTTP { + t.Errorf("http = %v, want %v", http, expectedHTTP) + } + }) + } + } + } +} diff --git a/cmd/tailscale/cli/configure.go b/cmd/tailscale/cli/configure.go index acb416755a586..da6278ce24330 100644 --- a/cmd/tailscale/cli/configure.go +++ b/cmd/tailscale/cli/configure.go @@ -10,6 +10,8 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" ) +var maybeJetKVMConfigureCmd func() *ffcli.Command // non-nil only on Linux/arm for JetKVM + func configureCmd() *ffcli.Command { return &ffcli.Command{ Name: "configure", @@ -29,6 +31,7 @@ services on the host to use Tailscale in more ways. synologyConfigureCertCmd(), ccall(maybeSysExtCmd), ccall(maybeVPNConfigCmd), + ccall(maybeJetKVMConfigureCmd), ), } } diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index 213a0166e2aa5..fb062fd17c7aa 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -6,6 +6,7 @@ package cli import ( "bufio" "bytes" + "cmp" "context" "encoding/binary" "encoding/json" @@ -102,6 +103,23 @@ func debugCmd() *ffcli.Command { return fs })(), }, + { + Name: "daemon-bus-events", + ShortUsage: "tailscale debug daemon-bus-events", + Exec: runDaemonBusEvents, + ShortHelp: "Watch events on the tailscaled bus", + }, + { + Name: "daemon-bus-graph", + ShortUsage: "tailscale debug daemon-bus-graph", + Exec: runDaemonBusGraph, + ShortHelp: "Print graph for the tailscaled bus", + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("debug-bus-graph") + fs.StringVar(&daemonBusGraphArgs.format, "format", "json", "output format [json/dot]") + return fs + })(), + }, { Name: "metrics", ShortUsage: "tailscale debug metrics", @@ -350,6 +368,12 @@ func debugCmd() *ffcli.Command { ShortHelp: "Print Go's runtime/debug.BuildInfo", Exec: runGoBuildInfo, }, + { + Name: "peer-relay-servers", + ShortUsage: "tailscale debug peer-relay-servers", + ShortHelp: "Print the current set of candidate peer relay servers", + Exec: runPeerRelayServers, + }, }...), } } @@ -784,6 +808,61 @@ func runDaemonLogs(ctx context.Context, args []string) error { } } +func runDaemonBusEvents(ctx context.Context, args []string) error { + for line, err := range localClient.StreamBusEvents(ctx) { + if err != nil { + return err + } + fmt.Printf("[%d][%q][from: %q][to: %q] %s\n", line.Count, line.Type, + line.From, line.To, line.Event) + } + return nil +} + +var daemonBusGraphArgs struct { + format string +} + +func runDaemonBusGraph(ctx context.Context, args []string) error { + graph, err := localClient.EventBusGraph(ctx) + if err != nil { + return err + } + if format := daemonBusGraphArgs.format; format != "json" && format != "dot" { + return fmt.Errorf("unrecognized output format %q", format) + } + if daemonBusGraphArgs.format == "dot" { + var topics eventbus.DebugTopics + if err := json.Unmarshal(graph, &topics); err != nil { + return fmt.Errorf("unable to parse json: %w", err) + } + fmt.Print(generateDOTGraph(topics.Topics)) + } else { + fmt.Print(string(graph)) + } + return nil +} + +// generateDOTGraph generates the DOT graph format based on the events +func generateDOTGraph(topics []eventbus.DebugTopic) string { + var sb strings.Builder + sb.WriteString("digraph event_bus {\n") + + for _, topic := range topics { + // If no subscribers, still ensure the topic is drawn + if len(topic.Subscribers) == 0 { + topic.Subscribers = append(topic.Subscribers, "no-subscribers") + } + for _, subscriber := range topic.Subscribers { + fmt.Fprintf(&sb, "\t%q -> %q [label=%q];\n", + topic.Publisher, subscriber, cmp.Or(topic.Name, "???")) + } + } + + sb.WriteString("}\n") + return sb.String() +} + var metricsArgs struct { watch bool } @@ -1310,3 +1389,17 @@ func runDebugResolve(ctx context.Context, args []string) error { } return nil } + +func runPeerRelayServers(ctx context.Context, args []string) error { + if len(args) > 0 { + return errors.New("unexpected arguments") + } + v, err := localClient.DebugResultJSON(ctx, "peer-relay-servers") + if err != nil { + return err + } + e := json.NewEncoder(os.Stdout) + e.SetIndent("", " ") + e.Encode(v) + return nil +} diff --git a/cmd/tailscale/cli/dns-query.go b/cmd/tailscale/cli/dns-query.go index da2d9d2a56d77..11f64453732fa 100644 --- a/cmd/tailscale/cli/dns-query.go +++ b/cmd/tailscale/cli/dns-query.go @@ -9,12 +9,31 @@ import ( "fmt" "net/netip" "os" + "strings" "text/tabwriter" + "github.com/peterbourgon/ff/v3/ffcli" "golang.org/x/net/dns/dnsmessage" "tailscale.com/types/dnstype" ) +var dnsQueryCmd = &ffcli.Command{ + Name: "query", + ShortUsage: "tailscale dns query [a|aaaa|cname|mx|ns|opt|ptr|srv|txt]", + Exec: runDNSQuery, + ShortHelp: "Perform a DNS query", + LongHelp: strings.TrimSpace(` +The 'tailscale dns query' subcommand performs a DNS query for the specified name +using the internal DNS forwarder (100.100.100.100). + +By default, the DNS query will request an A record. Another DNS record type can +be specified as the second parameter. + +The output also provides information about the resolver(s) used to resolve the +query. +`), +} + func runDNSQuery(ctx context.Context, args []string) error { if len(args) < 1 { return flag.ErrHelp diff --git a/cmd/tailscale/cli/dns-status.go b/cmd/tailscale/cli/dns-status.go index e487c66bc331c..8c18622ce45af 100644 --- a/cmd/tailscale/cli/dns-status.go +++ b/cmd/tailscale/cli/dns-status.go @@ -5,15 +5,77 @@ package cli import ( "context" + "flag" "fmt" "maps" "slices" "strings" + "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/ipn" "tailscale.com/types/netmap" ) +var dnsStatusCmd = &ffcli.Command{ + Name: "status", + ShortUsage: "tailscale dns status [--all]", + Exec: runDNSStatus, + ShortHelp: "Print the current DNS status and configuration", + LongHelp: strings.TrimSpace(` +The 'tailscale dns status' subcommand prints the current DNS status and +configuration, including: + +- Whether the built-in DNS forwarder is enabled. + +- The MagicDNS configuration provided by the coordination server. + +- Details on which resolver(s) Tailscale believes the system is using by + default. + +The --all flag can be used to output advanced debugging information, including +fallback resolvers, nameservers, certificate domains, extra records, and the +exit node filtered set. + +=== Contents of the MagicDNS configuration === + +The MagicDNS configuration is provided by the coordination server to the client +and includes the following components: + +- MagicDNS enablement status: Indicates whether MagicDNS is enabled across the + entire tailnet. + +- MagicDNS Suffix: The DNS suffix used for devices within your tailnet. + +- DNS Name: The DNS name that other devices in the tailnet can use to reach this + device. + +- Resolvers: The preferred DNS resolver(s) to be used for resolving queries, in + order of preference. If no resolvers are listed here, the system defaults are + used. + +- Split DNS Routes: Custom DNS resolvers may be used to resolve hostnames in + specific domains, this is also known as a 'Split DNS' configuration. The + mapping of domains to their respective resolvers is provided here. + +- Certificate Domains: The DNS names for which the coordination server will + assist in provisioning TLS certificates. + +- Extra Records: Additional DNS records that the coordination server might + provide to the internal DNS resolver. + +- Exit Node Filtered Set: DNS suffixes that the node, when acting as an exit + node DNS proxy, will not answer. + +For more information about the DNS functionality built into Tailscale, refer to +https://tailscale.com/kb/1054/dns. +`), + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("status") + fs.BoolVar(&dnsStatusArgs.all, "all", false, "outputs advanced debugging information") + return fs + })(), +} + // dnsStatusArgs are the arguments for the "dns status" subcommand. var dnsStatusArgs struct { all bool @@ -208,35 +270,3 @@ func fetchNetMap() (netMap *netmap.NetworkMap, err error) { } return notify.NetMap, nil } - -func dnsStatusLongHelp() string { - return `The 'tailscale dns status' subcommand prints the current DNS status and configuration, including: - -- Whether the built-in DNS forwarder is enabled. -- The MagicDNS configuration provided by the coordination server. -- Details on which resolver(s) Tailscale believes the system is using by default. - -The --all flag can be used to output advanced debugging information, including fallback resolvers, nameservers, certificate domains, extra records, and the exit node filtered set. - -=== Contents of the MagicDNS configuration === - -The MagicDNS configuration is provided by the coordination server to the client and includes the following components: - -- MagicDNS enablement status: Indicates whether MagicDNS is enabled across the entire tailnet. - -- MagicDNS Suffix: The DNS suffix used for devices within your tailnet. - -- DNS Name: The DNS name that other devices in the tailnet can use to reach this device. - -- Resolvers: The preferred DNS resolver(s) to be used for resolving queries, in order of preference. If no resolvers are listed here, the system defaults are used. - -- Split DNS Routes: Custom DNS resolvers may be used to resolve hostnames in specific domains, this is also known as a 'Split DNS' configuration. The mapping of domains to their respective resolvers is provided here. - -- Certificate Domains: The DNS names for which the coordination server will assist in provisioning TLS certificates. - -- Extra Records: Additional DNS records that the coordination server might provide to the internal DNS resolver. - -- Exit Node Filtered Set: DNS suffixes that the node, when acting as an exit node DNS proxy, will not answer. - -For more information about the DNS functionality built into Tailscale, refer to https://tailscale.com/kb/1054/dns.` -} diff --git a/cmd/tailscale/cli/dns.go b/cmd/tailscale/cli/dns.go index 402f0cedf0a1e..086abefd6b2bf 100644 --- a/cmd/tailscale/cli/dns.go +++ b/cmd/tailscale/cli/dns.go @@ -4,46 +4,32 @@ package cli import ( - "flag" + "strings" "github.com/peterbourgon/ff/v3/ffcli" ) var dnsCmd = &ffcli.Command{ - Name: "dns", - ShortHelp: "Diagnose the internal DNS forwarder", - LongHelp: dnsCmdLongHelp(), - ShortUsage: "tailscale dns [flags]", - UsageFunc: usageFuncNoDefaultValues, + Name: "dns", + ShortHelp: "Diagnose the internal DNS forwarder", + LongHelp: strings.TrimSpace(` +The 'tailscale dns' subcommand provides tools for diagnosing the internal DNS +forwarder (100.100.100.100). + +For more information about the DNS functionality built into Tailscale, refer to +https://tailscale.com/kb/1054/dns. +`), + ShortUsage: strings.Join([]string{ + dnsStatusCmd.ShortUsage, + dnsQueryCmd.ShortUsage, + }, "\n"), + UsageFunc: usageFuncNoDefaultValues, Subcommands: []*ffcli.Command{ - { - Name: "status", - ShortUsage: "tailscale dns status [--all]", - Exec: runDNSStatus, - ShortHelp: "Print the current DNS status and configuration", - LongHelp: dnsStatusLongHelp(), - FlagSet: (func() *flag.FlagSet { - fs := newFlagSet("status") - fs.BoolVar(&dnsStatusArgs.all, "all", false, "outputs advanced debugging information (fallback resolvers, nameservers, cert domains, extra records, and exit node filtered set)") - return fs - })(), - }, - { - Name: "query", - ShortUsage: "tailscale dns query [a|aaaa|cname|mx|ns|opt|ptr|srv|txt]", - Exec: runDNSQuery, - ShortHelp: "Perform a DNS query", - LongHelp: "The 'tailscale dns query' subcommand performs a DNS query for the specified name using the internal DNS forwarder (100.100.100.100).\n\nIt also provides information about the resolver(s) used to resolve the query.", - }, + dnsStatusCmd, + dnsQueryCmd, // TODO: implement `tailscale log` here // The above work is tracked in https://github.com/tailscale/tailscale/issues/13326 }, } - -func dnsCmdLongHelp() string { - return `The 'tailscale dns' subcommand provides tools for diagnosing the internal DNS forwarder (100.100.100.100). - -For more information about the DNS functionality built into Tailscale, refer to https://tailscale.com/kb/1054/dns.` -} diff --git a/cmd/tailscale/cli/exitnode.go b/cmd/tailscale/cli/exitnode.go index ad7a8ccee5b42..b153f096d6869 100644 --- a/cmd/tailscale/cli/exitnode.go +++ b/cmd/tailscale/cli/exitnode.go @@ -131,7 +131,7 @@ func runExitNodeList(ctx context.Context, args []string) error { for _, country := range filteredPeers.Countries { for _, city := range country.Cities { for _, peer := range city.Peers { - fmt.Fprintf(w, "\n %s\t%s\t%s\t%s\t%s\t", peer.TailscaleIPs[0], strings.Trim(peer.DNSName, "."), country.Name, city.Name, peerStatus(peer)) + fmt.Fprintf(w, "\n %s\t%s\t%s\t%s\t%s\t", peer.TailscaleIPs[0], strings.Trim(peer.DNSName, "."), cmp.Or(country.Name, "-"), cmp.Or(city.Name, "-"), peerStatus(peer)) } } } @@ -202,23 +202,16 @@ type filteredCity struct { Peers []*ipnstate.PeerStatus } -const noLocationData = "-" - -var noLocation = &tailcfg.Location{ - Country: noLocationData, - CountryCode: noLocationData, - City: noLocationData, - CityCode: noLocationData, -} - // filterFormatAndSortExitNodes filters and sorts exit nodes into // alphabetical order, by country, city and then by priority if // present. +// // If an exit node has location data, and the country has more than // one city, an `Any` city is added to the country that contains the // highest priority exit node within that country. +// // For exit nodes without location data, their country fields are -// defined as '-' to indicate that the data is not available. +// defined as the empty string to indicate that the data is not available. func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string) filteredExitNodes { // first get peers into some fixed order, as code below doesn't break ties // and our input comes from a random range-over-map. @@ -229,7 +222,10 @@ func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string) countries := make(map[string]*filteredCountry) cities := make(map[string]*filteredCity) for _, ps := range peers { - loc := cmp.Or(ps.Location, noLocation) + loc := ps.Location + if loc == nil { + loc = &tailcfg.Location{} + } if filterBy != "" && !strings.EqualFold(loc.Country, filterBy) { continue @@ -259,7 +255,7 @@ func filterFormatAndSortExitNodes(peers []*ipnstate.PeerStatus, filterBy string) } for _, country := range filteredExitNodes.Countries { - if country.Name == noLocationData { + if country.Name == "" { // Countries without location data should not // be filtered further. continue diff --git a/cmd/tailscale/cli/exitnode_test.go b/cmd/tailscale/cli/exitnode_test.go index 9d569a45a4615..cc38fd3a4d39e 100644 --- a/cmd/tailscale/cli/exitnode_test.go +++ b/cmd/tailscale/cli/exitnode_test.go @@ -74,10 +74,10 @@ func TestFilterFormatAndSortExitNodes(t *testing.T) { want := filteredExitNodes{ Countries: []*filteredCountry{ { - Name: noLocationData, + Name: "", Cities: []*filteredCity{ { - Name: noLocationData, + Name: "", Peers: []*ipnstate.PeerStatus{ ps[5], }, @@ -273,14 +273,20 @@ func TestSortByCountryName(t *testing.T) { Name: "Zimbabwe", }, { - Name: noLocationData, + Name: "", }, } sortByCountryName(fc) - if fc[0].Name != noLocationData { - t.Fatalf("sortByCountryName did not order countries by alphabetical order, got %v, want %v", fc[0].Name, noLocationData) + want := []string{"", "Albania", "Sweden", "Zimbabwe"} + var got []string + for _, c := range fc { + got = append(got, c.Name) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("sortByCountryName did not order countries by alphabetical order (-want +got):\n%s", diff) } } @@ -296,13 +302,19 @@ func TestSortByCityName(t *testing.T) { Name: "Squamish", }, { - Name: noLocationData, + Name: "", }, } sortByCityName(fc) - if fc[0].Name != noLocationData { - t.Fatalf("sortByCityName did not order cities by alphabetical order, got %v, want %v", fc[0].Name, noLocationData) + want := []string{"", "Goteborg", "Kingston", "Squamish"} + var got []string + for _, c := range fc { + got = append(got, c.Name) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("sortByCityName did not order countries by alphabetical order (-want +got):\n%s", diff) } } diff --git a/cmd/tailscale/cli/logout.go b/cmd/tailscale/cli/logout.go index 0c2007a66ab1b..fbc39473026a1 100644 --- a/cmd/tailscale/cli/logout.go +++ b/cmd/tailscale/cli/logout.go @@ -5,12 +5,18 @@ package cli import ( "context" + "flag" "fmt" "strings" "github.com/peterbourgon/ff/v3/ffcli" + "tailscale.com/client/tailscale/apitype" ) +var logoutArgs struct { + reason string +} + var logoutCmd = &ffcli.Command{ Name: "logout", ShortUsage: "tailscale logout", @@ -22,11 +28,17 @@ the current node key, forcing a future use of it to cause a reauthentication. `), Exec: runLogout, + FlagSet: (func() *flag.FlagSet { + fs := newFlagSet("logout") + fs.StringVar(&logoutArgs.reason, "reason", "", "reason for the logout, if required by a policy") + return fs + })(), } func runLogout(ctx context.Context, args []string) error { if len(args) > 0 { return fmt.Errorf("too many non-flag arguments: %q", args) } + ctx = apitype.RequestReasonKey.WithValue(ctx, logoutArgs.reason) return localClient.Logout(ctx) } diff --git a/cmd/tailscale/cli/network-lock.go b/cmd/tailscale/cli/network-lock.go index c7776707422ec..d19909576c090 100644 --- a/cmd/tailscale/cli/network-lock.go +++ b/cmd/tailscale/cli/network-lock.go @@ -17,12 +17,14 @@ import ( "strings" "time" + "github.com/mattn/go-isatty" "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/ipn/ipnstate" "tailscale.com/tka" "tailscale.com/tsconst" "tailscale.com/types/key" "tailscale.com/types/tkatype" + "tailscale.com/util/prompt" ) var netlockCmd = &ffcli.Command{ @@ -326,6 +328,9 @@ func runNetworkLockRemove(ctx context.Context, args []string) error { if !st.Enabled { return errors.New("tailnet lock is not enabled") } + if len(st.TrustedKeys) == 1 { + return errors.New("cannot remove the last trusted signing key; use 'tailscale lock disable' to disable tailnet lock instead, or add another signing key before removing one") + } if nlRemoveArgs.resign { // Validate we are not removing trust in ourselves while resigning. This is because @@ -366,6 +371,18 @@ func runNetworkLockRemove(ctx context.Context, args []string) error { } } } + } else { + if isatty.IsTerminal(os.Stdout.Fd()) { + fmt.Printf(`Warning +Removal of a signing key(s) without resigning nodes (--re-sign=false) +will cause any nodes signed by the the given key(s) to be locked out +of the Tailscale network. Proceed with caution. +`) + if !prompt.YesNo("Are you sure you want to remove the signing key(s)?") { + fmt.Printf("aborting removal of signing key(s)\n") + os.Exit(0) + } + } } return localClient.NetworkLockModify(ctx, nil, removeKeys) @@ -606,7 +623,7 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er printKey := func(key *tka.Key, prefix string) { fmt.Fprintf(&stanza, "%sType: %s\n", prefix, key.Kind.String()) if keyID, err := key.ID(); err == nil { - fmt.Fprintf(&stanza, "%sKeyID: %x\n", prefix, keyID) + fmt.Fprintf(&stanza, "%sKeyID: tlpub:%x\n", prefix, keyID) } else { // Older versions of the client shouldn't explode when they encounter an // unknown key type. @@ -622,16 +639,20 @@ func nlDescribeUpdate(update ipnstate.NetworkLockUpdate, color bool) (string, er return "", fmt.Errorf("decoding: %w", err) } - fmt.Fprintf(&stanza, "%supdate %x (%s)%s\n", terminalYellow, update.Hash, update.Change, terminalClear) + tkaHead, err := aum.Hash().MarshalText() + if err != nil { + return "", fmt.Errorf("decoding AUM hash: %w", err) + } + fmt.Fprintf(&stanza, "%supdate %s (%s)%s\n", terminalYellow, string(tkaHead), update.Change, terminalClear) switch update.Change { case tka.AUMAddKey.String(): printKey(aum.Key, "") case tka.AUMRemoveKey.String(): - fmt.Fprintf(&stanza, "KeyID: %x\n", aum.KeyID) + fmt.Fprintf(&stanza, "KeyID: tlpub:%x\n", aum.KeyID) case tka.AUMUpdateKey.String(): - fmt.Fprintf(&stanza, "KeyID: %x\n", aum.KeyID) + fmt.Fprintf(&stanza, "KeyID: tlpub:%x\n", aum.KeyID) if aum.Votes != nil { fmt.Fprintf(&stanza, "Votes: %d\n", aum.Votes) } diff --git a/cmd/tailscale/cli/ping.go b/cmd/tailscale/cli/ping.go index 3a909f30dee86..d438cb2286d4c 100644 --- a/cmd/tailscale/cli/ping.go +++ b/cmd/tailscale/cli/ping.go @@ -152,7 +152,9 @@ func runPing(ctx context.Context, args []string) error { } latency := time.Duration(pr.LatencySeconds * float64(time.Second)).Round(time.Millisecond) via := pr.Endpoint - if pr.DERPRegionID != 0 { + if pr.PeerRelay != "" { + via = fmt.Sprintf("peer-relay(%s)", pr.PeerRelay) + } else if pr.DERPRegionID != 0 { via = fmt.Sprintf("DERP(%s)", pr.DERPRegionCode) } if via == "" { diff --git a/cmd/tailscale/cli/risks.go b/cmd/tailscale/cli/risks.go index c36ffafaeb11a..9b03025a83a0b 100644 --- a/cmd/tailscale/cli/risks.go +++ b/cmd/tailscale/cli/risks.go @@ -4,15 +4,18 @@ package cli import ( + "context" "errors" "flag" "fmt" "os" "os/signal" + "runtime" "strings" "syscall" "time" + "tailscale.com/ipn" "tailscale.com/util/testenv" ) @@ -20,11 +23,12 @@ var ( riskTypes []string riskLoseSSH = registerRiskType("lose-ssh") riskMacAppConnector = registerRiskType("mac-app-connector") + riskStrictRPFilter = registerRiskType("linux-strict-rp-filter") riskAll = registerRiskType("all") ) const riskMacAppConnectorMessage = ` -You are trying to configure an app connector on macOS, which is not officially supported due to system limitations. This may result in performance and reliability issues. +You are trying to configure an app connector on macOS, which is not officially supported due to system limitations. This may result in performance and reliability issues. Do not use a macOS app connector for any mission-critical purposes. For the best experience, Linux is the only recommended platform for app connectors. ` @@ -89,3 +93,18 @@ func presentRiskToUser(riskType, riskMessage, acceptedRisks string) error { printf("\r%s\r", strings.Repeat(" ", msgLen)) return errAborted } + +// checkExitNodeRisk checks if the user is using an exit node on Linux and +// whether reverse path filtering is enabled. If so, it presents a risk message. +func checkExitNodeRisk(ctx context.Context, prefs *ipn.Prefs, acceptedRisks string) error { + if runtime.GOOS != "linux" { + return nil + } + if !prefs.ExitNodeIP.IsValid() && prefs.ExitNodeID == "" { + return nil + } + if err := localClient.CheckReversePathFiltering(ctx); err != nil { + return presentRiskToUser(riskStrictRPFilter, err.Error(), acceptedRisks) + } + return nil +} diff --git a/cmd/tailscale/cli/serve_legacy.go b/cmd/tailscale/cli/serve_legacy.go index 96629b5ad45ef..1a05d0543f58e 100644 --- a/cmd/tailscale/cli/serve_legacy.go +++ b/cmd/tailscale/cli/serve_legacy.go @@ -141,6 +141,8 @@ type localServeClient interface { QueryFeature(ctx context.Context, feature string) (*tailcfg.QueryFeatureResponse, error) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (*tailscale.IPNBusWatcher, error) IncrementCounter(ctx context.Context, name string, delta int) error + GetPrefs(ctx context.Context) (*ipn.Prefs, error) + EditPrefs(ctx context.Context, mp *ipn.MaskedPrefs) (*ipn.Prefs, error) } // serveEnv is the environment the serve command runs within. All I/O should be @@ -154,14 +156,16 @@ type serveEnv struct { json bool // output JSON (status only for now) // v2 specific flags - bg bool // background mode - setPath string // serve path - https uint // HTTP port - http uint // HTTP port - tcp uint // TCP port - tlsTerminatedTCP uint // a TLS terminated TCP port - subcmd serveMode // subcommand - yes bool // update without prompt + bg bgBoolFlag // background mode + setPath string // serve path + https uint // HTTP port + http uint // HTTP port + tcp uint // TCP port + tlsTerminatedTCP uint // a TLS terminated TCP port + subcmd serveMode // subcommand + yes bool // update without prompt + service tailcfg.ServiceName // service name + tun bool // redirect traffic to OS for service lc localServeClient // localClient interface, specific to serve @@ -354,12 +358,12 @@ func (e *serveEnv) handleWebServe(ctx context.Context, srvPort uint16, useTLS bo if err != nil { return err } - if sc.IsTCPForwardingOnPort(srvPort) { + if sc.IsTCPForwardingOnPort(srvPort, noService) { fmt.Fprintf(Stderr, "error: cannot serve web; already serving TCP\n") return errHelp } - sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS) + sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS, noService.String()) if !reflect.DeepEqual(cursc, sc) { if err := e.lc.SetServeConfig(ctx, sc); err != nil { @@ -411,11 +415,11 @@ func (e *serveEnv) handleWebServeRemove(ctx context.Context, srvPort uint16, mou if err != nil { return err } - if sc.IsTCPForwardingOnPort(srvPort) { + if sc.IsTCPForwardingOnPort(srvPort, noService) { return errors.New("cannot remove web handler; currently serving TCP") } hp := ipn.HostPort(net.JoinHostPort(dnsName, strconv.Itoa(int(srvPort)))) - if !sc.WebHandlerExists(hp, mount) { + if !sc.WebHandlerExists(noService, hp, mount) { return errors.New("error: handler does not exist") } sc.RemoveWebHandler(dnsName, srvPort, []string{mount}, false) @@ -550,15 +554,15 @@ func (e *serveEnv) handleTCPServe(ctx context.Context, srcType string, srcPort u fwdAddr := "127.0.0.1:" + dstPortStr - if sc.IsServingWeb(srcPort) { - return fmt.Errorf("cannot serve TCP; already serving web on %d", srcPort) - } - dnsName, err := e.getSelfDNSName(ctx) if err != nil { return err } + if sc.IsServingWeb(srcPort, noService) { + return fmt.Errorf("cannot serve TCP; already serving web on %d", srcPort) + } + sc.SetTCPForwarding(srcPort, fwdAddr, terminateTLS, dnsName) if !reflect.DeepEqual(cursc, sc) { @@ -581,11 +585,11 @@ func (e *serveEnv) handleTCPServeRemove(ctx context.Context, src uint16) error { if sc == nil { sc = new(ipn.ServeConfig) } - if sc.IsServingWeb(src) { + if sc.IsServingWeb(src, noService) { return fmt.Errorf("unable to remove; serving web, not TCP forwarding on serve port %d", src) } - if ph := sc.GetTCPPortHandler(src); ph != nil { - sc.RemoveTCPForwarding(src) + if ph := sc.GetTCPPortHandler(src, noService); ph != nil { + sc.RemoveTCPForwarding(noService, src) return e.lc.SetServeConfig(ctx, sc) } return errors.New("error: serve config does not exist") @@ -682,7 +686,7 @@ func (e *serveEnv) printWebStatusTree(sc *ipn.ServeConfig, hp ipn.HostPort) erro } scheme := "https" - if sc.IsServingHTTP(port) { + if sc.IsServingHTTP(port, noService) { scheme = "http" } diff --git a/cmd/tailscale/cli/serve_legacy_test.go b/cmd/tailscale/cli/serve_legacy_test.go index df68b5edd32a1..1ea76e72ca818 100644 --- a/cmd/tailscale/cli/serve_legacy_test.go +++ b/cmd/tailscale/cli/serve_legacy_test.go @@ -859,6 +859,7 @@ type fakeLocalServeClient struct { config *ipn.ServeConfig setCount int // counts calls to SetServeConfig queryFeatureResponse *mockQueryFeatureResponse // mock response to QueryFeature calls + prefs *ipn.Prefs // fake preferences, used to test GetPrefs and SetPrefs } // fakeStatus is a fake ipnstate.Status value for tests. @@ -875,6 +876,7 @@ var fakeStatus = &ipnstate.Status{ tailcfg.CapabilityFunnelPorts + "?ports=443,8443": nil, }, }, + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, } func (lc *fakeLocalServeClient) StatusWithoutPeers(ctx context.Context) (*ipnstate.Status, error) { @@ -891,6 +893,21 @@ func (lc *fakeLocalServeClient) SetServeConfig(ctx context.Context, config *ipn. return nil } +func (lc *fakeLocalServeClient) GetPrefs(ctx context.Context) (*ipn.Prefs, error) { + if lc.prefs == nil { + lc.prefs = ipn.NewPrefs() + } + return lc.prefs, nil +} + +func (lc *fakeLocalServeClient) EditPrefs(ctx context.Context, prefs *ipn.MaskedPrefs) (*ipn.Prefs, error) { + if lc.prefs == nil { + lc.prefs = ipn.NewPrefs() + } + lc.prefs.ApplyEdits(prefs) + return lc.prefs, nil +} + type mockQueryFeatureResponse struct { resp *tailcfg.QueryFeatureResponse err error diff --git a/cmd/tailscale/cli/serve_v2.go b/cmd/tailscale/cli/serve_v2.go index 3e173ce28d8c1..91a23697035a8 100644 --- a/cmd/tailscale/cli/serve_v2.go +++ b/cmd/tailscale/cli/serve_v2.go @@ -18,6 +18,7 @@ import ( "os/signal" "path" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -28,6 +29,7 @@ import ( "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" "tailscale.com/util/mak" + "tailscale.com/util/prompt" "tailscale.com/util/slicesx" "tailscale.com/version" ) @@ -40,6 +42,55 @@ type commandInfo struct { LongHelp string } +type serviceNameFlag struct { + Value *tailcfg.ServiceName +} + +func (s *serviceNameFlag) Set(sv string) error { + if sv == "" { + s.Value = new(tailcfg.ServiceName) + return nil + } + v := tailcfg.ServiceName(sv) + if err := v.Validate(); err != nil { + return fmt.Errorf("invalid service name: %q", sv) + } + *s.Value = v + return nil +} + +// String returns the string representation of service name. +func (s *serviceNameFlag) String() string { + return s.Value.String() +} + +type bgBoolFlag struct { + Value bool + IsSet bool // tracks if the flag was set by the user +} + +// Set sets the boolean flag and whether it's explicitly set by user based on the string value. +func (b *bgBoolFlag) Set(s string) error { + v, err := strconv.ParseBool(s) + if err != nil { + return err + } + b.Value = v + b.IsSet = true + return nil +} + +// This is a hack to make the flag package recognize that this is a boolean flag. +func (b *bgBoolFlag) IsBoolFlag() bool { return true } + +// String returns the string representation of the boolean flag. +func (b *bgBoolFlag) String() string { + if !b.IsSet { + return "default" + } + return strconv.FormatBool(b.Value) +} + var serveHelpCommon = strings.TrimSpace(` can be a file, directory, text, or most commonly the location to a service running on the local machine. The location to the location service can be expressed as a port number (e.g., 3000), @@ -72,8 +123,11 @@ const ( serveTypeHTTP serveTypeTCP serveTypeTLSTerminatedTCP + serveTypeTUN ) +const noService tailcfg.ServiceName = "" + var infoMap = map[serveMode]commandInfo{ serve: { Name: "serve", @@ -119,7 +173,7 @@ func newServeV2Command(e *serveEnv, subcmd serveMode) *ffcli.Command { Exec: e.runServeCombined(subcmd), FlagSet: e.newFlags("serve-set", func(fs *flag.FlagSet) { - fs.BoolVar(&e.bg, "bg", false, "Run the command as a background process (default false)") + fs.Var(&e.bg, "bg", "Run the command as a background process (default false, when --service is set defaults to true).") fs.StringVar(&e.setPath, "set-path", "", "Appends the specified path to the base URL for accessing the underlying service") fs.UintVar(&e.https, "https", 0, "Expose an HTTPS server at the specified port (default mode)") if subcmd == serve { @@ -127,7 +181,9 @@ func newServeV2Command(e *serveEnv, subcmd serveMode) *ffcli.Command { } fs.UintVar(&e.tcp, "tcp", 0, "Expose a TCP forwarder to forward raw TCP packets at the specified port") fs.UintVar(&e.tlsTerminatedTCP, "tls-terminated-tcp", 0, "Expose a TCP forwarder to forward TLS-terminated TCP packets at the specified port") + fs.Var(&serviceNameFlag{Value: &e.service}, "service", "Serve for a service with distinct virtual IP instead on node itself.") fs.BoolVar(&e.yes, "yes", false, "Update without interactive prompts (default false)") + fs.BoolVar(&e.tun, "tun", false, "Forward all traffic to the local machine (default false), only supported for services. Refer to docs for more information.") }), UsageFunc: usageFuncNoDefaultValues, Subcommands: []*ffcli.Command{ @@ -147,6 +203,33 @@ func newServeV2Command(e *serveEnv, subcmd serveMode) *ffcli.Command { Exec: e.runServeReset, FlagSet: e.newFlags("serve-reset", nil), }, + { + Name: "drain", + ShortUsage: fmt.Sprintf("tailscale %s drain ", info.Name), + ShortHelp: "Drain a service from the current node", + LongHelp: "Make the current node no longer accept new connections for the specified service.\n" + + "Existing connections will continue to work until they are closed, but no new connections will be accepted.\n" + + "Use this command to gracefully remove a service from the current node without disrupting existing connections.\n" + + " should be a service name (e.g., svc:my-service).", + Exec: e.runServeDrain, + }, + { + Name: "clear", + ShortUsage: fmt.Sprintf("tailscale %s clear ", info.Name), + ShortHelp: "Remove all config for a service", + LongHelp: "Remove all handlers configured for the specified service.", + Exec: e.runServeClear, + }, + { + Name: "advertise", + ShortUsage: fmt.Sprintf("tailscale %s advertise ", info.Name), + ShortHelp: "Advertise this node as a service proxy to the tailnet", + LongHelp: "Advertise this node as a service proxy to the tailnet. This command is used\n" + + "to make the current node be considered as a service host for a service. This is\n" + + "useful to bring a service back after it has been drained. (i.e. after running \n" + + "`tailscale serve drain `). This is not needed if you are using `tailscale serve` to initialize a service.", + Exec: e.runServeAdvertise, + }, }, } } @@ -161,9 +244,16 @@ func (e *serveEnv) validateArgs(subcmd serveMode, args []string) error { fmt.Fprint(e.stderr(), "\nPlease see https://tailscale.com/kb/1242/tailscale-serve for more information.\n") return errHelpFunc(subcmd) } + if len(args) == 0 && e.tun { + return nil + } if len(args) == 0 { return flag.ErrHelp } + if e.tun && len(args) > 1 { + fmt.Fprintln(e.stderr(), "Error: invalid argument format") + return errHelpFunc(subcmd) + } if len(args) > 2 { fmt.Fprintf(e.stderr(), "Error: invalid number of arguments (%d)\n", len(args)) return errHelpFunc(subcmd) @@ -205,7 +295,16 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { ctx, cancel := signal.NotifyContext(ctx, os.Interrupt) defer cancel() + forService := e.service != "" + if !e.bg.IsSet { + e.bg.Value = forService + } + funnel := subcmd == funnel + if forService && funnel { + return errors.New("Error: --service flag is not supported with funnel") + } + if funnel { // verify node has funnel capabilities if err := e.verifyFunnelEnabled(ctx, 443); err != nil { @@ -213,6 +312,10 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } } + if forService && !e.bg.Value { + return errors.New("Error: --service flag is only compatible with background mode") + } + mount, err := cleanURLPath(e.setPath) if err != nil { return fmt.Errorf("failed to clean the mount point: %w", err) @@ -238,6 +341,7 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { return fmt.Errorf("getting client status: %w", err) } dnsName := strings.TrimSuffix(st.Self.DNSName, ".") + magicDNSSuffix := st.CurrentTailnet.MagicDNSSuffix // set parent serve config to always be persisted // at the top level, but a nested config might be @@ -245,7 +349,7 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { // foreground or background. parentSC := sc - turnOff := "off" == args[len(args)-1] + turnOff := len(args) > 0 && "off" == args[len(args)-1] if !turnOff && srvType == serveTypeHTTPS { // Running serve with https requires that the tailnet has enabled // https cert provisioning. Send users through an interactive flow @@ -262,10 +366,19 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } var watcher *tailscale.IPNBusWatcher - wantFg := !e.bg && !turnOff + svcName := noService + + if forService { + svcName = e.service + dnsName = e.service.String() + } + if !forService && srvType == serveTypeTUN { + return errors.New("tun mode is only supported for services") + } + wantFg := !e.bg.Value && !turnOff if wantFg { // validate the config before creating a WatchIPNBus session - if err := e.validateConfig(parentSC, srvPort, srvType); err != nil { + if err := e.validateConfig(parentSC, srvPort, srvType, svcName); err != nil { return err } @@ -291,12 +404,20 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { var msg string if turnOff { - err = e.unsetServe(sc, dnsName, srvType, srvPort, mount) + // only unset serve when trying to unset with type and port flags. + err = e.unsetServe(sc, dnsName, srvType, srvPort, mount, magicDNSSuffix) } else { - if err := e.validateConfig(parentSC, srvPort, srvType); err != nil { + if err := e.validateConfig(parentSC, srvPort, srvType, svcName); err != nil { return err } - err = e.setServe(sc, st, dnsName, srvType, srvPort, mount, args[0], funnel) + if forService { + e.addServiceToPrefs(ctx, svcName) + } + target := "" + if len(args) > 0 { + target = args[0] + } + err = e.setServe(sc, dnsName, srvType, srvPort, mount, target, funnel, magicDNSSuffix) msg = e.messageForPort(sc, st, dnsName, srvType, srvPort) } if err != nil { @@ -331,22 +452,146 @@ func (e *serveEnv) runServeCombined(subcmd serveMode) execFunc { } } -const backgroundExistsMsg = "background configuration already exists, use `tailscale %s --%s=%d off` to remove the existing configuration" +func (e *serveEnv) addServiceToPrefs(ctx context.Context, serviceName tailcfg.ServiceName) error { + prefs, err := e.lc.GetPrefs(ctx) + if err != nil { + return fmt.Errorf("error getting prefs: %w", err) + } + advertisedServices := prefs.AdvertiseServices + if slices.Contains(advertisedServices, serviceName.String()) { + return nil // already advertised + } + advertisedServices = append(advertisedServices, serviceName.String()) + _, err = e.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: advertisedServices, + }, + }) + return err +} -func (e *serveEnv) validateConfig(sc *ipn.ServeConfig, port uint16, wantServe serveType) error { - sc, isFg := sc.FindConfig(port) - if sc == nil { +func (e *serveEnv) removeServiceFromPrefs(ctx context.Context, serviceName tailcfg.ServiceName) error { + prefs, err := e.lc.GetPrefs(ctx) + if err != nil { + return fmt.Errorf("error getting prefs: %w", err) + } + if len(prefs.AdvertiseServices) == 0 { + return nil // nothing to remove + } + initialLen := len(prefs.AdvertiseServices) + prefs.AdvertiseServices = slices.DeleteFunc(prefs.AdvertiseServices, func(s string) bool { return s == serviceName.String() }) + if initialLen == len(prefs.AdvertiseServices) { + return nil // serviceName not advertised + } + _, err = e.lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: prefs.AdvertiseServices, + }, + }) + return err +} + +func (e *serveEnv) runServeDrain(ctx context.Context, args []string) error { + if len(args) == 0 { + return errHelp + } + if len(args) != 1 { + fmt.Fprintf(Stderr, "error: invalid number of arguments\n\n") + return errHelp + } + svc := args[0] + svcName := tailcfg.ServiceName(svc) + if err := svcName.Validate(); err != nil { + return fmt.Errorf("invalid service name: %w", err) + } + return e.removeServiceFromPrefs(ctx, svcName) +} + +func (e *serveEnv) runServeClear(ctx context.Context, args []string) error { + if len(args) == 0 { + return errHelp + } + if len(args) != 1 { + fmt.Fprintf(Stderr, "error: invalid number of arguments\n\n") + return errHelp + } + svc := tailcfg.ServiceName(args[0]) + if err := svc.Validate(); err != nil { + return fmt.Errorf("invalid service name: %w", err) + } + sc, err := e.lc.GetServeConfig(ctx) + if err != nil { + return fmt.Errorf("error getting serve config: %w", err) + } + if _, ok := sc.Services[svc]; !ok { + log.Printf("service %s not found in serve config, nothing to clear", svc) return nil } - if isFg { - return errors.New("foreground already exists under this port") + delete(sc.Services, svc) + if err := e.removeServiceFromPrefs(ctx, svc); err != nil { + return fmt.Errorf("error removing service %s from prefs: %w", svc, err) + } + return e.lc.SetServeConfig(ctx, sc) +} + +func (e *serveEnv) runServeAdvertise(ctx context.Context, args []string) error { + if len(args) == 0 { + return fmt.Errorf("error: missing service name argument") } - if !e.bg { - return fmt.Errorf(backgroundExistsMsg, infoMap[e.subcmd].Name, wantServe.String(), port) + if len(args) != 1 { + fmt.Fprintf(Stderr, "error: invalid number of arguments\n\n") + return errHelp + } + svc := tailcfg.ServiceName(args[0]) + if err := svc.Validate(); err != nil { + return fmt.Errorf("invalid service name: %w", err) + } + return e.addServiceToPrefs(ctx, svc) +} + +const backgroundExistsMsg = "background configuration already exists, use `tailscale %s --%s=%d off` to remove the existing configuration" + +// validateConfig checks if the serve config is valid to serve the type wanted on the port. +// dnsName is a FQDN or a serviceName (with `svc:` prefix). +func (e *serveEnv) validateConfig(sc *ipn.ServeConfig, port uint16, wantServe serveType, svcName tailcfg.ServiceName) error { + var tcpHandlerForPort *ipn.TCPPortHandler + if svcName != noService { + svc := sc.Services[svcName] + if svc == nil { + return nil + } + if wantServe == serveTypeTUN && (svc.TCP != nil || svc.Web != nil) { + return errors.New("service already has a TCP or Web handler, cannot serve in TUN mode") + } + if svc.Tun && wantServe != serveTypeTUN { + return errors.New("service is already being served in TUN mode") + } + if svc.TCP[port] == nil { + return nil + } + tcpHandlerForPort = svc.TCP[port] + } else { + sc, isFg := sc.FindConfig(port) + if sc == nil { + return nil + } + if isFg { + return errors.New("foreground already exists under this port") + } + if !e.bg.Value { + return fmt.Errorf(backgroundExistsMsg, infoMap[e.subcmd].Name, wantServe.String(), port) + } + tcpHandlerForPort = sc.TCP[port] } - existingServe := serveFromPortHandler(sc.TCP[port]) + existingServe := serveFromPortHandler(tcpHandlerForPort) if wantServe != existingServe { - return fmt.Errorf("want %q but port is already serving %q", wantServe, existingServe) + target := svcName + if target == noService { + target = "machine" + } + return fmt.Errorf("want to serve %q but port is already serving %q for %q", wantServe, existingServe, target) } return nil } @@ -366,12 +611,12 @@ func serveFromPortHandler(tcp *ipn.TCPPortHandler) serveType { } } -func (e *serveEnv) setServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName string, srvType serveType, srvPort uint16, mount string, target string, allowFunnel bool) error { +func (e *serveEnv) setServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string, target string, allowFunnel bool, mds string) error { // update serve config based on the type switch srvType { case serveTypeHTTPS, serveTypeHTTP: useTLS := srvType == serveTypeHTTPS - err := e.applyWebServe(sc, dnsName, srvPort, useTLS, mount, target) + err := e.applyWebServe(sc, dnsName, srvPort, useTLS, mount, target, mds) if err != nil { return fmt.Errorf("failed apply web serve: %w", err) } @@ -379,45 +624,60 @@ func (e *serveEnv) setServe(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName st if e.setPath != "" { return fmt.Errorf("cannot mount a path for TCP serve") } - err := e.applyTCPServe(sc, dnsName, srvType, srvPort, target) if err != nil { return fmt.Errorf("failed to apply TCP serve: %w", err) } + case serveTypeTUN: + // Caller checks that TUN mode is only supported for services. + svcName := tailcfg.ServiceName(dnsName) + if _, ok := sc.Services[svcName]; !ok { + mak.Set(&sc.Services, svcName, new(ipn.ServiceConfig)) + } + sc.Services[svcName].Tun = true default: return fmt.Errorf("invalid type %q", srvType) } // update the serve config based on if funnel is enabled - e.applyFunnel(sc, dnsName, srvPort, allowFunnel) - + // Since funnel is not supported for services, we only apply it for node's serve. + if svcName := tailcfg.AsServiceName(dnsName); svcName == noService { + e.applyFunnel(sc, dnsName, srvPort, allowFunnel) + } return nil } var ( - msgFunnelAvailable = "Available on the internet:" - msgServeAvailable = "Available within your tailnet:" - msgRunningInBackground = "%s started and running in the background." - msgDisableProxy = "To disable the proxy, run: tailscale %s --%s=%d off" - msgToExit = "Press Ctrl+C to exit." + msgFunnelAvailable = "Available on the internet:" + msgServeAvailable = "Available within your tailnet:" + msgServiceWaitingApproval = "This machine is configured as a service proxy for %s, but approval from an admin is required. Once approved, it will be available in your Tailnet as:" + msgRunningInBackground = "%s started and running in the background." + msgRunningTunService = "IPv4 and IPv6 traffic to %s is being routed to your operating system." + msgDisableProxy = "To disable the proxy, run: tailscale %s --%s=%d off" + msgDisableServiceProxy = "To disable the proxy, run: tailscale serve --service=%s --%s=%d off" + msgDisableServiceTun = "To disable the service in TUN mode, run: tailscale serve --service=%s --tun off" + msgDisableService = "To remove config for the service, run: tailscale serve clear --service=%s" + msgToExit = "Press Ctrl+C to exit." ) // messageForPort returns a message for the given port based on the // serve config and status. func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsName string, srvType serveType, srvPort uint16) string { var output strings.Builder - - hp := ipn.HostPort(net.JoinHostPort(dnsName, strconv.Itoa(int(srvPort)))) - - if sc.AllowFunnel[hp] == true { - output.WriteString(msgFunnelAvailable) - } else { - output.WriteString(msgServeAvailable) + svcName := tailcfg.AsServiceName(dnsName) + forService := svcName != noService + var webConfig *ipn.WebServerConfig + var tcpHandler *ipn.TCPPortHandler + ips := st.TailscaleIPs + magicDNSSuffix := st.CurrentTailnet.MagicDNSSuffix + host := dnsName + if forService { + host = strings.Join([]string{svcName.WithoutPrefix(), magicDNSSuffix}, ".") } - output.WriteString("\n\n") + hp := ipn.HostPort(net.JoinHostPort(host, strconv.Itoa(int(srvPort)))) scheme := "https" - if sc.IsServingHTTP(srvPort) { + if sc.IsServingHTTP(srvPort, svcName) { scheme = "http" } @@ -438,37 +698,68 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN } return "", "" } + if forService { + serviceIPMaps, err := tailcfg.UnmarshalNodeCapJSON[tailcfg.ServiceIPMappings](st.Self.CapMap, tailcfg.NodeAttrServiceHost) + if err != nil || len(serviceIPMaps) == 0 || serviceIPMaps[0][svcName] == nil { + // The capmap does not contain IPs for this service yet. Usually this means + // the service hasn't been added to prefs and sent to control yet. + output.WriteString(fmt.Sprintf(msgServiceWaitingApproval, svcName.String())) + ips = nil + } else { + output.WriteString(msgServeAvailable) + ips = serviceIPMaps[0][svcName] + } + output.WriteString("\n\n") + svc := sc.Services[svcName] + if srvType == serveTypeTUN && svc.Tun { + output.WriteString(fmt.Sprintf(msgRunningTunService, host)) + output.WriteString("\n") + output.WriteString(fmt.Sprintf(msgDisableServiceTun, dnsName)) + output.WriteString("\n") + output.WriteString(fmt.Sprintf(msgDisableService, dnsName)) + return output.String() + } + if svc != nil { + webConfig = svc.Web[hp] + tcpHandler = svc.TCP[srvPort] + } + } else { + if sc.AllowFunnel[hp] == true { + output.WriteString(msgFunnelAvailable) + } else { + output.WriteString(msgServeAvailable) + } + output.WriteString("\n\n") + webConfig = sc.Web[hp] + tcpHandler = sc.TCP[srvPort] + } - if sc.Web[hp] != nil { - mounts := slicesx.MapKeys(sc.Web[hp].Handlers) + if webConfig != nil { + mounts := slicesx.MapKeys(webConfig.Handlers) sort.Slice(mounts, func(i, j int) bool { return len(mounts[i]) < len(mounts[j]) }) - for _, m := range mounts { - h := sc.Web[hp].Handlers[m] - t, d := srvTypeAndDesc(h) - output.WriteString(fmt.Sprintf("%s://%s%s%s\n", scheme, dnsName, portPart, m)) + t, d := srvTypeAndDesc(webConfig.Handlers[m]) + output.WriteString(fmt.Sprintf("%s://%s%s%s\n", scheme, host, portPart, m)) output.WriteString(fmt.Sprintf("%s %-5s %s\n\n", "|--", t, d)) } - } else if sc.TCP[srvPort] != nil { - h := sc.TCP[srvPort] + } else if tcpHandler != nil { tlsStatus := "TLS over TCP" - if h.TerminateTLS != "" { + if tcpHandler.TerminateTLS != "" { tlsStatus = "TLS terminated" } - output.WriteString(fmt.Sprintf("%s://%s%s\n", scheme, dnsName, portPart)) - output.WriteString(fmt.Sprintf("|-- tcp://%s (%s)\n", hp, tlsStatus)) - for _, a := range st.TailscaleIPs { + output.WriteString(fmt.Sprintf("|-- tcp://%s:%d (%s)\n", host, srvPort, tlsStatus)) + for _, a := range ips { ipp := net.JoinHostPort(a.String(), strconv.Itoa(int(srvPort))) output.WriteString(fmt.Sprintf("|-- tcp://%s\n", ipp)) } - output.WriteString(fmt.Sprintf("|--> tcp://%s\n", h.TCPForward)) + output.WriteString(fmt.Sprintf("|--> tcp://%s\n\n", tcpHandler.TCPForward)) } - if !e.bg { + if !forService && !e.bg.Value { output.WriteString(msgToExit) return output.String() } @@ -478,14 +769,19 @@ func (e *serveEnv) messageForPort(sc *ipn.ServeConfig, st *ipnstate.Status, dnsN output.WriteString(fmt.Sprintf(msgRunningInBackground, subCmdUpper)) output.WriteString("\n") - output.WriteString(fmt.Sprintf(msgDisableProxy, subCmd, srvType.String(), srvPort)) + if forService { + output.WriteString(fmt.Sprintf(msgDisableServiceProxy, dnsName, srvType.String(), srvPort)) + output.WriteString("\n") + output.WriteString(fmt.Sprintf(msgDisableService, dnsName)) + } else { + output.WriteString(fmt.Sprintf(msgDisableProxy, subCmd, srvType.String(), srvPort)) + } return output.String() } -func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, useTLS bool, mount, target string) error { +func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, useTLS bool, mount, target string, mds string) error { h := new(ipn.HTTPHandler) - switch { case strings.HasPrefix(target, "text:"): text := strings.TrimPrefix(target, "text:") @@ -521,11 +817,12 @@ func (e *serveEnv) applyWebServe(sc *ipn.ServeConfig, dnsName string, srvPort ui } // TODO: validation needs to check nested foreground configs - if sc.IsTCPForwardingOnPort(srvPort) { + svcName := tailcfg.AsServiceName(dnsName) + if sc.IsTCPForwardingOnPort(srvPort, svcName) { return errors.New("cannot serve web; already serving TCP") } - sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS) + sc.SetWebHandler(h, dnsName, srvPort, mount, useTLS, mds) return nil } @@ -552,8 +849,9 @@ func (e *serveEnv) applyTCPServe(sc *ipn.ServeConfig, dnsName string, srcType se } // TODO: needs to account for multiple configs from foreground mode - if sc.IsServingWeb(srcPort) { - return fmt.Errorf("cannot serve TCP; already serving web on %d", srcPort) + svcName := tailcfg.AsServiceName(dnsName) + if sc.IsServingWeb(srcPort, svcName) { + return fmt.Errorf("cannot serve TCP; already serving web on %d for %s", srcPort, dnsName) } sc.SetTCPForwarding(srcPort, dstURL.Host, terminateTLS, dnsName) @@ -577,18 +875,25 @@ func (e *serveEnv) applyFunnel(sc *ipn.ServeConfig, dnsName string, srvPort uint } // unsetServe removes the serve config for the given serve port. -func (e *serveEnv) unsetServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string) error { +// dnsName is a FQDN or a serviceName (with `svc:` prefix). mds +// is the Magic DNS suffix, which is used to recreate serve's host. +func (e *serveEnv) unsetServe(sc *ipn.ServeConfig, dnsName string, srvType serveType, srvPort uint16, mount string, mds string) error { switch srvType { case serveTypeHTTPS, serveTypeHTTP: - err := e.removeWebServe(sc, dnsName, srvPort, mount) + err := e.removeWebServe(sc, dnsName, srvPort, mount, mds) if err != nil { return fmt.Errorf("failed to remove web serve: %w", err) } case serveTypeTCP, serveTypeTLSTerminatedTCP: - err := e.removeTCPServe(sc, srvPort) + err := e.removeTCPServe(sc, dnsName, srvPort) if err != nil { return fmt.Errorf("failed to remove TCP serve: %w", err) } + case serveTypeTUN: + err := e.removeTunServe(sc, dnsName) + if err != nil { + return fmt.Errorf("failed to remove TUN serve: %w", err) + } default: return fmt.Errorf("invalid type %q", srvType) } @@ -619,11 +924,16 @@ func srvTypeAndPortFromFlags(e *serveEnv) (srvType serveType, srvPort uint16, er } } + if e.tun { + srcTypeCount++ + srvType = serveTypeTUN + } + if srcTypeCount > 1 { return 0, 0, fmt.Errorf("cannot serve multiple types for a single mount point") - } else if srcTypeCount == 0 { - srvType = serveTypeHTTPS - srvPort = 443 + } + if srcTypeCount == 0 { + return serveTypeHTTPS, 443, nil } return srvType, srvPort, nil @@ -726,59 +1036,100 @@ func isLegacyInvocation(subcmd serveMode, args []string) (string, bool) { // removeWebServe removes a web handler from the serve config // and removes funnel if no remaining mounts exist for the serve port. // The srvPort argument is the serving port and the mount argument is -// the mount point or registered path to remove. -func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, mount string) error { - if sc.IsTCPForwardingOnPort(srvPort) { - return errors.New("cannot remove web handler; currently serving TCP") +// the mount point or registered path to remove. mds is the Magic DNS suffix, +// which is used to recreate serve's host. +func (e *serveEnv) removeWebServe(sc *ipn.ServeConfig, dnsName string, srvPort uint16, mount string, mds string) error { + if sc == nil { + return nil } portStr := strconv.Itoa(int(srvPort)) - hp := ipn.HostPort(net.JoinHostPort(dnsName, portStr)) + hostName := dnsName + webServeMap := sc.Web + svcName := tailcfg.AsServiceName(dnsName) + forService := svcName != noService + if forService { + svc := sc.Services[svcName] + if svc == nil { + return errors.New("service does not exist") + } + hostName = strings.Join([]string{svcName.WithoutPrefix(), mds}, ".") + webServeMap = svc.Web + } + hp := ipn.HostPort(net.JoinHostPort(hostName, portStr)) + + if sc.IsTCPForwardingOnPort(srvPort, svcName) { + return errors.New("cannot remove web handler; currently serving TCP") + } var targetExists bool var mounts []string // mount is deduced from e.setPath but it is ambiguous as // to whether the user explicitly passed "/" or it was defaulted to. if e.setPath == "" { - targetExists = sc.Web[hp] != nil && len(sc.Web[hp].Handlers) > 0 + targetExists = webServeMap[hp] != nil && len(webServeMap[hp].Handlers) > 0 if targetExists { - for mount := range sc.Web[hp].Handlers { + for mount := range webServeMap[hp].Handlers { mounts = append(mounts, mount) } } } else { - targetExists = sc.WebHandlerExists(hp, mount) + targetExists = sc.WebHandlerExists(svcName, hp, mount) mounts = []string{mount} } if !targetExists { - return errors.New("error: handler does not exist") + return errors.New("handler does not exist") } if len(mounts) > 1 { msg := fmt.Sprintf("Are you sure you want to delete %d handlers under port %s?", len(mounts), portStr) - if !e.yes && !promptYesNo(msg) { + if !e.yes && !prompt.YesNo(msg) { return nil } } - sc.RemoveWebHandler(dnsName, srvPort, mounts, true) + if forService { + sc.RemoveServiceWebHandler(svcName, hostName, srvPort, mounts) + } else { + sc.RemoveWebHandler(dnsName, srvPort, mounts, true) + } return nil } // removeTCPServe removes the TCP forwarding configuration for the -// given srvPort, or serving port. -func (e *serveEnv) removeTCPServe(sc *ipn.ServeConfig, src uint16) error { +// given srvPort, or serving port for the given dnsName. +func (e *serveEnv) removeTCPServe(sc *ipn.ServeConfig, dnsName string, src uint16) error { if sc == nil { return nil } - if sc.GetTCPPortHandler(src) == nil { - return errors.New("error: serve config does not exist") + svcName := tailcfg.AsServiceName(dnsName) + if sc.GetTCPPortHandler(src, svcName) == nil { + return errors.New("serve config does not exist") } - if sc.IsServingWeb(src) { + if sc.IsServingWeb(src, svcName) { return fmt.Errorf("unable to remove; serving web, not TCP forwarding on serve port %d", src) } - sc.RemoveTCPForwarding(src) + sc.RemoveTCPForwarding(svcName, src) + return nil +} + +func (e *serveEnv) removeTunServe(sc *ipn.ServeConfig, dnsName string) error { + if sc == nil { + return nil + } + svcName := tailcfg.ServiceName(dnsName) + svc, ok := sc.Services[svcName] + if !ok || svc == nil { + return errors.New("service does not exist") + } + if !svc.Tun { + return errors.New("service is not being served in TUN mode") + } + delete(sc.Services, svcName) + if len(sc.Services) == 0 { + sc.Services = nil // clean up empty map + } return nil } diff --git a/cmd/tailscale/cli/serve_v2_test.go b/cmd/tailscale/cli/serve_v2_test.go index 5768127ad0421..1deeaf3eaa9b5 100644 --- a/cmd/tailscale/cli/serve_v2_test.go +++ b/cmd/tailscale/cli/serve_v2_test.go @@ -8,9 +8,11 @@ import ( "context" "encoding/json" "fmt" + "net/netip" "os" "path/filepath" "reflect" + "slices" "strconv" "strings" "testing" @@ -19,6 +21,7 @@ import ( "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" ) func TestServeDevConfigMutations(t *testing.T) { @@ -874,9 +877,10 @@ func TestValidateConfig(t *testing.T) { name string desc string cfg *ipn.ServeConfig + svc tailcfg.ServiceName servePort uint16 serveType serveType - bg bool + bg bgBoolFlag wantErr bool }{ { @@ -894,7 +898,7 @@ func TestValidateConfig(t *testing.T) { 443: {HTTPS: true}, }, }, - bg: true, + bg: bgBoolFlag{true, false}, servePort: 10000, serveType: serveTypeHTTPS, }, @@ -906,7 +910,7 @@ func TestValidateConfig(t *testing.T) { 443: {TCPForward: "http://localhost:4545"}, }, }, - bg: true, + bg: bgBoolFlag{true, false}, servePort: 443, serveType: serveTypeTCP, }, @@ -918,7 +922,7 @@ func TestValidateConfig(t *testing.T) { 443: {HTTPS: true}, }, }, - bg: true, + bg: bgBoolFlag{true, false}, servePort: 443, serveType: serveTypeHTTP, wantErr: true, @@ -957,12 +961,90 @@ func TestValidateConfig(t *testing.T) { serveType: serveTypeTCP, wantErr: true, }, + { + name: "new_service_tcp", + desc: "no error when adding a new service port", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + }, + }, + }, + svc: "svc:foo", + servePort: 8080, + serveType: serveTypeTCP, + }, + { + name: "override_service_tcp", + desc: "no error when overwriting a previous service port", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {TCPForward: "http://localhost:4545"}, + }, + }, + }, + }, + svc: "svc:foo", + servePort: 443, + serveType: serveTypeTCP, + }, + { + name: "override_service_tcp", + desc: "error when overwriting a previous service port with a different serve type", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {HTTPS: true}, + }, + }, + }, + }, + svc: "svc:foo", + servePort: 443, + serveType: serveTypeHTTP, + wantErr: true, + }, + { + name: "override_service_tcp", + desc: "error when setting previous tcp service to tun mode", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 443: {TCPForward: "http://localhost:4545"}, + }, + }, + }, + }, + svc: "svc:foo", + serveType: serveTypeTUN, + wantErr: true, + }, + { + name: "override_service_tun", + desc: "error when setting previous tun service to tcp forwarder", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + Tun: true, + }, + }, + }, + svc: "svc:foo", + serveType: serveTypeTCP, + servePort: 443, + wantErr: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { se := serveEnv{bg: tc.bg} - err := se.validateConfig(tc.cfg, tc.servePort, tc.serveType) + err := se.validateConfig(tc.cfg, tc.servePort, tc.serveType, tc.svc) if err == nil && tc.wantErr { t.Fatal("expected an error but got nil") } @@ -1017,6 +1099,13 @@ func TestSrcTypeFromFlags(t *testing.T) { expectedPort: 443, expectedErr: false, }, + { + name: "defaults to https, port 443 for service", + env: &serveEnv{service: "svc:foo"}, + expectedType: serveTypeHTTPS, + expectedPort: 443, + expectedErr: false, + }, { name: "multiple types set", env: &serveEnv{http: 80, https: 443}, @@ -1075,12 +1164,118 @@ func TestCleanURLPath(t *testing.T) { } } +func TestAddServiceToPrefs(t *testing.T) { + tests := []struct { + name string + svcName tailcfg.ServiceName + startServices []string + expected []string + }{ + { + name: "add service to empty prefs", + svcName: "svc:foo", + expected: []string{"svc:foo"}, + }, + { + name: "add service to existing prefs", + svcName: "svc:bar", + startServices: []string{"svc:foo"}, + expected: []string{"svc:foo", "svc:bar"}, + }, + { + name: "add existing service to prefs", + svcName: "svc:foo", + startServices: []string{"svc:foo"}, + expected: []string{"svc:foo"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lc := &fakeLocalServeClient{} + ctx := t.Context() + lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: tt.startServices, + }, + }) + e := &serveEnv{lc: lc, bg: bgBoolFlag{true, false}} + err := e.addServiceToPrefs(ctx, tt.svcName) + if err != nil { + t.Fatalf("addServiceToPrefs(%q) returned unexpected error: %v", tt.svcName, err) + } + if !slices.Equal(lc.prefs.AdvertiseServices, tt.expected) { + t.Errorf("addServiceToPrefs(%q) = %v, want %v", tt.svcName, lc.prefs.AdvertiseServices, tt.expected) + } + }) + } + +} + +func TestRemoveServiceFromPrefs(t *testing.T) { + tests := []struct { + name string + svcName tailcfg.ServiceName + startServices []string + expected []string + }{ + { + name: "remove service from empty prefs", + svcName: "svc:foo", + expected: []string{}, + }, + { + name: "remove existing service from prefs", + svcName: "svc:foo", + startServices: []string{"svc:foo"}, + expected: []string{}, + }, + { + name: "remove service not in prefs", + svcName: "svc:bar", + startServices: []string{"svc:foo"}, + expected: []string{"svc:foo"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lc := &fakeLocalServeClient{} + ctx := t.Context() + lc.EditPrefs(ctx, &ipn.MaskedPrefs{ + AdvertiseServicesSet: true, + Prefs: ipn.Prefs{ + AdvertiseServices: tt.startServices, + }, + }) + e := &serveEnv{lc: lc, bg: bgBoolFlag{true, false}} + err := e.removeServiceFromPrefs(ctx, tt.svcName) + if err != nil { + t.Fatalf("removeServiceFromPrefs(%q) returned unexpected error: %v", tt.svcName, err) + } + if !slices.Equal(lc.prefs.AdvertiseServices, tt.expected) { + t.Errorf("removeServiceFromPrefs(%q) = %v, want %v", tt.svcName, lc.prefs.AdvertiseServices, tt.expected) + } + }) + } +} + func TestMessageForPort(t *testing.T) { + svcIPMap := tailcfg.ServiceIPMappings{ + "svc:foo": []netip.Addr{ + netip.MustParseAddr("100.101.101.101"), + netip.MustParseAddr("fd7a:115c:a1e0:ab12:4843:cd96:6565:6565"), + }, + } + svcIPMapJSON, _ := json.Marshal(svcIPMap) + svcIPMapJSONRawMSG := tailcfg.RawMessage(svcIPMapJSON) + tests := []struct { name string subcmd serveMode serveConfig *ipn.ServeConfig status *ipnstate.Status + prefs *ipn.Prefs dnsName string srvType serveType srvPort uint16 @@ -1104,7 +1299,7 @@ func TestMessageForPort(t *testing.T) { "foo.test.ts.net:443": true, }, }, - status: &ipnstate.Status{}, + status: &ipnstate.Status{CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}}, dnsName: "foo.test.ts.net", srvType: serveTypeHTTPS, srvPort: 443, @@ -1133,7 +1328,7 @@ func TestMessageForPort(t *testing.T) { }, }, }, - status: &ipnstate.Status{}, + status: &ipnstate.Status{CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}}, dnsName: "foo.test.ts.net", srvType: serveTypeHTTP, srvPort: 80, @@ -1147,10 +1342,206 @@ func TestMessageForPort(t *testing.T) { fmt.Sprintf(msgDisableProxy, "serve", "http", 80), }, "\n"), }, + { + name: "serve service http", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{ + AdvertiseServices: []string{"svc:foo"}, + }, + dnsName: "svc:foo", + srvType: serveTypeHTTP, + srvPort: 80, + expected: strings.Join([]string{ + msgServeAvailable, + "", + "http://foo.test.ts.net/", + "|-- proxy http://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:foo", "http", 80), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, + { + name: "serve service no capmap", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{ + AdvertiseServices: []string{"svc:bar"}, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + expected: strings.Join([]string{ + fmt.Sprintf(msgServiceWaitingApproval, "svc:bar"), + "", + "http://bar.test.ts.net/", + "|-- proxy http://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:bar", "http", 80), + fmt.Sprintf(msgDisableService, "svc:bar"), + }, "\n"), + }, + { + name: "serve service https non-default port", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 2200: {HTTPS: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:2200": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{AdvertiseServices: []string{"svc:foo"}}, + dnsName: "svc:foo", + srvType: serveTypeHTTPS, + srvPort: 2200, + expected: strings.Join([]string{ + msgServeAvailable, + "", + "https://foo.test.ts.net:2200/", + "|-- proxy http://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:foo", "https", 2200), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, + { + name: "serve service TCPForward", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 2200: {TCPForward: "localhost:3000"}, + }, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{AdvertiseServices: []string{"svc:foo"}}, + dnsName: "svc:foo", + srvType: serveTypeTCP, + srvPort: 2200, + expected: strings.Join([]string{ + msgServeAvailable, + "", + "|-- tcp://foo.test.ts.net:2200 (TLS over TCP)", + "|-- tcp://100.101.101.101:2200", + "|-- tcp://[fd7a:115c:a1e0:ab12:4843:cd96:6565:6565]:2200", + "|--> tcp://localhost:3000", + "", + fmt.Sprintf(msgRunningInBackground, "Serve"), + fmt.Sprintf(msgDisableServiceProxy, "svc:foo", "tcp", 2200), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, + { + name: "serve service Tun", + subcmd: serve, + serveConfig: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:foo": { + Tun: true, + }, + }, + }, + status: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + Self: &ipnstate.PeerStatus{ + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrServiceHost: []tailcfg.RawMessage{svcIPMapJSONRawMSG}, + }, + }, + }, + prefs: &ipn.Prefs{AdvertiseServices: []string{"svc:foo"}}, + dnsName: "svc:foo", + srvType: serveTypeTUN, + expected: strings.Join([]string{ + msgServeAvailable, + "", + fmt.Sprintf(msgRunningTunService, "foo.test.ts.net"), + fmt.Sprintf(msgDisableServiceTun, "svc:foo"), + fmt.Sprintf(msgDisableService, "svc:foo"), + }, "\n"), + }, } for _, tt := range tests { - e := &serveEnv{bg: true, subcmd: tt.subcmd} + e := &serveEnv{bg: bgBoolFlag{true, false}, subcmd: tt.subcmd} t.Run(tt.name, func(t *testing.T) { actual := e.messageForPort(tt.serveConfig, tt.status, tt.dnsName, tt.srvType, tt.srvPort) @@ -1277,6 +1668,577 @@ func TestIsLegacyInvocation(t *testing.T) { } } +func TestSetServe(t *testing.T) { + e := &serveEnv{} + magicDNSSuffix := "test.ts.net" + tests := []struct { + name string + desc string + cfg *ipn.ServeConfig + st *ipnstate.Status + dnsName string + srvType serveType + srvPort uint16 + mountPath string + target string + allowFunnel bool + expected *ipn.ServeConfig + expectErr bool + }{ + { + name: "add new handler", + desc: "add a new http handler to empty config", + cfg: &ipn.ServeConfig{}, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3000", + expected: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + { + name: "update http handler", + desc: "update an existing http handler on the same port to same type", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + { + name: "update TCP handler", + desc: "update an existing TCP handler on the same port to a http handler", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "http://localhost:3000"}}, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expectErr: true, + }, + { + name: "add new service handler", + desc: "add a new service TCP handler to empty config", + cfg: &ipn.ServeConfig{}, + + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + target: "3000", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3000"}}, + }, + }, + }, + }, + { + name: "update service handler", + desc: "update an existing service TCP handler on the same port to same type", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3000"}}, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + target: "3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3001"}}, + }, + }, + }, + }, + { + name: "update service handler", + desc: "update an existing service TCP handler on the same port to a http handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {TCPForward: "127.0.0.1:3000"}}, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expectErr: true, + }, + { + name: "add new service handler", + desc: "add a new service HTTP handler to empty config", + cfg: &ipn.ServeConfig{}, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3000", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "update existing service handler", + desc: "update an existing service HTTP handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "add new service handler", + desc: "add a new service HTTP handler to existing service config", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 88, + mountPath: "/", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + 88: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + "bar.test.ts.net:88": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "add new service mount", + desc: "add a new service mount to existing service config", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{80: {HTTP: true}}, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mountPath: "/added", + target: "http://localhost:3001", + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + "/added": {Proxy: "http://localhost:3001"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "add new service handler", + desc: "add a new service handler in tun mode to empty config", + cfg: &ipn.ServeConfig{}, + dnsName: "svc:bar", + srvType: serveTypeTUN, + expected: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + Tun: true, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := e.setServe(tt.cfg, tt.dnsName, tt.srvType, tt.srvPort, tt.mountPath, tt.target, tt.allowFunnel, magicDNSSuffix) + if err != nil && !tt.expectErr { + t.Fatalf("got error: %v; did not expect error.", err) + } + if err == nil && tt.expectErr { + t.Fatalf("got no error; expected error.") + } + if !tt.expectErr && !reflect.DeepEqual(tt.cfg, tt.expected) { + svcName := tailcfg.ServiceName(tt.dnsName) + t.Fatalf("got: %v; expected: %v", tt.cfg.Services[svcName], tt.expected.Services[svcName]) + } + }) + } +} + +func TestUnsetServe(t *testing.T) { + tests := []struct { + name string + desc string + cfg *ipn.ServeConfig + st *ipnstate.Status + dnsName string + srvType serveType + srvPort uint16 + mount string + setServeEnv bool + serveEnv *serveEnv // if set, use this instead of the default serveEnv + expected *ipn.ServeConfig + expectErr bool + }{ + { + name: "unset http handler", + desc: "remove an existing http handler", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/", + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset service handler", + desc: "remove an existing service TCP handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/", + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset service handler tun", + desc: "remove an existing service handler in tun mode", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + Tun: true, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeTUN, + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset service handler tcp", + desc: "remove an existing service TCP handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {TCPForward: "11.11.11.11:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + expected: &ipn.ServeConfig{}, + expectErr: false, + }, + { + name: "unset http handler not found", + desc: "try to remove a non-existing http handler", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "bar.test.ts.net", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/abc", + expected: &ipn.ServeConfig{}, + expectErr: true, + }, + { + name: "unset service handler not found", + desc: "try to remove a non-existing service TCP handler", + + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeHTTP, + srvPort: 80, + mount: "/abc", + setServeEnv: true, + serveEnv: &serveEnv{setPath: "/abc"}, + expected: &ipn.ServeConfig{}, + expectErr: true, + }, + { + name: "unset service doesn't exist", + desc: "try to remove a non-existing service's handler", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {TCPForward: "11.11.11.11:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:foo", + srvType: serveTypeTCP, + srvPort: 80, + expectErr: true, + }, + { + name: "unset tcp while port is in use", + desc: "try to remove a TCP handler while the port is used for web", + cfg: &ipn.ServeConfig{ + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "foo:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "foo.test.ts.net", + srvType: serveTypeTCP, + srvPort: 80, + mount: "/", + expectErr: true, + }, + { + name: "unset service tcp while port is in use", + desc: "try to remove a service TCP handler while the port is used for web", + cfg: &ipn.ServeConfig{ + Services: map[tailcfg.ServiceName]*ipn.ServiceConfig{ + "svc:bar": { + TCP: map[uint16]*ipn.TCPPortHandler{ + 80: {HTTP: true}, + }, + Web: map[ipn.HostPort]*ipn.WebServerConfig{ + "bar.test.ts.net:80": { + Handlers: map[string]*ipn.HTTPHandler{ + "/": {Proxy: "http://localhost:3000"}, + }, + }, + }, + }, + }, + }, + st: &ipnstate.Status{ + CurrentTailnet: &ipnstate.TailnetStatus{MagicDNSSuffix: "test.ts.net"}, + }, + dnsName: "svc:bar", + srvType: serveTypeTCP, + srvPort: 80, + mount: "/", + expectErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &serveEnv{} + if tt.setServeEnv { + e = tt.serveEnv + } + err := e.unsetServe(tt.cfg, tt.dnsName, tt.srvType, tt.srvPort, tt.mount, tt.st.CurrentTailnet.MagicDNSSuffix) + if err != nil && !tt.expectErr { + t.Fatalf("got error: %v; did not expect error.", err) + } + if err == nil && tt.expectErr { + t.Fatalf("got no error; expected error.") + } + if !tt.expectErr && !reflect.DeepEqual(tt.cfg, tt.expected) { + t.Fatalf("got: %v; expected: %v", tt.cfg, tt.expected) + } + }) + } +} + // exactErrMsg returns an error checker that wants exactly the provided want error. // If optName is non-empty, it's used in the error message. func exactErrMsg(want error) func(error) string { diff --git a/cmd/tailscale/cli/set.go b/cmd/tailscale/cli/set.go index aa59666987532..f1b21995ec388 100644 --- a/cmd/tailscale/cli/set.go +++ b/cmd/tailscale/cli/set.go @@ -73,7 +73,7 @@ func newSetFlagSet(goos string, setArgs *setArgsT) *flag.FlagSet { setf.StringVar(&setArgs.profileName, "nickname", "", "nickname for the current account") setf.BoolVar(&setArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes") setf.BoolVar(&setArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel") - setf.StringVar(&setArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP or base name) for internet traffic, or empty string to not use an exit node") + setf.StringVar(&setArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP, base name, or auto:any) for internet traffic, or empty string to not use an exit node") setf.BoolVar(&setArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node") setf.BoolVar(&setArgs.shieldsUp, "shields-up", false, "don't allow incoming connections") setf.BoolVar(&setArgs.runSSH, "ssh", false, "run an SSH server, permitting access per tailnet admin's declared policy") @@ -173,7 +173,10 @@ func runSet(ctx context.Context, args []string) (retErr error) { } if setArgs.exitNodeIP != "" { - if err := maskedPrefs.Prefs.SetExitNodeIP(setArgs.exitNodeIP, st); err != nil { + if expr, useAutoExitNode := ipn.ParseAutoExitNodeString(setArgs.exitNodeIP); useAutoExitNode { + maskedPrefs.AutoExitNode = expr + maskedPrefs.AutoExitNodeSet = true + } else if err := maskedPrefs.Prefs.SetExitNodeIP(setArgs.exitNodeIP, st); err != nil { var e ipn.ExitNodeLocalIPError if errors.As(err, &e) { return fmt.Errorf("%w; did you mean --advertise-exit-node?", err) @@ -183,6 +186,9 @@ func runSet(ctx context.Context, args []string) (retErr error) { } warnOnAdvertiseRouts(ctx, &maskedPrefs.Prefs) + if err := checkExitNodeRisk(ctx, &maskedPrefs.Prefs, setArgs.acceptedRisks); err != nil { + return err + } var advertiseExitNodeSet, advertiseRoutesSet bool setFlagSet.Visit(func(f *flag.Flag) { updateMaskedPrefsFromUpOrSetFlag(maskedPrefs, f.Name) diff --git a/cmd/tailscale/cli/status.go b/cmd/tailscale/cli/status.go index e4dccc247fd54..726606109aa15 100644 --- a/cmd/tailscale/cli/status.go +++ b/cmd/tailscale/cli/status.go @@ -70,6 +70,8 @@ var statusArgs struct { peers bool // in CLI mode, show status of peer machines } +const mullvadTCD = "mullvad.ts.net." + func runStatus(ctx context.Context, args []string) error { if len(args) > 0 { return errors.New("unexpected non-flag arguments to 'tailscale status'") @@ -183,10 +185,12 @@ func runStatus(ctx context.Context, args []string) error { } else if ps.ExitNodeOption { f("offers exit node; ") } - if relay != "" && ps.CurAddr == "" { + if relay != "" && ps.CurAddr == "" && ps.PeerRelay == "" { f("relay %q", relay) } else if ps.CurAddr != "" { f("direct %s", ps.CurAddr) + } else if ps.PeerRelay != "" { + f("peer-relay %s", ps.PeerRelay) } if !ps.Online { f("; offline") @@ -210,9 +214,8 @@ func runStatus(ctx context.Context, args []string) error { if ps.ShareeNode { continue } - if ps.Location != nil && ps.ExitNodeOption && !ps.ExitNode { - // Location based exit nodes are only shown with the - // `exit-node list` command. + if ps.ExitNodeOption && !ps.ExitNode && strings.HasSuffix(ps.DNSName, mullvadTCD) { + // Mullvad exit nodes are only shown with the `exit-node list` command. locBasedExitNode = true continue } @@ -260,7 +263,7 @@ func printFunnelStatus(ctx context.Context) { } sni, portStr, _ := net.SplitHostPort(string(hp)) p, _ := strconv.ParseUint(portStr, 10, 16) - isTCP := sc.IsTCPForwardingOnPort(uint16(p)) + isTCP := sc.IsTCPForwardingOnPort(uint16(p), noService) url := "https://" if isTCP { url = "tcp://" diff --git a/cmd/tailscale/cli/up.go b/cmd/tailscale/cli/up.go index e4bb6f576759e..1863957d3f143 100644 --- a/cmd/tailscale/cli/up.go +++ b/cmd/tailscale/cli/up.go @@ -100,7 +100,7 @@ func newUpFlagSet(goos string, upArgs *upArgsT, cmd string) *flag.FlagSet { upf.BoolVar(&upArgs.acceptRoutes, "accept-routes", acceptRouteDefault(goos), "accept routes advertised by other Tailscale nodes") upf.BoolVar(&upArgs.acceptDNS, "accept-dns", true, "accept DNS configuration from the admin panel") upf.Var(notFalseVar{}, "host-routes", hidden+"install host routes to other Tailscale nodes (must be true as of Tailscale 1.67+)") - upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP or base name) for internet traffic, or empty string to not use an exit node") + upf.StringVar(&upArgs.exitNodeIP, "exit-node", "", "Tailscale exit node (IP, base name, or auto:any) for internet traffic, or empty string to not use an exit node") upf.BoolVar(&upArgs.exitNodeAllowLANAccess, "exit-node-allow-lan-access", false, "Allow direct access to the local network when routing traffic via an exit node") upf.BoolVar(&upArgs.shieldsUp, "shields-up", false, "don't allow incoming connections") upf.BoolVar(&upArgs.runSSH, "ssh", false, "run an SSH server, permitting access per tailnet admin's declared policy") @@ -278,7 +278,9 @@ func prefsFromUpArgs(upArgs upArgsT, warnf logger.Logf, st *ipnstate.Status, goo prefs.NetfilterMode = preftype.NetfilterOff } if upArgs.exitNodeIP != "" { - if err := prefs.SetExitNodeIP(upArgs.exitNodeIP, st); err != nil { + if expr, useAutoExitNode := ipn.ParseAutoExitNodeString(upArgs.exitNodeIP); useAutoExitNode { + prefs.AutoExitNode = expr + } else if err := prefs.SetExitNodeIP(upArgs.exitNodeIP, st); err != nil { var e ipn.ExitNodeLocalIPError if errors.As(err, &e) { return nil, fmt.Errorf("%w; did you mean --advertise-exit-node?", err) @@ -408,6 +410,9 @@ func updatePrefs(prefs, curPrefs *ipn.Prefs, env upCheckEnv) (simpleUp bool, jus if env.upArgs.reset { visitFlags = env.flagSet.VisitAll } + if prefs.AutoExitNode.IsSet() { + justEditMP.AutoExitNodeSet = true + } visitFlags(func(f *flag.Flag) { updateMaskedPrefsFromUpOrSetFlag(justEditMP, f.Name) }) @@ -481,6 +486,9 @@ func runUp(ctx context.Context, cmd string, args []string, upArgs upArgsT) (retE } warnOnAdvertiseRouts(ctx, prefs) + if err := checkExitNodeRisk(ctx, prefs, upArgs.acceptedRisks); err != nil { + return err + } curPrefs, err := localClient.GetPrefs(ctx) if err != nil { diff --git a/cmd/tailscale/cli/update.go b/cmd/tailscale/cli/update.go index 69d1aa97b43f7..7c0269f6a7687 100644 --- a/cmd/tailscale/cli/update.go +++ b/cmd/tailscale/cli/update.go @@ -9,10 +9,10 @@ import ( "flag" "fmt" "runtime" - "strings" "github.com/peterbourgon/ff/v3/ffcli" "tailscale.com/clientupdate" + "tailscale.com/util/prompt" "tailscale.com/version" "tailscale.com/version/distro" ) @@ -87,19 +87,5 @@ func confirmUpdate(ver string) bool { } msg := fmt.Sprintf("This will update Tailscale from %v to %v. Continue?", version.Short(), ver) - return promptYesNo(msg) -} - -// PromptYesNo takes a question and prompts the user to answer the -// question with a yes or no. It appends a [y/n] to the message. -func promptYesNo(msg string) bool { - fmt.Print(msg + " [y/n] ") - var resp string - fmt.Scanln(&resp) - resp = strings.ToLower(resp) - switch resp { - case "y", "yes", "sure": - return true - } - return false + return prompt.YesNo(msg) } diff --git a/cmd/tailscale/cli/web.go b/cmd/tailscale/cli/web.go index e209d388eb123..5e1821dd011eb 100644 --- a/cmd/tailscale/cli/web.go +++ b/cmd/tailscale/cli/web.go @@ -43,6 +43,7 @@ Tailscale, as opposed to a CLI or a native app. webf.BoolVar(&webArgs.cgi, "cgi", false, "run as CGI script") webf.StringVar(&webArgs.prefix, "prefix", "", "URL prefix added to requests (for cgi or reverse proxies)") webf.BoolVar(&webArgs.readonly, "readonly", false, "run web UI in read-only mode") + webf.StringVar(&webArgs.origin, "origin", "", "origin at which the web UI is served (if behind a reverse proxy or used with cgi)") return webf })(), Exec: runWeb, @@ -53,6 +54,7 @@ var webArgs struct { cgi bool prefix string readonly bool + origin string } func tlsConfigFromEnvironment() *tls.Config { @@ -115,6 +117,9 @@ func runWeb(ctx context.Context, args []string) error { if webArgs.readonly { opts.Mode = web.ReadOnlyServerMode } + if webArgs.origin != "" { + opts.OriginOverride = webArgs.origin + } webServer, err := web.NewServer(opts) if err != nil { log.Printf("tailscale.web: %v", err) diff --git a/cmd/tailscale/depaware.txt b/cmd/tailscale/depaware.txt index 03bf2f94ca4df..e44e20e8c92b2 100644 --- a/cmd/tailscale/depaware.txt +++ b/cmd/tailscale/depaware.txt @@ -27,8 +27,6 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ DW github.com/google/uuid from tailscale.com/clientupdate+ - github.com/gorilla/csrf from tailscale.com/client/web - github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink @@ -87,7 +85,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/control/controlhttp from tailscale.com/cmd/tailscale/cli tailscale.com/control/controlhttp/controlhttpcommon from tailscale.com/control/controlhttp tailscale.com/control/controlknobs from tailscale.com/net/portmapper - tailscale.com/derp from tailscale.com/derp/derphttp + tailscale.com/derp from tailscale.com/derp/derphttp+ tailscale.com/derp/derpconst from tailscale.com/derp+ tailscale.com/derp/derphttp from tailscale.com/net/netcheck tailscale.com/disco from tailscale.com/derp @@ -174,6 +172,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep tailscale.com/util/multierr from tailscale.com/control/controlhttp+ tailscale.com/util/must from tailscale.com/clientupdate/distsign+ tailscale.com/util/nocasemaps from tailscale.com/types/ipproto + tailscale.com/util/prompt from tailscale.com/cmd/tailscale/cli tailscale.com/util/quarantine from tailscale.com/cmd/tailscale/cli tailscale.com/util/rands from tailscale.com/tsweb tailscale.com/util/set from tailscale.com/derp+ @@ -319,12 +318,11 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/peterbourgon/ff/v3+ - encoding from encoding/gob+ + encoding from encoding/json+ encoding/asn1 from crypto/x509+ encoding/base32 from github.com/fxamacker/cbor/v2+ encoding/base64 from encoding/json+ encoding/binary from compress/gzip+ - encoding/gob from github.com/gorilla/securecookie encoding/hex from crypto/x509+ encoding/json from expvar+ encoding/pem from crypto/tls+ @@ -338,7 +336,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep hash/crc32 from compress/gzip+ hash/maphash from go4.org/mem html from html/template+ - html/template from github.com/gorilla/csrf+ + html/template from tailscale.com/util/eventbus image from github.com/skip2/go-qrcode+ image/color from github.com/skip2/go-qrcode+ image/png from github.com/skip2/go-qrcode @@ -372,7 +370,7 @@ tailscale.com/cmd/tailscale dependencies: (generated by github.com/tailscale/dep internal/runtime/math from internal/runtime/maps+ internal/runtime/sys from crypto/subtle+ L internal/runtime/syscall from runtime+ - internal/saferio from debug/pe+ + W internal/saferio from debug/pe internal/singleflight from net internal/stringslite from embed+ internal/sync from sync+ diff --git a/cmd/tailscaled/depaware.txt b/cmd/tailscaled/depaware.txt index 6de0ddc391794..7c4885a4be4c4 100644 --- a/cmd/tailscaled/depaware.txt +++ b/cmd/tailscaled/depaware.txt @@ -123,8 +123,6 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ DW github.com/google/uuid from tailscale.com/clientupdate+ - github.com/gorilla/csrf from tailscale.com/client/web - github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L 💣 github.com/illarion/gonotify/v3 from tailscale.com/net/dns L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 @@ -283,7 +281,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de tailscale.com/feature/tpm from tailscale.com/feature/condregister tailscale.com/feature/wakeonlan from tailscale.com/feature/condregister tailscale.com/health from tailscale.com/control/controlclient+ - tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ tailscale.com/hostinfo from tailscale.com/client/web+ tailscale.com/internal/noiseconn from tailscale.com/control/controlclient tailscale.com/ipn from tailscale.com/client/local+ @@ -476,7 +474,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ golang.org/x/crypto/nacl/box from tailscale.com/types/key - golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box + golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box+ golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ LD golang.org/x/crypto/ssh from github.com/pkg/sftp+ @@ -553,7 +551,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/internal/fips140/edwards25519/field from crypto/ecdh+ crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls + crypto/internal/fips140/mlkem from crypto/tls+ crypto/internal/fips140/nistec from crypto/elliptic+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa @@ -573,6 +571,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de crypto/internal/randutil from crypto/dsa+ crypto/internal/sysrand from crypto/internal/entropy+ crypto/md5 from crypto/tls+ + LD crypto/mlkem from golang.org/x/crypto/ssh crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls+ crypto/rsa from crypto/tls+ @@ -590,12 +589,11 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/tailscale/web-client-prebuilt+ - encoding from encoding/gob+ + encoding from encoding/json+ encoding/asn1 from crypto/x509+ encoding/base32 from github.com/fxamacker/cbor/v2+ encoding/base64 from encoding/json+ encoding/binary from compress/gzip+ - encoding/gob from github.com/gorilla/securecookie encoding/hex from crypto/x509+ encoding/json from expvar+ encoding/pem from crypto/tls+ @@ -609,7 +607,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de hash/crc32 from compress/gzip+ hash/maphash from go4.org/mem html from html/template+ - html/template from github.com/gorilla/csrf+ + html/template from tailscale.com/util/eventbus internal/abi from crypto/x509/internal/macos+ internal/asan from internal/runtime/maps+ internal/bisect from internal/godebug @@ -640,7 +638,7 @@ tailscale.com/cmd/tailscaled dependencies: (generated by github.com/tailscale/de internal/runtime/math from internal/runtime/maps+ internal/runtime/sys from crypto/subtle+ L internal/runtime/syscall from runtime+ - internal/saferio from debug/pe+ + W internal/saferio from debug/pe internal/singleflight from net internal/stringslite from embed+ internal/sync from sync+ diff --git a/cmd/tailscaled/tailscaled.go b/cmd/tailscaled/tailscaled.go index 87750bc5d5241..ab1590132ece6 100644 --- a/cmd/tailscaled/tailscaled.go +++ b/cmd/tailscaled/tailscaled.go @@ -64,6 +64,7 @@ import ( "tailscale.com/util/clientmetric" "tailscale.com/util/multierr" "tailscale.com/util/osshare" + "tailscale.com/util/syspolicy" "tailscale.com/version" "tailscale.com/version/distro" "tailscale.com/wgengine" @@ -126,6 +127,7 @@ var args struct { debug string port uint16 statepath string + encryptState bool statedir string socketpath string birdSocketPath string @@ -193,6 +195,7 @@ func main() { flag.StringVar(&args.tunname, "tun", defaultTunName(), `tunnel interface name; use "userspace-networking" (beta) to not use TUN`) flag.Var(flagtype.PortValue(&args.port, defaultPort()), "port", "UDP port to listen on for WireGuard and peer-to-peer traffic; 0 means automatically select") flag.StringVar(&args.statepath, "state", "", "absolute path of state file; use 'kube:' to use Kubernetes secrets or 'arn:aws:ssm:...' to store in AWS SSM; use 'mem:' to not store state and register as an ephemeral node. If empty and --statedir is provided, the default is /tailscaled.state. Default: "+paths.DefaultTailscaledStateFile()) + flag.BoolVar(&args.encryptState, "encrypt-state", defaultEncryptState(), "encrypt the state file on disk; uses TPM on Linux and Windows, on all other platforms this flag is not supported") flag.StringVar(&args.statedir, "statedir", "", "path to directory for storage of config state, TLS certs, temporary incoming Taildrop files, etc. If empty, it's derived from --state when possible.") flag.StringVar(&args.socketpath, "socket", paths.DefaultTailscaledSocket(), "path of the service unix socket") flag.StringVar(&args.birdSocketPath, "bird-socket", "", "path of the bird unix socket") @@ -254,20 +257,41 @@ func main() { // Only apply a default statepath when neither have been provided, so that a // user may specify only --statedir if they wish. if args.statepath == "" && args.statedir == "" { - if runtime.GOOS == "plan9" { - home, err := os.UserHomeDir() - if err != nil { - log.Fatalf("failed to get home directory: %v", err) - } - args.statedir = filepath.Join(home, "tailscale-state") - if err := os.MkdirAll(args.statedir, 0700); err != nil { - log.Fatalf("failed to create state directory: %v", err) + if paths.MakeAutomaticStateDir() { + d := paths.DefaultTailscaledStateDir() + if d != "" { + args.statedir = d + if err := os.MkdirAll(d, 0700); err != nil { + log.Fatalf("failed to create state directory: %v", err) + } } } else { args.statepath = paths.DefaultTailscaledStateFile() } } + if args.encryptState { + if runtime.GOOS != "linux" && runtime.GOOS != "windows" { + log.SetFlags(0) + log.Fatalf("--encrypt-state is not supported on %s", runtime.GOOS) + } + // Check if we have TPM support in this build. + if !store.HasKnownProviderPrefix(store.TPMPrefix + "/") { + log.SetFlags(0) + log.Fatal("--encrypt-state is not supported in this build of tailscaled") + } + // Check if we have TPM access. + if !hostinfo.New().TPM.Present() { + log.SetFlags(0) + log.Fatal("--encrypt-state is not supported on this device or a TPM is not accessible") + } + // Check for conflicting prefix in --state, like arn: or kube:. + if args.statepath != "" && store.HasKnownProviderPrefix(args.statepath) { + log.SetFlags(0) + log.Fatal("--encrypt-state can only be used with --state set to a local file path") + } + } + if args.disableLogs { envknob.SetNoLogsNoSupport() } @@ -315,13 +339,17 @@ func trySynologyMigration(p string) error { } func statePathOrDefault() string { + var path string if args.statepath != "" { - return args.statepath + path = args.statepath } - if args.statedir != "" { - return filepath.Join(args.statedir, "tailscaled.state") + if path == "" && args.statedir != "" { + path = filepath.Join(args.statedir, "tailscaled.state") } - return "" + if path != "" && !store.HasKnownProviderPrefix(path) && args.encryptState { + path = store.TPMPrefix + path + } + return path } // serverOptions is the configuration of the Tailscale node agent. @@ -719,6 +747,7 @@ func tryEngine(logf logger.Logf, sys *tsd.System, name string) (onlyNetstack boo Dialer: sys.Dialer.Get(), SetSubsystem: sys.Set, ControlKnobs: sys.ControlKnobs(), + EventBus: sys.Bus.Get(), DriveForLocal: driveimpl.NewFileSystemForLocal(logf), } @@ -973,3 +1002,15 @@ func applyIntegrationTestEnvKnob() { } } } + +func defaultEncryptState() bool { + if runtime.GOOS != "windows" && runtime.GOOS != "linux" { + // TPM encryption is only configurable on Windows and Linux. Other + // platforms either use system APIs and are not configurable + // (Android/Apple), or don't support any form of encryption yet + // (plan9/FreeBSD/etc). + return false + } + v, _ := syspolicy.GetBoolean(syspolicy.EncryptState, false) + return v +} diff --git a/cmd/testwrapper/testwrapper.go b/cmd/testwrapper/testwrapper.go index 53c1b1d05f7ca..173edee733f04 100644 --- a/cmd/testwrapper/testwrapper.go +++ b/cmd/testwrapper/testwrapper.go @@ -213,6 +213,16 @@ func main() { return } + // As a special case, if the packages looks like "sharded:1/2" then shell out to + // ./tool/listpkgs to cut up the package list pieces for each sharded builder. + if nOfM, ok := strings.CutPrefix(packages[0], "sharded:"); ok && len(packages) == 1 { + out, err := exec.Command("go", "run", "tailscale.com/tool/listpkgs", "-shard", nOfM, "./...").Output() + if err != nil { + log.Fatalf("failed to list packages for sharded test: %v", err) + } + packages = strings.Split(strings.TrimSpace(string(out)), "\n") + } + ctx := context.Background() type nextRun struct { tests []*packageTests diff --git a/cmd/tsconnect/wasm/wasm_js.go b/cmd/tsconnect/wasm/wasm_js.go index 779a87e49dec9..ebf7284aa0d43 100644 --- a/cmd/tsconnect/wasm/wasm_js.go +++ b/cmd/tsconnect/wasm/wasm_js.go @@ -27,6 +27,7 @@ import ( "golang.org/x/crypto/ssh" "tailscale.com/control/controlclient" "tailscale.com/ipn" + "tailscale.com/ipn/ipnauth" "tailscale.com/ipn/ipnlocal" "tailscale.com/ipn/ipnserver" "tailscale.com/ipn/store/mem" @@ -336,7 +337,7 @@ func (i *jsIPN) logout() { go func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - i.lb.Logout(ctx) + i.lb.Logout(ctx, ipnauth.Self) }() } diff --git a/cmd/tsidp/Dockerfile b/cmd/tsidp/Dockerfile deleted file mode 100644 index c4f352ed01839..0000000000000 --- a/cmd/tsidp/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Build stage -FROM golang:alpine AS builder - -# Install build dependencies -RUN apk add --no-cache git - -# Set working directory -WORKDIR /src - -# Copy only go.mod and go.sum first to leverage Docker caching -COPY go.mod go.sum ./ -RUN go mod download - -# Copy the entire repository -COPY . . - -# Build the tsidp binary -RUN go build -o /bin/tsidp ./cmd/tsidp - -# Final stage -FROM alpine:latest - -# Create necessary directories -RUN mkdir -p /var/lib/tsidp - -# Copy binary from builder stage -COPY --from=builder /bin/tsidp /app/tsidp - -# Set working directory -WORKDIR /app - -# Environment variables -ENV TAILSCALE_USE_WIP_CODE=1 \ - TS_HOSTNAME=idp \ - TS_STATE_DIR=/var/lib/tsidp - -# Expose the default port -EXPOSE 443 - -# Run the application -ENTRYPOINT ["/bin/sh", "-c", "/app/tsidp --hostname=${TS_HOSTNAME} --dir=${TS_STATE_DIR}"] diff --git a/cmd/tsidp/README.md b/cmd/tsidp/README.md index 61a81e8aee7ae..780d9ab95b037 100644 --- a/cmd/tsidp/README.md +++ b/cmd/tsidp/README.md @@ -1,6 +1,6 @@ # `tsidp` - Tailscale OpenID Connect (OIDC) Identity Provider -[![status: experimental](https://img.shields.io/badge/status-experimental-blue)](https://tailscale.com/kb/1167/release-stages/#experimental) +[![status: community project](https://img.shields.io/badge/status-community_project-blue)](https://tailscale.com/kb/1531/community-projects) `tsidp` is an OIDC Identity Provider (IdP) server that integrates with your Tailscale network. It allows you to use Tailscale identities for authentication in applications that support OpenID Connect, enabling single sign-on (SSO) capabilities within your tailnet. @@ -12,43 +12,38 @@ ## Installation using Docker -1. **Build the Docker Image** +### Building from Source - The Dockerfile uses a multi-stage build process to: - - Build the `tsidp` binary from source - - Create a minimal Alpine-based image with just the necessary components +```bash +# Clone the Tailscale repository +git clone https://github.com/tailscale/tailscale.git +cd tailscale - ```bash - # Clone the Tailscale repository - git clone https://github.com/tailscale/tailscale.git - cd tailscale - ``` +# Build and publish to your own registry +make publishdevtsidp REPO=ghcr.io/yourusername/tsidp TAGS=v0.0.1 PUSH=true +``` - ```bash - # Build the Docker image - docker build -t tsidp:latest -f cmd/tsidp/Dockerfile . - ``` +### Running the Container -2. **Run the Container** +Replace `YOUR_TAILSCALE_AUTHKEY` with your Tailscale authentication key: - Replace `YOUR_TAILSCALE_AUTHKEY` with your Tailscale authentication key. +```bash +docker run -d \ + --name tsidp \ + -p 443:443 \ + -e TS_AUTHKEY=YOUR_TAILSCALE_AUTHKEY \ + -e TAILSCALE_USE_WIP_CODE=1 \ + -v tsidp-data:/var/lib/tsidp \ + ghcr.io/yourusername/tsidp:v0.0.1 \ + tsidp --hostname=idp --dir=/var/lib/tsidp +``` - ```bash - docker run -d \ - --name tsidp \ - -p 443:443 \ - -e TS_AUTHKEY=YOUR_TAILSCALE_AUTHKEY \ - -e TS_HOSTNAME=idp \ - -v tsidp-data:/var/lib/tsidp \ - tsidp:latest - ``` +### Verify Installation +```bash +docker logs tsidp +``` -3. **Verify Installation** - ```bash - docker logs tsidp - ``` - - Visit `https://idp.tailnet.ts.net` to confirm the service is running. +Visit `https://idp.tailnet.ts.net` to confirm the service is running. ## Usage Example: Proxmox Integration @@ -94,7 +89,7 @@ The `tsidp` server supports several command-line flags: ## Support -This is an [experimental](https://tailscale.com/kb/1167/release-stages#experimental), work in progress feature. For issues or questions, file issues on the [GitHub repository](https://github.com/tailscale/tailscale) +This is an experimental, work in progress, [community project](https://tailscale.com/kb/1531/community-projects). For issues or questions, file issues on the [GitHub repository](https://github.com/tailscale/tailscale). ## License diff --git a/cmd/tsidp/depaware.txt b/cmd/tsidp/depaware.txt new file mode 100644 index 0000000000000..b28460352bb9f --- /dev/null +++ b/cmd/tsidp/depaware.txt @@ -0,0 +1,657 @@ +tailscale.com/cmd/tsidp dependencies: (generated by github.com/tailscale/depaware) + + filippo.io/edwards25519 from github.com/hdevalence/ed25519consensus + filippo.io/edwards25519/field from filippo.io/edwards25519 + W 💣 github.com/alexbrainman/sspi from github.com/alexbrainman/sspi/internal/common+ + W github.com/alexbrainman/sspi/internal/common from github.com/alexbrainman/sspi/negotiate + W 💣 github.com/alexbrainman/sspi/negotiate from tailscale.com/net/tshttpproxy + L github.com/aws/aws-sdk-go-v2/aws from github.com/aws/aws-sdk-go-v2/aws/defaults+ + L github.com/aws/aws-sdk-go-v2/aws/arn from tailscale.com/ipn/store/awsstore + L github.com/aws/aws-sdk-go-v2/aws/defaults from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/aws/middleware from github.com/aws/aws-sdk-go-v2/aws/retry+ + L github.com/aws/aws-sdk-go-v2/aws/protocol/query from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/aws/protocol/restjson from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/aws/protocol/xml from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/aws/ratelimit from github.com/aws/aws-sdk-go-v2/aws/retry + L github.com/aws/aws-sdk-go-v2/aws/retry from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client+ + L github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 from github.com/aws/aws-sdk-go-v2/aws/signer/v4 + L github.com/aws/aws-sdk-go-v2/aws/signer/v4 from github.com/aws/aws-sdk-go-v2/internal/auth/smithy+ + L github.com/aws/aws-sdk-go-v2/aws/transport/http from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/config from tailscale.com/ipn/store/awsstore + L github.com/aws/aws-sdk-go-v2/credentials from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/credentials/endpointcreds from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client from github.com/aws/aws-sdk-go-v2/credentials/endpointcreds + L github.com/aws/aws-sdk-go-v2/credentials/processcreds from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/credentials/ssocreds from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/credentials/stscreds from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/feature/ec2/imds from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config from github.com/aws/aws-sdk-go-v2/feature/ec2/imds + L github.com/aws/aws-sdk-go-v2/internal/auth from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ + L github.com/aws/aws-sdk-go-v2/internal/auth/smithy from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/internal/configsources from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/internal/context from github.com/aws/aws-sdk-go-v2/aws/retry+ + L github.com/aws/aws-sdk-go-v2/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 from github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints+ + L github.com/aws/aws-sdk-go-v2/internal/ini from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/aws-sdk-go-v2/internal/middleware from github.com/aws/aws-sdk-go-v2/service/sso+ + L github.com/aws/aws-sdk-go-v2/internal/rand from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/aws-sdk-go-v2/internal/sdk from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/aws-sdk-go-v2/internal/sdkio from github.com/aws/aws-sdk-go-v2/credentials/processcreds + L github.com/aws/aws-sdk-go-v2/internal/shareddefaults from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/internal/strings from github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 + L github.com/aws/aws-sdk-go-v2/internal/sync/singleflight from github.com/aws/aws-sdk-go-v2/aws + L github.com/aws/aws-sdk-go-v2/internal/timeconv from github.com/aws/aws-sdk-go-v2/aws/retry + L github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/service/internal/presigned-url from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/service/ssm from tailscale.com/ipn/store/awsstore + L github.com/aws/aws-sdk-go-v2/service/ssm/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm + L github.com/aws/aws-sdk-go-v2/service/ssm/types from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/aws-sdk-go-v2/service/sso from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sso + L github.com/aws/aws-sdk-go-v2/service/sso/types from github.com/aws/aws-sdk-go-v2/service/sso + L github.com/aws/aws-sdk-go-v2/service/ssooidc from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/service/ssooidc/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/ssooidc + L github.com/aws/aws-sdk-go-v2/service/ssooidc/types from github.com/aws/aws-sdk-go-v2/service/ssooidc + L github.com/aws/aws-sdk-go-v2/service/sts from github.com/aws/aws-sdk-go-v2/config+ + L github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/aws-sdk-go-v2/service/sts/types from github.com/aws/aws-sdk-go-v2/credentials/stscreds+ + L github.com/aws/smithy-go from github.com/aws/aws-sdk-go-v2/aws/protocol/restjson+ + L github.com/aws/smithy-go/auth from github.com/aws/aws-sdk-go-v2/internal/auth+ + L github.com/aws/smithy-go/auth/bearer from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/smithy-go/context from github.com/aws/smithy-go/auth/bearer + L github.com/aws/smithy-go/document from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/smithy-go/encoding from github.com/aws/smithy-go/encoding/json+ + L github.com/aws/smithy-go/encoding/httpbinding from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ + L github.com/aws/smithy-go/encoding/json from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/smithy-go/encoding/xml from github.com/aws/aws-sdk-go-v2/service/sts + L github.com/aws/smithy-go/endpoints from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/smithy-go/internal/sync/singleflight from github.com/aws/smithy-go/auth/bearer + L github.com/aws/smithy-go/io from github.com/aws/aws-sdk-go-v2/feature/ec2/imds+ + L github.com/aws/smithy-go/logging from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/smithy-go/metrics from github.com/aws/aws-sdk-go-v2/aws/retry+ + L github.com/aws/smithy-go/middleware from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/smithy-go/private/requestcompression from github.com/aws/aws-sdk-go-v2/config + L github.com/aws/smithy-go/ptr from github.com/aws/aws-sdk-go-v2/aws+ + L github.com/aws/smithy-go/rand from github.com/aws/aws-sdk-go-v2/aws/middleware+ + L github.com/aws/smithy-go/time from github.com/aws/aws-sdk-go-v2/service/ssm+ + L github.com/aws/smithy-go/tracing from github.com/aws/aws-sdk-go-v2/aws/middleware+ + L github.com/aws/smithy-go/transport/http from github.com/aws/aws-sdk-go-v2/aws/middleware+ + L github.com/aws/smithy-go/transport/http/internal/io from github.com/aws/smithy-go/transport/http + L github.com/aws/smithy-go/waiter from github.com/aws/aws-sdk-go-v2/service/ssm + github.com/coder/websocket from tailscale.com/util/eventbus + github.com/coder/websocket/internal/errd from github.com/coder/websocket + github.com/coder/websocket/internal/util from github.com/coder/websocket + github.com/coder/websocket/internal/xsync from github.com/coder/websocket + L github.com/coreos/go-iptables/iptables from tailscale.com/util/linuxfw + W 💣 github.com/dblohm7/wingoes from github.com/dblohm7/wingoes/com+ + W 💣 github.com/dblohm7/wingoes/com from tailscale.com/util/osdiag+ + W 💣 github.com/dblohm7/wingoes/com/automation from tailscale.com/util/osdiag/internal/wsc + W github.com/dblohm7/wingoes/internal from github.com/dblohm7/wingoes/com + W 💣 github.com/dblohm7/wingoes/pe from tailscale.com/util/osdiag+ + LW 💣 github.com/digitalocean/go-smbios/smbios from tailscale.com/posture + github.com/fxamacker/cbor/v2 from tailscale.com/tka + github.com/gaissmai/bart from tailscale.com/net/ipset+ + github.com/gaissmai/bart/internal/bitset from github.com/gaissmai/bart+ + github.com/gaissmai/bart/internal/sparse from github.com/gaissmai/bart + github.com/go-json-experiment/json from tailscale.com/types/opt+ + github.com/go-json-experiment/json/internal from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonflags from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonopts from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/internal/jsonwire from github.com/go-json-experiment/json+ + github.com/go-json-experiment/json/jsontext from github.com/go-json-experiment/json+ + W 💣 github.com/go-ole/go-ole from github.com/go-ole/go-ole/oleutil+ + W 💣 github.com/go-ole/go-ole/oleutil from tailscale.com/wgengine/winnet + L 💣 github.com/godbus/dbus/v5 from tailscale.com/net/dns + github.com/golang/groupcache/lru from tailscale.com/net/dnscache + github.com/google/btree from gvisor.dev/gvisor/pkg/tcpip/header+ + L github.com/google/nftables from tailscale.com/util/linuxfw + L 💣 github.com/google/nftables/alignedbuff from github.com/google/nftables/xt + L 💣 github.com/google/nftables/binaryutil from github.com/google/nftables+ + L github.com/google/nftables/expr from github.com/google/nftables+ + L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ + L github.com/google/nftables/xt from github.com/google/nftables/expr+ + DW github.com/google/uuid from github.com/prometheus-community/pro-bing+ + github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ + L 💣 github.com/illarion/gonotify/v3 from tailscale.com/net/dns + L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 + L github.com/jmespath/go-jmespath from github.com/aws/aws-sdk-go-v2/service/ssm + L 💣 github.com/jsimonetti/rtnetlink from tailscale.com/net/netmon + L github.com/jsimonetti/rtnetlink/internal/unix from github.com/jsimonetti/rtnetlink + github.com/klauspost/compress from github.com/klauspost/compress/zstd + github.com/klauspost/compress/fse from github.com/klauspost/compress/huff0 + github.com/klauspost/compress/huff0 from github.com/klauspost/compress/zstd + github.com/klauspost/compress/internal/cpuinfo from github.com/klauspost/compress/huff0+ + github.com/klauspost/compress/internal/snapref from github.com/klauspost/compress/zstd + github.com/klauspost/compress/zstd from tailscale.com/util/zstdframe + github.com/klauspost/compress/zstd/internal/xxhash from github.com/klauspost/compress/zstd + L github.com/mdlayher/genetlink from tailscale.com/net/tstun + L 💣 github.com/mdlayher/netlink from github.com/google/nftables+ + L 💣 github.com/mdlayher/netlink/nlenc from github.com/jsimonetti/rtnetlink+ + L github.com/mdlayher/netlink/nltest from github.com/google/nftables + L github.com/mdlayher/sdnotify from tailscale.com/util/systemd + L 💣 github.com/mdlayher/socket from github.com/mdlayher/netlink+ + github.com/miekg/dns from tailscale.com/net/dns/recursive + 💣 github.com/mitchellh/go-ps from tailscale.com/safesocket + D github.com/prometheus-community/pro-bing from tailscale.com/wgengine/netstack + L 💣 github.com/safchain/ethtool from tailscale.com/doctor/ethtool+ + W 💣 github.com/tailscale/certstore from tailscale.com/control/controlclient + W 💣 github.com/tailscale/go-winio from tailscale.com/safesocket + W 💣 github.com/tailscale/go-winio/internal/fs from github.com/tailscale/go-winio + W 💣 github.com/tailscale/go-winio/internal/socket from github.com/tailscale/go-winio + W github.com/tailscale/go-winio/internal/stringbuffer from github.com/tailscale/go-winio/internal/fs + W github.com/tailscale/go-winio/pkg/guid from github.com/tailscale/go-winio+ + github.com/tailscale/goupnp from github.com/tailscale/goupnp/dcps/internetgateway2+ + github.com/tailscale/goupnp/dcps/internetgateway2 from tailscale.com/net/portmapper + github.com/tailscale/goupnp/httpu from github.com/tailscale/goupnp+ + github.com/tailscale/goupnp/scpd from github.com/tailscale/goupnp + github.com/tailscale/goupnp/soap from github.com/tailscale/goupnp+ + github.com/tailscale/goupnp/ssdp from github.com/tailscale/goupnp + github.com/tailscale/hujson from tailscale.com/ipn/conffile + L 💣 github.com/tailscale/netlink from tailscale.com/net/routetable+ + L 💣 github.com/tailscale/netlink/nl from github.com/tailscale/netlink + github.com/tailscale/peercred from tailscale.com/ipn/ipnauth + github.com/tailscale/web-client-prebuilt from tailscale.com/client/web + 💣 github.com/tailscale/wireguard-go/conn from github.com/tailscale/wireguard-go/device+ + W 💣 github.com/tailscale/wireguard-go/conn/winrio from github.com/tailscale/wireguard-go/conn + 💣 github.com/tailscale/wireguard-go/device from tailscale.com/net/tstun+ + 💣 github.com/tailscale/wireguard-go/ipc from github.com/tailscale/wireguard-go/device + W 💣 github.com/tailscale/wireguard-go/ipc/namedpipe from github.com/tailscale/wireguard-go/ipc + github.com/tailscale/wireguard-go/ratelimiter from github.com/tailscale/wireguard-go/device + github.com/tailscale/wireguard-go/replay from github.com/tailscale/wireguard-go/device + github.com/tailscale/wireguard-go/rwcancel from github.com/tailscale/wireguard-go/device+ + github.com/tailscale/wireguard-go/tai64n from github.com/tailscale/wireguard-go/device + 💣 github.com/tailscale/wireguard-go/tun from github.com/tailscale/wireguard-go/device+ + L github.com/vishvananda/netns from github.com/tailscale/netlink+ + github.com/x448/float16 from github.com/fxamacker/cbor/v2 + 💣 go4.org/mem from tailscale.com/client/local+ + go4.org/netipx from tailscale.com/ipn/ipnlocal+ + W 💣 golang.zx2c4.com/wintun from github.com/tailscale/wireguard-go/tun + W 💣 golang.zx2c4.com/wireguard/windows/tunnel/winipcfg from tailscale.com/net/dns+ + gopkg.in/square/go-jose.v2 from gopkg.in/square/go-jose.v2/jwt+ + gopkg.in/square/go-jose.v2/cipher from gopkg.in/square/go-jose.v2 + gopkg.in/square/go-jose.v2/json from gopkg.in/square/go-jose.v2+ + gopkg.in/square/go-jose.v2/jwt from tailscale.com/cmd/tsidp + gvisor.dev/gvisor/pkg/atomicbitops from gvisor.dev/gvisor/pkg/buffer+ + gvisor.dev/gvisor/pkg/bits from gvisor.dev/gvisor/pkg/buffer + 💣 gvisor.dev/gvisor/pkg/buffer from gvisor.dev/gvisor/pkg/tcpip+ + gvisor.dev/gvisor/pkg/context from gvisor.dev/gvisor/pkg/refs + 💣 gvisor.dev/gvisor/pkg/gohacks from gvisor.dev/gvisor/pkg/state/wire+ + gvisor.dev/gvisor/pkg/linewriter from gvisor.dev/gvisor/pkg/log + gvisor.dev/gvisor/pkg/log from gvisor.dev/gvisor/pkg/context+ + gvisor.dev/gvisor/pkg/rand from gvisor.dev/gvisor/pkg/tcpip+ + gvisor.dev/gvisor/pkg/refs from gvisor.dev/gvisor/pkg/buffer+ + 💣 gvisor.dev/gvisor/pkg/sleep from gvisor.dev/gvisor/pkg/tcpip/transport/tcp + 💣 gvisor.dev/gvisor/pkg/state from gvisor.dev/gvisor/pkg/atomicbitops+ + gvisor.dev/gvisor/pkg/state/wire from gvisor.dev/gvisor/pkg/state + 💣 gvisor.dev/gvisor/pkg/sync from gvisor.dev/gvisor/pkg/atomicbitops+ + 💣 gvisor.dev/gvisor/pkg/sync/locking from gvisor.dev/gvisor/pkg/tcpip/stack + gvisor.dev/gvisor/pkg/tcpip from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ + gvisor.dev/gvisor/pkg/tcpip/adapters/gonet from tailscale.com/wgengine/netstack + 💣 gvisor.dev/gvisor/pkg/tcpip/checksum from gvisor.dev/gvisor/pkg/buffer+ + gvisor.dev/gvisor/pkg/tcpip/hash/jenkins from gvisor.dev/gvisor/pkg/tcpip/stack+ + gvisor.dev/gvisor/pkg/tcpip/header from gvisor.dev/gvisor/pkg/tcpip/header/parse+ + gvisor.dev/gvisor/pkg/tcpip/header/parse from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ + gvisor.dev/gvisor/pkg/tcpip/internal/tcp from gvisor.dev/gvisor/pkg/tcpip/transport/tcp + gvisor.dev/gvisor/pkg/tcpip/network/hash from gvisor.dev/gvisor/pkg/tcpip/network/ipv4 + gvisor.dev/gvisor/pkg/tcpip/network/internal/fragmentation from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ + gvisor.dev/gvisor/pkg/tcpip/network/internal/ip from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ + gvisor.dev/gvisor/pkg/tcpip/network/internal/multicast from gvisor.dev/gvisor/pkg/tcpip/network/ipv4+ + gvisor.dev/gvisor/pkg/tcpip/network/ipv4 from tailscale.com/wgengine/netstack + gvisor.dev/gvisor/pkg/tcpip/network/ipv6 from tailscale.com/wgengine/netstack + gvisor.dev/gvisor/pkg/tcpip/ports from gvisor.dev/gvisor/pkg/tcpip/stack+ + gvisor.dev/gvisor/pkg/tcpip/seqnum from gvisor.dev/gvisor/pkg/tcpip/header+ + 💣 gvisor.dev/gvisor/pkg/tcpip/stack from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ + gvisor.dev/gvisor/pkg/tcpip/stack/gro from tailscale.com/wgengine/netstack/gro + gvisor.dev/gvisor/pkg/tcpip/transport from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+ + gvisor.dev/gvisor/pkg/tcpip/transport/icmp from tailscale.com/wgengine/netstack + gvisor.dev/gvisor/pkg/tcpip/transport/internal/network from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+ + gvisor.dev/gvisor/pkg/tcpip/transport/internal/noop from gvisor.dev/gvisor/pkg/tcpip/transport/raw + gvisor.dev/gvisor/pkg/tcpip/transport/packet from gvisor.dev/gvisor/pkg/tcpip/transport/raw + gvisor.dev/gvisor/pkg/tcpip/transport/raw from gvisor.dev/gvisor/pkg/tcpip/transport/icmp+ + 💣 gvisor.dev/gvisor/pkg/tcpip/transport/tcp from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ + gvisor.dev/gvisor/pkg/tcpip/transport/tcpconntrack from gvisor.dev/gvisor/pkg/tcpip/stack + gvisor.dev/gvisor/pkg/tcpip/transport/udp from gvisor.dev/gvisor/pkg/tcpip/adapters/gonet+ + gvisor.dev/gvisor/pkg/waiter from gvisor.dev/gvisor/pkg/context+ + tailscale.com from tailscale.com/version + tailscale.com/appc from tailscale.com/ipn/ipnlocal + 💣 tailscale.com/atomicfile from tailscale.com/ipn+ + tailscale.com/client/local from tailscale.com/client/tailscale+ + tailscale.com/client/tailscale from tailscale.com/derp+ + tailscale.com/client/tailscale/apitype from tailscale.com/client/local+ + tailscale.com/client/web from tailscale.com/ipn/ipnlocal + tailscale.com/clientupdate from tailscale.com/client/web+ + LW tailscale.com/clientupdate/distsign from tailscale.com/clientupdate + tailscale.com/control/controlbase from tailscale.com/control/controlhttp+ + tailscale.com/control/controlclient from tailscale.com/ipn/ipnext+ + tailscale.com/control/controlhttp from tailscale.com/control/controlclient + tailscale.com/control/controlhttp/controlhttpcommon from tailscale.com/control/controlhttp + tailscale.com/control/controlknobs from tailscale.com/control/controlclient+ + tailscale.com/derp from tailscale.com/derp/derphttp+ + tailscale.com/derp/derpconst from tailscale.com/derp+ + tailscale.com/derp/derphttp from tailscale.com/ipn/localapi+ + tailscale.com/disco from tailscale.com/derp+ + tailscale.com/doctor from tailscale.com/ipn/ipnlocal + tailscale.com/doctor/ethtool from tailscale.com/ipn/ipnlocal + 💣 tailscale.com/doctor/permissions from tailscale.com/ipn/ipnlocal + tailscale.com/doctor/routetable from tailscale.com/ipn/ipnlocal + tailscale.com/drive from tailscale.com/client/local+ + tailscale.com/envknob from tailscale.com/client/local+ + tailscale.com/envknob/featureknob from tailscale.com/client/web+ + tailscale.com/feature from tailscale.com/ipn/ipnext+ + tailscale.com/health from tailscale.com/control/controlclient+ + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ + tailscale.com/hostinfo from tailscale.com/client/web+ + tailscale.com/internal/noiseconn from tailscale.com/control/controlclient + tailscale.com/ipn from tailscale.com/client/local+ + tailscale.com/ipn/conffile from tailscale.com/ipn/ipnlocal+ + 💣 tailscale.com/ipn/ipnauth from tailscale.com/ipn/ipnext+ + tailscale.com/ipn/ipnext from tailscale.com/ipn/ipnlocal + tailscale.com/ipn/ipnlocal from tailscale.com/ipn/localapi+ + tailscale.com/ipn/ipnstate from tailscale.com/client/local+ + tailscale.com/ipn/localapi from tailscale.com/tsnet + tailscale.com/ipn/policy from tailscale.com/ipn/ipnlocal + tailscale.com/ipn/store from tailscale.com/ipn/ipnlocal+ + L tailscale.com/ipn/store/awsstore from tailscale.com/ipn/store + L tailscale.com/ipn/store/kubestore from tailscale.com/ipn/store + tailscale.com/ipn/store/mem from tailscale.com/ipn/ipnlocal+ + L tailscale.com/kube/kubeapi from tailscale.com/ipn/store/kubestore+ + L tailscale.com/kube/kubeclient from tailscale.com/ipn/store/kubestore + tailscale.com/kube/kubetypes from tailscale.com/envknob+ + tailscale.com/licenses from tailscale.com/client/web + tailscale.com/log/filelogger from tailscale.com/logpolicy + tailscale.com/log/sockstatlog from tailscale.com/ipn/ipnlocal + tailscale.com/logpolicy from tailscale.com/ipn/ipnlocal+ + tailscale.com/logtail from tailscale.com/control/controlclient+ + tailscale.com/logtail/backoff from tailscale.com/control/controlclient+ + tailscale.com/logtail/filch from tailscale.com/log/sockstatlog+ + tailscale.com/metrics from tailscale.com/derp+ + tailscale.com/net/bakedroots from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/captivedetection from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/connstats from tailscale.com/net/tstun+ + tailscale.com/net/dns from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/dns/publicdns from tailscale.com/net/dns+ + tailscale.com/net/dns/recursive from tailscale.com/net/dnsfallback + tailscale.com/net/dns/resolvconffile from tailscale.com/net/dns+ + tailscale.com/net/dns/resolver from tailscale.com/net/dns+ + tailscale.com/net/dnscache from tailscale.com/control/controlclient+ + tailscale.com/net/dnsfallback from tailscale.com/control/controlclient+ + tailscale.com/net/flowtrack from tailscale.com/net/packet+ + tailscale.com/net/ipset from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/memnet from tailscale.com/tsnet + tailscale.com/net/netaddr from tailscale.com/ipn+ + tailscale.com/net/netcheck from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/neterror from tailscale.com/net/dns/resolver+ + tailscale.com/net/netkernelconf from tailscale.com/ipn/ipnlocal + tailscale.com/net/netknob from tailscale.com/logpolicy+ + 💣 tailscale.com/net/netmon from tailscale.com/control/controlclient+ + 💣 tailscale.com/net/netns from tailscale.com/derp/derphttp+ + W 💣 tailscale.com/net/netstat from tailscale.com/portlist + tailscale.com/net/netutil from tailscale.com/client/local+ + tailscale.com/net/netx from tailscale.com/control/controlclient+ + tailscale.com/net/packet from tailscale.com/ipn/ipnlocal+ + tailscale.com/net/packet/checksum from tailscale.com/net/tstun + tailscale.com/net/ping from tailscale.com/net/netcheck+ + tailscale.com/net/portmapper from tailscale.com/ipn/localapi+ + tailscale.com/net/proxymux from tailscale.com/tsnet + tailscale.com/net/routetable from tailscale.com/doctor/routetable + tailscale.com/net/socks5 from tailscale.com/tsnet + tailscale.com/net/sockstats from tailscale.com/control/controlclient+ + tailscale.com/net/stun from tailscale.com/ipn/localapi+ + L tailscale.com/net/tcpinfo from tailscale.com/derp + tailscale.com/net/tlsdial from tailscale.com/control/controlclient+ + tailscale.com/net/tlsdial/blockblame from tailscale.com/net/tlsdial + tailscale.com/net/tsaddr from tailscale.com/client/web+ + tailscale.com/net/tsdial from tailscale.com/control/controlclient+ + 💣 tailscale.com/net/tshttpproxy from tailscale.com/clientupdate/distsign+ + tailscale.com/net/tstun from tailscale.com/tsd+ + tailscale.com/net/udprelay/endpoint from tailscale.com/wgengine/magicsock + tailscale.com/omit from tailscale.com/ipn/conffile + tailscale.com/paths from tailscale.com/client/local+ + 💣 tailscale.com/portlist from tailscale.com/ipn/ipnlocal + tailscale.com/posture from tailscale.com/ipn/ipnlocal + tailscale.com/proxymap from tailscale.com/tsd+ + 💣 tailscale.com/safesocket from tailscale.com/client/local+ + tailscale.com/syncs from tailscale.com/control/controlhttp+ + tailscale.com/tailcfg from tailscale.com/client/local+ + tailscale.com/tempfork/acme from tailscale.com/ipn/ipnlocal + tailscale.com/tempfork/heap from tailscale.com/wgengine/magicsock + tailscale.com/tempfork/httprec from tailscale.com/control/controlclient + tailscale.com/tka from tailscale.com/client/local+ + tailscale.com/tsconst from tailscale.com/ipn/ipnlocal+ + tailscale.com/tsd from tailscale.com/ipn/ipnext+ + tailscale.com/tsnet from tailscale.com/cmd/tsidp + tailscale.com/tstime from tailscale.com/control/controlclient+ + tailscale.com/tstime/mono from tailscale.com/net/tstun+ + tailscale.com/tstime/rate from tailscale.com/derp+ + tailscale.com/tsweb from tailscale.com/util/eventbus + tailscale.com/tsweb/varz from tailscale.com/tsweb+ + tailscale.com/types/appctype from tailscale.com/ipn/ipnlocal + tailscale.com/types/bools from tailscale.com/tsnet + tailscale.com/types/dnstype from tailscale.com/client/local+ + tailscale.com/types/empty from tailscale.com/ipn+ + tailscale.com/types/ipproto from tailscale.com/ipn+ + tailscale.com/types/key from tailscale.com/client/local+ + tailscale.com/types/lazy from tailscale.com/clientupdate+ + tailscale.com/types/logger from tailscale.com/appc+ + tailscale.com/types/logid from tailscale.com/ipn/ipnlocal+ + tailscale.com/types/mapx from tailscale.com/ipn/ipnext + tailscale.com/types/netlogtype from tailscale.com/net/connstats+ + tailscale.com/types/netmap from tailscale.com/control/controlclient+ + tailscale.com/types/nettype from tailscale.com/ipn/localapi+ + tailscale.com/types/opt from tailscale.com/client/tailscale+ + tailscale.com/types/persist from tailscale.com/control/controlclient+ + tailscale.com/types/preftype from tailscale.com/ipn+ + tailscale.com/types/ptr from tailscale.com/control/controlclient+ + tailscale.com/types/result from tailscale.com/util/lineiter + tailscale.com/types/structs from tailscale.com/control/controlclient+ + tailscale.com/types/tkatype from tailscale.com/client/local+ + tailscale.com/types/views from tailscale.com/appc+ + tailscale.com/util/cibuild from tailscale.com/health + tailscale.com/util/clientmetric from tailscale.com/appc+ + tailscale.com/util/cloudenv from tailscale.com/hostinfo+ + tailscale.com/util/cmpver from tailscale.com/clientupdate+ + tailscale.com/util/ctxkey from tailscale.com/client/tailscale/apitype+ + 💣 tailscale.com/util/deephash from tailscale.com/ipn/ipnlocal+ + L 💣 tailscale.com/util/dirwalk from tailscale.com/metrics+ + tailscale.com/util/dnsname from tailscale.com/appc+ + tailscale.com/util/eventbus from tailscale.com/ipn/localapi+ + tailscale.com/util/execqueue from tailscale.com/appc+ + tailscale.com/util/goroutines from tailscale.com/ipn/ipnlocal + tailscale.com/util/groupmember from tailscale.com/client/web+ + 💣 tailscale.com/util/hashx from tailscale.com/util/deephash + tailscale.com/util/httpm from tailscale.com/client/tailscale+ + tailscale.com/util/lineiter from tailscale.com/hostinfo+ + L tailscale.com/util/linuxfw from tailscale.com/net/netns+ + tailscale.com/util/mak from tailscale.com/appc+ + tailscale.com/util/multierr from tailscale.com/control/controlclient+ + tailscale.com/util/must from tailscale.com/clientupdate/distsign+ + tailscale.com/util/nocasemaps from tailscale.com/types/ipproto + 💣 tailscale.com/util/osdiag from tailscale.com/ipn/localapi + W 💣 tailscale.com/util/osdiag/internal/wsc from tailscale.com/util/osdiag + tailscale.com/util/osuser from tailscale.com/ipn/ipnlocal + tailscale.com/util/race from tailscale.com/net/dns/resolver + tailscale.com/util/racebuild from tailscale.com/logpolicy + tailscale.com/util/rands from tailscale.com/cmd/tsidp+ + tailscale.com/util/ringbuffer from tailscale.com/wgengine/magicsock + tailscale.com/util/set from tailscale.com/control/controlclient+ + tailscale.com/util/singleflight from tailscale.com/control/controlclient+ + tailscale.com/util/slicesx from tailscale.com/appc+ + tailscale.com/util/syspolicy from tailscale.com/control/controlclient+ + tailscale.com/util/syspolicy/internal from tailscale.com/util/syspolicy+ + tailscale.com/util/syspolicy/internal/loggerx from tailscale.com/util/syspolicy+ + tailscale.com/util/syspolicy/internal/metrics from tailscale.com/util/syspolicy/source + tailscale.com/util/syspolicy/rsop from tailscale.com/ipn/ipnlocal+ + tailscale.com/util/syspolicy/setting from tailscale.com/client/local+ + tailscale.com/util/syspolicy/source from tailscale.com/util/syspolicy+ + tailscale.com/util/sysresources from tailscale.com/wgengine/magicsock + tailscale.com/util/systemd from tailscale.com/control/controlclient+ + tailscale.com/util/testenv from tailscale.com/control/controlclient+ + tailscale.com/util/truncate from tailscale.com/logtail + tailscale.com/util/usermetric from tailscale.com/health+ + tailscale.com/util/vizerror from tailscale.com/tailcfg+ + 💣 tailscale.com/util/winutil from tailscale.com/clientupdate+ + W 💣 tailscale.com/util/winutil/authenticode from tailscale.com/clientupdate+ + W 💣 tailscale.com/util/winutil/gp from tailscale.com/net/dns+ + W tailscale.com/util/winutil/policy from tailscale.com/ipn/ipnlocal + W 💣 tailscale.com/util/winutil/winenv from tailscale.com/hostinfo+ + tailscale.com/util/zstdframe from tailscale.com/control/controlclient+ + tailscale.com/version from tailscale.com/client/web+ + tailscale.com/version/distro from tailscale.com/client/web+ + tailscale.com/wgengine from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/filter from tailscale.com/control/controlclient+ + tailscale.com/wgengine/filter/filtertype from tailscale.com/types/netmap+ + 💣 tailscale.com/wgengine/magicsock from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/netlog from tailscale.com/wgengine + tailscale.com/wgengine/netstack from tailscale.com/tsnet + tailscale.com/wgengine/netstack/gro from tailscale.com/net/tstun+ + tailscale.com/wgengine/router from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/wgcfg from tailscale.com/ipn/ipnlocal+ + tailscale.com/wgengine/wgcfg/nmcfg from tailscale.com/ipn/ipnlocal + 💣 tailscale.com/wgengine/wgint from tailscale.com/wgengine+ + tailscale.com/wgengine/wglog from tailscale.com/wgengine + W 💣 tailscale.com/wgengine/winnet from tailscale.com/wgengine/router + golang.org/x/crypto/argon2 from tailscale.com/tka + golang.org/x/crypto/blake2b from golang.org/x/crypto/argon2+ + golang.org/x/crypto/blake2s from github.com/tailscale/wireguard-go/device+ + LD golang.org/x/crypto/blowfish from golang.org/x/crypto/ssh/internal/bcrypt_pbkdf + golang.org/x/crypto/chacha20 from golang.org/x/crypto/chacha20poly1305+ + golang.org/x/crypto/chacha20poly1305 from crypto/internal/hpke+ + golang.org/x/crypto/cryptobyte from crypto/ecdsa+ + golang.org/x/crypto/cryptobyte/asn1 from crypto/ecdsa+ + golang.org/x/crypto/curve25519 from github.com/tailscale/wireguard-go/device+ + golang.org/x/crypto/ed25519 from gopkg.in/square/go-jose.v2 + golang.org/x/crypto/hkdf from tailscale.com/control/controlbase + golang.org/x/crypto/internal/alias from golang.org/x/crypto/chacha20+ + golang.org/x/crypto/internal/poly1305 from golang.org/x/crypto/chacha20poly1305+ + golang.org/x/crypto/nacl/box from tailscale.com/types/key + golang.org/x/crypto/nacl/secretbox from golang.org/x/crypto/nacl/box + golang.org/x/crypto/pbkdf2 from gopkg.in/square/go-jose.v2 + golang.org/x/crypto/poly1305 from github.com/tailscale/wireguard-go/device + golang.org/x/crypto/salsa20/salsa from golang.org/x/crypto/nacl/box+ + LD golang.org/x/crypto/ssh from tailscale.com/ipn/ipnlocal + LD golang.org/x/crypto/ssh/internal/bcrypt_pbkdf from golang.org/x/crypto/ssh + golang.org/x/exp/constraints from github.com/dblohm7/wingoes/pe+ + golang.org/x/exp/maps from tailscale.com/ipn/store/mem+ + golang.org/x/net/bpf from github.com/mdlayher/genetlink+ + golang.org/x/net/dns/dnsmessage from net+ + golang.org/x/net/http/httpguts from golang.org/x/net/http2+ + golang.org/x/net/http/httpproxy from net/http+ + golang.org/x/net/http2 from golang.org/x/net/http2/h2c+ + golang.org/x/net/http2/h2c from tailscale.com/ipn/ipnlocal + golang.org/x/net/http2/hpack from golang.org/x/net/http2+ + golang.org/x/net/icmp from github.com/prometheus-community/pro-bing+ + golang.org/x/net/idna from golang.org/x/net/http/httpguts+ + golang.org/x/net/internal/httpcommon from golang.org/x/net/http2 + golang.org/x/net/internal/iana from golang.org/x/net/icmp+ + golang.org/x/net/internal/socket from golang.org/x/net/icmp+ + golang.org/x/net/internal/socks from golang.org/x/net/proxy + golang.org/x/net/ipv4 from github.com/miekg/dns+ + golang.org/x/net/ipv6 from github.com/miekg/dns+ + golang.org/x/net/proxy from tailscale.com/net/netns + D golang.org/x/net/route from net+ + golang.org/x/sync/errgroup from github.com/mdlayher/socket+ + golang.org/x/sys/cpu from github.com/tailscale/certstore+ + LD golang.org/x/sys/unix from github.com/google/nftables+ + W golang.org/x/sys/windows from github.com/dblohm7/wingoes+ + W golang.org/x/sys/windows/registry from github.com/dblohm7/wingoes+ + W golang.org/x/sys/windows/svc from golang.org/x/sys/windows/svc/mgr+ + W golang.org/x/sys/windows/svc/mgr from tailscale.com/util/winutil + golang.org/x/term from tailscale.com/logpolicy + golang.org/x/text/secure/bidirule from golang.org/x/net/idna + golang.org/x/text/transform from golang.org/x/text/secure/bidirule+ + golang.org/x/text/unicode/bidi from golang.org/x/net/idna+ + golang.org/x/text/unicode/norm from golang.org/x/net/idna + golang.org/x/time/rate from gvisor.dev/gvisor/pkg/log+ + archive/tar from tailscale.com/clientupdate + bufio from compress/flate+ + bytes from archive/tar+ + cmp from encoding/json+ + compress/flate from compress/gzip+ + compress/gzip from github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding+ + W compress/zlib from debug/pe + container/heap from gvisor.dev/gvisor/pkg/tcpip/transport/tcp + container/list from crypto/tls+ + context from crypto/tls+ + crypto from crypto/ecdh+ + crypto/aes from crypto/internal/hpke+ + crypto/cipher from crypto/aes+ + crypto/des from crypto/tls+ + crypto/dsa from crypto/x509+ + crypto/ecdh from crypto/ecdsa+ + crypto/ecdsa from crypto/tls+ + crypto/ed25519 from crypto/tls+ + crypto/elliptic from crypto/ecdsa+ + crypto/hmac from crypto/tls+ + crypto/internal/boring from crypto/aes+ + crypto/internal/boring/bbig from crypto/ecdsa+ + crypto/internal/boring/sig from crypto/internal/boring + crypto/internal/entropy from crypto/internal/fips140/drbg + crypto/internal/fips140 from crypto/internal/fips140/aes+ + crypto/internal/fips140/aes from crypto/aes+ + crypto/internal/fips140/aes/gcm from crypto/cipher+ + crypto/internal/fips140/alias from crypto/cipher+ + crypto/internal/fips140/bigmod from crypto/internal/fips140/ecdsa+ + crypto/internal/fips140/check from crypto/internal/fips140/aes+ + crypto/internal/fips140/drbg from crypto/internal/fips140/aes/gcm+ + crypto/internal/fips140/ecdh from crypto/ecdh + crypto/internal/fips140/ecdsa from crypto/ecdsa + crypto/internal/fips140/ed25519 from crypto/ed25519 + crypto/internal/fips140/edwards25519 from crypto/internal/fips140/ed25519 + crypto/internal/fips140/edwards25519/field from crypto/ecdh+ + crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ + crypto/internal/fips140/hmac from crypto/hmac+ + crypto/internal/fips140/mlkem from crypto/tls+ + crypto/internal/fips140/nistec from crypto/elliptic+ + crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec + crypto/internal/fips140/rsa from crypto/rsa + crypto/internal/fips140/sha256 from crypto/internal/fips140/check+ + crypto/internal/fips140/sha3 from crypto/internal/fips140/hmac+ + crypto/internal/fips140/sha512 from crypto/internal/fips140/ecdsa+ + crypto/internal/fips140/subtle from crypto/internal/fips140/aes+ + crypto/internal/fips140/tls12 from crypto/tls + crypto/internal/fips140/tls13 from crypto/tls + crypto/internal/fips140deps/byteorder from crypto/internal/fips140/aes+ + crypto/internal/fips140deps/cpu from crypto/internal/fips140/aes+ + crypto/internal/fips140deps/godebug from crypto/internal/fips140+ + crypto/internal/fips140hash from crypto/ecdsa+ + crypto/internal/fips140only from crypto/cipher+ + crypto/internal/hpke from crypto/tls + crypto/internal/impl from crypto/internal/fips140/aes+ + crypto/internal/randutil from crypto/dsa+ + crypto/internal/sysrand from crypto/internal/entropy+ + crypto/md5 from crypto/tls+ + LD crypto/mlkem from golang.org/x/crypto/ssh + crypto/rand from crypto/ed25519+ + crypto/rc4 from crypto/tls+ + crypto/rsa from crypto/tls+ + crypto/sha1 from crypto/tls+ + crypto/sha256 from crypto/tls+ + crypto/sha3 from crypto/internal/fips140hash + crypto/sha512 from crypto/ecdsa+ + crypto/subtle from crypto/cipher+ + crypto/tls from github.com/aws/aws-sdk-go-v2/aws/transport/http+ + crypto/tls/internal/fips140tls from crypto/tls + crypto/x509 from crypto/tls+ + D crypto/x509/internal/macos from crypto/x509 + crypto/x509/pkix from crypto/x509+ + DW database/sql/driver from github.com/google/uuid + W debug/dwarf from debug/pe + W debug/pe from github.com/dblohm7/wingoes/pe + embed from github.com/tailscale/web-client-prebuilt+ + encoding from encoding/json+ + encoding/asn1 from crypto/x509+ + encoding/base32 from github.com/fxamacker/cbor/v2+ + encoding/base64 from encoding/json+ + encoding/binary from compress/gzip+ + encoding/hex from crypto/x509+ + encoding/json from expvar+ + encoding/pem from crypto/tls+ + encoding/xml from github.com/aws/aws-sdk-go-v2/aws/protocol/xml+ + errors from archive/tar+ + expvar from tailscale.com/derp+ + flag from tailscale.com/cmd/tsidp+ + fmt from archive/tar+ + hash from compress/zlib+ + W hash/adler32 from compress/zlib + hash/crc32 from compress/gzip+ + hash/maphash from go4.org/mem + html from html/template+ + html/template from tailscale.com/util/eventbus+ + internal/abi from crypto/x509/internal/macos+ + internal/asan from internal/runtime/maps+ + internal/bisect from internal/godebug + internal/bytealg from bytes+ + internal/byteorder from crypto/cipher+ + internal/chacha8rand from math/rand/v2+ + internal/coverage/rtcov from runtime + internal/cpu from crypto/internal/fips140deps/cpu+ + internal/filepathlite from os+ + internal/fmtsort from fmt+ + internal/goarch from crypto/internal/fips140deps/cpu+ + internal/godebug from archive/tar+ + internal/godebugs from internal/godebug+ + internal/goexperiment from hash/maphash+ + internal/goos from crypto/x509+ + internal/itoa from internal/poll+ + internal/msan from internal/runtime/maps+ + internal/nettrace from net+ + internal/oserror from io/fs+ + internal/poll from net+ + internal/profile from net/http/pprof + internal/profilerecord from runtime+ + internal/race from internal/poll+ + internal/reflectlite from context+ + internal/runtime/atomic from internal/runtime/exithook+ + internal/runtime/exithook from runtime + internal/runtime/maps from reflect+ + internal/runtime/math from internal/runtime/maps+ + internal/runtime/sys from crypto/subtle+ + L internal/runtime/syscall from runtime+ + W internal/saferio from debug/pe + internal/singleflight from net + internal/stringslite from embed+ + internal/sync from sync+ + internal/syscall/execenv from os+ + LD internal/syscall/unix from crypto/internal/sysrand+ + W internal/syscall/windows from crypto/internal/sysrand+ + W internal/syscall/windows/registry from mime+ + W internal/syscall/windows/sysdll from internal/syscall/windows+ + internal/testlog from os + internal/unsafeheader from internal/reflectlite+ + io from archive/tar+ + io/fs from archive/tar+ + io/ioutil from github.com/aws/aws-sdk-go-v2/aws/protocol/query+ + iter from bytes+ + log from expvar+ + log/internal from log + maps from archive/tar+ + math from archive/tar+ + math/big from crypto/dsa+ + math/bits from bytes+ + math/rand from github.com/fxamacker/cbor/v2+ + math/rand/v2 from crypto/ecdsa+ + mime from mime/multipart+ + mime/multipart from net/http + mime/quotedprintable from mime/multipart + net from crypto/tls+ + net/http from expvar+ + net/http/httptrace from github.com/aws/smithy-go/transport/http+ + net/http/httputil from github.com/aws/smithy-go/transport/http+ + net/http/internal from net/http+ + net/http/internal/ascii from net/http+ + net/http/pprof from tailscale.com/ipn/localapi+ + net/netip from crypto/x509+ + net/textproto from github.com/aws/aws-sdk-go-v2/aws/signer/v4+ + net/url from crypto/x509+ + os from crypto/internal/sysrand+ + os/exec from github.com/aws/aws-sdk-go-v2/credentials/processcreds+ + os/signal from tailscale.com/cmd/tsidp + os/user from archive/tar+ + path from archive/tar+ + path/filepath from archive/tar+ + reflect from archive/tar+ + regexp from github.com/aws/aws-sdk-go-v2/internal/endpoints+ + regexp/syntax from regexp + runtime from archive/tar+ + runtime/debug from github.com/aws/aws-sdk-go-v2/internal/sync/singleflight+ + runtime/pprof from net/http/pprof+ + runtime/trace from net/http/pprof + slices from archive/tar+ + sort from compress/flate+ + strconv from archive/tar+ + strings from archive/tar+ + sync from archive/tar+ + sync/atomic from context+ + syscall from archive/tar+ + text/tabwriter from runtime/pprof + text/template from html/template + text/template/parse from html/template+ + time from archive/tar+ + unicode from bytes+ + unicode/utf16 from crypto/x509+ + unicode/utf8 from bufio+ + unique from net/netip + unsafe from bytes+ + weak from unique diff --git a/cmd/tsidp/tsidp.go b/cmd/tsidp/tsidp.go index e2b777fa1b68e..8df68cd744148 100644 --- a/cmd/tsidp/tsidp.go +++ b/cmd/tsidp/tsidp.go @@ -39,6 +39,7 @@ import ( "tailscale.com/client/local" "tailscale.com/client/tailscale/apitype" "tailscale.com/envknob" + "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/ipnstate" "tailscale.com/tailcfg" @@ -121,6 +122,7 @@ func main() { } defer cleanup() } else { + hostinfo.SetApp("tsidp") ts := &tsnet.Server{ Hostname: *flagHostname, Dir: *flagDir, @@ -161,16 +163,17 @@ func main() { } else { srv.serverURL = fmt.Sprintf("https://%s", strings.TrimSuffix(st.Self.DNSName, ".")) } - if *flagFunnel { - f, err := os.Open(funnelClientsFile) - if err == nil { - srv.funnelClients = make(map[string]*funnelClient) - if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil { - log.Fatalf("could not parse %s: %v", funnelClientsFile, err) - } - } else if !errors.Is(err, os.ErrNotExist) { - log.Fatalf("could not open %s: %v", funnelClientsFile, err) + + // Load funnel clients from disk if they exist, regardless of whether funnel is enabled + // This ensures OIDC clients persist across restarts + f, err := os.Open(funnelClientsFile) + if err == nil { + if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil { + log.Fatalf("could not parse %s: %v", funnelClientsFile, err) } + f.Close() + } else if !errors.Is(err, os.ErrNotExist) { + log.Fatalf("could not open %s: %v", funnelClientsFile, err) } log.Printf("Running tsidp at %s ...", srv.serverURL) @@ -267,7 +270,7 @@ func serveOnLocalTailscaled(ctx context.Context, lc *local.Client, st *ipnstate. foregroundSc.SetFunnel(serverURL, dstPort, shouldFunnel) foregroundSc.SetWebHandler(&ipn.HTTPHandler{ Proxy: fmt.Sprintf("https://%s", net.JoinHostPort(serverURL, strconv.Itoa(int(dstPort)))), - }, serverURL, uint16(*flagPort), "/", true) + }, serverURL, uint16(*flagPort), "/", true, st.CurrentTailnet.MagicDNSSuffix) err = lc.SetServeConfig(ctx, sc) if err != nil { return nil, watcherChan, fmt.Errorf("could not set serve config: %v", err) @@ -452,13 +455,7 @@ func (s *idpServer) newMux() *http.ServeMux { mux.HandleFunc("/userinfo", s.serveUserInfo) mux.HandleFunc("/token", s.serveToken) mux.HandleFunc("/clients/", s.serveClients) - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/" { - io.WriteString(w, "

Tailscale OIDC IdP

") - return - } - http.Error(w, "tsidp: not found", http.StatusNotFound) - }) + mux.HandleFunc("/", s.handleUI) return mux } @@ -795,7 +792,7 @@ type oidcTokenResponse struct { IDToken string `json:"id_token"` TokenType string `json:"token_type"` AccessToken string `json:"access_token"` - RefreshToken string `json:"refresh_token"` + RefreshToken string `json:"refresh_token,omitempty"` ExpiresIn int `json:"expires_in"` } diff --git a/cmd/tsidp/tsidp_test.go b/cmd/tsidp/tsidp_test.go index 76a11899187f7..e5465d3cfbf62 100644 --- a/cmd/tsidp/tsidp_test.go +++ b/cmd/tsidp/tsidp_test.go @@ -1,11 +1,13 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package main import ( "crypto/rand" "crypto/rsa" "encoding/json" + "errors" "fmt" "io" "log" @@ -13,6 +15,7 @@ import ( "net/http/httptest" "net/netip" "net/url" + "os" "reflect" "sort" "strings" @@ -824,3 +827,139 @@ func TestExtraUserInfo(t *testing.T) { }) } } + +func TestFunnelClientsPersistence(t *testing.T) { + testClients := map[string]*funnelClient{ + "test-client-1": { + ID: "test-client-1", + Secret: "test-secret-1", + Name: "Test Client 1", + RedirectURI: "https://example.com/callback", + }, + "test-client-2": { + ID: "test-client-2", + Secret: "test-secret-2", + Name: "Test Client 2", + RedirectURI: "https://example2.com/callback", + }, + } + + testData, err := json.Marshal(testClients) + if err != nil { + t.Fatalf("failed to marshal test data: %v", err) + } + + tmpFile := t.TempDir() + "/oidc-funnel-clients.json" + if err := os.WriteFile(tmpFile, testData, 0600); err != nil { + t.Fatalf("failed to write test file: %v", err) + } + + t.Run("step1_load_from_existing_file", func(t *testing.T) { + srv := &idpServer{} + + // Simulate the funnel clients loading logic from main() + srv.funnelClients = make(map[string]*funnelClient) + f, err := os.Open(tmpFile) + if err == nil { + if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil { + t.Fatalf("could not parse %s: %v", tmpFile, err) + } + f.Close() + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("could not open %s: %v", tmpFile, err) + } + + // Verify clients were loaded correctly + if len(srv.funnelClients) != 2 { + t.Errorf("expected 2 clients, got %d", len(srv.funnelClients)) + } + + client1, ok := srv.funnelClients["test-client-1"] + if !ok { + t.Error("expected test-client-1 to be loaded") + } else { + if client1.Name != "Test Client 1" { + t.Errorf("expected client name 'Test Client 1', got '%s'", client1.Name) + } + if client1.Secret != "test-secret-1" { + t.Errorf("expected client secret 'test-secret-1', got '%s'", client1.Secret) + } + } + }) + + t.Run("step2_initialize_empty_when_no_file", func(t *testing.T) { + nonExistentFile := t.TempDir() + "/non-existent.json" + + srv := &idpServer{} + + // Simulate the funnel clients loading logic from main() + srv.funnelClients = make(map[string]*funnelClient) + f, err := os.Open(nonExistentFile) + if err == nil { + if err := json.NewDecoder(f).Decode(&srv.funnelClients); err != nil { + t.Fatalf("could not parse %s: %v", nonExistentFile, err) + } + f.Close() + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("could not open %s: %v", nonExistentFile, err) + } + + // Verify map is initialized but empty + if srv.funnelClients == nil { + t.Error("expected funnelClients map to be initialized") + } + if len(srv.funnelClients) != 0 { + t.Errorf("expected empty map, got %d clients", len(srv.funnelClients)) + } + }) + + t.Run("step3_persist_and_reload_clients", func(t *testing.T) { + tmpFile2 := t.TempDir() + "/test-persistence.json" + + // Create initial server with one client + srv1 := &idpServer{ + funnelClients: make(map[string]*funnelClient), + } + srv1.funnelClients["new-client"] = &funnelClient{ + ID: "new-client", + Secret: "new-secret", + Name: "New Client", + RedirectURI: "https://new.example.com/callback", + } + + // Save clients to file (simulating saveFunnelClients) + data, err := json.Marshal(srv1.funnelClients) + if err != nil { + t.Fatalf("failed to marshal clients: %v", err) + } + if err := os.WriteFile(tmpFile2, data, 0600); err != nil { + t.Fatalf("failed to write clients file: %v", err) + } + + // Create new server instance and load clients + srv2 := &idpServer{} + srv2.funnelClients = make(map[string]*funnelClient) + f, err := os.Open(tmpFile2) + if err == nil { + if err := json.NewDecoder(f).Decode(&srv2.funnelClients); err != nil { + t.Fatalf("could not parse %s: %v", tmpFile2, err) + } + f.Close() + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("could not open %s: %v", tmpFile2, err) + } + + // Verify the client was persisted correctly + loadedClient, ok := srv2.funnelClients["new-client"] + if !ok { + t.Error("expected new-client to be loaded after persistence") + } else { + if loadedClient.Name != "New Client" { + t.Errorf("expected client name 'New Client', got '%s'", loadedClient.Name) + } + if loadedClient.Secret != "new-secret" { + t.Errorf("expected client secret 'new-secret', got '%s'", loadedClient.Secret) + } + } + }) +} diff --git a/cmd/tsidp/ui-edit.html b/cmd/tsidp/ui-edit.html new file mode 100644 index 0000000000000..d463981aa34df --- /dev/null +++ b/cmd/tsidp/ui-edit.html @@ -0,0 +1,199 @@ + + + + + {{if .IsNew}}Add New Client{{else}}Edit Client{{end}} - Tailscale OIDC Identity Provider + + + + + + + {{template "header"}} + +
+
+
+

+ {{if .IsNew}}Add New OIDC Client{{else}}Edit OIDC Client{{end}} +

+ ← Back to Clients +
+ + {{if .Success}} +
+ {{.Success}} +
+ {{end}} + + {{if .Error}} +
+ {{.Error}} +
+ {{end}} + + {{if and .Secret .IsNew}} +
+

Client Created Successfully!

+

⚠️ Save both the Client ID and Secret now! The secret will not be shown again.

+ +
+ +
+ + +
+
+ +
+ +
+ + +
+
+
+ {{end}} + + {{if and .Secret .IsEdit}} +
+

New Client Secret

+

⚠️ Save this secret now! It will not be shown again.

+
+ + +
+
+ {{end}} + +
+
+ + +
+ A descriptive name for this OIDC client (optional). +
+
+ +
+ + +
+ The URL where users will be redirected after authentication. +
+
+ + {{if .IsEdit}} +
+ + +
+ The client ID cannot be changed. +
+
+ {{end}} + +
+ + + {{if .IsEdit}} + + + + {{end}} +
+
+ + {{if .IsEdit}} +
+

Client Information

+
+
Client ID
+
{{.ID}}
+
Secret Status
+
+ {{if .HasSecret}} + Secret configured + {{else}} + No secret + {{end}} +
+
+
+ {{end}} +
+
+ + + + \ No newline at end of file diff --git a/cmd/tsidp/ui-header.html b/cmd/tsidp/ui-header.html new file mode 100644 index 0000000000000..68e9bc0df4d7e --- /dev/null +++ b/cmd/tsidp/ui-header.html @@ -0,0 +1,53 @@ +
+ +
\ No newline at end of file diff --git a/cmd/tsidp/ui-list.html b/cmd/tsidp/ui-list.html new file mode 100644 index 0000000000000..d45b883494403 --- /dev/null +++ b/cmd/tsidp/ui-list.html @@ -0,0 +1,73 @@ + + + + Tailscale OIDC Identity Provider + + + + + + {{template "header"}} + +
+
+
+

OIDC Clients

+ {{if .}} +

{{len .}} client{{if ne (len .) 1}}s{{end}} configured

+ {{end}} +
+ Add New Client +
+ + {{if .}} + + + + + + + + + + + + {{range .}} + + + + + + + + {{end}} + +
NameClient IDRedirect URIStatusActions
+ {{if .Name}} + {{.Name}} + {{else}} + Unnamed Client + {{end}} + + {{.ID}} + + {{.RedirectURI}} + + {{if .HasSecret}} + Active + {{else}} + No Secret + {{end}} + + Edit +
+ {{else}} +
+

No OIDC clients configured

+

Create your first OIDC client to get started with authentication.

+ Add New Client +
+ {{end}} +
+ + \ No newline at end of file diff --git a/cmd/tsidp/ui-style.css b/cmd/tsidp/ui-style.css new file mode 100644 index 0000000000000..148ec3030d0b3 --- /dev/null +++ b/cmd/tsidp/ui-style.css @@ -0,0 +1,446 @@ +:root { + --tw-text-opacity: 1; + --color-gray-100: 247 245 244; + --color-gray-200: 238 235 234; + --color-gray-500: 112 110 109; + --color-gray-700: 46 45 45; + --color-gray-800: 35 34 34; + --color-gray-900: 31 30 30; + --color-bg-app: rgb(var(--color-gray-900) / 1); + --color-border-base: rgb(var(--color-gray-200) / 1); + --color-primary: 59 130 246; + --color-primary-hover: 37 99 235; + --color-secondary: 107 114 128; + --color-secondary-hover: 75 85 99; + --color-success: 34 197 94; + --color-warning: 245 158 11; + --color-danger: 239 68 68; + --color-danger-hover: 220 38 38; +} + +* { + box-sizing: border-box; + padding: 0; + margin: 0; +} + +body { + font-family: Inter, -apple-system, BlinkMacSystemFont, Helvetica, Arial, + sans-serif; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + font-size: 16px; + line-height: 1.4; + margin: 0; + background-color: var(--color-bg-app); + color: rgb(var(--color-gray-200)); +} + +a { + text-decoration: none; + color: inherit; +} + +header { + margin-top: 40px; +} +header nav { + margin: 0 auto; + max-width: 1120px; + display: flex; + align-items: center; + justify-content: center; +} +header nav h1 { + display: inline; + font-weight: 600; + font-size: 1.125rem; + line-height: 1.75rem; + margin-left: 0.75rem; +} + +main { + margin: 40px auto 60px auto; + max-width: 1120px; + padding: 0 20px; +} + +/* Header actions */ +.header-actions { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 2rem; +} + +.header-actions h2 { + font-size: 1.5rem; + font-weight: 600; + margin: 0 0 0.25rem 0; +} + +.client-count { + font-size: 0.875rem; + color: rgb(var(--color-gray-500)); + margin: 0; +} + +/* Buttons */ +.btn { + display: inline-flex; + align-items: center; + padding: 8px 16px; + border-radius: 6px; + font-size: 14px; + font-weight: 500; + text-decoration: none; + border: none; + cursor: pointer; + transition: all 0.2s ease; +} + +.btn-small { + padding: 4px 8px; + font-size: 12px; +} + +.btn-primary { + background-color: rgb(var(--color-primary)); + color: white; +} + +.btn-primary:hover { + background-color: rgb(var(--color-primary-hover)); +} + +.btn-secondary { + background-color: rgb(var(--color-secondary)); + color: white; +} + +.btn-secondary:hover { + background-color: rgb(var(--color-secondary-hover)); +} + +.btn-success { + background-color: rgb(var(--color-success)); + color: white; +} + +.btn-warning { + background-color: rgb(var(--color-warning)); + color: white; +} + +.btn-danger { + background-color: rgb(var(--color-danger)); + color: white; +} + +.btn-danger:hover { + background-color: rgb(var(--color-danger-hover)); +} + +/* Tables */ +table { + width: 100%; + border-spacing: 0; + border: 1px solid rgb(var(--color-gray-700)); + border-bottom-width: 0; + border-radius: 8px; + overflow: hidden; +} + +td { + border: 0 solid rgb(var(--color-gray-700)); + border-bottom-width: 1px; + padding: 12px 16px; +} + +thead td { + text-transform: uppercase; + color: rgb(var(--color-gray-500) / var(--tw-text-opacity)); + font-size: 12px; + letter-spacing: 0.08em; + font-weight: 600; + background-color: rgb(var(--color-gray-800)); +} + +tbody tr:hover { + background-color: rgb(var(--color-gray-800)); +} + +/* Client display elements */ +.client-id { + font-family: "SF Mono", SFMono-Regular, ui-monospace, "DejaVu Sans Mono", + Menlo, Consolas, monospace; + font-size: 12px; + background-color: rgb(var(--color-gray-800)); + padding: 2px 6px; + border-radius: 4px; + color: rgb(var(--color-gray-200)); +} + +.redirect-uri { + font-size: 14px; + color: rgb(var(--color-gray-200)); + word-break: break-all; +} + +.status-active { + color: rgb(var(--color-success)); + font-weight: 500; +} + +.status-inactive { + color: rgb(var(--color-gray-500)); + font-weight: 500; +} + +.text-muted { + color: rgb(var(--color-gray-500)); +} + +/* Empty state */ +.empty-state { + text-align: center; + padding: 60px 20px; + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 8px; + background-color: rgb(var(--color-gray-800) / 0.5); +} + +.empty-state h3 { + font-size: 1.25rem; + font-weight: 600; + margin-bottom: 0.5rem; + color: rgb(var(--color-gray-200)); +} + +.empty-state p { + color: rgb(var(--color-gray-500)); + margin-bottom: 1.5rem; +} + +/* Forms */ +.form-container { + max-width: 600px; + margin: 0 auto; +} + +.form-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 2rem; +} + +.form-header h2 { + font-size: 1.5rem; + font-weight: 600; + margin: 0; +} + +.client-form { + background-color: rgb(var(--color-gray-800) / 0.5); + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 8px; + padding: 24px; + margin-bottom: 2rem; +} + +.form-group { + margin-bottom: 1.5rem; +} + +.form-group:last-child { + margin-bottom: 0; +} + +.form-group label { + display: block; + font-weight: 500; + margin-bottom: 0.5rem; + color: rgb(var(--color-gray-200)); +} + +.required { + color: rgb(var(--color-danger)); +} + +.form-input { + width: 100%; + padding: 10px 12px; + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 6px; + background-color: rgb(var(--color-gray-900)); + color: rgb(var(--color-gray-200)); + font-size: 14px; +} + +.form-input:focus { + outline: none; + border-color: rgb(var(--color-primary)); + box-shadow: 0 0 0 3px rgb(var(--color-primary) / 0.1); +} + +.form-input-readonly { + background-color: rgb(var(--color-gray-800)); + color: rgb(var(--color-gray-500)); +} + +.form-help { + font-size: 12px; + color: rgb(var(--color-gray-500)); + margin-top: 0.25rem; +} + +.form-actions { + display: flex; + gap: 1rem; + margin-top: 2rem; + padding-top: 1.5rem; + border-top: 1px solid rgb(var(--color-gray-700)); +} + +/* Alerts */ +.alert { + padding: 12px 16px; + border-radius: 6px; + margin-bottom: 1.5rem; + font-size: 14px; +} + +.alert-success { + background-color: rgb(var(--color-success) / 0.1); + border: 1px solid rgb(var(--color-success) / 0.3); + color: rgb(var(--color-success)); +} + +.alert-error { + background-color: rgb(var(--color-danger) / 0.1); + border: 1px solid rgb(var(--color-danger) / 0.3); + color: rgb(var(--color-danger)); +} + +/* Secret display */ +.secret-display { + background-color: rgb(var(--color-gray-800) / 0.5); + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 8px; + padding: 20px; + margin-bottom: 2rem; +} + +.secret-display h3 { + font-size: 1.125rem; + font-weight: 600; + margin-bottom: 0.5rem; + color: rgb(var(--color-gray-200)); +} + +.warning { + color: rgb(var(--color-warning)); + font-weight: 500; + margin-bottom: 1rem; +} + +.secret-field { + display: flex; + gap: 0.5rem; +} + +.secret-input { + flex: 1; + padding: 10px 12px; + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 6px; + background-color: rgb(var(--color-gray-900)); + color: rgb(var(--color-gray-200)); + font-family: "SF Mono", SFMono-Regular, ui-monospace, "DejaVu Sans Mono", + Menlo, Consolas, monospace; + font-size: 12px; +} + +/* Client info */ +.client-info { + background-color: rgb(var(--color-gray-800) / 0.5); + border: 1px solid rgb(var(--color-gray-700)); + border-radius: 8px; + padding: 20px; +} + +.client-info h3 { + font-size: 1.125rem; + font-weight: 600; + margin-bottom: 1rem; + color: rgb(var(--color-gray-200)); +} + +.client-info dl { + display: grid; + grid-template-columns: auto 1fr; + gap: 0.5rem 1rem; + border: none; + border-radius: 0; + padding: 0; +} + +.client-info dt { + font-weight: 600; + color: rgb(var(--color-gray-400)); + border: none; + padding: 0; +} + +.client-info dd { + color: rgb(var(--color-gray-200)); + border: none; + padding: 0; +} + +.client-info code { + font-family: "SF Mono", SFMono-Regular, ui-monospace, "DejaVu Sans Mono", + Menlo, Consolas, monospace; + font-size: 12px; + background-color: rgb(var(--color-gray-800)); + padding: 2px 6px; + border-radius: 4px; + color: rgb(var(--color-gray-200)); +} + +/* Responsive design */ +@media (max-width: 768px) { + .header-actions { + flex-direction: column; + align-items: stretch; + gap: 1rem; + } + + .form-header { + flex-direction: column; + align-items: stretch; + gap: 1rem; + } + + .form-actions { + flex-direction: column; + } + + .secret-field { + flex-direction: column; + } + + table { + font-size: 14px; + } + + td { + padding: 8px 12px; + } + + .client-id { + font-size: 10px; + } +} \ No newline at end of file diff --git a/cmd/tsidp/ui.go b/cmd/tsidp/ui.go new file mode 100644 index 0000000000000..d37b64990cac8 --- /dev/null +++ b/cmd/tsidp/ui.go @@ -0,0 +1,325 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "bytes" + _ "embed" + "html/template" + "log" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "tailscale.com/util/rands" +) + +//go:embed ui-header.html +var headerHTML string + +//go:embed ui-list.html +var listHTML string + +//go:embed ui-edit.html +var editHTML string + +//go:embed ui-style.css +var styleCSS string + +var headerTmpl = template.Must(template.New("header").Parse(headerHTML)) +var listTmpl = template.Must(headerTmpl.New("list").Parse(listHTML)) +var editTmpl = template.Must(headerTmpl.New("edit").Parse(editHTML)) + +var processStart = time.Now() + +func (s *idpServer) handleUI(w http.ResponseWriter, r *http.Request) { + if isFunnelRequest(r) { + http.Error(w, "tsidp: UI not available over Funnel", http.StatusNotFound) + return + } + + switch r.URL.Path { + case "/": + s.handleClientsList(w, r) + return + case "/new": + s.handleNewClient(w, r) + return + case "/style.css": + http.ServeContent(w, r, "ui-style.css", processStart, strings.NewReader(styleCSS)) + return + } + + if strings.HasPrefix(r.URL.Path, "/edit/") { + s.handleEditClient(w, r) + return + } + + http.Error(w, "tsidp: not found", http.StatusNotFound) +} + +func (s *idpServer) handleClientsList(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + clients := make([]clientDisplayData, 0, len(s.funnelClients)) + for _, c := range s.funnelClients { + clients = append(clients, clientDisplayData{ + ID: c.ID, + Name: c.Name, + RedirectURI: c.RedirectURI, + HasSecret: c.Secret != "", + }) + } + s.mu.Unlock() + + sort.Slice(clients, func(i, j int) bool { + if clients[i].Name != clients[j].Name { + return clients[i].Name < clients[j].Name + } + return clients[i].ID < clients[j].ID + }) + + var buf bytes.Buffer + if err := listTmpl.Execute(&buf, clients); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + buf.WriteTo(w) +} + +func (s *idpServer) handleNewClient(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + if err := s.renderClientForm(w, clientDisplayData{IsNew: true}); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + if r.Method == "POST" { + if err := r.ParseForm(); err != nil { + http.Error(w, "Failed to parse form", http.StatusBadRequest) + return + } + + name := strings.TrimSpace(r.FormValue("name")) + redirectURI := strings.TrimSpace(r.FormValue("redirect_uri")) + + baseData := clientDisplayData{ + IsNew: true, + Name: name, + RedirectURI: redirectURI, + } + + if errMsg := validateRedirectURI(redirectURI); errMsg != "" { + s.renderFormError(w, baseData, errMsg) + return + } + + clientID := rands.HexString(32) + clientSecret := rands.HexString(64) + newClient := funnelClient{ + ID: clientID, + Secret: clientSecret, + Name: name, + RedirectURI: redirectURI, + } + + s.mu.Lock() + if s.funnelClients == nil { + s.funnelClients = make(map[string]*funnelClient) + } + s.funnelClients[clientID] = &newClient + err := s.storeFunnelClientsLocked() + s.mu.Unlock() + + if err != nil { + log.Printf("could not write funnel clients db: %v", err) + s.renderFormError(w, baseData, "Failed to save client") + return + } + + successData := clientDisplayData{ + ID: clientID, + Name: name, + RedirectURI: redirectURI, + Secret: clientSecret, + IsNew: true, + } + s.renderFormSuccess(w, successData, "Client created successfully! Save the client secret - it won't be shown again.") + return + } + + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) +} + +func (s *idpServer) handleEditClient(w http.ResponseWriter, r *http.Request) { + clientID := strings.TrimPrefix(r.URL.Path, "/edit/") + if clientID == "" { + http.Error(w, "Client ID required", http.StatusBadRequest) + return + } + + s.mu.Lock() + client, exists := s.funnelClients[clientID] + s.mu.Unlock() + + if !exists { + http.Error(w, "Client not found", http.StatusNotFound) + return + } + + if r.Method == "GET" { + data := createEditBaseData(client, client.Name, client.RedirectURI) + if err := s.renderClientForm(w, data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + if r.Method == "POST" { + action := r.FormValue("action") + + if action == "delete" { + s.mu.Lock() + delete(s.funnelClients, clientID) + err := s.storeFunnelClientsLocked() + s.mu.Unlock() + + if err != nil { + log.Printf("could not write funnel clients db: %v", err) + s.mu.Lock() + s.funnelClients[clientID] = client + s.mu.Unlock() + + baseData := createEditBaseData(client, client.Name, client.RedirectURI) + s.renderFormError(w, baseData, "Failed to delete client. Please try again.") + return + } + + http.Redirect(w, r, "/", http.StatusSeeOther) + return + } + + if action == "regenerate_secret" { + newSecret := rands.HexString(64) + s.mu.Lock() + s.funnelClients[clientID].Secret = newSecret + err := s.storeFunnelClientsLocked() + s.mu.Unlock() + + baseData := createEditBaseData(client, client.Name, client.RedirectURI) + baseData.HasSecret = true + + if err != nil { + log.Printf("could not write funnel clients db: %v", err) + s.renderFormError(w, baseData, "Failed to regenerate secret") + return + } + + baseData.Secret = newSecret + s.renderFormSuccess(w, baseData, "New client secret generated! Save it - it won't be shown again.") + return + } + + if err := r.ParseForm(); err != nil { + http.Error(w, "Failed to parse form", http.StatusBadRequest) + return + } + + name := strings.TrimSpace(r.FormValue("name")) + redirectURI := strings.TrimSpace(r.FormValue("redirect_uri")) + baseData := createEditBaseData(client, name, redirectURI) + + if errMsg := validateRedirectURI(redirectURI); errMsg != "" { + s.renderFormError(w, baseData, errMsg) + return + } + + s.mu.Lock() + s.funnelClients[clientID].Name = name + s.funnelClients[clientID].RedirectURI = redirectURI + err := s.storeFunnelClientsLocked() + s.mu.Unlock() + + if err != nil { + log.Printf("could not write funnel clients db: %v", err) + s.renderFormError(w, baseData, "Failed to update client") + return + } + + s.renderFormSuccess(w, baseData, "Client updated successfully!") + return + } + + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) +} + +type clientDisplayData struct { + ID string + Name string + RedirectURI string + Secret string + HasSecret bool + IsNew bool + IsEdit bool + Success string + Error string +} + +func (s *idpServer) renderClientForm(w http.ResponseWriter, data clientDisplayData) error { + var buf bytes.Buffer + if err := editTmpl.Execute(&buf, data); err != nil { + return err + } + if _, err := buf.WriteTo(w); err != nil { + return err + } + return nil +} + +func (s *idpServer) renderFormError(w http.ResponseWriter, data clientDisplayData, errorMsg string) { + data.Error = errorMsg + if err := s.renderClientForm(w, data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func (s *idpServer) renderFormSuccess(w http.ResponseWriter, data clientDisplayData, successMsg string) { + data.Success = successMsg + if err := s.renderClientForm(w, data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func createEditBaseData(client *funnelClient, name, redirectURI string) clientDisplayData { + return clientDisplayData{ + ID: client.ID, + Name: name, + RedirectURI: redirectURI, + HasSecret: client.Secret != "", + IsEdit: true, + } +} + +func validateRedirectURI(redirectURI string) string { + if redirectURI == "" { + return "Redirect URI is required" + } + + u, err := url.Parse(redirectURI) + if err != nil { + return "Invalid URL format" + } + + if u.Scheme != "http" && u.Scheme != "https" { + return "Redirect URI must be a valid HTTP or HTTPS URL" + } + + if u.Host == "" { + return "Redirect URI must include a valid host" + } + + return "" +} diff --git a/control/controlclient/auto.go b/control/controlclient/auto.go index e0168c19db6c0..e6335e54d251b 100644 --- a/control/controlclient/auto.go +++ b/control/controlclient/auto.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "tailscale.com/health" "tailscale.com/logtail/backoff" "tailscale.com/net/sockstats" "tailscale.com/tailcfg" @@ -198,7 +199,11 @@ func NewNoStart(opts Options) (_ *Auto, err error) { c.mapCtx, c.mapCancel = context.WithCancel(context.Background()) c.mapCtx = sockstats.WithSockStats(c.mapCtx, sockstats.LabelControlClientAuto, opts.Logf) - c.unregisterHealthWatch = opts.HealthTracker.RegisterWatcher(direct.ReportHealthChange) + c.unregisterHealthWatch = opts.HealthTracker.RegisterWatcher(func(c health.Change) { + if c.WarnableChanged { + direct.ReportWarnableChange(c.Warnable, c.UnhealthyState) + } + }) return c, nil } diff --git a/control/controlclient/controlclient_test.go b/control/controlclient/controlclient_test.go index f8882a4e796ca..792c26955e5d1 100644 --- a/control/controlclient/controlclient_test.go +++ b/control/controlclient/controlclient_test.go @@ -4,13 +4,35 @@ package controlclient import ( + "context" + "crypto/tls" "errors" + "flag" "fmt" "io" + "net" + "net/http" + "net/netip" + "net/url" "reflect" "slices" + "sync/atomic" "testing" + "time" + "tailscale.com/control/controlknobs" + "tailscale.com/health" + "tailscale.com/net/bakedroots" + "tailscale.com/net/connectproxy" + "tailscale.com/net/netmon" + "tailscale.com/net/tsdial" + "tailscale.com/tailcfg" + "tailscale.com/tstest" + "tailscale.com/tstest/integration/testcontrol" + "tailscale.com/tstest/tlstest" + "tailscale.com/tstime" + "tailscale.com/types/key" + "tailscale.com/types/logger" "tailscale.com/types/netmap" "tailscale.com/types/persist" ) @@ -188,3 +210,206 @@ func isRetryableErrorForTest(err error) bool { } return false } + +var liveNetworkTest = flag.Bool("live-network-test", false, "run live network tests") + +func TestDirectProxyManual(t *testing.T) { + if !*liveNetworkTest { + t.Skip("skipping without --live-network-test") + } + + dialer := &tsdial.Dialer{} + dialer.SetNetMon(netmon.NewStatic()) + + opts := Options{ + Persist: persist.Persist{}, + GetMachinePrivateKey: func() (key.MachinePrivate, error) { + return key.NewMachine(), nil + }, + ServerURL: "https://controlplane.tailscale.com", + Clock: tstime.StdClock{}, + Hostinfo: &tailcfg.Hostinfo{ + BackendLogID: "test-backend-log-id", + }, + DiscoPublicKey: key.NewDisco().Public(), + Logf: t.Logf, + HealthTracker: &health.Tracker{}, + PopBrowserURL: func(url string) { + t.Logf("PopBrowserURL: %q", url) + }, + Dialer: dialer, + ControlKnobs: &controlknobs.Knobs{}, + } + d, err := NewDirect(opts) + if err != nil { + t.Fatalf("NewDirect: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + url, err := d.TryLogin(ctx, LoginEphemeral) + if err != nil { + t.Fatalf("TryLogin: %v", err) + } + t.Logf("URL: %q", url) +} + +func TestHTTPSNoProxy(t *testing.T) { testHTTPS(t, false) } + +// TestTLSWithProxy verifies we can connect to the control plane via +// an HTTPS proxy. +func TestHTTPSWithProxy(t *testing.T) { testHTTPS(t, true) } + +func testHTTPS(t *testing.T, withProxy bool) { + bakedroots.ResetForTest(t, tlstest.TestRootCA()) + + controlLn, err := tls.Listen("tcp", "127.0.0.1:0", tlstest.ControlPlane.ServerTLSConfig()) + if err != nil { + t.Fatal(err) + } + defer controlLn.Close() + + proxyLn, err := tls.Listen("tcp", "127.0.0.1:0", tlstest.ProxyServer.ServerTLSConfig()) + if err != nil { + t.Fatal(err) + } + defer proxyLn.Close() + + const requiredAuthKey = "hunter2" + const someUsername = "testuser" + const somePassword = "testpass" + + testControl := &testcontrol.Server{ + Logf: tstest.WhileTestRunningLogger(t), + RequireAuthKey: requiredAuthKey, + } + controlSrv := &http.Server{ + Handler: testControl, + ErrorLog: logger.StdLogger(t.Logf), + } + go controlSrv.Serve(controlLn) + + const fakeControlIP = "1.2.3.4" + const fakeProxyIP = "5.6.7.8" + + dialer := &tsdial.Dialer{} + dialer.SetNetMon(netmon.NewStatic()) + dialer.SetSystemDialerForTest(func(ctx context.Context, network, addr string) (net.Conn, error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("SplitHostPort(%q): %v", addr, err) + } + var d net.Dialer + if host == fakeControlIP { + return d.DialContext(ctx, network, controlLn.Addr().String()) + } + if host == fakeProxyIP { + return d.DialContext(ctx, network, proxyLn.Addr().String()) + } + return nil, fmt.Errorf("unexpected dial to %q", addr) + }) + + opts := Options{ + Persist: persist.Persist{}, + GetMachinePrivateKey: func() (key.MachinePrivate, error) { + return key.NewMachine(), nil + }, + AuthKey: requiredAuthKey, + ServerURL: "https://controlplane.tstest", + Clock: tstime.StdClock{}, + Hostinfo: &tailcfg.Hostinfo{ + BackendLogID: "test-backend-log-id", + }, + DiscoPublicKey: key.NewDisco().Public(), + Logf: t.Logf, + HealthTracker: &health.Tracker{}, + PopBrowserURL: func(url string) { + t.Logf("PopBrowserURL: %q", url) + }, + Dialer: dialer, + } + d, err := NewDirect(opts) + if err != nil { + t.Fatalf("NewDirect: %v", err) + } + + d.dnsCache.LookupIPForTest = func(ctx context.Context, host string) ([]netip.Addr, error) { + switch host { + case "controlplane.tstest": + return []netip.Addr{netip.MustParseAddr(fakeControlIP)}, nil + case "proxy.tstest": + if !withProxy { + t.Errorf("unexpected DNS lookup for %q with proxy disabled", host) + return nil, fmt.Errorf("unexpected DNS lookup for %q", host) + } + return []netip.Addr{netip.MustParseAddr(fakeProxyIP)}, nil + } + t.Errorf("unexpected DNS query for %q", host) + return []netip.Addr{}, nil + } + + var proxyReqs atomic.Int64 + if withProxy { + d.httpc.Transport.(*http.Transport).Proxy = func(req *http.Request) (*url.URL, error) { + t.Logf("using proxy for %q", req.URL) + u := &url.URL{ + Scheme: "https", + Host: "proxy.tstest:443", + User: url.UserPassword(someUsername, somePassword), + } + return u, nil + } + + connectProxy := &http.Server{ + Handler: connectProxyTo(t, "controlplane.tstest:443", controlLn.Addr().String(), &proxyReqs), + } + go connectProxy.Serve(proxyLn) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + url, err := d.TryLogin(ctx, LoginEphemeral) + if err != nil { + t.Fatalf("TryLogin: %v", err) + } + if url != "" { + t.Errorf("got URL %q, want empty", url) + } + + if withProxy { + if got, want := proxyReqs.Load(), int64(1); got != want { + t.Errorf("proxy CONNECT requests = %d; want %d", got, want) + } + } +} + +func connectProxyTo(t testing.TB, target, backendAddrPort string, reqs *atomic.Int64) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI != target { + t.Errorf("invalid CONNECT request to %q; want %q", r.RequestURI, target) + http.Error(w, "bad target", http.StatusBadRequest) + return + } + + r.Header.Set("Authorization", r.Header.Get("Proxy-Authorization")) // for the BasicAuth method. kinda trashy. + user, pass, ok := r.BasicAuth() + if !ok || user != "testuser" || pass != "testpass" { + t.Errorf("invalid CONNECT auth %q:%q; want %q:%q", user, pass, "testuser", "testpass") + http.Error(w, "bad auth", http.StatusUnauthorized) + return + } + + (&connectproxy.Handler{ + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + var d net.Dialer + c, err := d.DialContext(ctx, network, backendAddrPort) + if err == nil { + reqs.Add(1) + } + return c, err + }, + Logf: t.Logf, + }).ServeHTTP(w, r) + }) +} diff --git a/control/controlclient/direct.go b/control/controlclient/direct.go index ac799e2d916dc..4c9b04ce9b114 100644 --- a/control/controlclient/direct.go +++ b/control/controlclient/direct.go @@ -16,7 +16,6 @@ import ( "net" "net/http" "net/netip" - "net/url" "os" "reflect" "runtime" @@ -240,10 +239,6 @@ func NewDirect(opts Options) (*Direct, error) { opts.ControlKnobs = &controlknobs.Knobs{} } opts.ServerURL = strings.TrimRight(opts.ServerURL, "/") - serverURL, err := url.Parse(opts.ServerURL) - if err != nil { - return nil, err - } if opts.Clock == nil { opts.Clock = tstime.StdClock{} } @@ -273,7 +268,7 @@ func NewDirect(opts Options) (*Direct, error) { tr := http.DefaultTransport.(*http.Transport).Clone() tr.Proxy = tshttpproxy.ProxyFromEnvironment tshttpproxy.SetTransportGetProxyConnectHeader(tr) - tr.TLSClientConfig = tlsdial.Config(serverURL.Hostname(), opts.HealthTracker, tr.TLSClientConfig) + tr.TLSClientConfig = tlsdial.Config(opts.HealthTracker, tr.TLSClientConfig) var dialFunc netx.DialFunc dialFunc, interceptedDial = makeScreenTimeDetectingDialFunc(opts.Dialer.SystemDial) tr.DialContext = dnscache.Dialer(dialFunc, dnsCache) @@ -1623,9 +1618,9 @@ func postPingResult(start time.Time, logf logger.Logf, c *http.Client, pr *tailc return nil } -// ReportHealthChange reports to the control plane a change to this node's +// ReportWarnableChange reports to the control plane a change to this node's // health. w must be non-nil. us can be nil to indicate a healthy state for w. -func (c *Direct) ReportHealthChange(w *health.Warnable, us *health.UnhealthyState) { +func (c *Direct) ReportWarnableChange(w *health.Warnable, us *health.UnhealthyState) { if w == health.NetworkStatusWarnable || w == health.IPNStateWarnable || w == health.LoginStateWarnable { // We don't report these. These include things like the network is down // (in which case we can't report anyway) or the user wanted things diff --git a/control/controlclient/map.go b/control/controlclient/map.go index 3173040fe31d8..22cea5acaa2f7 100644 --- a/control/controlclient/map.go +++ b/control/controlclient/map.go @@ -6,7 +6,10 @@ package controlclient import ( "cmp" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" + "io" "maps" "net" "reflect" @@ -87,6 +90,7 @@ type mapSession struct { lastDomain string lastDomainAuditLogID string lastHealth []string + lastDisplayMessages map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage lastPopBrowserURL string lastTKAInfo *tailcfg.TKAInfo lastNetmapSummary string // from NetworkMap.VeryConcise @@ -409,6 +413,21 @@ func (ms *mapSession) updateStateFromResponse(resp *tailcfg.MapResponse) { if resp.Health != nil { ms.lastHealth = resp.Health } + if resp.DisplayMessages != nil { + if v, ok := resp.DisplayMessages["*"]; ok && v == nil { + ms.lastDisplayMessages = nil + } + for k, v := range resp.DisplayMessages { + if k == "*" { + continue + } + if v != nil { + mak.Set(&ms.lastDisplayMessages, k, *v) + } else { + delete(ms.lastDisplayMessages, k) + } + } + } if resp.TKAInfo != nil { ms.lastTKAInfo = resp.TKAInfo } @@ -828,6 +847,21 @@ func (ms *mapSession) sortedPeers() []tailcfg.NodeView { func (ms *mapSession) netmap() *netmap.NetworkMap { peerViews := ms.sortedPeers() + var msgs map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage + if len(ms.lastDisplayMessages) != 0 { + msgs = ms.lastDisplayMessages + } else if len(ms.lastHealth) > 0 { + // Convert all ms.lastHealth to the new [netmap.NetworkMap.DisplayMessages] + for _, h := range ms.lastHealth { + id := "health-" + strhash(h) // Unique ID in case there is more than one health message + mak.Set(&msgs, tailcfg.DisplayMessageID(id), tailcfg.DisplayMessage{ + Title: "Coordination server reports an issue", + Severity: tailcfg.SeverityMedium, + Text: "The coordination server is reporting a health issue: " + h, + }) + } + } + nm := &netmap.NetworkMap{ NodeKey: ms.publicNodeKey, PrivateKey: ms.privateNodeKey, @@ -842,7 +876,7 @@ func (ms *mapSession) netmap() *netmap.NetworkMap { SSHPolicy: ms.lastSSHPolicy, CollectServices: ms.collectServices, DERPMap: ms.lastDERPMap, - ControlHealth: ms.lastHealth, + DisplayMessages: msgs, TKAEnabled: ms.lastTKAInfo != nil && !ms.lastTKAInfo.Disabled, } @@ -868,5 +902,12 @@ func (ms *mapSession) netmap() *netmap.NetworkMap { if DevKnob.ForceProxyDNS() { nm.DNS.Proxied = true } + return nm } + +func strhash(h string) string { + s := sha256.New() + io.WriteString(s, h) + return hex.EncodeToString(s.Sum(nil)) +} diff --git a/control/controlclient/map_test.go b/control/controlclient/map_test.go index ccc57ae2b86a8..7e42f6f6a8b25 100644 --- a/control/controlclient/map_test.go +++ b/control/controlclient/map_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "net/netip" "reflect" "strings" @@ -15,6 +16,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "go4.org/mem" "tailscale.com/control/controlknobs" "tailscale.com/health" @@ -1138,8 +1140,190 @@ func BenchmarkMapSessionDelta(b *testing.B) { } } +// TestNetmapDisplayMessage checks that the various diff operations +// (add/update/delete/clear) for [tailcfg.DisplayMessage] in a +// [tailcfg.MapResponse] work as expected. +func TestNetmapDisplayMessage(t *testing.T) { + type test struct { + name string + initialState *tailcfg.MapResponse + mapResponse tailcfg.MapResponse + wantMessages map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage + } + + tests := []test{ + { + name: "basic-set", + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "test-message": { + Title: "Testing", + Text: "This is a test message", + Severity: tailcfg.SeverityHigh, + ImpactsConnectivity: true, + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "https://www.example.com", + Label: "Learn more", + }, + }, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-message": { + Title: "Testing", + Text: "This is a test message", + Severity: tailcfg.SeverityHigh, + ImpactsConnectivity: true, + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "https://www.example.com", + Label: "Learn more", + }, + }, + }, + }, + { + name: "delete-one", + initialState: &tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": nil, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "message-b": { + Title: "Message B", + }, + }, + }, + { + name: "update-one", + initialState: &tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A updated", + }, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A updated", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + { + name: "add-one", + initialState: &tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + }, + }, + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-b": { + Title: "Message B", + }, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + { + name: "delete-all", + initialState: &tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "*": nil, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{}, + }, + { + name: "delete-all-and-add", + initialState: &tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "message-a": { + Title: "Message A", + }, + "message-b": { + Title: "Message B", + }, + }, + }, + mapResponse: tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "*": nil, + "message-c": { + Title: "Message C", + }, + }, + }, + wantMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "message-c": { + Title: "Message C", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ms := newTestMapSession(t, nil) + + if test.initialState != nil { + ms.netmapForResponse(test.initialState) + } + + nm := ms.netmapForResponse(&test.mapResponse) + + if diff := cmp.Diff(test.wantMessages, nm.DisplayMessages, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("unexpected warnings (-want +got):\n%s", diff) + } + }) + } +} + // TestNetmapHealthIntegration checks that we get the expected health warnings -// from processing a map response and passing the NetworkMap to a health tracker +// from processing a [tailcfg.MapResponse] containing health messages and passing the +// [netmap.NetworkMap] to a [health.Tracker]. func TestNetmapHealthIntegration(t *testing.T) { ms := newTestMapSession(t, nil) ht := health.Tracker{} @@ -1148,23 +1332,89 @@ func TestNetmapHealthIntegration(t *testing.T) { ht.GotStreamedMapResponse() nm := ms.netmapForResponse(&tailcfg.MapResponse{ - Health: []string{"Test message"}, + Health: []string{ + "Test message", + "Another message", + }, }) - ht.SetControlHealth(nm.ControlHealth) + ht.SetControlHealth(nm.DisplayMessages) - state := ht.CurrentState() - warning, ok := state.Warnings["control-health"] + want := map[health.WarnableCode]health.UnhealthyState{ + "control-health.health-c0719e9a8d5d838d861dc6f675c899d2b309a3a65bb9fe6b11e5afcbf9a2c0b1": { + WarnableCode: "control-health.health-c0719e9a8d5d838d861dc6f675c899d2b309a3a65bb9fe6b11e5afcbf9a2c0b1", + Title: "Coordination server reports an issue", + Severity: health.SeverityMedium, + Text: "The coordination server is reporting a health issue: Test message", + }, + "control-health.health-1dc7017a73a3c55c0d6a8423e3813c7ab6562d9d3064c2ec6ac7822f61b1db9c": { + WarnableCode: "control-health.health-1dc7017a73a3c55c0d6a8423e3813c7ab6562d9d3064c2ec6ac7822f61b1db9c", + Title: "Coordination server reports an issue", + Severity: health.SeverityMedium, + Text: "The coordination server is reporting a health issue: Another message", + }, + } - if !ok { - t.Fatal("no warning found in current state with code 'control-health'") + got := maps.Clone(ht.CurrentState().Warnings) + for k := range got { + if !strings.HasPrefix(string(k), "control-health") { + delete(got, k) + } } - if got, want := warning.Title, "Coordination server reports an issue"; got != want { - t.Errorf("warning.Title = %q, want %q", got, want) + + if d := cmp.Diff(want, got); d != "" { + t.Fatalf("CurrentStatus().Warnings[\"control-health*\"] different than expected (-want +got)\n%s", d) } - if got, want := warning.Severity, health.SeverityMedium; got != want { - t.Errorf("warning.Severity = %s, want %s", got, want) +} + +// TestNetmapDisplayMessageIntegration checks that we get the expected health +// warnings from processing a [tailcfg.MapResponse] that contains DisplayMessages and +// passing the [netmap.NetworkMap] to a [health.Tracker]. +func TestNetmapDisplayMessageIntegration(t *testing.T) { + ms := newTestMapSession(t, nil) + ht := health.Tracker{} + + ht.SetIPNState("NeedsLogin", true) + ht.GotStreamedMapResponse() + baseWarnings := ht.CurrentState().Warnings + + nm := ms.netmapForResponse(&tailcfg.MapResponse{ + DisplayMessages: map[tailcfg.DisplayMessageID]*tailcfg.DisplayMessage{ + "test-message": { + Title: "Testing", + Text: "This is a test message", + Severity: tailcfg.SeverityHigh, + ImpactsConnectivity: true, + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "https://www.example.com", + Label: "Learn more", + }, + }, + }, + }) + ht.SetControlHealth(nm.DisplayMessages) + + state := ht.CurrentState() + + // Ignore warnings that aren't from the netmap + for k := range baseWarnings { + delete(state.Warnings, k) + } + + want := map[health.WarnableCode]health.UnhealthyState{ + "control-health.test-message": { + WarnableCode: "control-health.test-message", + Title: "Testing", + Text: "This is a test message", + Severity: health.SeverityHigh, + ImpactsConnectivity: true, + PrimaryAction: &health.UnhealthyStateAction{ + URL: "https://www.example.com", + Label: "Learn more", + }, + }, } - if got, want := warning.Text, "The coordination server is reporting an health issue: Test message"; got != want { - t.Errorf("warning.Text = %q, want %q", got, want) + + if diff := cmp.Diff(want, state.Warnings); diff != "" { + t.Errorf("unexpected message contents (-want +got):\n%s", diff) } } diff --git a/control/controlhttp/client.go b/control/controlhttp/client.go index 869bcb599c9f3..1bb60d672980d 100644 --- a/control/controlhttp/client.go +++ b/control/controlhttp/client.go @@ -534,7 +534,7 @@ func (a *Dialer) tryURLUpgrade(ctx context.Context, u *url.URL, optAddr netip.Ad // Disable HTTP2, since h2 can't do protocol switching. tr.TLSClientConfig.NextProtos = []string{} tr.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} - tr.TLSClientConfig = tlsdial.Config(a.Hostname, a.HealthTracker, tr.TLSClientConfig) + tr.TLSClientConfig = tlsdial.Config(a.HealthTracker, tr.TLSClientConfig) if !tr.TLSClientConfig.InsecureSkipVerify { panic("unexpected") // should be set by tlsdial.Config } diff --git a/derp/derp.go b/derp/derp.go index 65acd43210234..24c1ca65cfb3c 100644 --- a/derp/derp.go +++ b/derp/derp.go @@ -36,9 +36,13 @@ const ( frameHeaderLen = 1 + 4 // frameType byte + 4 byte length keyLen = 32 maxInfoLen = 1 << 20 - keepAlive = 60 * time.Second ) +// KeepAlive is the minimum frequency at which the DERP server sends +// keep alive frames. The server adds some jitter, so this timing is not +// exact, but 2x this value can be considered a missed keep alive. +const KeepAlive = 60 * time.Second + // ProtocolVersion is bumped whenever there's a wire-incompatible change. // - version 1 (zero on wire): consistent box headers, in use by employee dev nodes a bit // - version 2: received packets have src addrs in frameRecvPacket at beginning diff --git a/derp/derp_client.go b/derp/derp_client.go index 7a646fa517940..69f35db1e2791 100644 --- a/derp/derp_client.go +++ b/derp/derp_client.go @@ -30,7 +30,7 @@ type Client struct { logf logger.Logf nc Conn br *bufio.Reader - meshKey string + meshKey key.DERPMesh canAckPings bool isProber bool @@ -56,7 +56,7 @@ func (f clientOptFunc) update(o *clientOpt) { f(o) } // clientOpt are the options passed to newClient. type clientOpt struct { - MeshKey string + MeshKey key.DERPMesh ServerPub key.NodePublic CanAckPings bool IsProber bool @@ -66,7 +66,7 @@ type clientOpt struct { // access to join the mesh. // // An empty key means to not use a mesh key. -func MeshKey(key string) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.MeshKey = key }) } +func MeshKey(k key.DERPMesh) ClientOpt { return clientOptFunc(func(o *clientOpt) { o.MeshKey = k }) } // IsProber returns a ClientOpt to pass to the DERP server during connect to // declare that this client is a a prober. @@ -165,7 +165,7 @@ type clientInfo struct { // trusted clients. It's required to subscribe to the // connection list & forward packets. It's empty for regular // users. - MeshKey string `json:"meshKey,omitempty"` + MeshKey key.DERPMesh `json:"meshKey,omitempty,omitzero"` // Version is the DERP protocol version that the client was built with. // See the ProtocolVersion const. @@ -179,6 +179,17 @@ type clientInfo struct { IsProber bool `json:",omitempty"` } +// Equal reports if two clientInfo values are equal. +func (c *clientInfo) Equal(other *clientInfo) bool { + if c == nil || other == nil { + return c == other + } + if c.Version != other.Version || c.CanAckPings != other.CanAckPings || c.IsProber != other.IsProber { + return false + } + return c.MeshKey.Equal(other.MeshKey) +} + func (c *Client) sendClientKey() error { msg, err := json.Marshal(clientInfo{ Version: ProtocolVersion, diff --git a/derp/derp_server.go b/derp/derp_server.go index abda9da73a6fc..bd67e7eeca22f 100644 --- a/derp/derp_server.go +++ b/derp/derp_server.go @@ -134,7 +134,7 @@ type Server struct { publicKey key.NodePublic logf logger.Logf memSys0 uint64 // runtime.MemStats.Sys at start (or early-ish) - meshKey string + meshKey key.DERPMesh limitedLogf logger.Logf metaCert []byte // the encoded x509 cert to send after LetsEncrypt cert+intermediate dupPolicy dupPolicy @@ -464,8 +464,13 @@ func genDroppedCounters() { // amongst themselves. // // It must be called before serving begins. -func (s *Server) SetMeshKey(v string) { - s.meshKey = v +func (s *Server) SetMeshKey(v string) error { + k, err := key.ParseDERPMesh(v) + if err != nil { + return err + } + s.meshKey = k + return nil } // SetVerifyClients sets whether this DERP server verifies clients through tailscaled. @@ -506,10 +511,10 @@ func (s *Server) SetTCPWriteTimeout(d time.Duration) { } // HasMeshKey reports whether the server is configured with a mesh key. -func (s *Server) HasMeshKey() bool { return s.meshKey != "" } +func (s *Server) HasMeshKey() bool { return !s.meshKey.IsZero() } // MeshKey returns the configured mesh key, if any. -func (s *Server) MeshKey() string { return s.meshKey } +func (s *Server) MeshKey() key.DERPMesh { return s.meshKey } // PrivateKey returns the server's private key. func (s *Server) PrivateKey() key.NodePrivate { return s.privateKey } @@ -1355,7 +1360,15 @@ func (c *sclient) requestMeshUpdate() { // isMeshPeer reports whether the client is a trusted mesh peer // node in the DERP region. func (s *Server) isMeshPeer(info *clientInfo) bool { - return info != nil && info.MeshKey != "" && info.MeshKey == s.meshKey + // Compare mesh keys in constant time to prevent timing attacks. + // Since mesh keys are a fixed length, we don’t need to be concerned + // about timing attacks on client mesh keys that are the wrong length. + // See https://github.com/tailscale/corp/issues/28720 + if info == nil || info.MeshKey.IsZero() { + return false + } + + return s.meshKey.Equal(info.MeshKey) } // verifyClient checks whether the client is allowed to connect to the derper, @@ -1776,7 +1789,7 @@ func (c *sclient) sendLoop(ctx context.Context) error { defer c.onSendLoopDone() jitter := rand.N(5 * time.Second) - keepAliveTick, keepAliveTickChannel := c.s.clock.NewTicker(keepAlive + jitter) + keepAliveTick, keepAliveTickChannel := c.s.clock.NewTicker(KeepAlive + jitter) defer keepAliveTick.Stop() var werr error // last write error diff --git a/derp/derp_test.go b/derp/derp_test.go index c5a92bafae1dd..9d07e159b4584 100644 --- a/derp/derp_test.go +++ b/derp/derp_test.go @@ -20,6 +20,7 @@ import ( "os" "reflect" "strconv" + "strings" "sync" "testing" "time" @@ -33,21 +34,53 @@ import ( "tailscale.com/tstest" "tailscale.com/types/key" "tailscale.com/types/logger" + "tailscale.com/util/must" ) func TestClientInfoUnmarshal(t *testing.T) { - for i, in := range []string{ - `{"Version":5,"MeshKey":"abc"}`, - `{"version":5,"meshKey":"abc"}`, + for i, in := range map[string]struct { + json string + want *clientInfo + wantErr string + }{ + "empty": { + json: `{}`, + want: &clientInfo{}, + }, + "valid": { + json: `{"Version":5,"MeshKey":"6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8"}`, + want: &clientInfo{MeshKey: must.Get(key.ParseDERPMesh("6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8")), Version: 5}, + }, + "validLowerMeshKey": { + json: `{"version":5,"meshKey":"6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8"}`, + want: &clientInfo{MeshKey: must.Get(key.ParseDERPMesh("6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8")), Version: 5}, + }, + "invalidMeshKeyToShort": { + json: `{"version":5,"meshKey":"abcdefg"}`, + wantErr: "invalid mesh key", + }, + "invalidMeshKeyToLong": { + json: `{"version":5,"meshKey":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}`, + wantErr: "invalid mesh key", + }, } { - var got clientInfo - if err := json.Unmarshal([]byte(in), &got); err != nil { - t.Fatalf("[%d]: %v", i, err) - } - want := clientInfo{Version: 5, MeshKey: "abc"} - if got != want { - t.Errorf("[%d]: got %+v; want %+v", i, got, want) - } + t.Run(i, func(t *testing.T) { + t.Parallel() + var got clientInfo + err := json.Unmarshal([]byte(in.json), &got) + if in.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), in.wantErr) { + t.Errorf("Unmarshal(%q) = %v, want error containing %q", in.json, err, in.wantErr) + } + return + } + if err != nil { + t.Fatalf("Unmarshal(%q) = %v, want no error", in.json, err) + } + if !got.Equal(in.want) { + t.Errorf("Unmarshal(%q) = %+v, want %+v", in.json, got, in.want) + } + }) } } @@ -511,11 +544,13 @@ func (ts *testServer) close(t *testing.T) error { return nil } +const testMeshKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + func newTestServer(t *testing.T, ctx context.Context) *testServer { t.Helper() logf := logger.WithPrefix(t.Logf, "derp-server: ") s := NewServer(key.NewNode(), logf) - s.SetMeshKey("mesh-key") + s.SetMeshKey(testMeshKey) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -591,8 +626,12 @@ func newRegularClient(t *testing.T, ts *testServer, name string) *testClient { func newTestWatcher(t *testing.T, ts *testServer, name string) *testClient { return newTestClient(t, ts, name, func(nc net.Conn, priv key.NodePrivate, logf logger.Logf) (*Client, error) { + mk, err := key.ParseDERPMesh(testMeshKey) + if err != nil { + return nil, err + } brw := bufio.NewReadWriter(bufio.NewReader(nc), bufio.NewWriter(nc)) - c, err := NewClient(priv, nc, brw, logf, MeshKey("mesh-key")) + c, err := NewClient(priv, nc, brw, logf, MeshKey(mk)) if err != nil { return nil, err } @@ -1627,3 +1666,96 @@ func TestGetPerClientSendQueueDepth(t *testing.T) { }) } } + +func TestSetMeshKey(t *testing.T) { + for name, tt := range map[string]struct { + key string + want key.DERPMesh + wantErr bool + }{ + "clobber": { + key: testMeshKey, + wantErr: false, + }, + "invalid": { + key: "badf00d", + wantErr: true, + }, + } { + t.Run(name, func(t *testing.T) { + s := &Server{} + + err := s.SetMeshKey(tt.key) + if tt.wantErr { + if err == nil { + t.Fatalf("expected err") + } + return + } + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + want, err := key.ParseDERPMesh(tt.key) + if err != nil { + t.Fatal(err) + } + if !s.meshKey.Equal(want) { + t.Fatalf("got %v, want %v", s.meshKey, want) + } + }) + } +} + +func TestIsMeshPeer(t *testing.T) { + s := &Server{} + err := s.SetMeshKey(testMeshKey) + if err != nil { + t.Fatal(err) + } + for name, tt := range map[string]struct { + want bool + meshKey string + wantAllocs float64 + }{ + "nil": { + want: false, + wantAllocs: 0, + }, + "mismatch": { + meshKey: "6d529e9d4ef632d22d4a4214cb49da8f1ba1b72697061fb24e312984c35ec8d8", + want: false, + wantAllocs: 1, + }, + "match": { + meshKey: testMeshKey, + want: true, + wantAllocs: 0, + }, + } { + t.Run(name, func(t *testing.T) { + var got bool + var mKey key.DERPMesh + if tt.meshKey != "" { + mKey, err = key.ParseDERPMesh(tt.meshKey) + if err != nil { + t.Fatalf("ParseDERPMesh(%q) failed: %v", tt.meshKey, err) + } + } + + info := clientInfo{ + MeshKey: mKey, + } + allocs := testing.AllocsPerRun(1, func() { + got = s.isMeshPeer(&info) + }) + if got != tt.want { + t.Fatalf("got %t, want %t: info = %#v", got, tt.want, info) + } + + if allocs != tt.wantAllocs && tt.want { + t.Errorf("%f allocations, want %f", allocs, tt.wantAllocs) + } + }) + } +} diff --git a/derp/derphttp/derphttp_client.go b/derp/derphttp/derphttp_client.go index faa218ca25f0a..704b8175d07c6 100644 --- a/derp/derphttp/derphttp_client.go +++ b/derp/derphttp/derphttp_client.go @@ -57,7 +57,7 @@ type Client struct { TLSConfig *tls.Config // optional; nil means default HealthTracker *health.Tracker // optional; used if non-nil only DNSCache *dnscache.Resolver // optional; nil means no caching - MeshKey string // optional; for trusted clients + MeshKey key.DERPMesh // optional; for trusted clients IsProber bool // optional; for probers to optional declare themselves as such // WatchConnectionChanges is whether the client wishes to subscribe to @@ -647,7 +647,10 @@ func (c *Client) dialRegion(ctx context.Context, reg *tailcfg.DERPRegion) (net.C } func (c *Client) tlsClient(nc net.Conn, node *tailcfg.DERPNode) *tls.Conn { - tlsConf := tlsdial.Config(c.tlsServerName(node), c.HealthTracker, c.TLSConfig) + tlsConf := tlsdial.Config(c.HealthTracker, c.TLSConfig) + // node is allowed to be nil here, tlsServerName falls back to using the URL + // if node is nil. + tlsConf.ServerName = c.tlsServerName(node) if node != nil { if node.InsecureForTests { tlsConf.InsecureSkipVerify = true diff --git a/derp/derphttp/derphttp_test.go b/derp/derphttp/derphttp_test.go index cfb3676cda16f..bb33e60232357 100644 --- a/derp/derphttp/derphttp_test.go +++ b/derp/derphttp/derphttp_test.go @@ -7,16 +7,23 @@ import ( "bytes" "context" "crypto/tls" + "encoding/json" + "flag" "fmt" + "maps" "net" "net/http" "net/http/httptest" + "slices" + "strings" "sync" "testing" "time" "tailscale.com/derp" "tailscale.com/net/netmon" + "tailscale.com/net/netx" + "tailscale.com/tailcfg" "tailscale.com/types/key" ) @@ -212,6 +219,8 @@ func TestPing(t *testing.T) { } } +const testMeshKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + func newTestServer(t *testing.T, k key.NodePrivate) (serverURL string, s *derp.Server) { s = derp.NewServer(k, t.Logf) httpsrv := &http.Server{ @@ -224,7 +233,7 @@ func newTestServer(t *testing.T, k key.NodePrivate) (serverURL string, s *derp.S t.Fatal(err) } serverURL = "http://" + ln.Addr().String() - s.SetMeshKey("1234") + s.SetMeshKey(testMeshKey) go func() { if err := httpsrv.Serve(ln); err != nil { @@ -243,7 +252,11 @@ func newWatcherClient(t *testing.T, watcherPrivateKey key.NodePrivate, serverToW if err != nil { t.Fatal(err) } - c.MeshKey = "1234" + k, err := key.ParseDERPMesh(testMeshKey) + if err != nil { + t.Fatal(err) + } + c.MeshKey = k return } @@ -292,6 +305,7 @@ func TestBreakWatcherConnRecv(t *testing.T) { defer cancel() watcherChan := make(chan int, 1) + errChan := make(chan error, 1) // Start the watcher thread (which connects to the watched server) wg.Add(1) // To avoid using t.Logf after the test ends. See https://golang.org/issue/40343 @@ -305,8 +319,11 @@ func TestBreakWatcherConnRecv(t *testing.T) { watcherChan <- peers } remove := func(m derp.PeerGoneMessage) { t.Logf("remove: %v", m.Peer.ShortString()); peers-- } + notifyErr := func(err error) { + errChan <- err + } - watcher1.RunWatchConnectionLoop(ctx, serverPrivateKey1.Public(), t.Logf, add, remove) + watcher1.RunWatchConnectionLoop(ctx, serverPrivateKey1.Public(), t.Logf, add, remove, notifyErr) }() timer := time.NewTimer(5 * time.Second) @@ -320,6 +337,10 @@ func TestBreakWatcherConnRecv(t *testing.T) { if peers != 1 { t.Fatal("wrong number of peers added during watcher connection") } + case err := <-errChan: + if !strings.Contains(err.Error(), "use of closed network connection") { + t.Fatalf("expected notifyError connection error to contain 'use of closed network connection', got %v", err) + } case <-timer.C: t.Fatalf("watcher did not process the peer update") } @@ -363,6 +384,7 @@ func TestBreakWatcherConn(t *testing.T) { watcherChan := make(chan int, 1) breakerChan := make(chan bool, 1) + errorChan := make(chan error, 1) // Start the watcher thread (which connects to the watched server) wg.Add(1) // To avoid using t.Logf after the test ends. See https://golang.org/issue/40343 @@ -378,8 +400,11 @@ func TestBreakWatcherConn(t *testing.T) { <-breakerChan } remove := func(m derp.PeerGoneMessage) { t.Logf("remove: %v", m.Peer.ShortString()); peers-- } + notifyError := func(err error) { + errorChan <- err + } - watcher1.RunWatchConnectionLoop(ctx, serverPrivateKey1.Public(), t.Logf, add, remove) + watcher1.RunWatchConnectionLoop(ctx, serverPrivateKey1.Public(), t.Logf, add, remove, notifyError) }() timer := time.NewTimer(5 * time.Second) @@ -393,6 +418,10 @@ func TestBreakWatcherConn(t *testing.T) { if peers != 1 { t.Fatal("wrong number of peers added during watcher connection") } + case err := <-errorChan: + if !strings.Contains(err.Error(), "use of closed network connection") { + t.Fatalf("expected notifyError connection error to contain 'use of closed network connection', got %v", err) + } case <-timer.C: t.Fatalf("watcher did not process the peer update") } @@ -408,6 +437,7 @@ func TestBreakWatcherConn(t *testing.T) { func noopAdd(derp.PeerPresentMessage) {} func noopRemove(derp.PeerGoneMessage) {} +func noopNotifyError(error) {} func TestRunWatchConnectionLoopServeConnect(t *testing.T) { defer func() { testHookWatchLookConnectResult = nil }() @@ -435,7 +465,7 @@ func TestRunWatchConnectionLoopServeConnect(t *testing.T) { } return false } - watcher.RunWatchConnectionLoop(ctx, pub, t.Logf, noopAdd, noopRemove) + watcher.RunWatchConnectionLoop(ctx, pub, t.Logf, noopAdd, noopRemove, noopNotifyError) // Test connecting to the server with a zero value for ignoreServerKey, // so we should always connect. @@ -449,7 +479,7 @@ func TestRunWatchConnectionLoopServeConnect(t *testing.T) { } return false } - watcher.RunWatchConnectionLoop(ctx, key.NodePublic{}, t.Logf, noopAdd, noopRemove) + watcher.RunWatchConnectionLoop(ctx, key.NodePublic{}, t.Logf, noopAdd, noopRemove, noopNotifyError) } // verify that the LocalAddr method doesn't acquire the mutex. @@ -485,3 +515,114 @@ func TestProbe(t *testing.T) { } } } + +func TestNotifyError(t *testing.T) { + defer func() { testHookWatchLookConnectResult = nil }() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + priv := key.NewNode() + serverURL, s := newTestServer(t, priv) + defer s.Close() + + pub := priv.Public() + + // Test early error notification when c.connect fails. + watcher := newWatcherClient(t, priv, serverURL) + watcher.SetURLDialer(netx.DialFunc(func(ctx context.Context, network, addr string) (net.Conn, error) { + t.Helper() + return nil, fmt.Errorf("test error: %s", addr) + })) + defer watcher.Close() + + testHookWatchLookConnectResult = func(err error, wasSelfConnect bool) bool { + t.Helper() + if err == nil { + t.Fatal("expected error connecting to server, got nil") + } + if wasSelfConnect { + t.Error("wanted normal connect; got self connect") + } + return false + } + + errChan := make(chan error, 1) + notifyError := func(err error) { + errChan <- err + } + watcher.RunWatchConnectionLoop(ctx, pub, t.Logf, noopAdd, noopRemove, notifyError) + + select { + case err := <-errChan: + if !strings.Contains(err.Error(), "test") { + t.Errorf("expected test error, got %v", err) + } + case <-ctx.Done(): + t.Fatalf("context done before receiving error: %v", ctx.Err()) + } +} + +var liveNetworkTest = flag.Bool("live-net-tests", false, "run live network tests") + +func TestManualDial(t *testing.T) { + if !*liveNetworkTest { + t.Skip("skipping live network test without --live-net-tests") + } + dm := &tailcfg.DERPMap{} + res, err := http.Get("https://controlplane.tailscale.com/derpmap/default") + if err != nil { + t.Fatalf("fetching DERPMap: %v", err) + } + defer res.Body.Close() + if err := json.NewDecoder(res.Body).Decode(dm); err != nil { + t.Fatalf("decoding DERPMap: %v", err) + } + + region := slices.Sorted(maps.Keys(dm.Regions))[0] + + netMon := netmon.NewStatic() + rc := NewRegionClient(key.NewNode(), t.Logf, netMon, func() *tailcfg.DERPRegion { + return dm.Regions[region] + }) + defer rc.Close() + + if err := rc.Connect(context.Background()); err != nil { + t.Fatalf("rc.Connect: %v", err) + } +} + +func TestURLDial(t *testing.T) { + if !*liveNetworkTest { + t.Skip("skipping live network test without --live-net-tests") + } + dm := &tailcfg.DERPMap{} + res, err := http.Get("https://controlplane.tailscale.com/derpmap/default") + if err != nil { + t.Fatalf("fetching DERPMap: %v", err) + } + defer res.Body.Close() + if err := json.NewDecoder(res.Body).Decode(dm); err != nil { + t.Fatalf("decoding DERPMap: %v", err) + } + + // find a valid target DERP host to test against + var hostname string + for _, reg := range dm.Regions { + for _, node := range reg.Nodes { + if !node.STUNOnly && node.CanPort80 && node.CertName == "" || node.CertName == node.HostName { + hostname = node.HostName + break + } + } + if hostname != "" { + break + } + } + netMon := netmon.NewStatic() + c, err := NewClient(key.NewNode(), "https://"+hostname+"/", t.Logf, netMon) + defer c.Close() + + if err := c.Connect(context.Background()); err != nil { + t.Fatalf("rc.Connect: %v", err) + } +} diff --git a/derp/derphttp/mesh_client.go b/derp/derphttp/mesh_client.go index 66b8c166eeb37..c14a9a7e11111 100644 --- a/derp/derphttp/mesh_client.go +++ b/derp/derphttp/mesh_client.go @@ -31,6 +31,9 @@ var testHookWatchLookConnectResult func(connectError error, wasSelfConnect bool) // This behavior will likely change. Callers should do their own accounting // and dup suppression as needed. // +// If set the notifyError func is called with any error that occurs within the ctx +// main loop connection setup, or the inner loop receiving messages via RecvDetail. +// // infoLogf, if non-nil, is the logger to write periodic status updates about // how many peers are on the server. Error log output is set to the c's logger, // regardless of infoLogf's value. @@ -42,10 +45,11 @@ var testHookWatchLookConnectResult func(connectError error, wasSelfConnect bool) // initialized Client.WatchConnectionChanges to true. // // If the DERP connection breaks and reconnects, remove will be called for all -// previously seen peers, with Reason type PeerGoneReasonSynthetic. Those +// previously seen peers, with Reason type PeerGoneReasonMeshConnBroke. Those // clients are likely still connected and their add message will appear after // reconnect. -func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key.NodePublic, infoLogf logger.Logf, add func(derp.PeerPresentMessage), remove func(derp.PeerGoneMessage)) { +func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key.NodePublic, infoLogf logger.Logf, + add func(derp.PeerPresentMessage), remove func(derp.PeerGoneMessage), notifyError func(error)) { if !c.WatchConnectionChanges { if c.isStarted() { panic("invalid use of RunWatchConnectionLoop on already-started Client without setting Client.RunWatchConnectionLoop") @@ -121,6 +125,10 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key // Make sure we're connected before calling s.ServerPublicKey. _, _, err := c.connect(ctx, "RunWatchConnectionLoop") if err != nil { + logf("mesh connect: %v", err) + if notifyError != nil { + notifyError(err) + } if f := testHookWatchLookConnectResult; f != nil && !f(err, false) { return } @@ -141,6 +149,9 @@ func (c *Client) RunWatchConnectionLoop(ctx context.Context, ignoreServerKey key if err != nil { clear() logf("Recv: %v", err) + if notifyError != nil { + notifyError(err) + } sleep(retryInterval) break } diff --git a/disco/disco.go b/disco/disco.go index 0854eb4c0af5a..1689d2a93da77 100644 --- a/disco/disco.go +++ b/disco/disco.go @@ -42,13 +42,15 @@ const NonceLen = 24 type MessageType byte const ( - TypePing = MessageType(0x01) - TypePong = MessageType(0x02) - TypeCallMeMaybe = MessageType(0x03) - TypeBindUDPRelayEndpoint = MessageType(0x04) - TypeBindUDPRelayEndpointChallenge = MessageType(0x05) - TypeBindUDPRelayEndpointAnswer = MessageType(0x06) - TypeCallMeMaybeVia = MessageType(0x07) + TypePing = MessageType(0x01) + TypePong = MessageType(0x02) + TypeCallMeMaybe = MessageType(0x03) + TypeBindUDPRelayEndpoint = MessageType(0x04) + TypeBindUDPRelayEndpointChallenge = MessageType(0x05) + TypeBindUDPRelayEndpointAnswer = MessageType(0x06) + TypeCallMeMaybeVia = MessageType(0x07) + TypeAllocateUDPRelayEndpointRequest = MessageType(0x08) + TypeAllocateUDPRelayEndpointResponse = MessageType(0x09) ) const v0 = byte(0) @@ -97,6 +99,10 @@ func Parse(p []byte) (Message, error) { return parseBindUDPRelayEndpointAnswer(ver, p) case TypeCallMeMaybeVia: return parseCallMeMaybeVia(ver, p) + case TypeAllocateUDPRelayEndpointRequest: + return parseAllocateUDPRelayEndpointRequest(ver, p) + case TypeAllocateUDPRelayEndpointResponse: + return parseAllocateUDPRelayEndpointResponse(ver, p) default: return nil, fmt.Errorf("unknown message type 0x%02x", byte(t)) } @@ -321,82 +327,291 @@ const ( BindUDPRelayHandshakeStateAnswerReceived ) -// bindUDPRelayEndpointLen is the length of a marshalled BindUDPRelayEndpoint -// message, without the message header. -const bindUDPRelayEndpointLen = BindUDPRelayEndpointChallengeLen +// bindUDPRelayEndpointCommonLen is the length of a marshalled +// [BindUDPRelayEndpointCommon], without the message header. +const bindUDPRelayEndpointCommonLen = 72 + +// BindUDPRelayChallengeLen is the length of the Challenge field carried in +// [BindUDPRelayEndpointChallenge] & [BindUDPRelayEndpointAnswer] messages. +const BindUDPRelayChallengeLen = 32 + +// BindUDPRelayEndpointCommon contains fields that are common across all 3 +// UDP relay handshake message types. All 4 field values are expected to be +// consistent for the lifetime of a handshake besides Challenge, which is +// irrelevant in a [BindUDPRelayEndpoint] message. +type BindUDPRelayEndpointCommon struct { + // VNI is the Geneve header Virtual Network Identifier field value, which + // must match this disco-sealed value upon reception. If they are + // non-matching it indicates the cleartext Geneve header was tampered with + // and/or mangled. + VNI uint32 + // Generation represents the handshake generation. Clients must set a new, + // nonzero value at the start of every handshake. + Generation uint32 + // RemoteKey is the disco key of the remote peer participating over this + // relay endpoint. + RemoteKey key.DiscoPublic + // Challenge is set by the server in a [BindUDPRelayEndpointChallenge] + // message, and expected to be echoed back by the client in a + // [BindUDPRelayEndpointAnswer] message. Its value is irrelevant in a + // [BindUDPRelayEndpoint] message, where it simply serves a padding purpose + // ensuring all handshake messages are equal in size. + Challenge [BindUDPRelayChallengeLen]byte +} + +// encode encodes m in b. b must be at least bindUDPRelayEndpointCommonLen bytes +// long. +func (m *BindUDPRelayEndpointCommon) encode(b []byte) { + binary.BigEndian.PutUint32(b, m.VNI) + b = b[4:] + binary.BigEndian.PutUint32(b, m.Generation) + b = b[4:] + m.RemoteKey.AppendTo(b[:0]) + b = b[key.DiscoPublicRawLen:] + copy(b, m.Challenge[:]) +} + +// decode decodes m from b. +func (m *BindUDPRelayEndpointCommon) decode(b []byte) error { + if len(b) < bindUDPRelayEndpointCommonLen { + return errShort + } + m.VNI = binary.BigEndian.Uint32(b) + b = b[4:] + m.Generation = binary.BigEndian.Uint32(b) + b = b[4:] + m.RemoteKey = key.DiscoPublicFromRaw32(mem.B(b[:key.DiscoPublicRawLen])) + b = b[key.DiscoPublicRawLen:] + copy(m.Challenge[:], b[:BindUDPRelayChallengeLen]) + return nil +} // BindUDPRelayEndpoint is the first messaged transmitted from UDP relay client -// towards UDP relay server as part of the 3-way bind handshake. It is padded to -// match the length of BindUDPRelayEndpointChallenge. This message type is -// currently considered experimental and is not yet tied to a -// tailcfg.CapabilityVersion. +// towards UDP relay server as part of the 3-way bind handshake. type BindUDPRelayEndpoint struct { + BindUDPRelayEndpointCommon } func (m *BindUDPRelayEndpoint) AppendMarshal(b []byte) []byte { - ret, _ := appendMsgHeader(b, TypeBindUDPRelayEndpoint, v0, bindUDPRelayEndpointLen) + ret, d := appendMsgHeader(b, TypeBindUDPRelayEndpoint, v0, bindUDPRelayEndpointCommonLen) + m.BindUDPRelayEndpointCommon.encode(d) return ret } func parseBindUDPRelayEndpoint(ver uint8, p []byte) (m *BindUDPRelayEndpoint, err error) { m = new(BindUDPRelayEndpoint) + err = m.BindUDPRelayEndpointCommon.decode(p) + if err != nil { + return nil, err + } return m, nil } -// BindUDPRelayEndpointChallengeLen is the length of a marshalled -// BindUDPRelayEndpointChallenge message, without the message header. -const BindUDPRelayEndpointChallengeLen = 32 - // BindUDPRelayEndpointChallenge is transmitted from UDP relay server towards // UDP relay client in response to a BindUDPRelayEndpoint message as part of the -// 3-way bind handshake. This message type is currently considered experimental -// and is not yet tied to a tailcfg.CapabilityVersion. +// 3-way bind handshake. type BindUDPRelayEndpointChallenge struct { - Challenge [BindUDPRelayEndpointChallengeLen]byte + BindUDPRelayEndpointCommon } func (m *BindUDPRelayEndpointChallenge) AppendMarshal(b []byte) []byte { - ret, d := appendMsgHeader(b, TypeBindUDPRelayEndpointChallenge, v0, BindUDPRelayEndpointChallengeLen) - copy(d, m.Challenge[:]) + ret, d := appendMsgHeader(b, TypeBindUDPRelayEndpointChallenge, v0, bindUDPRelayEndpointCommonLen) + m.BindUDPRelayEndpointCommon.encode(d) return ret } func parseBindUDPRelayEndpointChallenge(ver uint8, p []byte) (m *BindUDPRelayEndpointChallenge, err error) { - if len(p) < BindUDPRelayEndpointChallengeLen { - return nil, errShort - } m = new(BindUDPRelayEndpointChallenge) - copy(m.Challenge[:], p[:]) + err = m.BindUDPRelayEndpointCommon.decode(p) + if err != nil { + return nil, err + } return m, nil } -// bindUDPRelayEndpointAnswerLen is the length of a marshalled -// BindUDPRelayEndpointAnswer message, without the message header. -const bindUDPRelayEndpointAnswerLen = BindUDPRelayEndpointChallengeLen - // BindUDPRelayEndpointAnswer is transmitted from UDP relay client to UDP relay -// server in response to a BindUDPRelayEndpointChallenge message. This message -// type is currently considered experimental and is not yet tied to a -// tailcfg.CapabilityVersion. +// server in response to a BindUDPRelayEndpointChallenge message. type BindUDPRelayEndpointAnswer struct { - Answer [bindUDPRelayEndpointAnswerLen]byte + BindUDPRelayEndpointCommon } func (m *BindUDPRelayEndpointAnswer) AppendMarshal(b []byte) []byte { - ret, d := appendMsgHeader(b, TypeBindUDPRelayEndpointAnswer, v0, bindUDPRelayEndpointAnswerLen) - copy(d, m.Answer[:]) + ret, d := appendMsgHeader(b, TypeBindUDPRelayEndpointAnswer, v0, bindUDPRelayEndpointCommonLen) + m.BindUDPRelayEndpointCommon.encode(d) return ret } func parseBindUDPRelayEndpointAnswer(ver uint8, p []byte) (m *BindUDPRelayEndpointAnswer, err error) { - if len(p) < bindUDPRelayEndpointAnswerLen { - return nil, errShort - } m = new(BindUDPRelayEndpointAnswer) - copy(m.Answer[:], p[:]) + err = m.BindUDPRelayEndpointCommon.decode(p) + if err != nil { + return nil, err + } return m, nil } +// AllocateUDPRelayEndpointRequest is a message sent only over DERP to request +// allocation of a relay endpoint on a [tailscale.com/net/udprelay.Server] +type AllocateUDPRelayEndpointRequest struct { + // ClientDisco are the Disco public keys of the clients that should be + // permitted to handshake with the endpoint. + ClientDisco [2]key.DiscoPublic + // Generation represents the allocation request generation. The server must + // echo it back in the [AllocateUDPRelayEndpointResponse] to enable request + // and response alignment client-side. + Generation uint32 +} + +// allocateUDPRelayEndpointRequestLen is the length of a marshaled +// [AllocateUDPRelayEndpointRequest] message without the message header. +const allocateUDPRelayEndpointRequestLen = key.DiscoPublicRawLen*2 + // ClientDisco + 4 // Generation + +func (m *AllocateUDPRelayEndpointRequest) AppendMarshal(b []byte) []byte { + ret, p := appendMsgHeader(b, TypeAllocateUDPRelayEndpointRequest, v0, allocateUDPRelayEndpointRequestLen) + for i := 0; i < len(m.ClientDisco); i++ { + disco := m.ClientDisco[i].AppendTo(nil) + copy(p, disco) + p = p[key.DiscoPublicRawLen:] + } + binary.BigEndian.PutUint32(p, m.Generation) + return ret +} + +func parseAllocateUDPRelayEndpointRequest(ver uint8, p []byte) (m *AllocateUDPRelayEndpointRequest, err error) { + m = new(AllocateUDPRelayEndpointRequest) + if ver != 0 { + return + } + if len(p) < allocateUDPRelayEndpointRequestLen { + return m, errShort + } + for i := 0; i < len(m.ClientDisco); i++ { + m.ClientDisco[i] = key.DiscoPublicFromRaw32(mem.B(p[:key.DiscoPublicRawLen])) + p = p[key.DiscoPublicRawLen:] + } + m.Generation = binary.BigEndian.Uint32(p) + return m, nil +} + +// AllocateUDPRelayEndpointResponse is a message sent only over DERP in response +// to a [AllocateUDPRelayEndpointRequest]. +type AllocateUDPRelayEndpointResponse struct { + // Generation represents the allocation request generation. The server must + // echo back the [AllocateUDPRelayEndpointRequest.Generation] here to enable + // request and response alignment client-side. + Generation uint32 + UDPRelayEndpoint +} + +func (m *AllocateUDPRelayEndpointResponse) AppendMarshal(b []byte) []byte { + endpointsLen := epLength * len(m.AddrPorts) + generationLen := 4 + ret, d := appendMsgHeader(b, TypeAllocateUDPRelayEndpointResponse, v0, generationLen+udpRelayEndpointLenMinusAddrPorts+endpointsLen) + binary.BigEndian.PutUint32(d, m.Generation) + m.encode(d[4:]) + return ret +} + +func parseAllocateUDPRelayEndpointResponse(ver uint8, p []byte) (m *AllocateUDPRelayEndpointResponse, err error) { + m = new(AllocateUDPRelayEndpointResponse) + if ver != 0 { + return m, nil + } + if len(p) < 4 { + return m, errShort + } + m.Generation = binary.BigEndian.Uint32(p) + err = m.decode(p[4:]) + return m, err +} + +const udpRelayEndpointLenMinusAddrPorts = key.DiscoPublicRawLen + // ServerDisco + (key.DiscoPublicRawLen * 2) + // ClientDisco + 8 + // LamportID + 4 + // VNI + 8 + // BindLifetime + 8 // SteadyStateLifetime + +// UDPRelayEndpoint is a mirror of [tailscale.com/net/udprelay/endpoint.ServerEndpoint], +// refer to it for field documentation. [UDPRelayEndpoint] is carried in both +// [CallMeMaybeVia] and [AllocateUDPRelayEndpointResponse] messages. +type UDPRelayEndpoint struct { + // ServerDisco is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.ServerDisco] + ServerDisco key.DiscoPublic + // ClientDisco is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.ClientDisco] + ClientDisco [2]key.DiscoPublic + // LamportID is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.LamportID] + LamportID uint64 + // VNI is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.VNI] + VNI uint32 + // BindLifetime is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.BindLifetime] + BindLifetime time.Duration + // SteadyStateLifetime is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.SteadyStateLifetime] + SteadyStateLifetime time.Duration + // AddrPorts is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.AddrPorts] + AddrPorts []netip.AddrPort +} + +// encode encodes m in b. b must be at least [udpRelayEndpointLenMinusAddrPorts] +// + [epLength] * len(m.AddrPorts) bytes long. +func (m *UDPRelayEndpoint) encode(b []byte) { + disco := m.ServerDisco.AppendTo(nil) + copy(b, disco) + b = b[key.DiscoPublicRawLen:] + for i := 0; i < len(m.ClientDisco); i++ { + disco = m.ClientDisco[i].AppendTo(nil) + copy(b, disco) + b = b[key.DiscoPublicRawLen:] + } + binary.BigEndian.PutUint64(b[:8], m.LamportID) + b = b[8:] + binary.BigEndian.PutUint32(b[:4], m.VNI) + b = b[4:] + binary.BigEndian.PutUint64(b[:8], uint64(m.BindLifetime)) + b = b[8:] + binary.BigEndian.PutUint64(b[:8], uint64(m.SteadyStateLifetime)) + b = b[8:] + for _, ipp := range m.AddrPorts { + a := ipp.Addr().As16() + copy(b, a[:]) + binary.BigEndian.PutUint16(b[16:18], ipp.Port()) + b = b[epLength:] + } +} + +// decode decodes m from b. +func (m *UDPRelayEndpoint) decode(b []byte) error { + if len(b) < udpRelayEndpointLenMinusAddrPorts+epLength || + (len(b)-udpRelayEndpointLenMinusAddrPorts)%epLength != 0 { + return errShort + } + m.ServerDisco = key.DiscoPublicFromRaw32(mem.B(b[:key.DiscoPublicRawLen])) + b = b[key.DiscoPublicRawLen:] + for i := 0; i < len(m.ClientDisco); i++ { + m.ClientDisco[i] = key.DiscoPublicFromRaw32(mem.B(b[:key.DiscoPublicRawLen])) + b = b[key.DiscoPublicRawLen:] + } + m.LamportID = binary.BigEndian.Uint64(b[:8]) + b = b[8:] + m.VNI = binary.BigEndian.Uint32(b[:4]) + b = b[4:] + m.BindLifetime = time.Duration(binary.BigEndian.Uint64(b[:8])) + b = b[8:] + m.SteadyStateLifetime = time.Duration(binary.BigEndian.Uint64(b[:8])) + b = b[8:] + m.AddrPorts = make([]netip.AddrPort, 0, len(b)-udpRelayEndpointLenMinusAddrPorts/epLength) + for len(b) > 0 { + var a [16]byte + copy(a[:], b) + m.AddrPorts = append(m.AddrPorts, netip.AddrPortFrom( + netip.AddrFrom16(a).Unmap(), + binary.BigEndian.Uint16(b[16:18]))) + b = b[epLength:] + } + return nil +} + // CallMeMaybeVia is a message sent only over DERP to request that the recipient // try to open up a magicsock path back to the sender. The 'Via' in // CallMeMaybeVia highlights that candidate paths are served through an @@ -412,78 +627,22 @@ func parseBindUDPRelayEndpointAnswer(ver uint8, p []byte) (m *BindUDPRelayEndpoi // The recipient may choose to not open a path back if it's already happy with // its path. Direct connections, e.g. [CallMeMaybe]-signaled, take priority over // CallMeMaybeVia paths. -// -// This message type is currently considered experimental and is not yet tied to -// a [tailscale.com/tailcfg.CapabilityVersion]. type CallMeMaybeVia struct { - // ServerDisco is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.ServerDisco] - ServerDisco key.DiscoPublic - // LamportID is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.LamportID] - LamportID uint64 - // VNI is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.VNI] - VNI uint32 - // BindLifetime is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.BindLifetime] - BindLifetime time.Duration - // SteadyStateLifetime is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.SteadyStateLifetime] - SteadyStateLifetime time.Duration - // AddrPorts is [tailscale.com/net/udprelay/endpoint.ServerEndpoint.AddrPorts] - AddrPorts []netip.AddrPort + UDPRelayEndpoint } -const cmmvDataLenMinusEndpoints = key.DiscoPublicRawLen + // ServerDisco - 8 + // LamportID - 4 + // VNI - 8 + // BindLifetime - 8 // SteadyStateLifetime - func (m *CallMeMaybeVia) AppendMarshal(b []byte) []byte { endpointsLen := epLength * len(m.AddrPorts) - ret, p := appendMsgHeader(b, TypeCallMeMaybeVia, v0, cmmvDataLenMinusEndpoints+endpointsLen) - disco := m.ServerDisco.AppendTo(nil) - copy(p, disco) - p = p[key.DiscoPublicRawLen:] - binary.BigEndian.PutUint64(p[:8], m.LamportID) - p = p[8:] - binary.BigEndian.PutUint32(p[:4], m.VNI) - p = p[4:] - binary.BigEndian.PutUint64(p[:8], uint64(m.BindLifetime)) - p = p[8:] - binary.BigEndian.PutUint64(p[:8], uint64(m.SteadyStateLifetime)) - p = p[8:] - for _, ipp := range m.AddrPorts { - a := ipp.Addr().As16() - copy(p, a[:]) - binary.BigEndian.PutUint16(p[16:18], ipp.Port()) - p = p[epLength:] - } + ret, p := appendMsgHeader(b, TypeCallMeMaybeVia, v0, udpRelayEndpointLenMinusAddrPorts+endpointsLen) + m.encode(p) return ret } func parseCallMeMaybeVia(ver uint8, p []byte) (m *CallMeMaybeVia, err error) { m = new(CallMeMaybeVia) - if len(p) < cmmvDataLenMinusEndpoints+epLength || - (len(p)-cmmvDataLenMinusEndpoints)%epLength != 0 || - ver != 0 { + if ver != 0 { return m, nil } - m.ServerDisco = key.DiscoPublicFromRaw32(mem.B(p[:key.DiscoPublicRawLen])) - p = p[key.DiscoPublicRawLen:] - m.LamportID = binary.BigEndian.Uint64(p[:8]) - p = p[8:] - m.VNI = binary.BigEndian.Uint32(p[:4]) - p = p[4:] - m.BindLifetime = time.Duration(binary.BigEndian.Uint64(p[:8])) - p = p[8:] - m.SteadyStateLifetime = time.Duration(binary.BigEndian.Uint64(p[:8])) - p = p[8:] - m.AddrPorts = make([]netip.AddrPort, 0, len(p)-cmmvDataLenMinusEndpoints/epLength) - for len(p) > 0 { - var a [16]byte - copy(a[:], p) - m.AddrPorts = append(m.AddrPorts, netip.AddrPortFrom( - netip.AddrFrom16(a).Unmap(), - binary.BigEndian.Uint16(p[16:18]))) - p = p[epLength:] - } - return m, nil + err = m.decode(p) + return m, err } diff --git a/disco/disco_test.go b/disco/disco_test.go index f2a29a744992f..71b68338a8c90 100644 --- a/disco/disco_test.go +++ b/disco/disco_test.go @@ -16,6 +16,28 @@ import ( ) func TestMarshalAndParse(t *testing.T) { + relayHandshakeCommon := BindUDPRelayEndpointCommon{ + VNI: 1, + Generation: 2, + RemoteKey: key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})), + Challenge: [BindUDPRelayChallengeLen]byte{ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + }, + } + + udpRelayEndpoint := UDPRelayEndpoint{ + ServerDisco: key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})), + ClientDisco: [2]key.DiscoPublic{key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 3: 3, 30: 30, 31: 31})), key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 4: 4, 30: 30, 31: 31}))}, + LamportID: 123, + VNI: 456, + BindLifetime: time.Second, + SteadyStateLifetime: time.Minute, + AddrPorts: []netip.AddrPort{ + netip.MustParseAddrPort("1.2.3.4:567"), + netip.MustParseAddrPort("[2001::3456]:789"), + }, + } + tests := []struct { name string want string @@ -86,41 +108,47 @@ func TestMarshalAndParse(t *testing.T) { }, { name: "bind_udp_relay_endpoint", - m: &BindUDPRelayEndpoint{}, - want: "04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + m: &BindUDPRelayEndpoint{ + relayHandshakeCommon, + }, + want: "04 00 00 00 00 01 00 00 00 02 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f", }, { name: "bind_udp_relay_endpoint_challenge", m: &BindUDPRelayEndpointChallenge{ - Challenge: [BindUDPRelayEndpointChallengeLen]byte{ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - }, + relayHandshakeCommon, }, - want: "05 00 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f", + want: "05 00 00 00 00 01 00 00 00 02 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f", }, { name: "bind_udp_relay_endpoint_answer", m: &BindUDPRelayEndpointAnswer{ - Answer: [bindUDPRelayEndpointAnswerLen]byte{ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - }, + relayHandshakeCommon, }, - want: "06 00 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f", + want: "06 00 00 00 00 01 00 00 00 02 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f", }, { name: "call_me_maybe_via", m: &CallMeMaybeVia{ - ServerDisco: key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 30: 30, 31: 31})), - LamportID: 123, - VNI: 456, - BindLifetime: time.Second, - SteadyStateLifetime: time.Minute, - AddrPorts: []netip.AddrPort{ - netip.MustParseAddrPort("1.2.3.4:567"), - netip.MustParseAddrPort("[2001::3456]:789"), - }, + UDPRelayEndpoint: udpRelayEndpoint, + }, + want: "07 00 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 00 00 00 00 00 00 7b 00 00 01 c8 00 00 00 00 3b 9a ca 00 00 00 00 0d f8 47 58 00 00 00 00 00 00 00 00 00 00 00 ff ff 01 02 03 04 02 37 20 01 00 00 00 00 00 00 00 00 00 00 00 00 34 56 03 15", + }, + { + name: "allocate_udp_relay_endpoint_request", + m: &AllocateUDPRelayEndpointRequest{ + ClientDisco: [2]key.DiscoPublic{key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 3: 3, 30: 30, 31: 31})), key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 2: 2, 4: 4, 30: 30, 31: 31}))}, + Generation: 1, + }, + want: "08 00 00 01 02 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 00 00 01", + }, + { + name: "allocate_udp_relay_endpoint_response", + m: &AllocateUDPRelayEndpointResponse{ + Generation: 1, + UDPRelayEndpoint: udpRelayEndpoint, }, - want: "07 00 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 00 00 00 00 00 00 7b 00 00 01 c8 00 00 00 00 3b 9a ca 00 00 00 00 0d f8 47 58 00 00 00 00 00 00 00 00 00 00 00 ff ff 01 02 03 04 02 37 20 01 00 00 00 00 00 00 00 00 00 00 00 00 34 56 03 15", + want: "09 00 00 00 00 01 00 01 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 01 02 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 1f 00 00 00 00 00 00 00 7b 00 00 01 c8 00 00 00 00 3b 9a ca 00 00 00 00 0d f8 47 58 00 00 00 00 00 00 00 00 00 00 00 ff ff 01 02 03 04 02 37 20 01 00 00 00 00 00 00 00 00 00 00 00 00 34 56 03 15", }, } for _, tt := range tests { diff --git a/docs/commit-messages.md b/docs/commit-messages.md index b3881eaeb9fbb..79b16e4c6f6f2 100644 --- a/docs/commit-messages.md +++ b/docs/commit-messages.md @@ -65,7 +65,7 @@ Notably, for the subject (the first line of description): | `foo/bar:fix memory leak` | BAD: no space after colon | | `foo/bar : fix memory leak` | BAD: space before colon | | `foo/bar: fix memory leak Fixes #123` | BAD: the "Fixes" shouldn't be part of the title | - | `!fixup reviewer feedback` | BAD: we don't check in fixup commits; the history should always bissect to a clean, working tree | + | `!fixup reviewer feedback` | BAD: we don't check in fixup commits; the history should always bisect to a clean, working tree | For the body (the rest of the description): diff --git a/docs/windows/policy/en-US/tailscale.adml b/docs/windows/policy/en-US/tailscale.adml index 62ff94da7096d..2e143d49c9c6c 100644 --- a/docs/windows/policy/en-US/tailscale.adml +++ b/docs/windows/policy/en-US/tailscale.adml @@ -19,9 +19,11 @@ Tailscale version 1.80.0 and later Tailscale version 1.82.0 and later Tailscale version 1.84.0 and later + Tailscale version 1.86.0 and later Tailscale UI customization Settings + Allowed Allowed (with audit) Not Allowed Require using a specific Tailscale coordination server @@ -67,8 +69,8 @@ If you disable or do not configure this policy setting, an interactive user logi See https://tailscale.com/kb/1315/mdm-keys#set-an-auth-key for more details.]]> Require using a specific Exit Node + Encrypt client state file stored on disk + @@ -314,6 +324,7 @@ If you disable this policy, the onboarding flow will never be shown.]]> + User override: Registration mode: diff --git a/docs/windows/policy/tailscale.admx b/docs/windows/policy/tailscale.admx index d97b24c36b5df..0da8aef42ded6 100644 --- a/docs/windows/policy/tailscale.admx +++ b/docs/windows/policy/tailscale.admx @@ -66,6 +66,10 @@ displayName="$(string.SINCE_V1_84)"> + + + @@ -111,6 +115,18 @@ + + + + + + + + + + + + @@ -365,5 +381,15 @@ + + + + + + + + + + diff --git a/drive/driveimpl/drive_test.go b/drive/driveimpl/drive_test.go index e7dd832918cec..cff55fbb2c858 100644 --- a/drive/driveimpl/drive_test.go +++ b/drive/driveimpl/drive_test.go @@ -524,7 +524,7 @@ func (s *system) addRemote(name string) string { for name, r := range s.remotes { remotes = append(remotes, &drive.Remote{ Name: name, - URL: fmt.Sprintf("http://%s", r.l.Addr()), + URL: func() string { return fmt.Sprintf("http://%s", r.l.Addr()) }, }) } s.local.fs.SetRemotes( diff --git a/drive/driveimpl/local_impl.go b/drive/driveimpl/local_impl.go index 8cdf60179aa0b..871d033431038 100644 --- a/drive/driveimpl/local_impl.go +++ b/drive/driveimpl/local_impl.go @@ -81,7 +81,7 @@ func (s *FileSystemForLocal) SetRemotes(domain string, remotes []*drive.Remote, Name: remote.Name, Available: remote.Available, }, - BaseURL: func() (string, error) { return remote.URL, nil }, + BaseURL: func() (string, error) { return remote.URL(), nil }, Transport: transport, }) } diff --git a/drive/local.go b/drive/local.go index aff79a57bd9b2..052efb3f97ecf 100644 --- a/drive/local.go +++ b/drive/local.go @@ -17,7 +17,7 @@ import ( // Remote represents a remote Taildrive node. type Remote struct { Name string - URL string + URL func() string Available func() bool } diff --git a/envknob/featureknob/featureknob.go b/envknob/featureknob/featureknob.go index e9b871f74a8c0..5a54a1c42978d 100644 --- a/envknob/featureknob/featureknob.go +++ b/envknob/featureknob/featureknob.go @@ -10,7 +10,6 @@ import ( "runtime" "tailscale.com/envknob" - "tailscale.com/hostinfo" "tailscale.com/version" "tailscale.com/version/distro" ) @@ -26,14 +25,6 @@ func CanRunTailscaleSSH() error { if distro.Get() == distro.QNAP && !envknob.UseWIPCode() { return errors.New("The Tailscale SSH server does not run on QNAP.") } - - // Setting SSH on Home Assistant causes trouble on startup - // (since the flag is not being passed to `tailscale up`). - // Although Tailscale SSH does work here, - // it's not terribly useful since it's running in a separate container. - if hostinfo.GetEnvType() == hostinfo.HomeAssistantAddOn { - return errors.New("The Tailscale SSH server does not run on HomeAssistant.") - } // otherwise okay case "darwin": // okay only in tailscaled mode for now. @@ -58,10 +49,5 @@ func CanUseExitNode() error { distro.QNAP: return errors.New("Tailscale exit nodes cannot be used on " + string(dist)) } - - if hostinfo.GetEnvType() == hostinfo.HomeAssistantAddOn { - return errors.New("Tailscale exit nodes cannot be used on HomeAssistant.") - } - return nil } diff --git a/feature/capture/dissector/ts-dissector.lua b/feature/capture/dissector/ts-dissector.lua index ad553d7674193..c2ee2b755f266 100644 --- a/feature/capture/dissector/ts-dissector.lua +++ b/feature/capture/dissector/ts-dissector.lua @@ -1,5 +1,5 @@ function hasbit(x, p) - return x % (p + p) >= p + return bit.band(x, p) ~= 0 end tsdebug_ll = Proto("tsdebug", "Tailscale debug") @@ -128,6 +128,10 @@ function tsdisco_frame.dissector(buffer, pinfo, tree) if message_type == 1 then subtree:add(DISCO_TYPE, "Ping") elseif message_type == 2 then subtree:add(DISCO_TYPE, "Pong") elseif message_type == 3 then subtree:add(DISCO_TYPE, "Call me maybe") + elseif message_type == 4 then subtree:add(DISCO_TYPE, "Bind UDP Relay Endpoint") + elseif message_type == 5 then subtree:add(DISCO_TYPE, "Bind UDP Relay Endpoint Challenge") + elseif message_type == 6 then subtree:add(DISCO_TYPE, "Bind UDP Relay Endpoint Answer") + elseif message_type == 7 then subtree:add(DISCO_TYPE, "Call me maybe via") end -- Message version diff --git a/feature/relayserver/relayserver.go b/feature/relayserver/relayserver.go index 96d21138edfbc..b90a6234508f2 100644 --- a/feature/relayserver/relayserver.go +++ b/feature/relayserver/relayserver.go @@ -6,25 +6,20 @@ package relayserver import ( - "encoding/json" - "errors" - "io" - "net/http" - "net/netip" "sync" - "tailscale.com/envknob" + "tailscale.com/disco" "tailscale.com/feature" "tailscale.com/ipn" "tailscale.com/ipn/ipnext" - "tailscale.com/ipn/ipnlocal" "tailscale.com/net/udprelay" "tailscale.com/net/udprelay/endpoint" "tailscale.com/tailcfg" "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/types/ptr" - "tailscale.com/util/httpm" + "tailscale.com/util/eventbus" + "tailscale.com/wgengine/magicsock" ) // featureName is the name of the feature implemented by this package. @@ -34,26 +29,30 @@ const featureName = "relayserver" func init() { feature.Register(featureName) ipnext.RegisterExtension(featureName, newExtension) - ipnlocal.RegisterPeerAPIHandler("/v0/relay/endpoint", handlePeerAPIRelayAllocateEndpoint) } // newExtension is an [ipnext.NewExtensionFn] that creates a new relay server // extension. It is registered with [ipnext.RegisterExtension] if the package is // imported. -func newExtension(logf logger.Logf, _ ipnext.SafeBackend) (ipnext.Extension, error) { - return &extension{logf: logger.WithPrefix(logf, featureName+": ")}, nil +func newExtension(logf logger.Logf, sb ipnext.SafeBackend) (ipnext.Extension, error) { + return &extension{ + logf: logger.WithPrefix(logf, featureName+": "), + bus: sb.Sys().Bus.Get(), + }, nil } // extension is an [ipnext.Extension] managing the relay server on platforms // that import this package. type extension struct { logf logger.Logf - - mu sync.Mutex // guards the following fields - shutdown bool - port *int // ipn.Prefs.RelayServerPort, nil if disabled - hasNodeAttrRelayServer bool // tailcfg.NodeAttrRelayServer - server relayServer // lazily initialized + bus *eventbus.Bus + + mu sync.Mutex // guards the following fields + shutdown bool + port *int // ipn.Prefs.RelayServerPort, nil if disabled + disconnectFromBusCh chan struct{} // non-nil if consumeEventbusTopics is running, closed to signal it to return + busDoneCh chan struct{} // non-nil if consumeEventbusTopics is running, closed when it returns + hasNodeAttrDisableRelayServer bool // tailcfg.NodeAttrDisableRelayServer } // relayServer is the interface of [udprelay.Server]. @@ -77,14 +76,27 @@ func (e *extension) Init(host ipnext.Host) error { return nil } +// handleBusLifetimeLocked handles the lifetime of consumeEventbusTopics. +func (e *extension) handleBusLifetimeLocked() { + busShouldBeRunning := !e.shutdown && e.port != nil && !e.hasNodeAttrDisableRelayServer + if !busShouldBeRunning { + e.disconnectFromBusLocked() + return + } + if e.busDoneCh != nil { + return // already running + } + port := *e.port + e.disconnectFromBusCh = make(chan struct{}) + e.busDoneCh = make(chan struct{}) + go e.consumeEventbusTopics(port) +} + func (e *extension) selfNodeViewChanged(nodeView tailcfg.NodeView) { e.mu.Lock() defer e.mu.Unlock() - e.hasNodeAttrRelayServer = nodeView.HasCap(tailcfg.NodeAttrRelayServer) - if !e.hasNodeAttrRelayServer && e.server != nil { - e.server.Close() - e.server = nil - } + e.hasNodeAttrDisableRelayServer = nodeView.HasCap(tailcfg.NodeAttrDisableRelayServer) + e.handleBusLifetimeLocked() } func (e *extension) profileStateChanged(_ ipn.LoginProfileView, prefs ipn.PrefsView, sameNode bool) { @@ -94,102 +106,85 @@ func (e *extension) profileStateChanged(_ ipn.LoginProfileView, prefs ipn.PrefsV enableOrDisableServer := ok != (e.port != nil) portChanged := ok && e.port != nil && newPort != *e.port if enableOrDisableServer || portChanged || !sameNode { - if e.server != nil { - e.server.Close() - e.server = nil - } + e.disconnectFromBusLocked() e.port = nil if ok { e.port = ptr.To(newPort) } } + e.handleBusLifetimeLocked() } -// Shutdown implements [ipnlocal.Extension]. -func (e *extension) Shutdown() error { - e.mu.Lock() - defer e.mu.Unlock() - e.shutdown = true - if e.server != nil { - e.server.Close() - e.server = nil - } - return nil -} +func (e *extension) consumeEventbusTopics(port int) { + defer close(e.busDoneCh) -func (e *extension) relayServerOrInit() (relayServer, error) { - e.mu.Lock() - defer e.mu.Unlock() - if e.shutdown { - return nil, errors.New("relay server is shutdown") - } - if e.server != nil { - return e.server, nil - } - if e.port == nil { - return nil, errors.New("relay server is not configured") - } - if !e.hasNodeAttrRelayServer { - return nil, errors.New("no relay:server node attribute") - } - if !envknob.UseWIPCode() { - return nil, errors.New("TAILSCALE_USE_WIP_CODE envvar is not set") - } - var err error - e.server, _, err = udprelay.NewServer(*e.port, []netip.Addr{netip.MustParseAddr("127.0.0.1")}) - if err != nil { - return nil, err - } - return e.server, nil -} + eventClient := e.bus.Client("relayserver.extension") + reqSub := eventbus.Subscribe[magicsock.UDPRelayAllocReq](eventClient) + respPub := eventbus.Publish[magicsock.UDPRelayAllocResp](eventClient) + defer eventClient.Close() -func handlePeerAPIRelayAllocateEndpoint(h ipnlocal.PeerAPIHandler, w http.ResponseWriter, r *http.Request) { - e, ok := ipnlocal.GetExt[*extension](h.LocalBackend()) - if !ok { - http.Error(w, "relay failed to initialize", http.StatusServiceUnavailable) - return - } - - httpErrAndLog := func(message string, code int) { - http.Error(w, message, code) - h.Logf("relayserver: request from %v returned code %d: %s", h.RemoteAddr(), code, message) - } - - if !h.PeerCaps().HasCapability(tailcfg.PeerCapabilityRelay) { - httpErrAndLog("relay not permitted", http.StatusForbidden) - return - } - - if r.Method != httpm.POST { - httpErrAndLog("only POST method is allowed", http.StatusMethodNotAllowed) - return + var rs relayServer // lazily initialized + defer func() { + if rs != nil { + rs.Close() + } + }() + for { + select { + case <-e.disconnectFromBusCh: + return + case <-reqSub.Done(): + // If reqSub is done, the eventClient has been closed, which is a + // signal to return. + return + case req := <-reqSub.Events(): + if rs == nil { + var err error + rs, err = udprelay.NewServer(e.logf, port, nil) + if err != nil { + e.logf("error initializing server: %v", err) + continue + } + } + se, err := rs.AllocateEndpoint(req.Message.ClientDisco[0], req.Message.ClientDisco[1]) + if err != nil { + e.logf("error allocating endpoint: %v", err) + continue + } + respPub.Publish(magicsock.UDPRelayAllocResp{ + ReqRxFromNodeKey: req.RxFromNodeKey, + ReqRxFromDiscoKey: req.RxFromDiscoKey, + Message: &disco.AllocateUDPRelayEndpointResponse{ + Generation: req.Message.Generation, + UDPRelayEndpoint: disco.UDPRelayEndpoint{ + ServerDisco: se.ServerDisco, + ClientDisco: se.ClientDisco, + LamportID: se.LamportID, + VNI: se.VNI, + BindLifetime: se.BindLifetime.Duration, + SteadyStateLifetime: se.SteadyStateLifetime.Duration, + AddrPorts: se.AddrPorts, + }, + }, + }) + } } +} - var allocateEndpointReq struct { - DiscoKeys []key.DiscoPublic - } - err := json.NewDecoder(io.LimitReader(r.Body, 512)).Decode(&allocateEndpointReq) - if err != nil { - httpErrAndLog(err.Error(), http.StatusBadRequest) - return - } - if len(allocateEndpointReq.DiscoKeys) != 2 { - httpErrAndLog("2 disco public keys must be supplied", http.StatusBadRequest) - return +func (e *extension) disconnectFromBusLocked() { + if e.busDoneCh != nil { + close(e.disconnectFromBusCh) + <-e.busDoneCh + e.busDoneCh = nil + e.disconnectFromBusCh = nil } +} - rs, err := e.relayServerOrInit() - if err != nil { - httpErrAndLog(err.Error(), http.StatusServiceUnavailable) - return - } - ep, err := rs.AllocateEndpoint(allocateEndpointReq.DiscoKeys[0], allocateEndpointReq.DiscoKeys[1]) - if err != nil { - httpErrAndLog(err.Error(), http.StatusInternalServerError) - return - } - err = json.NewEncoder(w).Encode(&ep) - if err != nil { - httpErrAndLog(err.Error(), http.StatusInternalServerError) - } +// Shutdown implements [ipnlocal.Extension]. +func (e *extension) Shutdown() error { + e.mu.Lock() + defer e.mu.Unlock() + e.disconnectFromBusLocked() + e.shutdown = true + return nil } diff --git a/feature/relayserver/relayserver_test.go b/feature/relayserver/relayserver_test.go index cc7f05f67fbdd..d3fc36a83674a 100644 --- a/feature/relayserver/relayserver_test.go +++ b/feature/relayserver/relayserver_test.go @@ -4,117 +4,105 @@ package relayserver import ( - "errors" "testing" "tailscale.com/ipn" - "tailscale.com/net/udprelay/endpoint" - "tailscale.com/types/key" + "tailscale.com/tsd" "tailscale.com/types/ptr" + "tailscale.com/util/eventbus" ) -type fakeRelayServer struct{} - -func (f *fakeRelayServer) Close() error { return nil } - -func (f *fakeRelayServer) AllocateEndpoint(_, _ key.DiscoPublic) (endpoint.ServerEndpoint, error) { - return endpoint.ServerEndpoint{}, errors.New("fake relay server") -} - func Test_extension_profileStateChanged(t *testing.T) { prefsWithPortOne := ipn.Prefs{RelayServerPort: ptr.To(1)} prefsWithNilPort := ipn.Prefs{RelayServerPort: nil} type fields struct { - server relayServer - port *int + port *int } type args struct { prefs ipn.PrefsView sameNode bool } tests := []struct { - name string - fields fields - args args - wantPort *int - wantNilServer bool + name string + fields fields + args args + wantPort *int + wantBusRunning bool }{ { - name: "no changes non-nil server", + name: "no changes non-nil port", fields: fields{ - server: &fakeRelayServer{}, - port: ptr.To(1), + port: ptr.To(1), }, args: args{ prefs: prefsWithPortOne.View(), sameNode: true, }, - wantPort: ptr.To(1), - wantNilServer: false, + wantPort: ptr.To(1), + wantBusRunning: true, }, { name: "prefs port nil", fields: fields{ - server: &fakeRelayServer{}, - port: ptr.To(1), + port: ptr.To(1), }, args: args{ prefs: prefsWithNilPort.View(), sameNode: true, }, - wantPort: nil, - wantNilServer: true, + wantPort: nil, + wantBusRunning: false, }, { name: "prefs port changed", fields: fields{ - server: &fakeRelayServer{}, - port: ptr.To(2), + port: ptr.To(2), }, args: args{ prefs: prefsWithPortOne.View(), sameNode: true, }, - wantPort: ptr.To(1), - wantNilServer: true, + wantPort: ptr.To(1), + wantBusRunning: true, }, { name: "sameNode false", fields: fields{ - server: &fakeRelayServer{}, - port: ptr.To(1), + port: ptr.To(1), }, args: args{ prefs: prefsWithPortOne.View(), sameNode: false, }, - wantPort: ptr.To(1), - wantNilServer: true, + wantPort: ptr.To(1), + wantBusRunning: true, }, { name: "prefs port non-nil extension port nil", fields: fields{ - server: nil, - port: nil, + port: nil, }, args: args{ prefs: prefsWithPortOne.View(), sameNode: false, }, - wantPort: ptr.To(1), - wantNilServer: true, + wantPort: ptr.To(1), + wantBusRunning: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + sys := tsd.NewSystem() + bus := sys.Bus.Get() e := &extension{ - port: tt.fields.port, - server: tt.fields.server, + port: tt.fields.port, + bus: bus, } + defer e.disconnectFromBusLocked() e.profileStateChanged(ipn.LoginProfileView{}, tt.args.prefs, tt.args.sameNode) - if tt.wantNilServer != (e.server == nil) { - t.Errorf("wantNilServer: %v != (e.server == nil): %v", tt.wantNilServer, e.server == nil) + if tt.wantBusRunning != (e.busDoneCh != nil) { + t.Errorf("wantBusRunning: %v != (e.busDoneCh != nil): %v", tt.wantBusRunning, e.busDoneCh != nil) } if (tt.wantPort == nil) != (e.port == nil) { t.Errorf("(tt.wantPort == nil): %v != (e.port == nil): %v", tt.wantPort == nil, e.port == nil) @@ -124,3 +112,59 @@ func Test_extension_profileStateChanged(t *testing.T) { }) } } + +func Test_extension_handleBusLifetimeLocked(t *testing.T) { + tests := []struct { + name string + shutdown bool + port *int + busDoneCh chan struct{} + hasNodeAttrDisableRelayServer bool + wantBusRunning bool + }{ + { + name: "want running", + shutdown: false, + port: ptr.To(1), + hasNodeAttrDisableRelayServer: false, + wantBusRunning: true, + }, + { + name: "shutdown true", + shutdown: true, + port: ptr.To(1), + hasNodeAttrDisableRelayServer: false, + wantBusRunning: false, + }, + { + name: "port nil", + shutdown: false, + port: nil, + hasNodeAttrDisableRelayServer: false, + wantBusRunning: false, + }, + { + name: "hasNodeAttrDisableRelayServer true", + shutdown: false, + port: nil, + hasNodeAttrDisableRelayServer: true, + wantBusRunning: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &extension{ + bus: eventbus.New(), + shutdown: tt.shutdown, + port: tt.port, + busDoneCh: tt.busDoneCh, + hasNodeAttrDisableRelayServer: tt.hasNodeAttrDisableRelayServer, + } + e.handleBusLifetimeLocked() + defer e.disconnectFromBusLocked() + if tt.wantBusRunning != (e.busDoneCh != nil) { + t.Errorf("wantBusRunning: %v != (e.busDoneCh != nil): %v", tt.wantBusRunning, e.busDoneCh != nil) + } + }) + } +} diff --git a/feature/taildrop/delete.go b/feature/taildrop/delete.go index e9c8d7f1c90fa..0b7259879f941 100644 --- a/feature/taildrop/delete.go +++ b/feature/taildrop/delete.go @@ -6,9 +6,7 @@ package taildrop import ( "container/list" "context" - "io/fs" "os" - "path/filepath" "strings" "sync" "time" @@ -28,7 +26,6 @@ const deleteDelay = time.Hour type fileDeleter struct { logf logger.Logf clock tstime.DefaultClock - dir string event func(string) // called for certain events; for testing only mu sync.Mutex @@ -39,6 +36,7 @@ type fileDeleter struct { group syncs.WaitGroup shutdownCtx context.Context shutdown context.CancelFunc + fs FileOps // must be used for all filesystem operations } // deleteFile is a specific file to delete after deleteDelay. @@ -50,15 +48,14 @@ type deleteFile struct { func (d *fileDeleter) Init(m *manager, eventHook func(string)) { d.logf = m.opts.Logf d.clock = m.opts.Clock - d.dir = m.opts.Dir d.event = eventHook + d.fs = m.opts.fileOps d.byName = make(map[string]*list.Element) d.emptySignal = make(chan struct{}) d.shutdownCtx, d.shutdown = context.WithCancel(context.Background()) // From a cold-start, load the list of partial and deleted files. - // // Only run this if we have ever received at least one file // to avoid ever touching the taildrop directory on systems (e.g., MacOS) // that pop up a security dialog window upon first access. @@ -71,38 +68,45 @@ func (d *fileDeleter) Init(m *manager, eventHook func(string)) { d.group.Go(func() { d.event("start full-scan") defer d.event("end full-scan") - rangeDir(d.dir, func(de fs.DirEntry) bool { + + if d.fs == nil { + d.logf("deleter: nil FileOps") + } + + files, err := d.fs.ListFiles() + if err != nil { + d.logf("deleter: ListDir error: %v", err) + return + } + for _, filename := range files { switch { case d.shutdownCtx.Err() != nil: - return false // terminate early - case !de.Type().IsRegular(): - return true - case strings.HasSuffix(de.Name(), partialSuffix): + return // terminate early + case strings.HasSuffix(filename, partialSuffix): // Only enqueue the file for deletion if there is no active put. - nameID := strings.TrimSuffix(de.Name(), partialSuffix) + nameID := strings.TrimSuffix(filename, partialSuffix) if i := strings.LastIndexByte(nameID, '.'); i > 0 { key := incomingFileKey{clientID(nameID[i+len("."):]), nameID[:i]} m.incomingFiles.LoadFunc(key, func(_ *incomingFile, loaded bool) { if !loaded { - d.Insert(de.Name()) + d.Insert(filename) } }) } else { - d.Insert(de.Name()) + d.Insert(filename) } - case strings.HasSuffix(de.Name(), deletedSuffix): + case strings.HasSuffix(filename, deletedSuffix): // Best-effort immediate deletion of deleted files. - name := strings.TrimSuffix(de.Name(), deletedSuffix) - if os.Remove(filepath.Join(d.dir, name)) == nil { - if os.Remove(filepath.Join(d.dir, de.Name())) == nil { - break + name := strings.TrimSuffix(filename, deletedSuffix) + if d.fs.Remove(name) == nil { + if d.fs.Remove(filename) == nil { + continue } } - // Otherwise, enqueue the file for later deletion. - d.Insert(de.Name()) + // Otherwise enqueue for later deletion. + d.Insert(filename) } - return true - }) + } }) } @@ -149,13 +153,13 @@ func (d *fileDeleter) waitAndDelete(wait time.Duration) { // Delete the expired file. if name, ok := strings.CutSuffix(file.name, deletedSuffix); ok { - if err := os.Remove(filepath.Join(d.dir, name)); err != nil && !os.IsNotExist(err) { + if err := d.fs.Remove(name); err != nil && !os.IsNotExist(err) { d.logf("could not delete: %v", redactError(err)) failed = append(failed, elem) continue } } - if err := os.Remove(filepath.Join(d.dir, file.name)); err != nil && !os.IsNotExist(err) { + if err := d.fs.Remove(file.name); err != nil && !os.IsNotExist(err) { d.logf("could not delete: %v", redactError(err)) failed = append(failed, elem) continue diff --git a/feature/taildrop/delete_test.go b/feature/taildrop/delete_test.go index 7a58de55c2492..36950f58288cb 100644 --- a/feature/taildrop/delete_test.go +++ b/feature/taildrop/delete_test.go @@ -5,7 +5,6 @@ package taildrop import ( "os" - "path/filepath" "slices" "testing" "time" @@ -20,11 +19,20 @@ import ( func TestDeleter(t *testing.T) { dir := t.TempDir() - must.Do(touchFile(filepath.Join(dir, "foo.partial"))) - must.Do(touchFile(filepath.Join(dir, "bar.partial"))) - must.Do(touchFile(filepath.Join(dir, "fizz"))) - must.Do(touchFile(filepath.Join(dir, "fizz.deleted"))) - must.Do(touchFile(filepath.Join(dir, "buzz.deleted"))) // lacks a matching "buzz" file + var m manager + var fd fileDeleter + m.opts.Logf = t.Logf + m.opts.Clock = tstime.DefaultClock{Clock: tstest.NewClock(tstest.ClockOpts{ + Start: time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), + })} + m.opts.State = must.Get(mem.New(nil, "")) + m.opts.fileOps, _ = newFileOps(dir) + + must.Do(m.touchFile("foo.partial")) + must.Do(m.touchFile("bar.partial")) + must.Do(m.touchFile("fizz")) + must.Do(m.touchFile("fizz.deleted")) + must.Do(m.touchFile("buzz.deleted")) // lacks a matching "buzz" file checkDirectory := func(want ...string) { t.Helper() @@ -69,12 +77,10 @@ func TestDeleter(t *testing.T) { } eventHook := func(event string) { eventsChan <- event } - var m manager - var fd fileDeleter m.opts.Logf = t.Logf m.opts.Clock = tstime.DefaultClock{Clock: clock} - m.opts.Dir = dir m.opts.State = must.Get(mem.New(nil, "")) + m.opts.fileOps, _ = newFileOps(dir) must.Do(m.opts.State.WriteState(ipn.TaildropReceivedKey, []byte{1})) fd.Init(&m, eventHook) defer fd.Shutdown() @@ -100,17 +106,17 @@ func TestDeleter(t *testing.T) { checkEvents("end waitAndDelete") checkDirectory() - must.Do(touchFile(filepath.Join(dir, "one.partial"))) + must.Do(m.touchFile("one.partial")) insert("one.partial") checkEvents("start waitAndDelete") advance(deleteDelay / 4) - must.Do(touchFile(filepath.Join(dir, "two.partial"))) + must.Do(m.touchFile("two.partial")) insert("two.partial") advance(deleteDelay / 4) - must.Do(touchFile(filepath.Join(dir, "three.partial"))) + must.Do(m.touchFile("three.partial")) insert("three.partial") advance(deleteDelay / 4) - must.Do(touchFile(filepath.Join(dir, "four.partial"))) + must.Do(m.touchFile("four.partial")) insert("four.partial") advance(deleteDelay / 4) @@ -145,8 +151,8 @@ func TestDeleterInitWithoutTaildrop(t *testing.T) { var m manager var fd fileDeleter m.opts.Logf = t.Logf - m.opts.Dir = t.TempDir() m.opts.State = must.Get(mem.New(nil, "")) + m.opts.fileOps, _ = newFileOps(t.TempDir()) fd.Init(&m, func(event string) { t.Errorf("unexpected event: %v", event) }) fd.Shutdown() } diff --git a/feature/taildrop/ext.go b/feature/taildrop/ext.go index c11fe3af427a1..f8f45b53fae26 100644 --- a/feature/taildrop/ext.go +++ b/feature/taildrop/ext.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "maps" - "os" "path/filepath" "runtime" "slices" @@ -75,7 +74,7 @@ type Extension struct { // FileOps abstracts platform-specific file operations needed for file transfers. // This is currently being used for Android to use the Storage Access Framework. - FileOps FileOps + fileOps FileOps nodeBackendForTest ipnext.NodeBackend // if non-nil, pretend we're this node state for tests @@ -89,30 +88,6 @@ type Extension struct { outgoingFiles map[string]*ipn.OutgoingFile } -// safDirectoryPrefix is used to determine if the directory is managed via SAF. -const SafDirectoryPrefix = "content://" - -// PutMode controls how Manager.PutFile writes files to storage. -// -// PutModeDirect – write files directly to a filesystem path (default). -// PutModeAndroidSAF – use Android’s Storage Access Framework (SAF), where -// the OS manages the underlying directory permissions. -type PutMode int - -const ( - PutModeDirect PutMode = iota - PutModeAndroidSAF -) - -// FileOps defines platform-specific file operations. -type FileOps interface { - OpenFileWriter(filename string) (io.WriteCloser, string, error) - - // RenamePartialFile finalizes a partial file. - // It returns the new SAF URI as a string and an error. - RenamePartialFile(partialUri, targetDirUri, targetName string) (string, error) -} - func (e *Extension) Name() string { return "taildrop" } @@ -176,23 +151,34 @@ func (e *Extension) onChangeProfile(profile ipn.LoginProfileView, _ ipn.PrefsVie return } - // If we have a netmap, create a taildrop manager. - fileRoot, isDirectFileMode := e.fileRoot(uid, activeLogin) - if fileRoot == "" { - e.logf("no Taildrop directory configured") - } - mode := PutModeDirect - if e.directFileRoot != "" && strings.HasPrefix(e.directFileRoot, SafDirectoryPrefix) { - mode = PutModeAndroidSAF + // Use the provided [FileOps] implementation (typically for SAF access on Android), + // or create an [fsFileOps] instance rooted at fileRoot. + // + // A non-nil [FileOps] also implies that we are in DirectFileMode. + fops := e.fileOps + isDirectFileMode := fops != nil + if fops == nil { + var fileRoot string + if fileRoot, isDirectFileMode = e.fileRoot(uid, activeLogin); fileRoot == "" { + e.logf("no Taildrop directory configured") + e.setMgrLocked(nil) + return + } + + var err error + if fops, err = newFileOps(fileRoot); err != nil { + e.logf("taildrop: cannot create FileOps: %v", err) + e.setMgrLocked(nil) + return + } } + e.setMgrLocked(managerOptions{ Logf: e.logf, Clock: tstime.DefaultClock{Clock: e.sb.Clock()}, State: e.stateStore, - Dir: fileRoot, DirectFileMode: isDirectFileMode, - FileOps: e.FileOps, - Mode: mode, + fileOps: fops, SendFileNotify: e.sendFileNotify, }.New()) } @@ -221,12 +207,7 @@ func (e *Extension) fileRoot(uid tailcfg.UserID, activeLogin string) (root strin baseDir := fmt.Sprintf("%s-uid-%d", strings.ReplaceAll(activeLogin, "@", "-"), uid) - dir := filepath.Join(varRoot, "files", baseDir) - if err := os.MkdirAll(dir, 0700); err != nil { - e.logf("Taildrop disabled; error making directory: %v", err) - return "", false - } - return dir, false + return filepath.Join(varRoot, "files", baseDir), false } // hasCapFileSharing reports whether the current node has the file sharing diff --git a/feature/taildrop/fileops.go b/feature/taildrop/fileops.go new file mode 100644 index 0000000000000..14f76067a8094 --- /dev/null +++ b/feature/taildrop/fileops.go @@ -0,0 +1,41 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package taildrop + +import ( + "io" + "io/fs" + "os" +) + +// FileOps abstracts over both local‐FS paths and Android SAF URIs. +type FileOps interface { + // OpenWriter creates or truncates a file named relative to the receiver's root, + // seeking to the specified offset. If the file does not exist, it is created with mode perm + // on platforms that support it. + // + // It returns an [io.WriteCloser] and the file's absolute path, or an error. + // This call may block. Callers should avoid holding locks when calling OpenWriter. + OpenWriter(name string, offset int64, perm os.FileMode) (wc io.WriteCloser, path string, err error) + + // Remove deletes a file or directory relative to the receiver's root. + // It returns [io.ErrNotExist] if the file or directory does not exist. + Remove(name string) error + + // Rename atomically renames oldPath to a new file named newName, + // returning the full new path or an error. + Rename(oldPath, newName string) (newPath string, err error) + + // ListFiles returns just the basenames of all regular files + // in the root directory. + ListFiles() ([]string, error) + + // Stat returns the FileInfo for the given name or an error. + Stat(name string) (fs.FileInfo, error) + + // OpenReader opens the given basename for the given name or an error. + OpenReader(name string) (io.ReadCloser, error) +} + +var newFileOps func(dir string) (FileOps, error) diff --git a/feature/taildrop/fileops_fs.go b/feature/taildrop/fileops_fs.go new file mode 100644 index 0000000000000..4fecbe4af6bbb --- /dev/null +++ b/feature/taildrop/fileops_fs.go @@ -0,0 +1,221 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause +//go:build !android + +package taildrop + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "sync" + "unicode/utf8" +) + +var renameMu sync.Mutex + +// fsFileOps implements FileOps using the local filesystem rooted at a directory. +// It is used on non-Android platforms. +type fsFileOps struct{ rootDir string } + +func init() { + newFileOps = func(dir string) (FileOps, error) { + if dir == "" { + return nil, errors.New("rootDir cannot be empty") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("mkdir %q: %w", dir, err) + } + return fsFileOps{rootDir: dir}, nil + } +} + +func (f fsFileOps) OpenWriter(name string, offset int64, perm os.FileMode) (io.WriteCloser, string, error) { + path, err := joinDir(f.rootDir, name) + if err != nil { + return nil, "", err + } + if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, "", err + } + fi, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, perm) + if err != nil { + return nil, "", err + } + if offset != 0 { + curr, err := fi.Seek(0, io.SeekEnd) + if err != nil { + fi.Close() + return nil, "", err + } + if offset < 0 || offset > curr { + fi.Close() + return nil, "", fmt.Errorf("offset %d out of range", offset) + } + if _, err := fi.Seek(offset, io.SeekStart); err != nil { + fi.Close() + return nil, "", err + } + if err := fi.Truncate(offset); err != nil { + fi.Close() + return nil, "", err + } + } + return fi, path, nil +} + +func (f fsFileOps) Remove(name string) error { + path, err := joinDir(f.rootDir, name) + if err != nil { + return err + } + return os.Remove(path) +} + +// Rename moves the partial file into its final name. +// newName must be a base name (not absolute or containing path separators). +// It will retry up to 10 times, de-dup same-checksum files, etc. +func (f fsFileOps) Rename(oldPath, newName string) (newPath string, err error) { + var dst string + if filepath.IsAbs(newName) || strings.ContainsRune(newName, os.PathSeparator) { + return "", fmt.Errorf("invalid newName %q: must not be an absolute path or contain path separators", newName) + } + + dst = filepath.Join(f.rootDir, newName) + + if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { + return "", err + } + + st, err := os.Stat(oldPath) + if err != nil { + return "", err + } + wantSize := st.Size() + + const maxRetries = 10 + for i := 0; i < maxRetries; i++ { + renameMu.Lock() + fi, statErr := os.Stat(dst) + // Atomically rename the partial file as the destination file if it doesn't exist. + // Otherwise, it returns the length of the current destination file. + // The operation is atomic. + if os.IsNotExist(statErr) { + err = os.Rename(oldPath, dst) + renameMu.Unlock() + if err != nil { + return "", err + } + return dst, nil + } + if statErr != nil { + renameMu.Unlock() + return "", statErr + } + gotSize := fi.Size() + renameMu.Unlock() + + // Avoid the final rename if a destination file has the same contents. + // + // Note: this is best effort and copying files from iOS from the Media Library + // results in processing on the iOS side which means the size and shas of the + // same file can be different. + if gotSize == wantSize { + sumP, err := sha256File(oldPath) + if err != nil { + return "", err + } + sumD, err := sha256File(dst) + if err != nil { + return "", err + } + if bytes.Equal(sumP[:], sumD[:]) { + if err := os.Remove(oldPath); err != nil { + return "", err + } + return dst, nil + } + } + + // Choose a new destination filename and try again. + dst = filepath.Join(filepath.Dir(dst), nextFilename(filepath.Base(dst))) + } + + return "", fmt.Errorf("too many retries trying to rename %q to %q", oldPath, newName) +} + +// sha256File computes the SHA‑256 of a file. +func sha256File(path string) (sum [sha256.Size]byte, _ error) { + f, err := os.Open(path) + if err != nil { + return sum, err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return sum, err + } + copy(sum[:], h.Sum(nil)) + return sum, nil +} + +func (f fsFileOps) ListFiles() ([]string, error) { + entries, err := os.ReadDir(f.rootDir) + if err != nil { + return nil, err + } + var names []string + for _, e := range entries { + if e.Type().IsRegular() { + names = append(names, e.Name()) + } + } + return names, nil +} + +func (f fsFileOps) Stat(name string) (fs.FileInfo, error) { + path, err := joinDir(f.rootDir, name) + if err != nil { + return nil, err + } + return os.Stat(path) +} + +func (f fsFileOps) OpenReader(name string) (io.ReadCloser, error) { + path, err := joinDir(f.rootDir, name) + if err != nil { + return nil, err + } + return os.Open(path) +} + +// joinDir is like [filepath.Join] but returns an error if baseName is too long, +// is a relative path instead of a basename, or is otherwise invalid or unsafe for incoming files. +func joinDir(dir, baseName string) (string, error) { + if !utf8.ValidString(baseName) || + strings.TrimSpace(baseName) != baseName || + len(baseName) > 255 { + return "", ErrInvalidFileName + } + // TODO: validate unicode normalization form too? Varies by platform. + clean := path.Clean(baseName) + if clean != baseName || clean == "." || clean == ".." { + return "", ErrInvalidFileName + } + for _, r := range baseName { + if !validFilenameRune(r) { + return "", ErrInvalidFileName + } + } + if !filepath.IsLocal(baseName) { + return "", ErrInvalidFileName + } + return filepath.Join(dir, baseName), nil +} diff --git a/feature/taildrop/paths.go b/feature/taildrop/paths.go index 22d01160cff8e..79dc37d8f0699 100644 --- a/feature/taildrop/paths.go +++ b/feature/taildrop/paths.go @@ -21,7 +21,7 @@ func (e *Extension) SetDirectFileRoot(root string) { // SetFileOps sets the platform specific file operations. This is used // to call Android's Storage Access Framework APIs. func (e *Extension) SetFileOps(fileOps FileOps) { - e.FileOps = fileOps + e.fileOps = fileOps } func (e *Extension) setPlatformDefaultDirectFileRoot() { diff --git a/feature/taildrop/peerapi_test.go b/feature/taildrop/peerapi_test.go index 1a003b6eddca7..6339973544453 100644 --- a/feature/taildrop/peerapi_test.go +++ b/feature/taildrop/peerapi_test.go @@ -24,6 +24,7 @@ import ( "tailscale.com/tstest" "tailscale.com/tstime" "tailscale.com/types/logger" + "tailscale.com/util/must" ) // peerAPIHandler serves the PeerAPI for a source specific client. @@ -93,7 +94,16 @@ func bodyContains(sub string) check { func fileHasSize(name string, size int) check { return func(t *testing.T, e *peerAPITestEnv) { - root := e.taildrop.Dir() + fsImpl, ok := e.taildrop.opts.fileOps.(*fsFileOps) + if !ok { + t.Skip("fileHasSize only supported on fsFileOps backend") + return + } + root := fsImpl.rootDir + if root == "" { + t.Errorf("no rootdir; can't check whether %q has size %v", name, size) + return + } if root == "" { t.Errorf("no rootdir; can't check whether %q has size %v", name, size) return @@ -109,12 +119,12 @@ func fileHasSize(name string, size int) check { func fileHasContents(name string, want string) check { return func(t *testing.T, e *peerAPITestEnv) { - root := e.taildrop.Dir() - if root == "" { - t.Errorf("no rootdir; can't check contents of %q", name) + fsImpl, ok := e.taildrop.opts.fileOps.(*fsFileOps) + if !ok { + t.Skip("fileHasContents only supported on fsFileOps backend") return } - path := filepath.Join(root, name) + path := filepath.Join(fsImpl.rootDir, name) got, err := os.ReadFile(path) if err != nil { t.Errorf("fileHasContents: %v", err) @@ -172,9 +182,10 @@ func TestHandlePeerAPI(t *testing.T) { reqs: []*http.Request{httptest.NewRequest("PUT", "/v0/put/foo", nil)}, checks: checks( httpStatus(http.StatusForbidden), - bodyContains("Taildrop disabled; no storage directory"), + bodyContains("Taildrop disabled"), ), }, + { name: "bad_method", isSelf: true, @@ -471,14 +482,18 @@ func TestHandlePeerAPI(t *testing.T) { selfNode.CapMap = tailcfg.NodeCapMap{tailcfg.CapabilityDebug: nil} } var rootDir string + var fo FileOps if !tt.omitRoot { - rootDir = t.TempDir() + var err error + if fo, err = newFileOps(t.TempDir()); err != nil { + t.Fatalf("newFileOps: %v", err) + } } var e peerAPITestEnv e.taildrop = managerOptions{ - Logf: e.logBuf.Logf, - Dir: rootDir, + Logf: e.logBuf.Logf, + fileOps: fo, }.New() ext := &fakeExtension{ @@ -490,9 +505,7 @@ func TestHandlePeerAPI(t *testing.T) { e.ph = &peerAPIHandler{ isSelf: tt.isSelf, selfNode: selfNode.View(), - peerNode: (&tailcfg.Node{ - ComputedName: "some-peer-name", - }).View(), + peerNode: (&tailcfg.Node{ComputedName: "some-peer-name"}).View(), } for _, req := range tt.reqs { e.rr = httptest.NewRecorder() @@ -526,8 +539,8 @@ func TestHandlePeerAPI(t *testing.T) { func TestFileDeleteRace(t *testing.T) { dir := t.TempDir() taildropMgr := managerOptions{ - Logf: t.Logf, - Dir: dir, + Logf: t.Logf, + fileOps: must.Get(newFileOps(dir)), }.New() ph := &peerAPIHandler{ diff --git a/feature/taildrop/resume.go b/feature/taildrop/resume.go index 211a1ff6b68dd..20ef527a6da55 100644 --- a/feature/taildrop/resume.go +++ b/feature/taildrop/resume.go @@ -9,7 +9,6 @@ import ( "encoding/hex" "fmt" "io" - "io/fs" "os" "strings" ) @@ -51,19 +50,20 @@ func (cs *checksum) UnmarshalText(b []byte) error { // PartialFiles returns a list of partial files in [Handler.Dir] // that were sent (or is actively being sent) by the provided id. -func (m *manager) PartialFiles(id clientID) (ret []string, err error) { - if m == nil || m.opts.Dir == "" { +func (m *manager) PartialFiles(id clientID) ([]string, error) { + if m == nil || m.opts.fileOps == nil { return nil, ErrNoTaildrop } - suffix := id.partialSuffix() - if err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { - if name := de.Name(); strings.HasSuffix(name, suffix) { - ret = append(ret, name) + files, err := m.opts.fileOps.ListFiles() + if err != nil { + return nil, redactError(err) + } + var ret []string + for _, filename := range files { + if strings.HasSuffix(filename, suffix) { + ret = append(ret, filename) } - return true - }); err != nil { - return ret, redactError(err) } return ret, nil } @@ -73,17 +73,13 @@ func (m *manager) PartialFiles(id clientID) (ret []string, err error) { // It returns (BlockChecksum{}, io.EOF) when the stream is complete. // It is the caller's responsibility to call close. func (m *manager) HashPartialFile(id clientID, baseName string) (next func() (blockChecksum, error), close func() error, err error) { - if m == nil || m.opts.Dir == "" { + if m == nil || m.opts.fileOps == nil { return nil, nil, ErrNoTaildrop } noopNext := func() (blockChecksum, error) { return blockChecksum{}, io.EOF } noopClose := func() error { return nil } - dstFile, err := joinDir(m.opts.Dir, baseName) - if err != nil { - return nil, nil, err - } - f, err := os.Open(dstFile + id.partialSuffix()) + f, err := m.opts.fileOps.OpenReader(baseName + id.partialSuffix()) if err != nil { if os.IsNotExist(err) { return noopNext, noopClose, nil diff --git a/feature/taildrop/resume_test.go b/feature/taildrop/resume_test.go index dac3c657bfb58..4e59d401dcc53 100644 --- a/feature/taildrop/resume_test.go +++ b/feature/taildrop/resume_test.go @@ -8,6 +8,7 @@ import ( "io" "math/rand" "os" + "path/filepath" "testing" "testing/iotest" @@ -19,7 +20,9 @@ func TestResume(t *testing.T) { defer func() { blockSize = oldBlockSize }() blockSize = 256 - m := managerOptions{Logf: t.Logf, Dir: t.TempDir()}.New() + dir := t.TempDir() + + m := managerOptions{Logf: t.Logf, fileOps: must.Get(newFileOps(dir))}.New() defer m.Shutdown() rn := rand.New(rand.NewSource(0)) @@ -37,7 +40,7 @@ func TestResume(t *testing.T) { must.Do(close()) // Windows wants the file handle to be closed to rename it. must.Get(m.PutFile("", "foo", r, offset, -1)) - got := must.Get(os.ReadFile(must.Get(joinDir(m.opts.Dir, "foo")))) + got := must.Get(os.ReadFile(filepath.Join(dir, "foo"))) if !bytes.Equal(got, want) { t.Errorf("content mismatches") } @@ -66,7 +69,7 @@ func TestResume(t *testing.T) { t.Fatalf("too many iterations to complete the test") } } - got := must.Get(os.ReadFile(must.Get(joinDir(m.opts.Dir, "bar")))) + got := must.Get(os.ReadFile(filepath.Join(dir, "bar"))) if !bytes.Equal(got, want) { t.Errorf("content mismatches") } diff --git a/feature/taildrop/retrieve.go b/feature/taildrop/retrieve.go index 6fb97519363bc..b048a1b3b5f9d 100644 --- a/feature/taildrop/retrieve.go +++ b/feature/taildrop/retrieve.go @@ -9,19 +9,19 @@ import ( "io" "io/fs" "os" - "path/filepath" "runtime" "sort" "time" "tailscale.com/client/tailscale/apitype" "tailscale.com/logtail/backoff" + "tailscale.com/util/set" ) // HasFilesWaiting reports whether any files are buffered in [Handler.Dir]. // This always returns false when [Handler.DirectFileMode] is false. -func (m *manager) HasFilesWaiting() (has bool) { - if m == nil || m.opts.Dir == "" || m.opts.DirectFileMode { +func (m *manager) HasFilesWaiting() bool { + if m == nil || m.opts.fileOps == nil || m.opts.DirectFileMode { return false } @@ -30,63 +30,66 @@ func (m *manager) HasFilesWaiting() (has bool) { // has-files-or-not values as the macOS/iOS client might // in the future use+delete the files directly. So only // keep this negative cache. - totalReceived := m.totalReceived.Load() - if totalReceived == m.emptySince.Load() { + total := m.totalReceived.Load() + if total == m.emptySince.Load() { return false } - // Check whether there is at least one one waiting file. - err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { - name := de.Name() - if isPartialOrDeleted(name) || !de.Type().IsRegular() { - return true + files, err := m.opts.fileOps.ListFiles() + if err != nil { + return false + } + + // Build a set of filenames present in Dir + fileSet := set.Of(files...) + + for _, filename := range files { + if isPartialOrDeleted(filename) { + continue } - _, err := os.Stat(filepath.Join(m.opts.Dir, name+deletedSuffix)) - if os.IsNotExist(err) { - has = true - return false + if fileSet.Contains(filename + deletedSuffix) { + continue // already handled } + // Found at least one downloadable file return true - }) - - // If there are no more waiting files, record totalReceived as emptySince - // so that we can short-circuit the expensive directory traversal - // if no files have been received after the start of this call. - if err == nil && !has { - m.emptySince.Store(totalReceived) } - return has + + // No waiting files → update negative‑result cache + m.emptySince.Store(total) + return false } // WaitingFiles returns the list of files that have been sent by a // peer that are waiting in [Handler.Dir]. // This always returns nil when [Handler.DirectFileMode] is false. -func (m *manager) WaitingFiles() (ret []apitype.WaitingFile, err error) { - if m == nil || m.opts.Dir == "" { +func (m *manager) WaitingFiles() ([]apitype.WaitingFile, error) { + if m == nil || m.opts.fileOps == nil { return nil, ErrNoTaildrop } if m.opts.DirectFileMode { return nil, nil } - if err := rangeDir(m.opts.Dir, func(de fs.DirEntry) bool { - name := de.Name() - if isPartialOrDeleted(name) || !de.Type().IsRegular() { - return true + names, err := m.opts.fileOps.ListFiles() + if err != nil { + return nil, redactError(err) + } + var ret []apitype.WaitingFile + for _, name := range names { + if isPartialOrDeleted(name) { + continue } - _, err := os.Stat(filepath.Join(m.opts.Dir, name+deletedSuffix)) - if os.IsNotExist(err) { - fi, err := de.Info() - if err != nil { - return true - } - ret = append(ret, apitype.WaitingFile{ - Name: filepath.Base(name), - Size: fi.Size(), - }) + // A corresponding .deleted marker means the file was already handled. + if _, err := m.opts.fileOps.Stat(name + deletedSuffix); err == nil { + continue } - return true - }); err != nil { - return nil, redactError(err) + fi, err := m.opts.fileOps.Stat(name) + if err != nil { + continue + } + ret = append(ret, apitype.WaitingFile{ + Name: name, + Size: fi.Size(), + }) } sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) return ret, nil @@ -95,21 +98,18 @@ func (m *manager) WaitingFiles() (ret []apitype.WaitingFile, err error) { // DeleteFile deletes a file of the given baseName from [Handler.Dir]. // This method is only allowed when [Handler.DirectFileMode] is false. func (m *manager) DeleteFile(baseName string) error { - if m == nil || m.opts.Dir == "" { + if m == nil || m.opts.fileOps == nil { return ErrNoTaildrop } if m.opts.DirectFileMode { return errors.New("deletes not allowed in direct mode") } - path, err := joinDir(m.opts.Dir, baseName) - if err != nil { - return err - } + var bo *backoff.Backoff logf := m.opts.Logf t0 := m.opts.Clock.Now() for { - err := os.Remove(path) + err := m.opts.fileOps.Remove(baseName) if err != nil && !os.IsNotExist(err) { err = redactError(err) // Put a retry loop around deletes on Windows. @@ -129,7 +129,7 @@ func (m *manager) DeleteFile(baseName string) error { bo.BackOff(context.Background(), err) continue } - if err := touchFile(path + deletedSuffix); err != nil { + if err := m.touchFile(baseName + deletedSuffix); err != nil { logf("peerapi: failed to leave deleted marker: %v", err) } m.deleter.Insert(baseName + deletedSuffix) @@ -141,35 +141,31 @@ func (m *manager) DeleteFile(baseName string) error { } } -func touchFile(path string) error { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) +func (m *manager) touchFile(name string) error { + wc, _, err := m.opts.fileOps.OpenWriter(name /* offset= */, 0, 0666) if err != nil { return redactError(err) } - return f.Close() + return wc.Close() } // OpenFile opens a file of the given baseName from [Handler.Dir]. // This method is only allowed when [Handler.DirectFileMode] is false. func (m *manager) OpenFile(baseName string) (rc io.ReadCloser, size int64, err error) { - if m == nil || m.opts.Dir == "" { + if m == nil || m.opts.fileOps == nil { return nil, 0, ErrNoTaildrop } if m.opts.DirectFileMode { return nil, 0, errors.New("opens not allowed in direct mode") } - path, err := joinDir(m.opts.Dir, baseName) - if err != nil { - return nil, 0, err - } - if _, err := os.Stat(path + deletedSuffix); err == nil { - return nil, 0, redactError(&fs.PathError{Op: "open", Path: path, Err: fs.ErrNotExist}) + if _, err := m.opts.fileOps.Stat(baseName + deletedSuffix); err == nil { + return nil, 0, redactError(&fs.PathError{Op: "open", Path: baseName, Err: fs.ErrNotExist}) } - f, err := os.Open(path) + f, err := m.opts.fileOps.OpenReader(baseName) if err != nil { return nil, 0, redactError(err) } - fi, err := f.Stat() + fi, err := m.opts.fileOps.Stat(baseName) if err != nil { f.Close() return nil, 0, redactError(err) diff --git a/feature/taildrop/send.go b/feature/taildrop/send.go index 59a1701da6f0d..32ba5f6f0d644 100644 --- a/feature/taildrop/send.go +++ b/feature/taildrop/send.go @@ -4,11 +4,8 @@ package taildrop import ( - "crypto/sha256" "fmt" "io" - "os" - "path/filepath" "sync" "time" @@ -73,9 +70,10 @@ func (f *incomingFile) Write(p []byte) (n int, err error) { // specific partial file. This allows the client to determine whether to resume // a partial file. While resuming, PutFile may be called again with a non-zero // offset to specify where to resume receiving data at. -func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, length int64) (int64, error) { +func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, length int64) (fileLength int64, err error) { + switch { - case m == nil || m.opts.Dir == "": + case m == nil || m.opts.fileOps == nil: return 0, ErrNoTaildrop case !envknob.CanTaildrop(): return 0, ErrNoTaildrop @@ -83,47 +81,47 @@ func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, len return 0, ErrNotAccessible } - //Compute dstPath & avoid mid‑upload deletion - var dstPath string - if m.opts.Mode == PutModeDirect { - var err error - dstPath, err = joinDir(m.opts.Dir, baseName) + if err := validateBaseName(baseName); err != nil { + return 0, err + } + + // and make sure we don't delete it while uploading: + m.deleter.Remove(baseName) + + // Create (if not already) the partial file with read-write permissions. + partialName := baseName + id.partialSuffix() + wc, partialPath, err := m.opts.fileOps.OpenWriter(partialName, offset, 0o666) + if err != nil { + return 0, m.redactAndLogError("Create", err) + } + defer func() { + wc.Close() if err != nil { - return 0, err + m.deleter.Insert(partialName) // mark partial file for eventual deletion } - } else { - // In SAF mode, we simply use the baseName as the destination "path" - // (the actual directory is managed by SAF). - dstPath = baseName - } - m.deleter.Remove(filepath.Base(dstPath)) // avoid deleting the partial file while receiving + }() // Check whether there is an in-progress transfer for the file. - partialFileKey := incomingFileKey{id, baseName} - inFile, loaded := m.incomingFiles.LoadOrInit(partialFileKey, func() *incomingFile { - return &incomingFile{ + inFileKey := incomingFileKey{id, baseName} + inFile, loaded := m.incomingFiles.LoadOrInit(inFileKey, func() *incomingFile { + inFile := &incomingFile{ clock: m.opts.Clock, started: m.opts.Clock.Now(), size: length, sendFileNotify: m.opts.SendFileNotify, } + if m.opts.DirectFileMode { + inFile.partialPath = partialPath + } + return inFile }) + + inFile.w = wc + if loaded { return 0, ErrFileExists } - defer m.incomingFiles.Delete(partialFileKey) - - // Open writer & populate inFile paths - wc, partialPath, err := m.openWriterAndPaths(id, m.opts.Mode, inFile, baseName, dstPath, offset) - if err != nil { - return 0, m.redactAndLogError("Create", err) - } - defer func() { - wc.Close() - if err != nil { - m.deleter.Insert(filepath.Base(partialPath)) // mark partial file for eventual deletion - } - }() + defer m.incomingFiles.Delete(inFileKey) // Record that we have started to receive at least one file. // This is used by the deleter upon a cold-start to scan the directory @@ -148,220 +146,26 @@ func (m *manager) PutFile(id clientID, baseName string, r io.Reader, offset, len return 0, m.redactAndLogError("Close", err) } - fileLength := offset + copyLength + fileLength = offset + copyLength inFile.mu.Lock() inFile.done = true inFile.mu.Unlock() - // Finalize rename - switch m.opts.Mode { - case PutModeDirect: - var finalDst string - finalDst, err = m.finalizeDirect(inFile, partialPath, dstPath, fileLength) - if err != nil { - return 0, m.redactAndLogError("Rename", err) - } - inFile.finalPath = finalDst - - case PutModeAndroidSAF: - if err = m.finalizeSAF(partialPath, baseName); err != nil { - return 0, m.redactAndLogError("Rename", err) - } + // 6) Finalize (rename/move) the partial into place via FileOps.Rename + finalPath, err := m.opts.fileOps.Rename(partialPath, baseName) + if err != nil { + return 0, m.redactAndLogError("Rename", err) } + inFile.finalPath = finalPath m.totalReceived.Add(1) m.opts.SendFileNotify() return fileLength, nil } -// openWriterAndPaths opens the correct writer, seeks/truncates if needed, -// and sets inFile.partialPath & inFile.finalPath for later cleanup/rename. -// The caller is responsible for closing the file on completion. -func (m *manager) openWriterAndPaths( - id clientID, - mode PutMode, - inFile *incomingFile, - baseName string, - dstPath string, - offset int64, -) (wc io.WriteCloser, partialPath string, err error) { - switch mode { - - case PutModeDirect: - partialPath = dstPath + id.partialSuffix() - f, err := os.OpenFile(partialPath, os.O_CREATE|os.O_RDWR, 0o666) - if err != nil { - return nil, "", m.redactAndLogError("Create", err) - } - if offset != 0 { - curr, err := f.Seek(0, io.SeekEnd) - if err != nil { - f.Close() - return nil, "", m.redactAndLogError("Seek", err) - } - if offset < 0 || offset > curr { - f.Close() - return nil, "", m.redactAndLogError("Seek", fmt.Errorf("offset %d out of range", offset)) - } - if _, err := f.Seek(offset, io.SeekStart); err != nil { - f.Close() - return nil, "", m.redactAndLogError("Seek", err) - } - if err := f.Truncate(offset); err != nil { - f.Close() - return nil, "", m.redactAndLogError("Truncate", err) - } - } - inFile.w = f - wc = f - inFile.partialPath = partialPath - inFile.finalPath = dstPath - return wc, partialPath, nil - - case PutModeAndroidSAF: - if m.opts.FileOps == nil { - return nil, "", m.redactAndLogError("Create (SAF)", fmt.Errorf("missing FileOps")) - } - writer, uri, err := m.opts.FileOps.OpenFileWriter(baseName) - if err != nil { - return nil, "", m.redactAndLogError("Create (SAF)", fmt.Errorf("failed to open file for writing via SAF")) - } - if writer == nil || uri == "" { - return nil, "", fmt.Errorf("invalid SAF writer or URI") - } - // SAF mode does not support resuming, so enforce offset == 0. - if offset != 0 { - writer.Close() - return nil, "", m.redactAndLogError("Seek", fmt.Errorf("resuming is not supported in SAF mode")) - } - inFile.w = writer - wc = writer - partialPath = uri - inFile.partialPath = uri - inFile.finalPath = baseName - return wc, partialPath, nil - - default: - return nil, "", fmt.Errorf("unsupported PutMode: %v", mode) - } -} - -// finalizeDirect atomically renames or dedups the partial file, retrying -// under new names up to 10 times. It returns the final path that succeeded. -func (m *manager) finalizeDirect( - inFile *incomingFile, - partialPath string, - initialDst string, - fileLength int64, -) (string, error) { - var ( - once sync.Once - cachedSum [sha256.Size]byte - cacheErr error - computeSum = func() ([sha256.Size]byte, error) { - once.Do(func() { cachedSum, cacheErr = sha256File(partialPath) }) - return cachedSum, cacheErr - } - ) - - dstPath := initialDst - const maxRetries = 10 - for i := 0; i < maxRetries; i++ { - // Atomically rename the partial file as the destination file if it doesn't exist. - // Otherwise, it returns the length of the current destination file. - // The operation is atomic. - lengthOnDisk, err := func() (int64, error) { - m.renameMu.Lock() - defer m.renameMu.Unlock() - fi, statErr := os.Stat(dstPath) - if os.IsNotExist(statErr) { - // dst missing → rename partial into place - return -1, os.Rename(partialPath, dstPath) - } - if statErr != nil { - return -1, statErr - } - return fi.Size(), nil - }() - if err != nil { - return "", err - } - if lengthOnDisk < 0 { - // successfully moved - inFile.finalPath = dstPath - return dstPath, nil - } - - // Avoid the final rename if a destination file has the same contents. - // - // Note: this is best effort and copying files from iOS from the Media Library - // results in processing on the iOS side which means the size and shas of the - // same file can be different. - if lengthOnDisk == fileLength { - partSum, err := computeSum() - if err != nil { - return "", err - } - dstSum, err := sha256File(dstPath) - if err != nil { - return "", err - } - if partSum == dstSum { - // same content → drop the partial - if err := os.Remove(partialPath); err != nil { - return "", err - } - inFile.finalPath = dstPath - return dstPath, nil - } - } - - // Choose a new destination filename and try again. - dstPath = nextFilename(dstPath) - } - - return "", fmt.Errorf("too many retries trying to rename a partial file %q", initialDst) -} - -// finalizeSAF retries RenamePartialFile up to 10 times, generating a new -// name on each failure until the SAF URI changes. -func (m *manager) finalizeSAF( - partialPath, finalName string, -) error { - if m.opts.FileOps == nil { - return fmt.Errorf("missing FileOps for SAF finalize") - } - const maxTries = 10 - name := finalName - for i := 0; i < maxTries; i++ { - newURI, err := m.opts.FileOps.RenamePartialFile(partialPath, m.opts.Dir, name) - if err != nil { - return err - } - if newURI != "" && newURI != name { - return nil - } - name = nextFilename(name) - } - return fmt.Errorf("failed to finalize SAF file after %d retries", maxTries) -} - func (m *manager) redactAndLogError(stage string, err error) error { err = redactError(err) m.opts.Logf("put %s error: %v", stage, err) return err } - -func sha256File(file string) (out [sha256.Size]byte, err error) { - h := sha256.New() - f, err := os.Open(file) - if err != nil { - return out, err - } - defer f.Close() - if _, err := io.Copy(h, f); err != nil { - return out, err - } - return [sha256.Size]byte(h.Sum(nil)), nil -} diff --git a/feature/taildrop/send_test.go b/feature/taildrop/send_test.go index 8edb704172fc5..9ffa5fccc0a36 100644 --- a/feature/taildrop/send_test.go +++ b/feature/taildrop/send_test.go @@ -4,123 +4,64 @@ package taildrop import ( - "bytes" - "fmt" - "io" "os" "path/filepath" + "strings" "testing" "tailscale.com/tstime" + "tailscale.com/util/must" ) -// nopWriteCloser is a no-op io.WriteCloser wrapping a bytes.Buffer. -type nopWriteCloser struct{ *bytes.Buffer } - -func (nwc nopWriteCloser) Close() error { return nil } - -// mockFileOps implements just enough of the FileOps interface for SAF tests. -type mockFileOps struct { - writes *bytes.Buffer - renameOK bool -} - -func (m *mockFileOps) OpenFileWriter(name string) (io.WriteCloser, string, error) { - m.writes = new(bytes.Buffer) - return nopWriteCloser{m.writes}, "uri://" + name + ".partial", nil -} - -func (m *mockFileOps) RenamePartialFile(partialPath, dir, finalName string) (string, error) { - if !m.renameOK { - m.renameOK = true - return "uri://" + finalName, nil - } - return "", io.ErrUnexpectedEOF -} - func TestPutFile(t *testing.T) { const content = "hello, world" tests := []struct { - name string - mode PutMode - setup func(t *testing.T) (*manager, string, *mockFileOps) - wantFile string + name string + directFileMode bool }{ - { - name: "PutModeDirect", - mode: PutModeDirect, - setup: func(t *testing.T) (*manager, string, *mockFileOps) { - dir := t.TempDir() - opts := managerOptions{ - Logf: t.Logf, - Clock: tstime.DefaultClock{}, - State: nil, - Dir: dir, - Mode: PutModeDirect, - DirectFileMode: true, - SendFileNotify: func() {}, - } - mgr := opts.New() - return mgr, dir, nil - }, - wantFile: "file.txt", - }, - { - name: "PutModeAndroidSAF", - mode: PutModeAndroidSAF, - setup: func(t *testing.T) (*manager, string, *mockFileOps) { - // SAF still needs a non-empty Dir to pass the guard. - dir := t.TempDir() - mops := &mockFileOps{} - opts := managerOptions{ - Logf: t.Logf, - Clock: tstime.DefaultClock{}, - State: nil, - Dir: dir, - Mode: PutModeAndroidSAF, - FileOps: mops, - DirectFileMode: true, - SendFileNotify: func() {}, - } - mgr := opts.New() - return mgr, dir, mops - }, - wantFile: "file.txt", - }, + {"DirectFileMode", true}, + {"NonDirectFileMode", false}, } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - mgr, dir, mops := tc.setup(t) - id := clientID(fmt.Sprint(0)) - reader := bytes.NewReader([]byte(content)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + mgr := managerOptions{ + Logf: t.Logf, + Clock: tstime.DefaultClock{}, + State: nil, + fileOps: must.Get(newFileOps(dir)), + DirectFileMode: tt.directFileMode, + SendFileNotify: func() {}, + }.New() - n, err := mgr.PutFile(id, "file.txt", reader, 0, int64(len(content))) + id := clientID("0") + n, err := mgr.PutFile(id, "file.txt", strings.NewReader(content), 0, int64(len(content))) if err != nil { - t.Fatalf("PutFile(%s) error: %v", tc.name, err) + t.Fatalf("PutFile error: %v", err) } if n != int64(len(content)) { t.Errorf("wrote %d bytes; want %d", n, len(content)) } - switch tc.mode { - case PutModeDirect: - path := filepath.Join(dir, tc.wantFile) - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("ReadFile error: %v", err) - } - if got := string(data); got != content { - t.Errorf("file contents = %q; want %q", got, content) - } + path := filepath.Join(dir, "file.txt") - case PutModeAndroidSAF: - if mops.writes == nil { - t.Fatal("SAF writer was never created") - } - if got := mops.writes.String(); got != content { - t.Errorf("SAF writes = %q; want %q", got, content) + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile %q: %v", path, err) + } + if string(got) != content { + t.Errorf("file contents = %q; want %q", string(got), content) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.Contains(entry.Name(), ".partial") { + t.Errorf("unexpected partial file left behind: %s", entry.Name()) } } }) diff --git a/feature/taildrop/taildrop.go b/feature/taildrop/taildrop.go index 2dfa415bbf0cc..6c3deaed1b538 100644 --- a/feature/taildrop/taildrop.go +++ b/feature/taildrop/taildrop.go @@ -12,8 +12,6 @@ package taildrop import ( "errors" "hash/adler32" - "io" - "io/fs" "os" "path" "path/filepath" @@ -21,7 +19,6 @@ import ( "sort" "strconv" "strings" - "sync" "sync/atomic" "unicode" "unicode/utf8" @@ -72,11 +69,6 @@ type managerOptions struct { Clock tstime.DefaultClock // may be nil State ipn.StateStore // may be nil - // Dir is the directory to store received files. - // This main either be the final location for the files - // or just a temporary staging directory (see DirectFileMode). - Dir string - // DirectFileMode reports whether we are writing files // directly to a download directory, rather than writing them to // a temporary staging directory. @@ -91,9 +83,10 @@ type managerOptions struct { // copy them out, and then delete them. DirectFileMode bool - FileOps FileOps - - Mode PutMode + // FileOps abstracts platform-specific file operations needed for file transfers. + // Android's implementation uses the Storage Access Framework, and other platforms + // use fsFileOps. + fileOps FileOps // SendFileNotify is called periodically while a file is actively // receiving the contents for the file. There is a final call @@ -111,9 +104,6 @@ type manager struct { // deleter managers asynchronous deletion of files. deleter fileDeleter - // renameMu is used to protect os.Rename calls so that they are atomic. - renameMu sync.Mutex - // totalReceived counts the cumulative total of received files. totalReceived atomic.Int64 // emptySince specifies that there were no waiting files @@ -137,11 +127,6 @@ func (opts managerOptions) New() *manager { return m } -// Dir returns the directory. -func (m *manager) Dir() string { - return m.opts.Dir -} - // Shutdown shuts down the Manager. // It blocks until all spawned goroutines have stopped running. func (m *manager) Shutdown() { @@ -172,57 +157,29 @@ func isPartialOrDeleted(s string) bool { return strings.HasSuffix(s, deletedSuffix) || strings.HasSuffix(s, partialSuffix) } -func joinDir(dir, baseName string) (fullPath string, err error) { - if !utf8.ValidString(baseName) { - return "", ErrInvalidFileName - } - if strings.TrimSpace(baseName) != baseName { - return "", ErrInvalidFileName - } - if len(baseName) > 255 { - return "", ErrInvalidFileName +func validateBaseName(name string) error { + if !utf8.ValidString(name) || + strings.TrimSpace(name) != name || + len(name) > 255 { + return ErrInvalidFileName } // TODO: validate unicode normalization form too? Varies by platform. - clean := path.Clean(baseName) - if clean != baseName || - clean == "." || clean == ".." || - isPartialOrDeleted(clean) { - return "", ErrInvalidFileName + clean := path.Clean(name) + if clean != name || clean == "." || clean == ".." { + return ErrInvalidFileName } - for _, r := range baseName { + if isPartialOrDeleted(name) { + return ErrInvalidFileName + } + for _, r := range name { if !validFilenameRune(r) { - return "", ErrInvalidFileName + return ErrInvalidFileName } } - if !filepath.IsLocal(baseName) { - return "", ErrInvalidFileName - } - return filepath.Join(dir, baseName), nil -} - -// rangeDir iterates over the contents of a directory, calling fn for each entry. -// It continues iterating while fn returns true. -// It reports the number of entries seen. -func rangeDir(dir string, fn func(fs.DirEntry) bool) error { - f, err := os.Open(dir) - if err != nil { - return err - } - defer f.Close() - for { - des, err := f.ReadDir(10) - for _, de := range des { - if !fn(de) { - return nil - } - } - if err != nil { - if err == io.EOF { - return nil - } - return err - } + if !filepath.IsLocal(name) { + return ErrInvalidFileName } + return nil } // IncomingFiles returns a list of active incoming files. diff --git a/feature/taildrop/taildrop_test.go b/feature/taildrop/taildrop_test.go index da0bd2f430579..0d77273f0aab0 100644 --- a/feature/taildrop/taildrop_test.go +++ b/feature/taildrop/taildrop_test.go @@ -4,40 +4,10 @@ package taildrop import ( - "path/filepath" "strings" "testing" ) -func TestJoinDir(t *testing.T) { - dir := t.TempDir() - tests := []struct { - in string - want string // just relative to m.Dir - wantOk bool - }{ - {"", "", false}, - {"foo", "foo", true}, - {"./foo", "", false}, - {"../foo", "", false}, - {"foo/bar", "", false}, - {"😋", "😋", true}, - {"\xde\xad\xbe\xef", "", false}, - {"foo.partial", "", false}, - {"foo.deleted", "", false}, - {strings.Repeat("a", 1024), "", false}, - {"foo:bar", "", false}, - } - for _, tt := range tests { - got, gotErr := joinDir(dir, tt.in) - got, _ = filepath.Rel(dir, got) - gotOk := gotErr == nil - if got != tt.want || gotOk != tt.wantOk { - t.Errorf("joinDir(%q) = (%v, %v), want (%v, %v)", tt.in, got, gotOk, tt.want, tt.wantOk) - } - } -} - func TestNextFilename(t *testing.T) { tests := []struct { in string @@ -67,3 +37,29 @@ func TestNextFilename(t *testing.T) { } } } + +func TestValidateBaseName(t *testing.T) { + tests := []struct { + in string + wantOk bool + }{ + {"", false}, + {"foo", true}, + {"./foo", false}, + {"../foo", false}, + {"foo/bar", false}, + {"😋", true}, + {"\xde\xad\xbe\xef", false}, + {"foo.partial", false}, + {"foo.deleted", false}, + {strings.Repeat("a", 1024), false}, + {"foo:bar", false}, + } + for _, tt := range tests { + err := validateBaseName(tt.in) + gotOk := err == nil + if gotOk != tt.wantOk { + t.Errorf("validateBaseName(%q) = %v, wantOk = %v", tt.in, err, tt.wantOk) + } + } +} diff --git a/feature/tpm/tpm.go b/feature/tpm/tpm.go index 18e56ae891ee1..0260cca586e13 100644 --- a/feature/tpm/tpm.go +++ b/feature/tpm/tpm.go @@ -5,14 +5,30 @@ package tpm import ( + "bytes" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "iter" + "log" + "os" + "path/filepath" "slices" + "strings" "sync" "github.com/google/go-tpm/tpm2" "github.com/google/go-tpm/tpm2/transport" + "golang.org/x/crypto/nacl/secretbox" + "tailscale.com/atomicfile" "tailscale.com/feature" "tailscale.com/hostinfo" + "tailscale.com/ipn" + "tailscale.com/ipn/store" + "tailscale.com/paths" "tailscale.com/tailcfg" + "tailscale.com/types/logger" ) var infoOnce = sync.OnceValue(info) @@ -22,10 +38,18 @@ func init() { hostinfo.RegisterHostinfoNewHook(func(hi *tailcfg.Hostinfo) { hi.TPM = infoOnce() }) + store.Register(store.TPMPrefix, newStore) } -//lint:ignore U1000 used in Linux and Windows builds only -func infoFromCapabilities(tpm transport.TPM) *tailcfg.TPMInfo { +func info() *tailcfg.TPMInfo { + tpm, err := open() + if err != nil { + log.Printf("TPM: error opening: %v", err) + return nil + } + log.Printf("TPM: successfully opened") + defer tpm.Close() + info := new(tailcfg.TPMInfo) toStr := func(s *string) func(*tailcfg.TPMInfo, uint32) { return func(info *tailcfg.TPMInfo, value uint32) { @@ -52,10 +76,12 @@ func infoFromCapabilities(tpm transport.TPM) *tailcfg.TPMInfo { PropertyCount: 1, }.Execute(tpm) if err != nil { + log.Printf("TPM: GetCapability %v: %v", cap.prop, err) continue } props, err := resp.CapabilityData.Data.TPMProperties() if err != nil { + log.Printf("TPM: GetCapability %v: %v", cap.prop, err) continue } if len(props.TPMProperty) == 0 { @@ -81,3 +107,317 @@ func propToString(v uint32) string { // Delete any non-printable ASCII characters. return string(slices.DeleteFunc(chars, func(b byte) bool { return b < ' ' || b > '~' })) } + +func newStore(logf logger.Logf, path string) (ipn.StateStore, error) { + path = strings.TrimPrefix(path, store.TPMPrefix) + if err := paths.MkStateDir(filepath.Dir(path)); err != nil { + return nil, fmt.Errorf("creating state directory: %w", err) + } + var parsed map[ipn.StateKey][]byte + bs, err := os.ReadFile(path) + if err != nil { + if !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to open %q: %w", path, err) + } + logf("tpm.newStore: initializing state file") + + var key [32]byte + // crypto/rand.Read never returns an error. + rand.Read(key[:]) + + store := &tpmStore{ + logf: logf, + path: path, + key: key, + cache: make(map[ipn.StateKey][]byte), + } + if err := store.writeSealed(); err != nil { + return nil, fmt.Errorf("failed to write initial state file: %w", err) + } + return store, nil + } + + // State file exists, unseal and parse it. + var sealed encryptedData + if err := json.Unmarshal(bs, &sealed); err != nil { + return nil, fmt.Errorf("failed to unmarshal state file: %w", err) + } + if len(sealed.Data) == 0 || sealed.Key == nil || len(sealed.Nonce) == 0 { + return nil, fmt.Errorf("state file %q has not been TPM-sealed or is corrupt", path) + } + data, err := unseal(logf, sealed) + if err != nil { + return nil, fmt.Errorf("failed to unseal state file: %w", err) + } + if err := json.Unmarshal(data.Data, &parsed); err != nil { + return nil, fmt.Errorf("failed to parse state file: %w", err) + } + return &tpmStore{ + logf: logf, + path: path, + key: data.Key, + cache: parsed, + }, nil +} + +// tpmStore is an ipn.StateStore that stores the state in a secretbox-encrypted +// file using a TPM-sealed symmetric key. +type tpmStore struct { + ipn.EncryptedStateStore + + logf logger.Logf + path string + key [32]byte + + mu sync.RWMutex + cache map[ipn.StateKey][]byte +} + +func (s *tpmStore) ReadState(k ipn.StateKey) ([]byte, error) { + s.mu.RLock() + defer s.mu.RUnlock() + v, ok := s.cache[k] + if !ok { + return nil, ipn.ErrStateNotExist + } + return bytes.Clone(v), nil +} + +func (s *tpmStore) WriteState(k ipn.StateKey, bs []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + if bytes.Equal(s.cache[k], bs) { + return nil + } + s.cache[k] = bytes.Clone(bs) + + return s.writeSealed() +} + +func (s *tpmStore) writeSealed() error { + bs, err := json.Marshal(s.cache) + if err != nil { + return err + } + sealed, err := seal(s.logf, decryptedData{Key: s.key, Data: bs}) + if err != nil { + return fmt.Errorf("failed to seal state file: %w", err) + } + buf, err := json.Marshal(sealed) + if err != nil { + return err + } + return atomicfile.WriteFile(s.path, buf, 0600) +} + +func (s *tpmStore) All() iter.Seq2[ipn.StateKey, []byte] { + return func(yield func(ipn.StateKey, []byte) bool) { + s.mu.Lock() + defer s.mu.Unlock() + + for k, v := range s.cache { + if !yield(k, v) { + break + } + } + } +} + +// Ensure tpmStore implements store.ExportableStore for migration to/from +// store.FileStore. +var _ store.ExportableStore = (*tpmStore)(nil) + +// The nested levels of encoding and encryption are confusing, so here's what's +// going on in plain English. +// +// Not all TPM devices support symmetric encryption (TPM2_EncryptDecrypt2) +// natively, but they do support "sealing" small values (see +// tpmSeal/tpmUnseal). The size limit is too small for the actual state file, +// so we seal a symmetric key instead. This symmetric key is then used to seal +// the actual data using nacl/secretbox. +// Confusingly, both TPMs and secretbox use "seal" terminology. +// +// tpmSeal/tpmUnseal do the lower-level sealing of small []byte blobs, which we +// use to seal a 32-byte secretbox key. +// +// seal/unseal do the higher-level sealing of store data using secretbox, and +// also sealing of the symmetric key using TPM. + +// decryptedData contains the fully decrypted raw data along with the symmetric +// key used for secretbox. This struct should only live in memory and never get +// stored to disk! +type decryptedData struct { + Key [32]byte + Data []byte +} + +func (decryptedData) MarshalJSON() ([]byte, error) { + return nil, errors.New("[unexpected]: decryptedData should never get JSON-marshaled!") +} + +// encryptedData contains the secretbox-sealed data and nonce, along with a +// TPM-sealed key. All fields are required. +type encryptedData struct { + Key *tpmSealedData `json:"key"` + Nonce []byte `json:"nonce"` + Data []byte `json:"data"` +} + +func seal(logf logger.Logf, dec decryptedData) (*encryptedData, error) { + var nonce [24]byte + // crypto/rand.Read never returns an error. + rand.Read(nonce[:]) + + sealedData := secretbox.Seal(nil, dec.Data, &nonce, &dec.Key) + sealedKey, err := tpmSeal(logf, dec.Key[:]) + if err != nil { + return nil, fmt.Errorf("failed to seal encryption key to TPM: %w", err) + } + + return &encryptedData{ + Key: sealedKey, + Nonce: nonce[:], + Data: sealedData, + }, nil +} + +func unseal(logf logger.Logf, data encryptedData) (*decryptedData, error) { + if len(data.Nonce) != 24 { + return nil, fmt.Errorf("nonce should be 24 bytes long, got %d", len(data.Nonce)) + } + + unsealedKey, err := tpmUnseal(logf, data.Key) + if err != nil { + return nil, fmt.Errorf("failed to unseal encryption key with TPM: %w", err) + } + if len(unsealedKey) != 32 { + return nil, fmt.Errorf("unsealed key should be 32 bytes long, got %d", len(unsealedKey)) + } + unsealedData, ok := secretbox.Open(nil, data.Data, (*[24]byte)(data.Nonce), (*[32]byte)(unsealedKey)) + if !ok { + return nil, errors.New("failed to unseal data") + } + + return &decryptedData{ + Key: *(*[32]byte)(unsealedKey), + Data: unsealedData, + }, nil +} + +type tpmSealedData struct { + Private []byte + Public []byte +} + +// withSRK runs fn with the loaded Storage Root Key (SRK) handle. The SRK is +// flushed after fn returns. +func withSRK(logf logger.Logf, tpm transport.TPM, fn func(srk tpm2.AuthHandle) error) error { + srkCmd := tpm2.CreatePrimary{ + PrimaryHandle: tpm2.TPMRHOwner, + InPublic: tpm2.New2B(tpm2.ECCSRKTemplate), + } + srkRes, err := srkCmd.Execute(tpm) + if err != nil { + return fmt.Errorf("tpm2.CreatePrimary: %w", err) + } + defer func() { + cmd := tpm2.FlushContext{FlushHandle: srkRes.ObjectHandle} + if _, err := cmd.Execute(tpm); err != nil { + logf("tpm2.FlushContext: failed to flush SRK handle: %v", err) + } + }() + + return fn(tpm2.AuthHandle{ + Handle: srkRes.ObjectHandle, + Name: srkRes.Name, + Auth: tpm2.HMAC(tpm2.TPMAlgSHA256, 32), + }) +} + +// tpmSeal seals the data using SRK of the local TPM. +func tpmSeal(logf logger.Logf, data []byte) (*tpmSealedData, error) { + tpm, err := open() + if err != nil { + return nil, fmt.Errorf("opening TPM: %w", err) + } + defer tpm.Close() + + var res *tpmSealedData + err = withSRK(logf, tpm, func(srk tpm2.AuthHandle) error { + sealCmd := tpm2.Create{ + ParentHandle: srk, + InSensitive: tpm2.TPM2BSensitiveCreate{ + Sensitive: &tpm2.TPMSSensitiveCreate{ + Data: tpm2.NewTPMUSensitiveCreate(&tpm2.TPM2BSensitiveData{ + Buffer: data, + }), + }, + }, + InPublic: tpm2.New2B(tpm2.TPMTPublic{ + Type: tpm2.TPMAlgKeyedHash, + NameAlg: tpm2.TPMAlgSHA256, + ObjectAttributes: tpm2.TPMAObject{ + FixedTPM: true, + FixedParent: true, + UserWithAuth: true, + }, + }), + } + sealRes, err := sealCmd.Execute(tpm) + if err != nil { + return fmt.Errorf("tpm2.Create: %w", err) + } + + res = &tpmSealedData{ + Private: sealRes.OutPrivate.Buffer, + Public: sealRes.OutPublic.Bytes(), + } + return nil + }) + return res, err +} + +// tpmUnseal unseals the data using SRK of the local TPM. +func tpmUnseal(logf logger.Logf, data *tpmSealedData) ([]byte, error) { + tpm, err := open() + if err != nil { + return nil, fmt.Errorf("opening TPM: %w", err) + } + defer tpm.Close() + + var res []byte + err = withSRK(logf, tpm, func(srk tpm2.AuthHandle) error { + // Load the sealed object into the TPM first under SRK. + loadCmd := tpm2.Load{ + ParentHandle: srk, + InPrivate: tpm2.TPM2BPrivate{Buffer: data.Private}, + InPublic: tpm2.BytesAs2B[tpm2.TPMTPublic](data.Public), + } + loadRes, err := loadCmd.Execute(tpm) + if err != nil { + return fmt.Errorf("tpm2.Load: %w", err) + } + defer func() { + cmd := tpm2.FlushContext{FlushHandle: loadRes.ObjectHandle} + if _, err := cmd.Execute(tpm); err != nil { + log.Printf("tpm2.FlushContext: failed to flush loaded sealed blob handle: %v", err) + } + }() + + // Then unseal the object. + unsealCmd := tpm2.Unseal{ + ItemHandle: tpm2.NamedHandle{ + Handle: loadRes.ObjectHandle, + Name: loadRes.Name, + }, + } + unsealRes, err := unsealCmd.Execute(tpm) + if err != nil { + return fmt.Errorf("tpm2.Unseal: %w", err) + } + res = unsealRes.OutData.Buffer + + return nil + }) + return res, err +} diff --git a/feature/tpm/tpm_linux.go b/feature/tpm/tpm_linux.go index a90c0e153962f..6c8131e8d8a28 100644 --- a/feature/tpm/tpm_linux.go +++ b/feature/tpm/tpm_linux.go @@ -4,15 +4,14 @@ package tpm import ( + "github.com/google/go-tpm/tpm2/transport" "github.com/google/go-tpm/tpm2/transport/linuxtpm" - "tailscale.com/tailcfg" ) -func info() *tailcfg.TPMInfo { - t, err := linuxtpm.Open("/dev/tpm0") - if err != nil { - return nil +func open() (transport.TPMCloser, error) { + tpm, err := linuxtpm.Open("/dev/tpmrm0") + if err == nil { + return tpm, nil } - defer t.Close() - return infoFromCapabilities(t) + return linuxtpm.Open("/dev/tpm0") } diff --git a/feature/tpm/tpm_other.go b/feature/tpm/tpm_other.go index ba7c67621eafb..108b2c057e4bd 100644 --- a/feature/tpm/tpm_other.go +++ b/feature/tpm/tpm_other.go @@ -5,8 +5,12 @@ package tpm -import "tailscale.com/tailcfg" +import ( + "errors" -func info() *tailcfg.TPMInfo { - return nil + "github.com/google/go-tpm/tpm2/transport" +) + +func open() (transport.TPMCloser, error) { + return nil, errors.New("TPM not supported on this platform") } diff --git a/feature/tpm/tpm_test.go b/feature/tpm/tpm_test.go index fc0fc178c1a79..f4497f8c72732 100644 --- a/feature/tpm/tpm_test.go +++ b/feature/tpm/tpm_test.go @@ -3,7 +3,27 @@ package tpm -import "testing" +import ( + "bytes" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "iter" + "maps" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "tailscale.com/ipn" + "tailscale.com/ipn/store" + "tailscale.com/types/logger" + "tailscale.com/util/mak" +) func TestPropToString(t *testing.T) { for prop, want := range map[uint32]string{ @@ -17,3 +37,310 @@ func TestPropToString(t *testing.T) { } } } + +func skipWithoutTPM(t testing.TB) { + if !tpmSupported() { + t.Skip("TPM not available") + } +} + +func TestSealUnseal(t *testing.T) { + skipWithoutTPM(t) + + data := make([]byte, 100*1024) + rand.Read(data) + var key [32]byte + rand.Read(key[:]) + + sealed, err := seal(t.Logf, decryptedData{Key: key, Data: data}) + if err != nil { + t.Fatalf("seal: %v", err) + } + if bytes.Contains(sealed.Data, data) { + t.Fatalf("sealed data %q contains original input %q", sealed.Data, data) + } + + unsealed, err := unseal(t.Logf, *sealed) + if err != nil { + t.Fatalf("unseal: %v", err) + } + if !bytes.Equal(data, unsealed.Data) { + t.Errorf("got unsealed data: %q, want: %q", unsealed, data) + } + if key != unsealed.Key { + t.Errorf("got unsealed key: %q, want: %q", unsealed.Key, key) + } +} + +func TestStore(t *testing.T) { + skipWithoutTPM(t) + + path := store.TPMPrefix + filepath.Join(t.TempDir(), "state") + store, err := newStore(t.Logf, path) + if err != nil { + t.Fatal(err) + } + + checkState := func(t *testing.T, store ipn.StateStore, k ipn.StateKey, want []byte) { + got, err := store.ReadState(k) + if err != nil { + t.Errorf("ReadState(%q): %v", k, err) + } + if !bytes.Equal(want, got) { + t.Errorf("ReadState(%q): got %q, want %q", k, got, want) + } + } + + k1, k2 := ipn.StateKey("k1"), ipn.StateKey("k2") + v1, v2 := []byte("v1"), []byte("v2") + + t.Run("read-non-existent-key", func(t *testing.T) { + _, err := store.ReadState(k1) + if !errors.Is(err, ipn.ErrStateNotExist) { + t.Errorf("ReadState succeeded, want %v", ipn.ErrStateNotExist) + } + }) + + t.Run("read-write-k1", func(t *testing.T) { + if err := store.WriteState(k1, v1); err != nil { + t.Errorf("WriteState(%q, %q): %v", k1, v1, err) + } + checkState(t, store, k1, v1) + }) + + t.Run("read-write-k2", func(t *testing.T) { + if err := store.WriteState(k2, v2); err != nil { + t.Errorf("WriteState(%q, %q): %v", k2, v2, err) + } + checkState(t, store, k2, v2) + }) + + t.Run("update-k2", func(t *testing.T) { + v2 = []byte("new v2") + if err := store.WriteState(k2, v2); err != nil { + t.Errorf("WriteState(%q, %q): %v", k2, v2, err) + } + checkState(t, store, k2, v2) + }) + + t.Run("reopen-store", func(t *testing.T) { + store, err := newStore(t.Logf, path) + if err != nil { + t.Fatal(err) + } + checkState(t, store, k1, v1) + checkState(t, store, k2, v2) + }) +} + +func BenchmarkStore(b *testing.B) { + skipWithoutTPM(b) + b.StopTimer() + + stores := make(map[string]ipn.StateStore) + key := ipn.StateKey(b.Name()) + + // Set up tpmStore + tpmStore, err := newStore(b.Logf, filepath.Join(b.TempDir(), "tpm.store")) + if err != nil { + b.Fatal(err) + } + if err := tpmStore.WriteState(key, []byte("-1")); err != nil { + b.Fatal(err) + } + stores["tpmStore"] = tpmStore + + // Set up FileStore + fileStore, err := store.NewFileStore(b.Logf, filepath.Join(b.TempDir(), "file.store")) + if err != nil { + b.Fatal(err) + } + if err := fileStore.WriteState(key, []byte("-1")); err != nil { + b.Fatal(err) + } + stores["fileStore"] = fileStore + + b.StartTimer() + + for name, store := range stores { + b.Run(name, func(b *testing.B) { + b.Run("write-noop", func(b *testing.B) { + for range b.N { + if err := store.WriteState(key, []byte("-1")); err != nil { + b.Fatal(err) + } + } + }) + b.Run("write", func(b *testing.B) { + for i := range b.N { + if err := store.WriteState(key, []byte(strconv.Itoa(i))); err != nil { + b.Fatal(err) + } + } + }) + b.Run("read", func(b *testing.B) { + for range b.N { + if _, err := store.ReadState(key); err != nil { + b.Fatal(err) + } + } + }) + }) + } +} + +func TestMigrateStateToTPM(t *testing.T) { + if !tpmSupported() { + t.Logf("using mock tpmseal provider") + store.RegisterForTest(t, store.TPMPrefix, newMockTPMSeal) + } + + storePath := filepath.Join(t.TempDir(), "store") + // Make sure migration doesn't cause a failure when no state file exists. + if _, err := store.New(t.Logf, store.TPMPrefix+storePath); err != nil { + t.Fatalf("store.New failed for new tpmseal store: %v", err) + } + os.Remove(storePath) + + initial, err := store.New(t.Logf, storePath) + if err != nil { + t.Fatalf("store.New failed for new file store: %v", err) + } + + // Populate initial state file. + content := map[ipn.StateKey][]byte{ + "foo": []byte("bar"), + "baz": []byte("qux"), + } + for k, v := range content { + if err := initial.WriteState(k, v); err != nil { + t.Fatal(err) + } + } + // Expected file keys for plaintext and sealed versions of state. + keysPlaintext := []string{"foo", "baz"} + keysTPMSeal := []string{"key", "nonce", "data"} + + for _, tt := range []struct { + desc string + path string + wantKeys []string + }{ + { + desc: "plaintext-to-plaintext", + path: storePath, + wantKeys: keysPlaintext, + }, + { + desc: "plaintext-to-tpmseal", + path: store.TPMPrefix + storePath, + wantKeys: keysTPMSeal, + }, + { + desc: "tpmseal-to-tpmseal", + path: store.TPMPrefix + storePath, + wantKeys: keysTPMSeal, + }, + { + desc: "tpmseal-to-plaintext", + path: storePath, + wantKeys: keysPlaintext, + }, + } { + t.Run(tt.desc, func(t *testing.T) { + s, err := store.New(t.Logf, tt.path) + if err != nil { + t.Fatalf("migration failed: %v", err) + } + gotContent := maps.Collect(s.(interface { + All() iter.Seq2[ipn.StateKey, []byte] + }).All()) + if diff := cmp.Diff(content, gotContent); diff != "" { + t.Errorf("unexpected content after migration, diff:\n%s", diff) + } + + buf, err := os.ReadFile(storePath) + if err != nil { + t.Fatal(err) + } + var data map[string]any + if err := json.Unmarshal(buf, &data); err != nil { + t.Fatal(err) + } + gotKeys := slices.Collect(maps.Keys(data)) + slices.Sort(gotKeys) + slices.Sort(tt.wantKeys) + if diff := cmp.Diff(gotKeys, tt.wantKeys); diff != "" { + t.Errorf("unexpected content keys after migration, diff:\n%s", diff) + } + }) + } +} + +func tpmSupported() bool { + tpm, err := open() + if err != nil { + return false + } + tpm.Close() + return true +} + +type mockTPMSealProvider struct { + path string + data map[ipn.StateKey][]byte +} + +func newMockTPMSeal(logf logger.Logf, path string) (ipn.StateStore, error) { + path, ok := strings.CutPrefix(path, store.TPMPrefix) + if !ok { + return nil, fmt.Errorf("%q missing tpmseal: prefix", path) + } + s := &mockTPMSealProvider{path: path} + buf, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return s, s.flushState() + } + if err != nil { + return nil, err + } + var data struct { + Key string + Nonce string + Data map[ipn.StateKey][]byte + } + if err := json.Unmarshal(buf, &data); err != nil { + return nil, err + } + if data.Key == "" || data.Nonce == "" { + return nil, fmt.Errorf("%q missing key or nonce", path) + } + s.data = data.Data + return s, nil +} + +func (p *mockTPMSealProvider) ReadState(k ipn.StateKey) ([]byte, error) { + return p.data[k], nil +} + +func (p *mockTPMSealProvider) WriteState(k ipn.StateKey, v []byte) error { + mak.Set(&p.data, k, v) + return p.flushState() +} + +func (p *mockTPMSealProvider) All() iter.Seq2[ipn.StateKey, []byte] { + return maps.All(p.data) +} + +func (p *mockTPMSealProvider) flushState() error { + data := map[string]any{ + "key": "foo", + "nonce": "bar", + "data": p.data, + } + buf, err := json.Marshal(data) + if err != nil { + return err + } + return os.WriteFile(p.path, buf, 0600) +} diff --git a/feature/tpm/tpm_windows.go b/feature/tpm/tpm_windows.go index 578d687af5739..429d20cb879f7 100644 --- a/feature/tpm/tpm_windows.go +++ b/feature/tpm/tpm_windows.go @@ -4,15 +4,10 @@ package tpm import ( + "github.com/google/go-tpm/tpm2/transport" "github.com/google/go-tpm/tpm2/transport/windowstpm" - "tailscale.com/tailcfg" ) -func info() *tailcfg.TPMInfo { - t, err := windowstpm.Open() - if err != nil { - return nil - } - defer t.Close() - return infoFromCapabilities(t) +func open() (transport.TPMCloser, error) { + return windowstpm.Open() } diff --git a/flake.lock b/flake.lock index 05b0f303e6433..87f234e3ecab1 100644 --- a/flake.lock +++ b/flake.lock @@ -36,11 +36,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1743938762, - "narHash": "sha256-UgFYn8sGv9B8PoFpUfCa43CjMZBl1x/ShQhRDHBFQdI=", + "lastModified": 1753151930, + "narHash": "sha256-XSQy6wRKHhRe//iVY5lS/ZpI/Jn6crWI8fQzl647wCg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "74a40410369a1c35ee09b8a1abee6f4acbedc059", + "rev": "83e677f31c84212343f4cc553bab85c2efcad60a", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 2f920bfd40ba5..17d263a8dd3c9 100644 --- a/flake.nix +++ b/flake.nix @@ -130,4 +130,5 @@ in flake-utils.lib.eachDefaultSystem (system: flakeForSystem nixpkgs system); } -# nix-direnv cache busting line: sha256-av4kr09rjNRmag94ziNjJuI/cg8b8lAD3Tk24t/ezH4= +# nix-direnv cache busting line: sha256-4QTSspHLYJfzlontQ7msXyOB5gzq7ZwSvWmKuYY5klA= + diff --git a/go.mod b/go.mod index f346b1e4095dd..3d7514158f069 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module tailscale.com -go 1.24.0 +go 1.24.4 require ( filippo.io/mkcert v1.4.4 @@ -51,6 +51,7 @@ require ( github.com/goreleaser/nfpm/v2 v2.33.1 github.com/hashicorp/go-hclog v1.6.2 github.com/hashicorp/raft v1.7.2 + github.com/hashicorp/raft-boltdb/v2 v2.3.1 github.com/hdevalence/ed25519consensus v0.2.0 github.com/illarion/gonotify/v3 v3.0.2 github.com/inetaf/tcpproxy v0.0.0-20250203165043-ded522cbd03f @@ -89,7 +90,7 @@ require ( github.com/tailscale/setec v0.0.0-20250205144240-8898a29c3fbb github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 - github.com/tailscale/wireguard-go v0.0.0-20250304000100-91a0587fb251 + github.com/tailscale/wireguard-go v0.0.0-20250716170648-1d0488a3d7da github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e github.com/tc-hib/winres v0.2.1 github.com/tcnksm/go-httpstat v0.2.0 @@ -99,16 +100,16 @@ require ( go.uber.org/zap v1.27.0 go4.org/mem v0.0.0-20240501181205-ae6ca9944745 go4.org/netipx v0.0.0-20231129151722-fdeea329fbba - golang.org/x/crypto v0.37.0 + golang.org/x/crypto v0.38.0 golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac - golang.org/x/mod v0.23.0 - golang.org/x/net v0.36.0 - golang.org/x/oauth2 v0.26.0 - golang.org/x/sync v0.13.0 - golang.org/x/sys v0.32.0 - golang.org/x/term v0.31.0 - golang.org/x/time v0.10.0 - golang.org/x/tools v0.30.0 + golang.org/x/mod v0.24.0 + golang.org/x/net v0.40.0 + golang.org/x/oauth2 v0.30.0 + golang.org/x/sync v0.14.0 + golang.org/x/sys v0.33.0 + golang.org/x/term v0.32.0 + golang.org/x/time v0.11.0 + golang.org/x/tools v0.33.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 golang.zx2c4.com/wireguard/windows v0.5.3 gopkg.in/square/go-jose.v2 v2.6.0 @@ -135,6 +136,7 @@ require ( github.com/alecthomas/go-check-sumtype v0.1.4 // indirect github.com/alexkohler/nakedret/v2 v2.0.4 // indirect github.com/armon/go-metrics v0.4.1 // indirect + github.com/boltdb/bolt v1.3.1 // indirect github.com/bombsimon/wsl/v4 v4.2.1 // indirect github.com/butuzov/mirror v1.1.0 // indirect github.com/catenacyber/perfsprint v0.7.1 // indirect @@ -166,6 +168,7 @@ require ( github.com/ykadowak/zerologlint v0.1.5 // indirect go-simpler.org/musttag v0.9.0 // indirect go-simpler.org/sloglint v0.5.0 // indirect + go.etcd.io/bbolt v1.3.11 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect go.opentelemetry.io/otel v1.33.0 // indirect @@ -222,7 +225,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charithe/durationcheck v0.0.10 // indirect github.com/chavacava/garif v0.1.0 // indirect - github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudflare/circl v1.6.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/curioswitch/go-reassign v0.2.0 // indirect github.com/daixiang0/gci v0.12.3 // indirect @@ -392,8 +395,8 @@ require ( gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect - golang.org/x/image v0.24.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/image v0.27.0 // indirect + golang.org/x/text v0.25.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.mod.sri b/go.mod.sri index 6c8357e0468ba..845086191699a 100644 --- a/go.mod.sri +++ b/go.mod.sri @@ -1 +1 @@ -sha256-av4kr09rjNRmag94ziNjJuI/cg8b8lAD3Tk24t/ezH4= \ No newline at end of file +sha256-4QTSspHLYJfzlontQ7msXyOB5gzq7ZwSvWmKuYY5klA= diff --git a/go.sum b/go.sum index bdbae11bb2e3a..995b930100ff9 100644 --- a/go.sum +++ b/go.sum @@ -180,6 +180,8 @@ github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bombsimon/wsl/v4 v4.2.1 h1:Cxg6u+XDWff75SIFFmNsqnIOgob+Q9hG6y/ioKbRFiM= github.com/bombsimon/wsl/v4 v4.2.1/go.mod h1:Xu/kDxGZTofQcDGCtQe9KCzhHphIe0fDuyWTxER9Feo= github.com/bramvdbogaerde/go-scp v1.4.0 h1:jKMwpwCbcX1KyvDbm/PDJuXcMuNVlLGi0Q0reuzjyKY= @@ -224,8 +226,8 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp github.com/ckaznocha/intrange v0.1.0 h1:ZiGBhvrdsKpoEfzh9CjBfDSZof6QB0ORY5tXasUtiew= github.com/ckaznocha/intrange v0.1.0/go.mod h1:Vwa9Ekex2BrEQMg6zlrWwbs/FtYw7eS5838Q7UjK7TQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= @@ -555,6 +557,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY= github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0= github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= @@ -571,6 +575,10 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/raft v1.7.2 h1:pyvxhfJ4R8VIAlHKvLoKQWElZspsCVT6YWuxVxsPAgc= github.com/hashicorp/raft v1.7.2/go.mod h1:DfvCGFxpAUPE0L4Uc8JLlTPtc3GzSbdH0MTJCLgnmJQ= +github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKcZ+qp2VlaaZsYZfLOmIiuJNpEi48Rl8u9cQ= +github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0= +github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc= +github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE= github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU= github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -967,8 +975,8 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 h1:U github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M= github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y= -github.com/tailscale/wireguard-go v0.0.0-20250304000100-91a0587fb251 h1:h/41LFTrwMxB9Xvvug0kRdQCU5TlV1+pAMQw0ZtDE3U= -github.com/tailscale/wireguard-go v0.0.0-20250304000100-91a0587fb251/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= +github.com/tailscale/wireguard-go v0.0.0-20250716170648-1d0488a3d7da h1:jVRUZPRs9sqyKlYHHzHjAqKN+6e/Vog6NpHYeNPJqOw= +github.com/tailscale/wireguard-go v0.0.0-20250716170648-1d0488a3d7da/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek= github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg= github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA= @@ -1046,6 +1054,8 @@ go-simpler.org/musttag v0.9.0 h1:Dzt6/tyP9ONr5g9h9P3cnYWCxeBFRkd0uJL/w+1Mxos= go-simpler.org/musttag v0.9.0/go.mod h1:gA9nThnalvNSKpEoyp3Ko4/vCX2xTpqKoUtNqXOnVR4= go-simpler.org/sloglint v0.5.0 h1:2YCcd+YMuYpuqthCgubcF5lBSjb6berc5VMOYUHKrpY= go-simpler.org/sloglint v0.5.0/go.mod h1:EUknX5s8iXqf18KQxKnaBHUPVriiPnOrPjjJcsaTcSQ= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -1091,8 +1101,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1111,8 +1121,8 @@ golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9 golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ= -golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8= +golang.org/x/image v0.27.0 h1:C8gA4oWU/tKkdCfYT6T2u4faJu3MeNS5O8UPWlPF61w= +golang.org/x/image v0.27.0/go.mod h1:xbdrClrAUway1MUTEZDq9mz/UpRwYAkFFNUslZtcB+g= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1140,8 +1150,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= -golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1181,16 +1191,16 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.26.0 h1:afQXWNNaeC4nvZ0Ed9XvCCzXM6UHJG7iCg0W4fPqSBE= -golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1204,8 +1214,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1268,16 +1278,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= -golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1288,13 +1298,13 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= -golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1359,8 +1369,8 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= -golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go.toolchain.rev b/go.toolchain.rev index e8ede337c11cb..33aa564236c3e 100644 --- a/go.toolchain.rev +++ b/go.toolchain.rev @@ -1 +1 @@ -982da8f24fa0504f2214f24b0d68b2febd5983f8 +1cd3bf1a6eaf559aa8c00e749289559c884cef09 diff --git a/health/health.go b/health/health.go index 1ec2bcc9b0dd1..05887043814ea 100644 --- a/health/health.go +++ b/health/health.go @@ -88,34 +88,35 @@ type Tracker struct { // sysErr maps subsystems to their current error (or nil if the subsystem is healthy) // Deprecated: using Warnables should be preferred sysErr map[Subsystem]error - watchers set.HandleSet[func(*Warnable, *UnhealthyState)] // opt func to run if error state changes + watchers set.HandleSet[func(Change)] // opt func to run if error state changes timer tstime.TimerController latestVersion *tailcfg.ClientVersion // or nil checkForUpdates bool applyUpdates opt.Bool - inMapPoll bool - inMapPollSince time.Time - lastMapPollEndedAt time.Time - lastStreamedMapResponse time.Time - lastNoiseDial time.Time - derpHomeRegion int - derpHomeless bool - derpRegionConnected map[int]bool - derpRegionHealthProblem map[int]string - derpRegionLastFrame map[int]time.Time - derpMap *tailcfg.DERPMap // last DERP map from control, could be nil if never received one - lastMapRequestHeard time.Time // time we got a 200 from control for a MapRequest - ipnState string - ipnWantRunning bool - ipnWantRunningLastTrue time.Time // when ipnWantRunning last changed false -> true - anyInterfaceUp opt.Bool // empty means unknown (assume true) - controlHealth []string - lastLoginErr error - localLogConfigErr error - tlsConnectionErrors map[string]error // map[ServerName]error - metricHealthMessage *metrics.MultiLabelMap[metricHealthMessageLabel] + inMapPoll bool + inMapPollSince time.Time + lastMapPollEndedAt time.Time + lastStreamedMapResponse time.Time + lastNoiseDial time.Time + derpHomeRegion int + derpHomeless bool + derpRegionConnected map[int]bool + derpRegionHealthProblem map[int]string + derpRegionLastFrame map[int]time.Time + derpMap *tailcfg.DERPMap // last DERP map from control, could be nil if never received one + lastMapRequestHeard time.Time // time we got a 200 from control for a MapRequest + ipnState string + ipnWantRunning bool + ipnWantRunningLastTrue time.Time // when ipnWantRunning last changed false -> true + anyInterfaceUp opt.Bool // empty means unknown (assume true) + lastNotifiedControlMessages map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage // latest control messages processed, kept for change detection + controlMessages map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage // latest control messages received + lastLoginErr error + localLogConfigErr error + tlsConnectionErrors map[string]error // map[ServerName]error + metricHealthMessage *metrics.MultiLabelMap[metricHealthMessageLabel] } func (t *Tracker) now() time.Time { @@ -207,13 +208,15 @@ func unregister(w *Warnable) { // the program. type WarnableCode string -// A Warnable is something that we might want to warn the user about, or not. A Warnable is either -// in an healthy or unhealth state. A Warnable is unhealthy if the Tracker knows about a WarningState -// affecting the Warnable. -// In most cases, Warnables are components of the backend (for instance, "DNS" or "Magicsock"). -// Warnables are similar to the Subsystem type previously used in this package, but they provide -// a unique identifying code for each Warnable, along with more metadata that makes it easier for -// a GUI to display the Warnable in a user-friendly way. +// A Warnable is something that we might want to warn the user about, or not. A +// Warnable is either in a healthy or unhealthy state. A Warnable is unhealthy if +// the Tracker knows about a WarningState affecting the Warnable. +// +// In most cases, Warnables are components of the backend (for instance, "DNS" +// or "Magicsock"). Warnables are similar to the Subsystem type previously used +// in this package, but they provide a unique identifying code for each +// Warnable, along with more metadata that makes it easier for a GUI to display +// the Warnable in a user-friendly way. type Warnable struct { // Code is a string that uniquely identifies this Warnable across the entire Tailscale backend, // and can be mapped to a user-displayable localized string. @@ -409,12 +412,18 @@ func (t *Tracker) setUnhealthyLocked(w *Warnable, args Args) { prevWs := t.warnableVal[w] mak.Set(&t.warnableVal, w, ws) if !ws.Equal(prevWs) { + + change := Change{ + WarnableChanged: true, + Warnable: w, + UnhealthyState: w.unhealthyState(ws), + } for _, cb := range t.watchers { // If the Warnable has been unhealthy for more than its TimeToVisible, the callback should be // executed immediately. Otherwise, the callback should be enqueued to run once the Warnable // becomes visible. if w.IsVisible(ws, t.now) { - cb(w, w.unhealthyState(ws)) + cb(change) continue } @@ -427,7 +436,7 @@ func (t *Tracker) setUnhealthyLocked(w *Warnable, args Args) { // Check if the Warnable is still unhealthy, as it could have become healthy between the time // the timer was set for and the time it was executed. if t.warnableVal[w] != nil { - cb(w, w.unhealthyState(ws)) + cb(change) delete(t.pendingVisibleTimers, w) } }) @@ -460,8 +469,23 @@ func (t *Tracker) setHealthyLocked(w *Warnable) { delete(t.pendingVisibleTimers, w) } + change := Change{ + WarnableChanged: true, + Warnable: w, + } + for _, cb := range t.watchers { + cb(change) + } +} + +// notifyWatchersControlChangedLocked calls each watcher to signal that control +// health messages have changed (and should be fetched via CurrentState). +func (t *Tracker) notifyWatchersControlChangedLocked() { + change := Change{ + ControlHealthChanged: true, + } for _, cb := range t.watchers { - cb(w, nil) + cb(change) } } @@ -488,23 +512,57 @@ func (t *Tracker) AppendWarnableDebugFlags(base []string) []string { return ret } -// RegisterWatcher adds a function that will be called whenever the health state of any Warnable changes. -// If a Warnable becomes unhealthy or its unhealthy state is updated, the callback will be called with its -// current Representation. -// If a Warnable becomes healthy, the callback will be called with ws set to nil. -// The provided callback function will be executed in its own goroutine. The returned function can be used -// to unregister the callback. -func (t *Tracker) RegisterWatcher(cb func(w *Warnable, r *UnhealthyState)) (unregister func()) { - return t.registerSyncWatcher(func(w *Warnable, r *UnhealthyState) { - go cb(w, r) +// Change is used to communicate a change to health. This could either be due to +// a Warnable changing from health to unhealthy (or vice-versa), or because the +// health messages received from the control-plane have changed. +// +// Exactly one *Changed field will be true. +type Change struct { + // ControlHealthChanged indicates it was health messages from the + // control-plane server that changed. + ControlHealthChanged bool + + // WarnableChanged indicates it was a client Warnable which changed state. + WarnableChanged bool + // Warnable is whose health changed, as indicated in UnhealthyState. + Warnable *Warnable + // UnhealthyState is set if the changed Warnable is now unhealthy, or nil + // if Warnable is now healthy. + UnhealthyState *UnhealthyState +} + +// RegisterWatcher adds a function that will be called its own goroutine +// whenever the health state of any client [Warnable] or control-plane health +// messages changes. The returned function can be used to unregister the +// callback. +// +// If a client [Warnable] becomes unhealthy or its unhealthy state is updated, +// the callback will be called with WarnableChanged set to true and the Warnable +// and its UnhealthyState: +// +// go cb(Change{WarnableChanged: true, Warnable: w, UnhealthyState: us}) +// +// If a Warnable becomes healthy, the callback will be called with +// WarnableChanged set to true, the Warnable set, and UnhealthyState set to nil: +// +// go cb(Change{WarnableChanged: true, Warnable: w, UnhealthyState: nil}) +// +// If the health messages from the control-plane change, the callback will be +// called with ControlHealthChanged set to true. Recipients can fetch the set of +// control-plane health messages by calling [Tracker.CurrentState]: +// +// go cb(Change{ControlHealthChanged: true}) +func (t *Tracker) RegisterWatcher(cb func(Change)) (unregister func()) { + return t.registerSyncWatcher(func(c Change) { + go cb(c) }) } // registerSyncWatcher adds a function that will be called whenever the health -// state of any Warnable changes. The provided callback function will be -// executed synchronously. Call RegisterWatcher to register any callbacks that -// won't return from execution immediately. -func (t *Tracker) registerSyncWatcher(cb func(w *Warnable, r *UnhealthyState)) (unregister func()) { +// state changes. The provided callback function will be executed synchronously. +// Call RegisterWatcher to register any callbacks that won't return from +// execution immediately. +func (t *Tracker) registerSyncWatcher(cb func(c Change)) (unregister func()) { if t.nil() { return func() {} } @@ -512,7 +570,7 @@ func (t *Tracker) registerSyncWatcher(cb func(w *Warnable, r *UnhealthyState)) ( t.mu.Lock() defer t.mu.Unlock() if t.watchers == nil { - t.watchers = set.HandleSet[func(*Warnable, *UnhealthyState)]{} + t.watchers = set.HandleSet[func(Change)]{} } handle := t.watchers.Add(cb) if t.timer == nil { @@ -659,13 +717,15 @@ func (t *Tracker) updateLegacyErrorWarnableLocked(key Subsystem, err error) { } } -func (t *Tracker) SetControlHealth(problems []string) { +func (t *Tracker) SetControlHealth(problems map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage) { if t.nil() { return } t.mu.Lock() defer t.mu.Unlock() - t.controlHealth = problems + + t.controlMessages = problems + t.selfCheckLocked() } @@ -961,11 +1021,11 @@ func (t *Tracker) OverallError() error { return t.multiErrLocked() } -// Strings() returns a string array containing the Text of all Warnings -// currently known to the Tracker. These strings can be presented to the -// user, although ideally you would use the Code property on each Warning -// to show a localized version of them instead. -// This function is here for legacy compatibility purposes and is deprecated. +// Strings() returns a string array containing the Text of all Warnings and +// ControlHealth messages currently known to the Tracker. These strings can be +// presented to the user, although ideally you would use the Code property on +// each Warning to show a localized version of them instead. This function is +// here for legacy compatibility purposes and is deprecated. func (t *Tracker) Strings() []string { if t.nil() { return nil @@ -991,6 +1051,24 @@ func (t *Tracker) stringsLocked() []string { result = append(result, w.Text(ws.Args)) } } + + warnLen := len(result) + for _, c := range t.controlMessages { + var msg string + if c.Title != "" && c.Text != "" { + msg = c.Title + ": " + c.Text + } else if c.Title != "" { + msg = c.Title + "." + } else if c.Text != "" { + msg = c.Text + } + if c.PrimaryAction != nil { + msg = msg + " " + c.PrimaryAction.Label + ": " + c.PrimaryAction.URL + } + result = append(result, msg) + } + sort.Strings(result[warnLen:]) + return result } @@ -1171,14 +1249,10 @@ func (t *Tracker) updateBuiltinWarnablesLocked() { t.setHealthyLocked(derpRegionErrorWarnable) } - if len(t.controlHealth) > 0 { - for _, s := range t.controlHealth { - t.setUnhealthyLocked(controlHealthWarnable, Args{ - ArgError: s, - }) - } - } else { - t.setHealthyLocked(controlHealthWarnable) + // Check if control health messages have changed + if !maps.EqualFunc(t.lastNotifiedControlMessages, t.controlMessages, tailcfg.DisplayMessage.Equal) { + t.lastNotifiedControlMessages = t.controlMessages + t.notifyWatchersControlChangedLocked() } if err := envknob.ApplyDiskConfigError(); err != nil { diff --git a/health/health_test.go b/health/health_test.go index aa39045817ce2..53f012ecffd55 100644 --- a/health/health_test.go +++ b/health/health_test.go @@ -5,12 +5,14 @@ package health import ( "fmt" + "maps" "reflect" "slices" "strconv" "testing" "time" + "github.com/google/go-cmp/cmp" "tailscale.com/tailcfg" "tailscale.com/tstest" "tailscale.com/types/opt" @@ -25,6 +27,7 @@ func TestAppendWarnableDebugFlags(t *testing.T) { w := Register(&Warnable{ Code: WarnableCode(fmt.Sprintf("warnable-code-%d", i)), MapDebugFlag: fmt.Sprint(i), + Text: StaticMessage(""), }) defer unregister(w) if i%2 == 0 { @@ -114,7 +117,9 @@ func TestWatcher(t *testing.T) { becameUnhealthy := make(chan struct{}) becameHealthy := make(chan struct{}) - watcherFunc := func(w *Warnable, us *UnhealthyState) { + watcherFunc := func(c Change) { + w := c.Warnable + us := c.UnhealthyState if w != testWarnable { t.Fatalf("watcherFunc was called, but with an unexpected Warnable: %v, want: %v", w, testWarnable) } @@ -184,7 +189,9 @@ func TestSetUnhealthyWithTimeToVisible(t *testing.T) { becameUnhealthy := make(chan struct{}) becameHealthy := make(chan struct{}) - watchFunc := func(w *Warnable, us *UnhealthyState) { + watchFunc := func(c Change) { + w := c.Warnable + us := c.UnhealthyState if w != mw { t.Fatalf("watcherFunc was called, but with an unexpected Warnable: %v, want: %v", w, w) } @@ -457,60 +464,179 @@ func TestControlHealth(t *testing.T) { ht.SetIPNState("NeedsLogin", true) ht.GotStreamedMapResponse() - ht.SetControlHealth([]string{"Test message"}) - state := ht.CurrentState() - warning, ok := state.Warnings["control-health"] + baseWarns := ht.CurrentState().Warnings + baseStrs := ht.Strings() - if !ok { - t.Fatal("no warning found in current state with code 'control-health'") - } - if got, want := warning.Title, "Coordination server reports an issue"; got != want { - t.Errorf("warning.Title = %q, want %q", got, want) - } - if got, want := warning.Severity, SeverityMedium; got != want { - t.Errorf("warning.Severity = %s, want %s", got, want) - } - if got, want := warning.Text, "The coordination server is reporting an health issue: Test message"; got != want { - t.Errorf("warning.Text = %q, want %q", got, want) + msgs := map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": { + Title: "Control health message", + Text: "Extra help.", + }, + "title": { + Title: "Control health title only", + }, + "with-action": { + Title: "Control health message", + Text: "Extra help.", + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "http://www.example.com", + Label: "Learn more", + }, + }, } -} - -func TestControlHealthNotifiesOnChange(t *testing.T) { - ht := Tracker{} - ht.SetIPNState("NeedsLogin", true) - ht.GotStreamedMapResponse() - - gotNotified := false - ht.registerSyncWatcher(func(_ *Warnable, _ *UnhealthyState) { - gotNotified = true + ht.SetControlHealth(msgs) + + t.Run("Warnings", func(t *testing.T) { + wantWarns := map[WarnableCode]UnhealthyState{ + "control-health.test": { + WarnableCode: "control-health.test", + Severity: SeverityMedium, + Title: "Control health message", + Text: "Extra help.", + }, + "control-health.title": { + WarnableCode: "control-health.title", + Severity: SeverityMedium, + Title: "Control health title only", + }, + "control-health.with-action": { + WarnableCode: "control-health.with-action", + Severity: SeverityMedium, + Title: "Control health message", + Text: "Extra help.", + PrimaryAction: &UnhealthyStateAction{ + URL: "http://www.example.com", + Label: "Learn more", + }, + }, + } + state := ht.CurrentState() + gotWarns := maps.Clone(state.Warnings) + for k := range gotWarns { + if _, inBase := baseWarns[k]; inBase { + delete(gotWarns, k) + } + } + if diff := cmp.Diff(wantWarns, gotWarns); diff != "" { + t.Fatalf(`CurrentState().Warnings["control-health-*"] wrong (-want +got):\n%s`, diff) + } }) - ht.SetControlHealth([]string{"Test message"}) + t.Run("Strings()", func(t *testing.T) { + wantStrs := []string{ + "Control health message: Extra help.", + "Control health message: Extra help. Learn more: http://www.example.com", + "Control health title only.", + } + var gotStrs []string + for _, s := range ht.Strings() { + if !slices.Contains(baseStrs, s) { + gotStrs = append(gotStrs, s) + } + } + if diff := cmp.Diff(wantStrs, gotStrs); diff != "" { + t.Fatalf(`Strings() wrong (-want +got):\n%s`, diff) + } + }) - if !gotNotified { - t.Errorf("watcher did not get called, want it to be called") - } + t.Run("tailscaled_health_messages", func(t *testing.T) { + var r usermetric.Registry + ht.SetMetricsRegistry(&r) + + got := ht.metricHealthMessage.Get(metricHealthMessageLabel{ + Type: MetricLabelWarning, + }).String() + want := strconv.Itoa( + len(msgs) + len(baseStrs), + ) + if got != want { + t.Errorf("metricsHealthMessage.Get(warning) = %q, want %q", got, want) + } + }) } -func TestControlHealthNoNotifyOnUnchanged(t *testing.T) { - ht := Tracker{} - ht.SetIPNState("NeedsLogin", true) - ht.GotStreamedMapResponse() +func TestControlHealthNotifies(t *testing.T) { + type test struct { + name string + initialState map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage + newState map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage + wantNotify bool + } + tests := []test{ + { + name: "no-change", + initialState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": {}, + }, + newState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": {}, + }, + wantNotify: false, + }, + { + name: "on-set", + initialState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{}, + newState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": {}, + }, + wantNotify: true, + }, + { + name: "details-change", + initialState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": { + Title: "Title", + }, + }, + newState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": { + Title: "Updated title", + }, + }, + wantNotify: true, + }, + { + name: "action-changes", + initialState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": { + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "http://www.example.com/a/123456", + Label: "Sign in", + }, + }, + }, + newState: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test": { + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "http://www.example.com/a/abcdefg", + Label: "Sign in", + }, + }, + }, + wantNotify: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ht := Tracker{} + ht.SetIPNState("NeedsLogin", true) + ht.GotStreamedMapResponse() - // Set up an existing control health issue - ht.SetControlHealth([]string{"Test message"}) + if len(test.initialState) != 0 { + ht.SetControlHealth(test.initialState) + } - // Now register our watcher - gotNotified := false - ht.registerSyncWatcher(func(_ *Warnable, _ *UnhealthyState) { - gotNotified = true - }) + gotNotified := false + ht.registerSyncWatcher(func(_ Change) { + gotNotified = true + }) - // Send the same control health message again - should not notify - ht.SetControlHealth([]string{"Test message"}) + ht.SetControlHealth(test.newState) - if gotNotified { - t.Errorf("watcher got called, want it to not be called") + if gotNotified != test.wantNotify { + t.Errorf("notified: got %v, want %v", gotNotified, test.wantNotify) + } + }) } } @@ -519,11 +645,13 @@ func TestControlHealthIgnoredOutsideMapPoll(t *testing.T) { ht.SetIPNState("NeedsLogin", true) gotNotified := false - ht.registerSyncWatcher(func(_ *Warnable, _ *UnhealthyState) { + ht.registerSyncWatcher(func(_ Change) { gotNotified = true }) - ht.SetControlHealth([]string{"Test message"}) + ht.SetControlHealth(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "control-health": {}, + }) state := ht.CurrentState() _, ok := state.Warnings["control-health"] diff --git a/health/healthmsg/healthmsg.go b/health/healthmsg/healthmsg.go index 6c237678eaf62..2384103738cf3 100644 --- a/health/healthmsg/healthmsg.go +++ b/health/healthmsg/healthmsg.go @@ -12,4 +12,5 @@ const ( TailscaleSSHOnBut = "Tailscale SSH enabled, but " // + ... something from caller LockedOut = "this node is locked out; it will not have connectivity until it is signed. For more info, see https://tailscale.com/s/locked-out" WarnExitNodeUsage = "The following issues on your machine will likely make usage of exit nodes impossible" + DisableRPFilter = "Please set rp_filter=2 instead of rp_filter=1; see https://github.com/tailscale/tailscale/issues/3310" ) diff --git a/health/state.go b/health/state.go index c06f6ef59c8ed..b5e6a8a3894d8 100644 --- a/health/state.go +++ b/health/state.go @@ -5,6 +5,8 @@ package health import ( "time" + + "tailscale.com/tailcfg" ) // State contains the health status of the backend, and is @@ -21,16 +23,26 @@ type State struct { } // UnhealthyState contains information to be shown to the user to inform them -// that a Warnable is currently unhealthy. +// that a [Warnable] is currently unhealthy or [tailcfg.DisplayMessage] is being +// sent from the control-plane. type UnhealthyState struct { WarnableCode WarnableCode Severity Severity Title string Text string - BrokenSince *time.Time `json:",omitempty"` - Args Args `json:",omitempty"` - DependsOn []WarnableCode `json:",omitempty"` - ImpactsConnectivity bool `json:",omitempty"` + BrokenSince *time.Time `json:",omitempty"` + Args Args `json:",omitempty"` + DependsOn []WarnableCode `json:",omitempty"` + ImpactsConnectivity bool `json:",omitempty"` + PrimaryAction *UnhealthyStateAction `json:",omitempty"` +} + +// UnhealthyStateAction represents an action (URL and link) to be presented to +// the user associated with an [UnhealthyState]. Analogous to +// [tailcfg.DisplayMessageAction]. +type UnhealthyStateAction struct { + URL string + Label string } // unhealthyState returns a unhealthyState of the Warnable given its current warningState. @@ -98,11 +110,42 @@ func (t *Tracker) CurrentState() *State { wm[w.Code] = *w.unhealthyState(ws) } + for id, msg := range t.lastNotifiedControlMessages { + state := UnhealthyState{ + WarnableCode: WarnableCode("control-health." + id), + Severity: severityFromTailcfg(msg.Severity), + Title: msg.Title, + Text: msg.Text, + ImpactsConnectivity: msg.ImpactsConnectivity, + // TODO(tailscale/corp#27759): DependsOn? + } + + if msg.PrimaryAction != nil { + state.PrimaryAction = &UnhealthyStateAction{ + URL: msg.PrimaryAction.URL, + Label: msg.PrimaryAction.Label, + } + } + + wm[state.WarnableCode] = state + } + return &State{ Warnings: wm, } } +func severityFromTailcfg(s tailcfg.DisplayMessageSeverity) Severity { + switch s { + case tailcfg.SeverityHigh: + return SeverityHigh + case tailcfg.SeverityLow: + return SeverityLow + default: + return SeverityMedium + } +} + // isEffectivelyHealthyLocked reports whether w is effectively healthy. // That means it's either actually healthy or it has a dependency that // that's unhealthy, so we should treat w as healthy to not spam users diff --git a/health/warnings.go b/health/warnings.go index 7a21f9695ff6d..3997e66b39ad0 100644 --- a/health/warnings.go +++ b/health/warnings.go @@ -238,16 +238,6 @@ var applyDiskConfigWarnable = Register(&Warnable{ }, }) -// controlHealthWarnable is a Warnable that warns the user that the coordination server is reporting an health issue. -var controlHealthWarnable = Register(&Warnable{ - Code: "control-health", - Title: "Coordination server reports an issue", - Severity: SeverityMedium, - Text: func(args Args) string { - return fmt.Sprintf("The coordination server is reporting an health issue: %v", args[ArgError]) - }, -}) - // warmingUpWarnableDuration is the duration for which the warmingUpWarnable is reported by the backend after the user // has changed ipnWantRunning to true from false. const warmingUpWarnableDuration = 5 * time.Second diff --git a/internal/client/tailscale/vip_service.go b/internal/client/tailscale/vip_service.go index 64fcfdf5e86d6..48c59ce4569da 100644 --- a/internal/client/tailscale/vip_service.go +++ b/internal/client/tailscale/vip_service.go @@ -36,6 +36,11 @@ type VIPService struct { Tags []string `json:"tags,omitempty"` } +// VIPServiceList represents the JSON response to the list VIP Services API. +type VIPServiceList struct { + VIPServices []VIPService `json:"vipServices"` +} + // GetVIPService retrieves a VIPService by its name. It returns 404 if the VIPService is not found. func (client *Client) GetVIPService(ctx context.Context, name tailcfg.ServiceName) (*VIPService, error) { path := client.BuildTailnetURL("vip-services", name.String()) @@ -59,6 +64,29 @@ func (client *Client) GetVIPService(ctx context.Context, name tailcfg.ServiceNam return svc, nil } +// ListVIPServices retrieves all existing Services and returns them as a list. +func (client *Client) ListVIPServices(ctx context.Context) (*VIPServiceList, error) { + path := client.BuildTailnetURL("vip-services") + req, err := http.NewRequestWithContext(ctx, httpm.GET, path, nil) + if err != nil { + return nil, fmt.Errorf("error creating new HTTP request: %w", err) + } + b, resp, err := SendRequest(client, req) + if err != nil { + return nil, fmt.Errorf("error making Tailsale API request: %w", err) + } + // If status code was not successful, return the error. + // TODO: Change the check for the StatusCode to include other 2XX success codes. + if resp.StatusCode != http.StatusOK { + return nil, HandleErrorResponse(b, resp) + } + result := &VIPServiceList{} + if err := json.Unmarshal(b, result); err != nil { + return nil, err + } + return result, nil +} + // CreateOrUpdateVIPService creates or updates a VIPService by its name. Caller must ensure that, if the // VIPService already exists, the VIPService is fetched first to ensure that any auto-allocated IP addresses are not // lost during the update. If the VIPService was created without any IP addresses explicitly set (so that they were diff --git a/ipn/backend.go b/ipn/backend.go index 3e956f4734f5f..ab01d2fdef57a 100644 --- a/ipn/backend.go +++ b/ipn/backend.go @@ -81,6 +81,8 @@ const ( NotifyInitialHealthState NotifyWatchOpt = 1 << 7 // if set, the first Notify message (sent immediately) will contain the current health.State of the client NotifyRateLimit NotifyWatchOpt = 1 << 8 // if set, rate limit spammy netmap updates to every few seconds + + NotifyHealthActions NotifyWatchOpt = 1 << 9 // if set, include PrimaryActions in health.State. Otherwise append the action URL to the text ) // Notify is a communication from a backend (e.g. tailscaled) to a frontend diff --git a/ipn/ipn_clone.go b/ipn/ipn_clone.go index 65438444e162f..3d67efc6fd33b 100644 --- a/ipn/ipn_clone.go +++ b/ipn/ipn_clone.go @@ -74,6 +74,7 @@ var _PrefsCloneNeedsRegeneration = Prefs(struct { RouteAll bool ExitNodeID tailcfg.StableNodeID ExitNodeIP netip.Addr + AutoExitNode ExitNodeExpression InternalExitNodePrior tailcfg.StableNodeID ExitNodeAllowLANAccess bool CorpDNS bool diff --git a/ipn/ipn_view.go b/ipn/ipn_view.go index 871270b8564f1..1d31ced9d3847 100644 --- a/ipn/ipn_view.go +++ b/ipn/ipn_view.go @@ -135,6 +135,7 @@ func (v PrefsView) ControlURL() string { return v.ж.Co func (v PrefsView) RouteAll() bool { return v.ж.RouteAll } func (v PrefsView) ExitNodeID() tailcfg.StableNodeID { return v.ж.ExitNodeID } func (v PrefsView) ExitNodeIP() netip.Addr { return v.ж.ExitNodeIP } +func (v PrefsView) AutoExitNode() ExitNodeExpression { return v.ж.AutoExitNode } func (v PrefsView) InternalExitNodePrior() tailcfg.StableNodeID { return v.ж.InternalExitNodePrior } func (v PrefsView) ExitNodeAllowLANAccess() bool { return v.ж.ExitNodeAllowLANAccess } func (v PrefsView) CorpDNS() bool { return v.ж.CorpDNS } @@ -179,6 +180,7 @@ var _PrefsViewNeedsRegeneration = Prefs(struct { RouteAll bool ExitNodeID tailcfg.StableNodeID ExitNodeIP netip.Addr + AutoExitNode ExitNodeExpression InternalExitNodePrior tailcfg.StableNodeID ExitNodeAllowLANAccess bool CorpDNS bool diff --git a/ipn/ipnauth/self.go b/ipn/ipnauth/self.go index 9b430dc6d915e..adee0696458d6 100644 --- a/ipn/ipnauth/self.go +++ b/ipn/ipnauth/self.go @@ -13,6 +13,11 @@ import ( // has unlimited access. var Self Actor = unrestricted{} +// TODO is a caller identity used when the operation is performed on behalf of a user, +// rather than by tailscaled itself, but the surrounding function is not yet extended +// to accept an [Actor] parameter. It grants the same unrestricted access as [Self]. +var TODO Actor = unrestricted{} + // unrestricted is an [Actor] that has unlimited access to the currently running // tailscaled instance. It's typically used for operations performed by tailscaled // on its own, or upon a request from the control plane, rather on behalf of a user. @@ -49,3 +54,10 @@ func (unrestricted) IsLocalSystem() bool { return false } // Deprecated: this method exists for compatibility with the current (as of 2025-01-28) // permission model and will be removed as we progress on tailscale/corp#18342. func (unrestricted) IsLocalAdmin(operatorUID string) bool { return false } + +// IsTailscaled reports whether the given Actor represents Tailscaled itself, +// such as [Self] or a [TODO] placeholder actor. +func IsTailscaled(a Actor) bool { + _, ok := a.(unrestricted) + return ok +} diff --git a/ipn/ipnlocal/c2n.go b/ipn/ipnlocal/c2n.go index b3379475164ae..4b91c3cb9453d 100644 --- a/ipn/ipnlocal/c2n.go +++ b/ipn/ipnlocal/c2n.go @@ -240,13 +240,14 @@ func handleC2NAppConnectorDomainRoutesGet(b *LocalBackend, w http.ResponseWriter b.logf("c2n: GET /appconnector/routes received") var res tailcfg.C2NAppConnectorDomainRoutesResponse - if b.appConnector == nil { + appConnector := b.AppConnector() + if appConnector == nil { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(res) return } - res.Domains = b.appConnector.DomainRoutes() + res.Domains = appConnector.DomainRoutes() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(res) @@ -442,7 +443,7 @@ func findCmdTailscale() (string, error) { } case "windows": ts = filepath.Join(filepath.Dir(self), "tailscale.exe") - case "freebsd": + case "freebsd", "openbsd": if self == "/usr/local/bin/tailscaled" { ts = "/usr/local/bin/tailscale" } diff --git a/ipn/ipnlocal/drive.go b/ipn/ipnlocal/drive.go index a06ea5e8c41ba..d77481903fc09 100644 --- a/ipn/ipnlocal/drive.go +++ b/ipn/ipnlocal/drive.go @@ -306,20 +306,29 @@ func (b *LocalBackend) updateDrivePeersLocked(nm *netmap.NetworkMap) { } func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Remote { + b.logf("[v1] taildrive: setting up drive remotes from peers") driveRemotes := make([]*drive.Remote, 0, len(nm.Peers)) for _, p := range nm.Peers { - peerID := p.ID() - url := fmt.Sprintf("%s/%s", peerAPIBase(nm, p), taildrivePrefix[1:]) + peer := p + peerID := peer.ID() + peerKey := peer.Key().ShortString() + b.logf("[v1] taildrive: appending remote for peer %s", peerKey) driveRemotes = append(driveRemotes, &drive.Remote{ Name: p.DisplayName(false), - URL: url, + URL: func() string { + url := fmt.Sprintf("%s/%s", b.currentNode().PeerAPIBase(peer), taildrivePrefix[1:]) + b.logf("[v2] taildrive: url for peer %s: %s", peerKey, url) + return url + }, Available: func() bool { // Peers are available to Taildrive if: // - They are online + // - Their PeerAPI is reachable // - They are allowed to share at least one folder with us cn := b.currentNode() - peer, ok := cn.PeerByID(peerID) + peer, ok := cn.NodeByID(peerID) if !ok { + b.logf("[v2] taildrive: Available(): peer %s not found", peerKey) return false } @@ -332,14 +341,22 @@ func (b *LocalBackend) driveRemotesFromPeers(nm *netmap.NetworkMap) []*drive.Rem // The netmap.Peers slice is not updated in all cases. // It should be fixed now that we use PeerByIDOk. if !peer.Online().Get() { + b.logf("[v2] taildrive: Available(): peer %s offline", peerKey) + return false + } + + if b.currentNode().PeerAPIBase(peer) == "" { + b.logf("[v2] taildrive: Available(): peer %s PeerAPI unreachable", peerKey) return false } // Check that the peer is allowed to share with us. if cn.PeerHasCap(peer, tailcfg.PeerCapabilityTaildriveSharer) { + b.logf("[v2] taildrive: Available(): peer %s available", peerKey) return true } + b.logf("[v2] taildrive: Available(): peer %s not allowed to share", peerKey) return false }, }) diff --git a/ipn/ipnlocal/extension_host_test.go b/ipn/ipnlocal/extension_host_test.go index f655c477fcb36..509833ff6de46 100644 --- a/ipn/ipnlocal/extension_host_test.go +++ b/ipn/ipnlocal/extension_host_test.go @@ -1230,7 +1230,7 @@ func (e *testExtension) InitCalled() bool { func (e *testExtension) Shutdown() (err error) { e.t.Helper() e.mu.Lock() - e.mu.Unlock() + defer e.mu.Unlock() if e.ShutdownHook != nil { err = e.ShutdownHook(e) } diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 0c9bb22a3fb61..83224c6fd11fc 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -11,6 +11,7 @@ import ( "context" "crypto/sha256" "encoding/base64" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -98,6 +99,7 @@ import ( "tailscale.com/util/clientmetric" "tailscale.com/util/deephash" "tailscale.com/util/dnsname" + "tailscale.com/util/eventbus" "tailscale.com/util/goroutines" "tailscale.com/util/httpm" "tailscale.com/util/mak" @@ -168,6 +170,21 @@ type watchSession struct { var metricCaptivePortalDetected = clientmetric.NewCounter("captiveportal_detected") +var ( + // errShutdown indicates that the [LocalBackend.Shutdown] was called. + errShutdown = errors.New("shutting down") + + // errNodeContextChanged indicates that [LocalBackend] has switched + // to a different [localNodeContext], usually due to a profile change. + // It is used as a context cancellation cause for the old context + // and can be returned when an operation is performed on it. + errNodeContextChanged = errors.New("profile changed") + + // errManagedByPolicy indicates the operation is blocked + // because the target state is managed by a GP/MDM policy. + errManagedByPolicy = errors.New("managed by policy") +) + // LocalBackend is the glue between the major pieces of the Tailscale // network software: the cloud control plane (via controlclient), the // network data plane (via wgengine), and the user-facing UIs and CLIs @@ -180,11 +197,11 @@ var metricCaptivePortalDetected = clientmetric.NewCounter("captiveportal_detecte // state machine generates events back out to zero or more components. type LocalBackend struct { // Elements that are thread-safe or constant after construction. - ctx context.Context // canceled by [LocalBackend.Shutdown] - ctxCancel context.CancelFunc // cancels ctx - logf logger.Logf // general logging - keyLogf logger.Logf // for printing list of peers on change - statsLogf logger.Logf // for printing peers stats on change + ctx context.Context // canceled by [LocalBackend.Shutdown] + ctxCancel context.CancelCauseFunc // cancels ctx + logf logger.Logf // general logging + keyLogf logger.Logf // for printing list of peers on change + statsLogf logger.Logf // for printing peers stats on change sys *tsd.System health *health.Tracker // always non-nil metrics metrics @@ -200,14 +217,14 @@ type LocalBackend struct { portpollOnce sync.Once // guards starting readPoller varRoot string // or empty if SetVarRoot never called logFlushFunc func() // or nil if SetLogFlusher wasn't called - em *expiryManager // non-nil; TODO(nickkhyl): move to nodeContext - sshAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeContext + em *expiryManager // non-nil; TODO(nickkhyl): move to nodeBackend + sshAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeBackend // webClientAtomicBool controls whether the web client is running. This should // be true unless the disable-web-client node attribute has been set. - webClientAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeContext + webClientAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeBackend // exposeRemoteWebClientAtomicBool controls whether the web client is exposed over // Tailscale on port 5252. - exposeRemoteWebClientAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeContext + exposeRemoteWebClientAtomicBool atomic.Bool // TODO(nickkhyl): move to nodeBackend shutdownCalled bool // if Shutdown has been called debugSink packet.CaptureSink sockstatLogger *sockstatlog.Logger @@ -228,10 +245,10 @@ type LocalBackend struct { // is never called. getTCPHandlerForFunnelFlow func(srcAddr netip.AddrPort, dstPort uint16) (handler func(net.Conn)) - containsViaIPFuncAtomic syncs.AtomicValue[func(netip.Addr) bool] // TODO(nickkhyl): move to nodeContext - shouldInterceptTCPPortAtomic syncs.AtomicValue[func(uint16) bool] // TODO(nickkhyl): move to nodeContext - shouldInterceptVIPServicesTCPPortAtomic syncs.AtomicValue[func(netip.AddrPort) bool] // TODO(nickkhyl): move to nodeContext - numClientStatusCalls atomic.Uint32 // TODO(nickkhyl): move to nodeContext + containsViaIPFuncAtomic syncs.AtomicValue[func(netip.Addr) bool] // TODO(nickkhyl): move to nodeBackend + shouldInterceptTCPPortAtomic syncs.AtomicValue[func(uint16) bool] // TODO(nickkhyl): move to nodeBackend + shouldInterceptVIPServicesTCPPortAtomic syncs.AtomicValue[func(netip.AddrPort) bool] // TODO(nickkhyl): move to nodeBackend + numClientStatusCalls atomic.Uint32 // TODO(nickkhyl): move to nodeBackend // goTracker accounts for all goroutines started by LocalBacked, primarily // for testing and graceful shutdown purposes. @@ -256,7 +273,7 @@ type LocalBackend struct { // // It is safe for reading with or without holding b.mu, but mutating it in place // or creating a new one must be done with b.mu held. If both mutexes must be held, - // the LocalBackend's mutex must be acquired first before acquiring the nodeContext's mutex. + // the LocalBackend's mutex must be acquired first before acquiring the nodeBackend's mutex. // // We intend to relax this in the future and only require holding b.mu when replacing it, // but that requires a better (strictly ordered?) state machine and better management @@ -265,30 +282,30 @@ type LocalBackend struct { conf *conffile.Config // latest parsed config, or nil if not in declarative mode pm *profileManager // mu guards access - filterHash deephash.Sum // TODO(nickkhyl): move to nodeContext + filterHash deephash.Sum // TODO(nickkhyl): move to nodeBackend httpTestClient *http.Client // for controlclient. nil by default, used by tests. ccGen clientGen // function for producing controlclient; lazily populated sshServer SSHServer // or nil, initialized lazily. appConnector *appc.AppConnector // or nil, initialized when configured. // notifyCancel cancels notifications to the current SetNotifyCallback. notifyCancel context.CancelFunc - cc controlclient.Client // TODO(nickkhyl): move to nodeContext - ccAuto *controlclient.Auto // if cc is of type *controlclient.Auto; TODO(nickkhyl): move to nodeContext + cc controlclient.Client // TODO(nickkhyl): move to nodeBackend + ccAuto *controlclient.Auto // if cc is of type *controlclient.Auto; TODO(nickkhyl): move to nodeBackend machinePrivKey key.MachinePrivate - tka *tkaState // TODO(nickkhyl): move to nodeContext - state ipn.State // TODO(nickkhyl): move to nodeContext + tka *tkaState // TODO(nickkhyl): move to nodeBackend + state ipn.State // TODO(nickkhyl): move to nodeBackend capTailnetLock bool // whether netMap contains the tailnet lock capability // hostinfo is mutated in-place while mu is held. - hostinfo *tailcfg.Hostinfo // TODO(nickkhyl): move to nodeContext - nmExpiryTimer tstime.TimerController // for updating netMap on node expiry; can be nil; TODO(nickkhyl): move to nodeContext - activeLogin string // last logged LoginName from netMap; TODO(nickkhyl): move to nodeContext (or remove? it's in [ipn.LoginProfile]). + hostinfo *tailcfg.Hostinfo // TODO(nickkhyl): move to nodeBackend + nmExpiryTimer tstime.TimerController // for updating netMap on node expiry; can be nil; TODO(nickkhyl): move to nodeBackend + activeLogin string // last logged LoginName from netMap; TODO(nickkhyl): move to nodeBackend (or remove? it's in [ipn.LoginProfile]). engineStatus ipn.EngineStatus endpoints []tailcfg.Endpoint blocked bool - keyExpired bool // TODO(nickkhyl): move to nodeContext - authURL string // non-empty if not Running; TODO(nickkhyl): move to nodeContext - authURLTime time.Time // when the authURL was received from the control server; TODO(nickkhyl): move to nodeContext - authActor ipnauth.Actor // an actor who called [LocalBackend.StartLoginInteractive] last, or nil; TODO(nickkhyl): move to nodeContext + keyExpired bool // TODO(nickkhyl): move to nodeBackend + authURL string // non-empty if not Running; TODO(nickkhyl): move to nodeBackend + authURLTime time.Time // when the authURL was received from the control server; TODO(nickkhyl): move to nodeBackend + authActor ipnauth.Actor // an actor who called [LocalBackend.StartLoginInteractive] last, or nil; TODO(nickkhyl): move to nodeBackend egg bool prevIfState *netmon.State peerAPIServer *peerAPIServer // or nil @@ -305,7 +322,7 @@ type LocalBackend struct { lastSelfUpdateState ipnstate.SelfUpdateStatus // capForcedNetfilter is the netfilter that control instructs Linux clients // to use, unless overridden locally. - capForcedNetfilter string // TODO(nickkhyl): move to nodeContext + capForcedNetfilter string // TODO(nickkhyl): move to nodeBackend // offlineAutoUpdateCancel stops offline auto-updates when called. It // should be used via stopOfflineAutoUpdate and // maybeStartOfflineAutoUpdate. It is nil when offline auto-updates are @@ -317,7 +334,7 @@ type LocalBackend struct { // ServeConfig fields. (also guarded by mu) lastServeConfJSON mem.RO // last JSON that was parsed into serveConfig serveConfig ipn.ServeConfigView // or !Valid if none - ipVIPServiceMap netmap.IPServiceMappings // map of VIPService IPs to their corresponding service names; TODO(nickkhyl): move to nodeContext + ipVIPServiceMap netmap.IPServiceMappings // map of VIPService IPs to their corresponding service names; TODO(nickkhyl): move to nodeBackend webClient webClient webClientListeners map[netip.AddrPort]*localListener // listeners for local web client traffic @@ -332,7 +349,7 @@ type LocalBackend struct { // dialPlan is any dial plan that we've received from the control // server during a previous connection; it is cleared on logout. - dialPlan atomic.Pointer[tailcfg.ControlDialPlan] // TODO(nickkhyl): maybe move to nodeContext? + dialPlan atomic.Pointer[tailcfg.ControlDialPlan] // TODO(nickkhyl): maybe move to nodeBackend? // tkaSyncLock is used to make tkaSyncIfNeeded an exclusive // section. This is needed to stop two map-responses in quick succession @@ -398,6 +415,19 @@ type LocalBackend struct { // reconnectTimer is used to schedule a reconnect by setting [ipn.Prefs.WantRunning] // to true after a delay, or nil if no reconnect is scheduled. reconnectTimer tstime.TimerController + + // overrideExitNodePolicy is whether the user has overridden the exit node policy + // by manually selecting an exit node, as allowed by [syspolicy.AllowExitNodeOverride]. + // + // If true, the [syspolicy.ExitNodeID] and [syspolicy.ExitNodeIP] policy settings are ignored, + // and the suggested exit node is not applied automatically. + // + // It is cleared when the user switches back to the state required by policy (typically, auto:any), + // or when switching profiles, connecting/disconnecting Tailscale, restarting the client, + // or on similar events. + // + // See tailscale/corp#29969. + overrideExitNodePolicy bool } // HealthTracker returns the health tracker for the backend. @@ -463,7 +493,7 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo envknob.LogCurrent(logf) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancelCause(context.Background()) clock := tstime.StdClock{} // Until we transition to a Running state, use a canceled context for @@ -503,7 +533,10 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo captiveCancel: nil, // so that we start checkCaptivePortalLoop when Running needsCaptiveDetection: make(chan bool), } - b.currentNodeAtomic.Store(newNodeBackend()) + nb := newNodeBackend(ctx, b.sys.Bus.Get()) + b.currentNodeAtomic.Store(nb) + nb.ready() + mConn.SetNetInfoCallback(b.setNetInfo) if sys.InitialConfig != nil { @@ -585,9 +618,18 @@ func (b *LocalBackend) currentNode() *nodeBackend { if v := b.currentNodeAtomic.Load(); v != nil || !testenv.InTest() { return v } - // Auto-init one in tests for LocalBackend created without the NewLocalBackend constructor... - v := newNodeBackend() - b.currentNodeAtomic.CompareAndSwap(nil, v) + // Auto-init [nodeBackend] in tests for LocalBackend created without the + // NewLocalBackend() constructor. Same reasoning for checking b.sys. + var bus *eventbus.Bus + if b.sys == nil { + bus = eventbus.New() + } else { + bus = b.sys.Bus.Get() + } + v := newNodeBackend(cmp.Or(b.ctx, context.Background()), bus) + if b.currentNodeAtomic.CompareAndSwap(nil, v) { + v.ready() + } return b.currentNodeAtomic.Load() } @@ -888,13 +930,14 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) { hadPAC := b.prevIfState.HasPAC() b.prevIfState = ifst b.pauseOrResumeControlClientLocked() - if delta.Major && shouldAutoExitNode() { + prefs := b.pm.CurrentPrefs() + if delta.Major && prefs.AutoExitNode().IsSet() { b.refreshAutoExitNode = true } var needReconfig bool // If the network changed and we're using an exit node and allowing LAN access, we may need to reconfigure. - if delta.Major && b.pm.CurrentPrefs().ExitNodeID() != "" && b.pm.CurrentPrefs().ExitNodeAllowLANAccess() { + if delta.Major && prefs.ExitNodeID() != "" && prefs.ExitNodeAllowLANAccess() { b.logf("linkChange: in state %v; updating LAN routes", b.state) needReconfig = true } @@ -917,8 +960,8 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) { // If the local network configuration has changed, our filter may // need updating to tweak default routes. - b.updateFilterLocked(b.pm.CurrentPrefs()) - updateExitNodeUsageWarning(b.pm.CurrentPrefs(), delta.New, b.health) + b.updateFilterLocked(prefs) + updateExitNodeUsageWarning(prefs, delta.New, b.health) cn := b.currentNode() nm := cn.NetMap() @@ -933,11 +976,15 @@ func (b *LocalBackend) linkChange(delta *netmon.ChangeDelta) { } } -func (b *LocalBackend) onHealthChange(w *health.Warnable, us *health.UnhealthyState) { - if us == nil { - b.logf("health(warnable=%s): ok", w.Code) - } else { - b.logf("health(warnable=%s): error: %s", w.Code, us.Text) +func (b *LocalBackend) onHealthChange(change health.Change) { + if change.WarnableChanged { + w := change.Warnable + us := change.UnhealthyState + if us == nil { + b.logf("health(warnable=%s): ok", w.Code) + } else { + b.logf("health(warnable=%s): error: %s", w.Code, us.Text) + } } // Whenever health changes, send the current health state to the frontend. @@ -1044,7 +1091,7 @@ func (b *LocalBackend) Shutdown() { ctx, cancel := context.WithTimeout(b.ctx, 5*time.Second) defer cancel() t0 := time.Now() - err := b.Logout(ctx) // best effort + err := b.Logout(ctx, ipnauth.Self) // best effort td := time.Since(t0).Round(time.Millisecond) if err != nil { b.logf("failed to log out ephemeral node on shutdown after %v: %v", td, err) @@ -1085,8 +1132,9 @@ func (b *LocalBackend) Shutdown() { if cc != nil { cc.Shutdown() } + b.ctxCancel(errShutdown) + b.currentNode().shutdown(errShutdown) extHost.Shutdown() - b.ctxCancel() b.e.Close() <-b.e.Done() b.awaitNoGoroutinesInTest() @@ -1362,7 +1410,7 @@ func profileFromView(v tailcfg.UserProfileView) tailcfg.UserProfile { func (b *LocalBackend) WhoIsNodeKey(k key.NodePublic) (n tailcfg.NodeView, u tailcfg.UserProfile, ok bool) { cn := b.currentNode() if nid, ok := cn.NodeByKey(k); ok { - if n, ok := cn.PeerByID(nid); ok { + if n, ok := cn.NodeByID(nid); ok { up, ok := cn.NetMap().UserProfiles[n.User()] u = profileFromView(up) return n, u, ok @@ -1428,13 +1476,9 @@ func (b *LocalBackend) WhoIs(proto string, ipp netip.AddrPort) (n tailcfg.NodeVi if nm == nil { return failf("no netmap") } - n, ok = cn.PeerByID(nid) + n, ok = cn.NodeByID(nid) if !ok { - // Check if this the self-node, which would not appear in peers. - if !nm.SelfNode.Valid() || nid != nm.SelfNode.ID() { - return zero, u, false - } - n = nm.SelfNode + return zero, u, false } up, ok := cn.UserByID(n.User()) if !ok { @@ -1598,17 +1642,11 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control prefsChanged = true } } - if shouldAutoExitNode() { - // Re-evaluate exit node suggestion in case circumstances have changed. - _, err := b.suggestExitNodeLocked(curNetMap) - if err != nil && !errors.Is(err, ErrNoPreferredDERP) { - b.logf("SetControlClientStatus failed to select auto exit node: %v", err) - } - } - if applySysPolicy(prefs, b.lastSuggestedExitNode, b.overrideAlwaysOn) { - prefsChanged = true - } - if setExitNodeID(prefs, curNetMap) { + // We primarily need this to apply syspolicy to the prefs if an implicit profile + // switch is about to happen. + // TODO(nickkhyl): remove this once we improve handling of implicit profile switching + // in tailscale/corp#28014 and we apply syspolicy when the switch actually happens. + if b.reconcilePrefsLocked(prefs) { prefsChanged = true } @@ -1627,6 +1665,8 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control }); err != nil { b.logf("Failed to save new controlclient state: %v", err) } + + b.sendToLocked(ipn.Notify{Prefs: ptr.To(prefs.View())}, allClients) } // initTKALocked is dependent on CurrentProfile.ID, which is initialized @@ -1669,16 +1709,14 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control b.mu.Unlock() // Now complete the lock-free parts of what we started while locked. - if prefsChanged { - b.send(ipn.Notify{Prefs: ptr.To(prefs.View())}) - } - if st.NetMap != nil { if envknob.NoLogsNoSupport() && st.NetMap.HasCap(tailcfg.CapabilityDataPlaneAuditLogs) { msg := "tailnet requires logging to be enabled. Remove --no-logs-no-support from tailscaled command line." b.health.SetLocalLogConfigHealth(errors.New(msg)) // Connecting to this tailnet without logging is forbidden; boot us outta here. b.mu.Lock() + // Get the current prefs again, since we unlocked above. + prefs := b.pm.CurrentPrefs().AsStruct() prefs.WantRunning = false p := prefs.View() if err := b.pm.SetPrefs(p, ipn.NetworkProfile{ @@ -1711,6 +1749,16 @@ func (b *LocalBackend) SetControlClientStatus(c controlclient.Client, st control b.health.SetDERPMap(st.NetMap.DERPMap) b.send(ipn.Notify{NetMap: st.NetMap}) + + // Check and update the exit node if needed, now that we have a new netmap. + // + // This must happen after the netmap change is sent via [ipn.Notify], + // so the GUI can correctly display the exit node if it has changed + // since the last netmap was sent. + // + // Otherwise, it might briefly show the exit node as offline and display a warning, + // if the node wasn't online or wasn't advertising default routes in the previous netmap. + b.RefreshExitNode() } if st.URL != "" { b.logf("Received auth URL: %.20v...", st.URL) @@ -1773,9 +1821,11 @@ var preferencePolicies = []preferencePolicyInfo{ }, } -// applySysPolicy overwrites configured preferences with policies that may be +// applySysPolicyLocked overwrites configured preferences with policies that may be // configured by the system administrator in an OS-specific way. -func applySysPolicy(prefs *ipn.Prefs, lastSuggestedExitNode tailcfg.StableNodeID, overrideAlwaysOn bool) (anyChange bool) { +// +// b.mu must be held. +func (b *LocalBackend) applySysPolicyLocked(prefs *ipn.Prefs) (anyChange bool) { if controlURL, err := syspolicy.GetString(syspolicy.ControlURL, prefs.ControlURL); err == nil && prefs.ControlURL != controlURL { prefs.ControlURL = controlURL anyChange = true @@ -1812,32 +1862,12 @@ func applySysPolicy(prefs *ipn.Prefs, lastSuggestedExitNode tailcfg.StableNodeID } } - if exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, ""); exitNodeIDStr != "" { - exitNodeID := tailcfg.StableNodeID(exitNodeIDStr) - if shouldAutoExitNode() && lastSuggestedExitNode != "" { - exitNodeID = lastSuggestedExitNode - } - // Note: when exitNodeIDStr == "auto" && lastSuggestedExitNode == "", - // then exitNodeID is now "auto" which will never match a peer's node ID. - // When there is no a peer matching the node ID, traffic will blackhole, - // preventing accidental non-exit-node usage when a policy is in effect that requires an exit node. - if prefs.ExitNodeID != exitNodeID || prefs.ExitNodeIP.IsValid() { - anyChange = true - } - prefs.ExitNodeID = exitNodeID - prefs.ExitNodeIP = netip.Addr{} - } else if exitNodeIPStr, _ := syspolicy.GetString(syspolicy.ExitNodeIP, ""); exitNodeIPStr != "" { - exitNodeIP, err := netip.ParseAddr(exitNodeIPStr) - if exitNodeIP.IsValid() && err == nil { - if prefs.ExitNodeID != "" || prefs.ExitNodeIP != exitNodeIP { - anyChange = true - } - prefs.ExitNodeID = "" - prefs.ExitNodeIP = exitNodeIP - } + // Only apply the exit node policy if the user hasn't overridden it. + if !b.overrideExitNodePolicy && b.applyExitNodeSysPolicyLocked(prefs) { + anyChange = true } - if alwaysOn, _ := syspolicy.GetBoolean(syspolicy.AlwaysOn, false); alwaysOn && !overrideAlwaysOn && !prefs.WantRunning { + if alwaysOn, _ := syspolicy.GetBoolean(syspolicy.AlwaysOn, false); alwaysOn && !b.overrideAlwaysOn && !prefs.WantRunning { prefs.WantRunning = true anyChange = true } @@ -1856,27 +1886,90 @@ func applySysPolicy(prefs *ipn.Prefs, lastSuggestedExitNode tailcfg.StableNodeID return anyChange } +// applyExitNodeSysPolicyLocked applies the exit node policy settings to prefs +// and reports whether any change was made. +// +// b.mu must be held. +func (b *LocalBackend) applyExitNodeSysPolicyLocked(prefs *ipn.Prefs) (anyChange bool) { + if exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, ""); exitNodeIDStr != "" { + exitNodeID := tailcfg.StableNodeID(exitNodeIDStr) + + // Try to parse the policy setting value as an "auto:"-prefixed [ipn.ExitNodeExpression], + // and update prefs if it differs from the current one. + // This includes cases where it was previously an expression but no longer is, + // or where it wasn't before but now is. + autoExitNode, useAutoExitNode := ipn.ParseAutoExitNodeString(exitNodeID) + if prefs.AutoExitNode != autoExitNode { + prefs.AutoExitNode = autoExitNode + anyChange = true + } + // Additionally, if the specified exit node ID is an expression, + // meaning an exit node is required but we don't yet have a valid exit node ID, + // we should set exitNodeID to a value that is never a valid [tailcfg.StableNodeID], + // to install a blackhole route and prevent accidental non-exit-node usage + // until the expression is evaluated and an actual exit node is selected. + // We use "auto:any" for this purpose, primarily for compatibility with + // older clients (in case a user downgrades to an earlier version) + // and GUIs/CLIs that have special handling for it. + if useAutoExitNode { + exitNodeID = unresolvedExitNodeID + } + + // If the current exit node ID doesn't match the one enforced by the policy setting, + // and the policy either requires a specific exit node ID, + // or requires an auto exit node ID and the current one isn't allowed, + // then update the exit node ID. + if prefs.ExitNodeID != exitNodeID { + if !useAutoExitNode || !isAllowedAutoExitNodeID(prefs.ExitNodeID) { + prefs.ExitNodeID = exitNodeID + anyChange = true + } + } + + // If the exit node IP is set, clear it. When ExitNodeIP is set in the prefs, + // it takes precedence over the ExitNodeID. + if prefs.ExitNodeIP.IsValid() { + prefs.ExitNodeIP = netip.Addr{} + anyChange = true + } + } else if exitNodeIPStr, _ := syspolicy.GetString(syspolicy.ExitNodeIP, ""); exitNodeIPStr != "" { + if prefs.AutoExitNode != "" { + prefs.AutoExitNode = "" // mutually exclusive with ExitNodeIP + anyChange = true + } + if exitNodeIP, err := netip.ParseAddr(exitNodeIPStr); err == nil { + if prefs.ExitNodeID != "" || prefs.ExitNodeIP != exitNodeIP { + anyChange = true + } + prefs.ExitNodeID = "" + prefs.ExitNodeIP = exitNodeIP + } + } + + return anyChange +} + // registerSysPolicyWatch subscribes to syspolicy change notifications // and immediately applies the effective syspolicy settings to the current profile. func (b *LocalBackend) registerSysPolicyWatch() (unregister func(), err error) { if unregister, err = syspolicy.RegisterChangeCallback(b.sysPolicyChanged); err != nil { return nil, fmt.Errorf("syspolicy: LocalBacked failed to register policy change callback: %v", err) } - if prefs, anyChange := b.applySysPolicy(); anyChange { + if prefs, anyChange := b.reconcilePrefs(); anyChange { b.logf("syspolicy: changed initial profile prefs: %v", prefs.Pretty()) } b.refreshAllowedSuggestions() return unregister, nil } -// applySysPolicy overwrites the current profile's preferences with policies +// reconcilePrefs overwrites the current profile's preferences with policies // that may be configured by the system administrator in an OS-specific way. // // b.mu must not be held. -func (b *LocalBackend) applySysPolicy() (_ ipn.PrefsView, anyChange bool) { +func (b *LocalBackend) reconcilePrefs() (_ ipn.PrefsView, anyChange bool) { unlock := b.lockAndGetUnlock() prefs := b.pm.CurrentPrefs().AsStruct() - if !applySysPolicy(prefs, b.lastSuggestedExitNode, b.overrideAlwaysOn) { + if !b.reconcilePrefsLocked(prefs) { unlock.UnlockEarly() return prefs.View(), false } @@ -1886,7 +1979,7 @@ func (b *LocalBackend) applySysPolicy() (_ ipn.PrefsView, anyChange bool) { // sysPolicyChanged is a callback triggered by syspolicy when it detects // a change in one or more syspolicy settings. func (b *LocalBackend) sysPolicyChanged(policy *rsop.PolicyChange) { - if policy.HasChanged(syspolicy.AlwaysOn) || policy.HasChanged(syspolicy.AlwaysOnOverrideWithReason) { + if policy.HasChangedAnyOf(syspolicy.AlwaysOn, syspolicy.AlwaysOnOverrideWithReason) { // If the AlwaysOn or the AlwaysOnOverrideWithReason policy has changed, // we should reset the overrideAlwaysOn flag, as the override might // no longer be valid. @@ -1895,20 +1988,25 @@ func (b *LocalBackend) sysPolicyChanged(policy *rsop.PolicyChange) { b.mu.Unlock() } + if policy.HasChangedAnyOf(syspolicy.ExitNodeID, syspolicy.ExitNodeIP, syspolicy.AllowExitNodeOverride) { + // Reset the exit node override if a policy that enforces exit node usage + // or allows the user to override automatic exit node selection has changed. + b.mu.Lock() + b.overrideExitNodePolicy = false + b.mu.Unlock() + } + if policy.HasChanged(syspolicy.AllowedSuggestedExitNodes) { b.refreshAllowedSuggestions() // Re-evaluate exit node suggestion now that the policy setting has changed. - b.mu.Lock() - _, err := b.suggestExitNodeLocked(nil) - b.mu.Unlock() - if err != nil && !errors.Is(err, ErrNoPreferredDERP) { + if _, err := b.SuggestExitNode(); err != nil && !errors.Is(err, ErrNoPreferredDERP) { b.logf("failed to select auto exit node: %v", err) } // If [syspolicy.ExitNodeID] is set to `auto:any`, the suggested exit node ID // will be used when [applySysPolicy] updates the current profile's prefs. } - if prefs, anyChange := b.applySysPolicy(); anyChange { + if prefs, anyChange := b.reconcilePrefs(); anyChange { b.logf("syspolicy: changed profile prefs: %v", prefs.Pretty()) } } @@ -1917,10 +2015,6 @@ var _ controlclient.NetmapDeltaUpdater = (*LocalBackend)(nil) // UpdateNetmapDelta implements controlclient.NetmapDeltaUpdater. func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) { - if !b.MagicConn().UpdateNetmapDelta(muts) { - return false - } - var notify *ipn.Notify // non-nil if we need to send a Notify defer func() { if notify != nil { @@ -1936,18 +2030,18 @@ func (b *LocalBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo // If auto exit nodes are enabled and our exit node went offline, // we need to schedule picking a new one. // TODO(nickkhyl): move the auto exit node logic to a feature package. - if shouldAutoExitNode() { - exitNodeID := b.pm.prefs.ExitNodeID() + if prefs := b.pm.CurrentPrefs(); prefs.AutoExitNode().IsSet() { + exitNodeID := prefs.ExitNodeID() for _, m := range muts { mo, ok := m.(netmap.NodeMutationOnline) if !ok || mo.Online { continue } - n, ok := cn.PeerByID(m.NodeIDBeingMutated()) + n, ok := cn.NodeByID(m.NodeIDBeingMutated()) if !ok || n.StableID() != exitNodeID { continue } - b.goTracker.Go(b.pickNewAutoExitNode) + b.goTracker.Go(b.RefreshExitNode) break } } @@ -1980,30 +2074,56 @@ func mutationsAreWorthyOfTellingIPNBus(muts []netmap.NodeMutation) bool { return false } -// pickNewAutoExitNode picks a new automatic exit node if needed. -func (b *LocalBackend) pickNewAutoExitNode() { - unlock := b.lockAndGetUnlock() - defer unlock() - - newPrefs := b.setAutoExitNodeIDLockedOnEntry(unlock) - if !newPrefs.Valid() { - // Unchanged. - return +// resolveAutoExitNodeLocked computes a suggested exit node and updates prefs +// to use it if AutoExitNode is enabled, and reports whether prefs was mutated. +// +// b.mu must be held. +func (b *LocalBackend) resolveAutoExitNodeLocked(prefs *ipn.Prefs) (prefsChanged bool) { + // As of 2025-07-08, the only supported auto exit node expression is [ipn.AnyExitNode]. + // + // However, to maintain forward compatibility with future auto exit node expressions, + // we treat any non-empty AutoExitNode as [ipn.AnyExitNode]. + // + // If and when we support additional auto exit node expressions, this method should be updated + // to handle them appropriately, while still falling back to [ipn.AnyExitNode] or a more appropriate + // default for unknown (or partially supported) expressions. + if !prefs.AutoExitNode.IsSet() { + return false } - - b.send(ipn.Notify{Prefs: &newPrefs}) + if _, err := b.suggestExitNodeLocked(); err != nil && !errors.Is(err, ErrNoPreferredDERP) { + b.logf("failed to select auto exit node: %v", err) // non-fatal, see below + } + var newExitNodeID tailcfg.StableNodeID + if !b.lastSuggestedExitNode.IsZero() { + // If we have a suggested exit node, use it. + newExitNodeID = b.lastSuggestedExitNode + } else if isAllowedAutoExitNodeID(prefs.ExitNodeID) { + // If we don't have a suggested exit node, but the prefs already + // specify an allowed auto exit node ID, retain it. + newExitNodeID = prefs.ExitNodeID + } else { + // Otherwise, use [unresolvedExitNodeID] to install a blackhole route, + // preventing traffic from leaking to the local network until an actual + // exit node is selected. + newExitNodeID = unresolvedExitNodeID + } + if prefs.ExitNodeID != newExitNodeID { + prefs.ExitNodeID = newExitNodeID + prefsChanged = true + } + if prefs.ExitNodeIP.IsValid() { + prefs.ExitNodeIP = netip.Addr{} + prefsChanged = true + } + return prefsChanged } -// setExitNodeID updates prefs to reference an exit node by ID, rather +// resolveExitNodeIPLocked updates prefs to reference an exit node by ID, rather // than by IP. It returns whether prefs was mutated. -func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap) (prefsChanged bool) { - if nm == nil { - // No netmap, can't resolve anything. - return false - } - - // If we have a desired IP on file, try to find the corresponding - // node. +// +// b.mu must be held. +func (b *LocalBackend) resolveExitNodeIPLocked(prefs *ipn.Prefs) (prefsChanged bool) { + // If we have a desired IP on file, try to find the corresponding node. if !prefs.ExitNodeIP.IsValid() { return false } @@ -2014,20 +2134,19 @@ func setExitNodeID(prefs *ipn.Prefs, nm *netmap.NetworkMap) (prefsChanged bool) prefsChanged = true } - oldExitNodeID := prefs.ExitNodeID - for _, peer := range nm.Peers { - for _, addr := range peer.Addresses().All() { - if !addr.IsSingleIP() || addr.Addr() != prefs.ExitNodeIP { - continue - } + cn := b.currentNode() + if nid, ok := cn.NodeByAddr(prefs.ExitNodeIP); ok { + if node, ok := cn.NodeByID(nid); ok { // Found the node being referenced, upgrade prefs to // reference it directly for next time. - prefs.ExitNodeID = peer.StableID() + prefs.ExitNodeID = node.StableID() prefs.ExitNodeIP = netip.Addr{} - return prefsChanged || oldExitNodeID != prefs.ExitNodeID + // Cleared ExitNodeIP, so prefs changed + // even if the ID stayed the same. + prefsChanged = true + } } - return prefsChanged } @@ -2223,6 +2342,7 @@ func (b *LocalBackend) Start(opts ipn.Options) error { hostinfo.Userspace.Set(b.sys.IsNetstack()) hostinfo.UserspaceRouter.Set(b.sys.IsNetstackRouter()) hostinfo.AppConnector.Set(b.appConnector != nil) + hostinfo.StateEncrypted = b.stateEncrypted() b.logf.JSON(1, "Hostinfo", hostinfo) // TODO(apenwarr): avoid the need to reinit controlclient. @@ -2242,26 +2362,32 @@ func (b *LocalBackend) Start(opts ipn.Options) error { b.setStateLocked(ipn.NoState) cn := b.currentNode() + + prefsChanged := false + newPrefs := b.pm.CurrentPrefs().AsStruct() if opts.UpdatePrefs != nil { - oldPrefs := b.pm.CurrentPrefs() - newPrefs := opts.UpdatePrefs.Clone() - newPrefs.Persist = oldPrefs.Persist().AsStruct() - pv := newPrefs.View() - if err := b.pm.SetPrefs(pv, cn.NetworkProfile()); err != nil { - b.logf("failed to save UpdatePrefs state: %v", err) + newPrefs = opts.UpdatePrefs.Clone() + prefsChanged = true + } + // Apply any syspolicy overrides, resolve exit node ID, etc. + // As of 2025-07-03, this is primarily needed in two cases: + // - when opts.UpdatePrefs is not nil + // - when Always Mode is enabled and we need to set WantRunning to true + if b.reconcilePrefsLocked(newPrefs) { + prefsChanged = true + } + if prefsChanged { + // Neither opts.UpdatePrefs nor prefs reconciliation + // is allowed to modify Persist; retain the old value. + newPrefs.Persist = b.pm.CurrentPrefs().Persist().AsStruct() + if err := b.pm.SetPrefs(newPrefs.View(), cn.NetworkProfile()); err != nil { + b.logf("failed to save updated and reconciled prefs: %v", err) } } + prefs := newPrefs.View() // Reset the always-on override whenever Start is called. b.resetAlwaysOnOverrideLocked() - // And also apply syspolicy settings to the current profile. - // This is important in two cases: when opts.UpdatePrefs is not nil, - // and when Always Mode is enabled and we need to set WantRunning to true. - if newp := b.pm.CurrentPrefs().AsStruct(); applySysPolicy(newp, b.lastSuggestedExitNode, b.overrideAlwaysOn) { - setExitNodeID(newp, cn.NetMap()) - b.pm.setPrefsNoPermCheck(newp.View()) - } - prefs := b.pm.CurrentPrefs() b.setAtomicValuesFromPrefsLocked(prefs) wantRunning := prefs.WantRunning() @@ -2949,28 +3075,19 @@ func (b *LocalBackend) WatchNotifications(ctx context.Context, mask ipn.NotifyWa b.WatchNotificationsAs(ctx, nil, mask, onWatchAdded, fn) } -// WatchNotificationsAs is like WatchNotifications but takes an [ipnauth.Actor] +// WatchNotificationsAs is like [LocalBackend.WatchNotifications] but takes an [ipnauth.Actor] // as an additional parameter. If non-nil, the specified callback is invoked // only for notifications relevant to this actor. func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.Actor, mask ipn.NotifyWatchOpt, onWatchAdded func(), fn func(roNotify *ipn.Notify) (keepGoing bool)) { ch := make(chan *ipn.Notify, 128) sessionID := rands.HexString(16) - origFn := fn if mask&ipn.NotifyNoPrivateKeys != 0 { - fn = func(n *ipn.Notify) bool { - if n.NetMap == nil || n.NetMap.PrivateKey.IsZero() { - return origFn(n) - } - - // The netmap in n is shared across all watchers, so to mutate it for a - // single watcher we have to clone the notify and the netmap. We can - // make shallow clones, at least. - nm2 := *n.NetMap - n2 := *n - n2.NetMap = &nm2 - n2.NetMap.PrivateKey = key.NodePrivate{} - return origFn(&n2) - } + fn = filterPrivateKeys(fn) + } + if mask&ipn.NotifyHealthActions == 0 { + // if UI does not support PrimaryAction in health warnings, append + // action URLs to the warning text instead. + fn = appendHealthActions(fn) } var ini *ipn.Notify @@ -3061,6 +3178,53 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A sender.Run(ctx, ch) } +// filterPrivateKeys returns an IPN listener func that wraps the supplied IPN +// listener and zeroes out the PrivateKey in the NetMap passed to the wrapped +// listener. +func filterPrivateKeys(fn func(roNotify *ipn.Notify) (keepGoing bool)) func(*ipn.Notify) bool { + return func(n *ipn.Notify) bool { + if n.NetMap == nil || n.NetMap.PrivateKey.IsZero() { + return fn(n) + } + + // The netmap in n is shared across all watchers, so to mutate it for a + // single watcher we have to clone the notify and the netmap. We can + // make shallow clones, at least. + nm2 := *n.NetMap + n2 := *n + n2.NetMap = &nm2 + n2.NetMap.PrivateKey = key.NodePrivate{} + return fn(&n2) + } +} + +// appendHealthActions returns an IPN listener func that wraps the supplied IPN +// listener func and transforms health messages passed to the wrapped listener. +// If health messages with PrimaryActions are present, it appends the label & +// url in the PrimaryAction to the text of the message. For use for clients that +// do not process the PrimaryAction. +func appendHealthActions(fn func(roNotify *ipn.Notify) (keepGoing bool)) func(*ipn.Notify) bool { + return func(n *ipn.Notify) bool { + if n.Health == nil || len(n.Health.Warnings) == 0 { + return fn(n) + } + + // Shallow clone the notify and health so we can mutate them + h2 := *n.Health + n2 := *n + n2.Health = &h2 + n2.Health.Warnings = make(map[health.WarnableCode]health.UnhealthyState, len(n.Health.Warnings)) + for k, v := range n.Health.Warnings { + if v.PrimaryAction != nil { + v.Text = fmt.Sprintf("%s %s: %s", v.Text, v.PrimaryAction.Label, v.PrimaryAction.URL) + v.PrimaryAction = nil + } + n2.Health.Warnings[k] = v + } + return fn(&n2) + } +} + // pollRequestEngineStatus calls b.e.RequestStatus every 2 seconds until ctx // is done. func (b *LocalBackend) pollRequestEngineStatus(ctx context.Context) { @@ -3290,6 +3454,16 @@ func (b *LocalBackend) popBrowserAuthNow(url string, keyExpired bool, recipient // // b.mu must *not* be held. func (b *LocalBackend) validPopBrowserURL(urlStr string) bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.validPopBrowserURLLocked(urlStr) +} + +// validPopBrowserURLLocked reports whether urlStr is a valid value for a +// control server to send in a *URL field. +// +// b.mu must be held. +func (b *LocalBackend) validPopBrowserURLLocked(urlStr string) bool { if urlStr == "" { return false } @@ -3297,7 +3471,7 @@ func (b *LocalBackend) validPopBrowserURL(urlStr string) bool { if err != nil { return false } - serverURL := b.Prefs().ControlURLOrDefault() + serverURL := b.sanitizedPrefsLocked().ControlURLOrDefault() if ipn.IsLoginServerSynonym(serverURL) { // When connected to the official Tailscale control plane, only allow // URLs from tailscale.com or its subdomains. @@ -3364,12 +3538,14 @@ func (b *LocalBackend) onTailnetDefaultAutoUpdate(au bool) { b.logf("using tailnet default auto-update setting: %v", au) prefsClone := prefs.AsStruct() prefsClone.AutoUpdate.Apply = opt.NewBool(au) - _, err := b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ - Prefs: *prefsClone, - AutoUpdateSet: ipn.AutoUpdatePrefsMask{ - ApplySet: true, - }, - }, unlock) + _, err := b.editPrefsLockedOnEntry( + ipnauth.Self, + &ipn.MaskedPrefs{ + Prefs: *prefsClone, + AutoUpdateSet: ipn.AutoUpdatePrefsMask{ + ApplySet: true, + }, + }, unlock) if err != nil { b.logf("failed to apply tailnet-wide default for auto-updates (%v): %v", au, err) return @@ -4065,9 +4241,8 @@ func updateExitNodeUsageWarning(p ipn.PrefsView, state *netmon.State, healthTrac var msg string if p.ExitNodeIP().IsValid() || p.ExitNodeID() != "" { warn, _ := netutil.CheckReversePathFiltering(state) - const comment = "please set rp_filter=2 instead of rp_filter=1; see https://github.com/tailscale/tailscale/issues/3310" if len(warn) > 0 { - msg = fmt.Sprintf("%s: %v, %s", healthmsg.WarnExitNodeUsage, warn, comment) + msg = fmt.Sprintf("%s: %v, %s", healthmsg.WarnExitNodeUsage, warn, healthmsg.DisableRPFilter) } } if len(msg) > 0 { @@ -4112,7 +4287,7 @@ func (b *LocalBackend) checkAutoUpdatePrefsLocked(p *ipn.Prefs) error { // On success, it returns the resulting prefs (or current prefs, in the case of no change). // Setting the value to false when use of an exit node is already false is not an error, // nor is true when the exit node is already in use. -func (b *LocalBackend) SetUseExitNodeEnabled(v bool) (ipn.PrefsView, error) { +func (b *LocalBackend) SetUseExitNodeEnabled(actor ipnauth.Actor, v bool) (ipn.PrefsView, error) { unlock := b.lockAndGetUnlock() defer unlock() @@ -4137,22 +4312,33 @@ func (b *LocalBackend) SetUseExitNodeEnabled(v bool) (ipn.PrefsView, error) { mp := &ipn.MaskedPrefs{} if v { mp.ExitNodeIDSet = true - mp.ExitNodeID = tailcfg.StableNodeID(p0.InternalExitNodePrior()) + mp.ExitNodeID = p0.InternalExitNodePrior() + if expr, ok := ipn.ParseAutoExitNodeString(mp.ExitNodeID); ok { + mp.AutoExitNodeSet = true + mp.AutoExitNode = expr + mp.ExitNodeID = unresolvedExitNodeID + } } else { mp.ExitNodeIDSet = true mp.ExitNodeID = "" + mp.AutoExitNodeSet = true + mp.AutoExitNode = "" mp.InternalExitNodePriorSet = true - mp.InternalExitNodePrior = p0.ExitNodeID() + if p0.AutoExitNode().IsSet() { + mp.InternalExitNodePrior = tailcfg.StableNodeID(ipn.AutoExitNodePrefix + p0.AutoExitNode()) + } else { + mp.InternalExitNodePrior = p0.ExitNodeID() + } } - return b.editPrefsLockedOnEntry(mp, unlock) + return b.editPrefsLockedOnEntry(actor, mp, unlock) } // MaybeClearAppConnector clears the routes from any AppConnector if // AdvertiseRoutes has been set in the MaskedPrefs. func (b *LocalBackend) MaybeClearAppConnector(mp *ipn.MaskedPrefs) error { var err error - if b.appConnector != nil && mp.AdvertiseRoutesSet { - err = b.appConnector.ClearRoutes() + if ac := b.AppConnector(); ac != nil && mp.AdvertiseRoutesSet { + err = ac.ClearRoutes() if err != nil { b.logf("appc: clear routes error: %v", err) } @@ -4173,23 +4359,138 @@ func (b *LocalBackend) EditPrefsAs(mp *ipn.MaskedPrefs, actor ipnauth.Actor) (ip return ipn.PrefsView{}, errors.New("can't set Internal fields") } - // Zeroing the ExitNodeId via localAPI must also zero the prior exit node. - if mp.ExitNodeIDSet && mp.ExitNodeID == "" { - mp.InternalExitNodePrior = "" - mp.InternalExitNodePriorSet = true + return b.editPrefsLockedOnEntry(actor, mp, b.lockAndGetUnlock()) +} + +// checkEditPrefsAccessLocked checks whether the current user has access +// to apply the changes in mp to the given prefs. +// +// It returns an error if the user is not allowed, or nil otherwise. +// +// b.mu must be held. +func (b *LocalBackend) checkEditPrefsAccessLocked(actor ipnauth.Actor, prefs ipn.PrefsView, mp *ipn.MaskedPrefs) error { + var errs []error + + if mp.RunSSHSet && mp.RunSSH && !envknob.CanSSHD() { + errs = append(errs, errors.New("Tailscale SSH server administratively disabled")) } - // Acquire the lock before checking the profile access to prevent - // TOCTOU issues caused by the current profile changing between the - // check and the actual edit. - unlock := b.lockAndGetUnlock() - defer unlock() + // Check if the user is allowed to disconnect Tailscale. if mp.WantRunningSet && !mp.WantRunning && b.pm.CurrentPrefs().WantRunning() { if err := actor.CheckProfileAccess(b.pm.CurrentProfile(), ipnauth.Disconnect, b.extHost.AuditLogger()); err != nil { - b.logf("check profile access failed: %v", err) - return ipn.PrefsView{}, err + errs = append(errs, err) } + } + + // Prevent users from changing exit node preferences + // when exit node usage is managed by policy. + if mp.ExitNodeIDSet || mp.ExitNodeIPSet || mp.AutoExitNodeSet { + isManaged, err := syspolicy.HasAnyOf(syspolicy.ExitNodeID, syspolicy.ExitNodeIP) + if err != nil { + err = fmt.Errorf("policy check failed: %w", err) + } else if isManaged { + // Allow users to override ExitNode policy settings and select an exit node manually + // if permitted by [syspolicy.AllowExitNodeOverride]. + // + // Disabling exit node usage entirely is not allowed. + allowExitNodeOverride, _ := syspolicy.GetBoolean(syspolicy.AllowExitNodeOverride, false) + if !allowExitNodeOverride || b.changeDisablesExitNodeLocked(prefs, mp) { + err = errManagedByPolicy + } + } + if err != nil { + errs = append(errs, fmt.Errorf("exit node cannot be changed: %w", err)) + } + } + + return multierr.New(errs...) +} + +// changeDisablesExitNodeLocked reports whether applying the change +// to the given prefs would disable exit node usage. +// +// In other words, it returns true if prefs.ExitNodeID is non-empty +// initially, but would become empty after applying the given change. +// +// It applies the same adjustments and resolves the exit node in the prefs +// as done during actual edits. While not optimal performance-wise, +// changing the exit node via LocalAPI isn't a hot path, and reusing +// the same logic ensures consistency and simplifies maintenance. +// +// b.mu must be held. +func (b *LocalBackend) changeDisablesExitNodeLocked(prefs ipn.PrefsView, change *ipn.MaskedPrefs) bool { + if !change.AutoExitNodeSet && !change.ExitNodeIDSet && !change.ExitNodeIPSet { + // The change does not affect exit node usage. + return false + } + + if prefs.ExitNodeID() == "" { + // Exit node usage is already disabled. + // Note that we do not check for ExitNodeIP here. + // If ExitNodeIP hasn't been resolved to a node, + // it's not enabled yet. + return false + } + + // First, apply the adjustments to a copy of the changes, + // e.g., clear AutoExitNode if ExitNodeID is set. + tmpChange := ptr.To(*change) + tmpChange.Prefs = *change.Prefs.Clone() + b.adjustEditPrefsLocked(prefs, tmpChange) + + // Then apply the adjusted changes to a copy of the current prefs, + // and resolve the exit node in the prefs. + tmpPrefs := prefs.AsStruct() + tmpPrefs.ApplyEdits(tmpChange) + b.resolveExitNodeInPrefsLocked(tmpPrefs) + + // If ExitNodeID is empty after applying the changes, + // but wasn't empty before, then the change disables + // exit node usage. + return tmpPrefs.ExitNodeID == "" +} + +// adjustEditPrefsLocked applies additional changes to mp if necessary, +// such as zeroing out mutually exclusive fields. +// +// It must not assume that the changes in mp will actually be applied. +// +// b.mu must be held. +func (b *LocalBackend) adjustEditPrefsLocked(prefs ipn.PrefsView, mp *ipn.MaskedPrefs) { + // Zeroing the ExitNodeID via localAPI must also zero the prior exit node. + if mp.ExitNodeIDSet && mp.ExitNodeID == "" && !mp.InternalExitNodePriorSet { + mp.InternalExitNodePrior = "" + mp.InternalExitNodePriorSet = true + } + + // Clear ExitNodeID if AutoExitNode is disabled and ExitNodeID is still unresolved. + if mp.AutoExitNodeSet && mp.AutoExitNode == "" && prefs.ExitNodeID() == unresolvedExitNodeID { + mp.ExitNodeIDSet = true + mp.ExitNodeID = "" + } + + // Disable automatic exit node selection if the user explicitly sets + // ExitNodeID or ExitNodeIP. + if (mp.ExitNodeIDSet || mp.ExitNodeIPSet) && !mp.AutoExitNodeSet { + mp.AutoExitNodeSet = true + mp.AutoExitNode = "" + } +} + +// onEditPrefsLocked is called when prefs are edited (typically, via LocalAPI), +// just before the changes in newPrefs are set for the current profile. +// +// The changes in mp have been allowed, but the resulting [ipn.Prefs] +// have not yet been applied and may be subject to reconciliation +// by [LocalBackend.reconcilePrefsLocked], either before or after being set. +// +// This method handles preference edits, typically initiated by the user, +// as opposed to reconfiguring the backend when the final prefs are set. +// +// b.mu must be held; mp must not be mutated by this method. +func (b *LocalBackend) onEditPrefsLocked(_ ipnauth.Actor, mp *ipn.MaskedPrefs, oldPrefs, newPrefs ipn.PrefsView) { + if mp.WantRunningSet && !mp.WantRunning && oldPrefs.WantRunning() { // If a user has enough rights to disconnect, such as when [syspolicy.AlwaysOn] // is disabled, or [syspolicy.AlwaysOnOverrideWithReason] is also set and the user // provides a reason for disconnecting, then we should not force the "always on" @@ -4201,7 +4502,34 @@ func (b *LocalBackend) EditPrefsAs(mp *ipn.MaskedPrefs, actor ipnauth.Actor) (ip } } - return b.editPrefsLockedOnEntry(mp, unlock) + if oldPrefs.WantRunning() != newPrefs.WantRunning() { + // Connecting to or disconnecting from Tailscale clears the override, + // unless the user is also explicitly changing the exit node (see below). + b.overrideExitNodePolicy = false + } + if mp.AutoExitNodeSet || mp.ExitNodeIDSet || mp.ExitNodeIPSet { + if allowExitNodeOverride, _ := syspolicy.GetBoolean(syspolicy.AllowExitNodeOverride, false); allowExitNodeOverride { + // If applying exit node policy settings to the new prefs results in no change, + // the user is not overriding the policy. Otherwise, it is an override. + b.overrideExitNodePolicy = b.applyExitNodeSysPolicyLocked(newPrefs.AsStruct()) + } else { + // Overrides are not allowed; clear the override flag. + b.overrideExitNodePolicy = false + } + } + + // This is recorded here in the EditPrefs path, not the setPrefs path on purpose. + // recordForEdit records metrics related to edits and changes, not the final state. + // If, in the future, we want to record gauge-metrics related to the state of prefs, + // that should be done in the setPrefs path. + e := prefsMetricsEditEvent{ + change: mp, + pNew: newPrefs, + pOld: oldPrefs, + node: b.currentNode(), + lastSuggestedExitNode: b.lastSuggestedExitNode, + } + e.record() } // startReconnectTimerLocked sets a timer to automatically set WantRunning to true @@ -4238,7 +4566,7 @@ func (b *LocalBackend) startReconnectTimerLocked(d time.Duration) { } mp := &ipn.MaskedPrefs{WantRunningSet: true, Prefs: ipn.Prefs{WantRunning: true}} - if _, err := b.editPrefsLockedOnEntry(mp, unlock); err != nil { + if _, err := b.editPrefsLockedOnEntry(ipnauth.Self, mp, unlock); err != nil { b.logf("failed to automatically reconnect as %q after %v: %v", cp.Name(), d, err) } else { b.logf("automatically reconnected as %q after %v", cp.Name(), d) @@ -4269,16 +4597,27 @@ func (b *LocalBackend) stopReconnectTimerLocked() { // Warning: b.mu must be held on entry, but it unlocks it on the way out. // TODO(bradfitz): redo the locking on all these weird methods like this. -func (b *LocalBackend) editPrefsLockedOnEntry(mp *ipn.MaskedPrefs, unlock unlockOnce) (ipn.PrefsView, error) { +func (b *LocalBackend) editPrefsLockedOnEntry(actor ipnauth.Actor, mp *ipn.MaskedPrefs, unlock unlockOnce) (ipn.PrefsView, error) { defer unlock() // for error paths + p0 := b.pm.CurrentPrefs() + + // Check if the changes in mp are allowed. + if err := b.checkEditPrefsAccessLocked(actor, p0, mp); err != nil { + b.logf("EditPrefs(%v): %v", mp.Pretty(), err) + return ipn.PrefsView{}, err + } + + // Apply additional changes to mp if necessary, + // such as clearing mutually exclusive fields. + b.adjustEditPrefsLocked(p0, mp) + if mp.EggSet { mp.EggSet = false b.egg = true b.goTracker.Go(b.doSetHostinfoFilterServices) } - p0 := b.pm.CurrentPrefs() p1 := b.pm.CurrentPrefs().AsStruct() p1.ApplyEdits(mp) @@ -4286,29 +4625,18 @@ func (b *LocalBackend) editPrefsLockedOnEntry(mp *ipn.MaskedPrefs, unlock unlock b.logf("EditPrefs check error: %v", err) return ipn.PrefsView{}, err } - if p1.RunSSH && !envknob.CanSSHD() { - b.logf("EditPrefs requests SSH, but disabled by envknob; returning error") - return ipn.PrefsView{}, errors.New("Tailscale SSH server administratively disabled.") - } + if p1.View().Equals(p0) { return stripKeysFromPrefs(p0), nil } b.logf("EditPrefs: %v", mp.Pretty()) - newPrefs := b.setPrefsLockedOnEntry(p1, unlock) - // This is recorded here in the EditPrefs path, not the setPrefs path on purpose. - // recordForEdit records metrics related to edits and changes, not the final state. - // If, in the future, we want to record gauge-metrics related to the state of prefs, - // that should be done in the setPrefs path. - e := prefsMetricsEditEvent{ - change: mp, - pNew: p1.View(), - pOld: p0, - node: b.currentNode(), - lastSuggestedExitNode: b.lastSuggestedExitNode, - } - e.record() + // Perform any actions required when prefs are edited (typically by a user), + // before the modified prefs are actually set for the current profile. + b.onEditPrefsLocked(actor, mp, p0, p1.View()) + + newPrefs := b.setPrefsLockedOnEntry(p1, unlock) // Note: don't perform any actions for the new prefs here. Not // every prefs change goes through EditPrefs. Put your actions @@ -4375,12 +4703,11 @@ func (b *LocalBackend) setPrefsLockedOnEntry(newp *ipn.Prefs, unlock unlockOnce) if oldp.Valid() { newp.Persist = oldp.Persist().AsStruct() // caller isn't allowed to override this } - // applySysPolicy returns whether it updated newp, - // but everything in this function treats b.prefs as completely new + // Apply reconciliation to the prefs, such as policy overrides, + // exit node resolution, and so on. The call returns whether it updated + // newp, but everything in this function treats newp as completely new // anyway, so its return value can be ignored here. - applySysPolicy(newp, b.lastSuggestedExitNode, b.overrideAlwaysOn) - // setExitNodeID does likewise. No-op if no exit node resolution is needed. - setExitNodeID(newp, netMap) + b.reconcilePrefsLocked(newp) // We do this to avoid holding the lock while doing everything else. @@ -4756,9 +5083,7 @@ func (b *LocalBackend) readvertiseAppConnectorRoutes() { // // Grab a copy of the field, since b.mu only guards access to the // b.appConnector field itself. - b.mu.Lock() - appConnector := b.appConnector - b.mu.Unlock() + appConnector := b.AppConnector() if appConnector == nil { return @@ -4788,6 +5113,11 @@ func (b *LocalBackend) readvertiseAppConnectorRoutes() { // updates are not currently blocked, based on the cached netmap and // user prefs. func (b *LocalBackend) authReconfig() { + // Wait for magicsock to process pending [eventbus] events, + // such as netmap updates. This should be completed before + // wireguard-go is reconfigured. See tailscale/tailscale#16369. + b.MagicConn().Synchronize() + b.mu.Lock() blocked := b.blocked prefs := b.pm.CurrentPrefs() @@ -5287,6 +5617,24 @@ func (b *LocalBackend) applyPrefsToHostinfoLocked(hi *tailcfg.Hostinfo, prefs ip // WireIngress. hi.WireIngress = b.shouldWireInactiveIngressLocked() hi.AppConnector.Set(prefs.AppConnector().Advertise) + + // The [tailcfg.Hostinfo.ExitNodeID] field tells control which exit node + // was selected, if any. + // + // If auto exit node is enabled (via [ipn.Prefs.AutoExitNode] or + // [syspolicy.ExitNodeID]), or an exit node is specified by ExitNodeIP + // instead of ExitNodeID , and we don't yet have enough info to resolve + // it (usually due to missing netmap or net report), then ExitNodeID in + // the prefs may be invalid (typically, [unresolvedExitNodeID]) until + // the netmap is available. + // + // In this case, we shouldn't update the Hostinfo with the bogus + // ExitNodeID here; [LocalBackend.ResolveExitNode] will be called once + // the netmap and/or net report have been received to both pick the exit + // node and notify control of the change. + if sid := prefs.ExitNodeID(); sid != unresolvedExitNodeID { + hi.ExitNodeID = prefs.ExitNodeID() + } } // enterState transitions the backend into newState, updating internal @@ -5683,7 +6031,7 @@ func (b *LocalBackend) ShouldHandleViaIP(ip netip.Addr) bool { // Logout logs out the current profile, if any, and waits for the logout to // complete. -func (b *LocalBackend) Logout(ctx context.Context) error { +func (b *LocalBackend) Logout(ctx context.Context, actor ipnauth.Actor) error { unlock := b.lockAndGetUnlock() defer unlock() @@ -5697,11 +6045,13 @@ func (b *LocalBackend) Logout(ctx context.Context) error { // delete it later. profile := b.pm.CurrentProfile() - _, err := b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ - WantRunningSet: true, - LoggedOutSet: true, - Prefs: ipn.Prefs{WantRunning: false, LoggedOut: true}, - }, unlock) + _, err := b.editPrefsLockedOnEntry( + actor, + &ipn.MaskedPrefs{ + WantRunningSet: true, + LoggedOutSet: true, + Prefs: ipn.Prefs{WantRunning: false, LoggedOut: true}, + }, unlock) if err != nil { return err } @@ -5773,40 +6123,94 @@ func (b *LocalBackend) setNetInfo(ni *tailcfg.NetInfo) { } cc.SetNetInfo(ni) if refresh { - unlock := b.lockAndGetUnlock() - defer unlock() - b.setAutoExitNodeIDLockedOnEntry(unlock) + b.RefreshExitNode() } } -func (b *LocalBackend) setAutoExitNodeIDLockedOnEntry(unlock unlockOnce) (newPrefs ipn.PrefsView) { - var zero ipn.PrefsView - defer unlock() +// RefreshExitNode determines which exit node to use based on the current +// prefs and netmap and switches to it if needed. +func (b *LocalBackend) RefreshExitNode() { + if b.resolveExitNode() { + b.authReconfig() + } +} - prefs := b.pm.CurrentPrefs() - if !prefs.Valid() { - b.logf("[unexpected]: received tailnet exit node ID pref change callback but current prefs are nil") - return zero +// resolveExitNode determines which exit node to use based on the current prefs +// and netmap. It updates the exit node ID in the prefs if needed, updates the +// exit node ID in the hostinfo if needed, sends a notification to clients, and +// returns true if the exit node has changed. +// +// It is the caller's responsibility to reconfigure routes and actually +// start using the selected exit node, if needed. +// +// b.mu must not be held. +func (b *LocalBackend) resolveExitNode() (changed bool) { + b.mu.Lock() + defer b.mu.Unlock() + + nm := b.currentNode().NetMap() + prefs := b.pm.CurrentPrefs().AsStruct() + if !b.resolveExitNodeInPrefsLocked(prefs) { + return } - prefsClone := prefs.AsStruct() - newSuggestion, err := b.suggestExitNodeLocked(nil) - if err != nil { - b.logf("setAutoExitNodeID: %v", err) - return zero + + if err := b.pm.SetPrefs(prefs.View(), ipn.NetworkProfile{ + MagicDNSName: nm.MagicDNSSuffix(), + DomainName: nm.DomainName(), + }); err != nil { + b.logf("failed to save exit node changes: %v", err) } - if prefsClone.ExitNodeID == newSuggestion.ID { - return zero + + // Send the resolved exit node to control via [tailcfg.Hostinfo]. + // [LocalBackend.applyPrefsToHostinfoLocked] usually sets the Hostinfo, + // but it deferred until this point because there was a bogus ExitNodeID + // in the prefs. + // + // TODO(sfllaw): Mutating b.hostinfo here is undesirable, mutating + // in-place doubly so. + sid := prefs.ExitNodeID + if sid != unresolvedExitNodeID && b.hostinfo.ExitNodeID != sid { + b.hostinfo.ExitNodeID = sid + b.goTracker.Go(b.doSetHostinfoFilterServices) } - prefsClone.ExitNodeID = newSuggestion.ID - newPrefs, err = b.editPrefsLockedOnEntry(&ipn.MaskedPrefs{ - Prefs: *prefsClone, - ExitNodeIDSet: true, - }, unlock) - if err != nil { - b.logf("setAutoExitNodeID: failed to apply exit node ID preference: %v", err) - return zero + + b.sendToLocked(ipn.Notify{Prefs: ptr.To(prefs.View())}, allClients) + return true +} + +// reconcilePrefsLocked applies policy overrides, exit node resolution, +// and other post-processing to the prefs, and reports whether the prefs +// were modified as a result. +// +// It must not perform any reconfiguration, as the prefs are not yet effective. +// +// b.mu must be held. +func (b *LocalBackend) reconcilePrefsLocked(prefs *ipn.Prefs) (changed bool) { + if b.applySysPolicyLocked(prefs) { + changed = true + } + if b.resolveExitNodeInPrefsLocked(prefs) { + changed = true + } + if changed { + b.logf("prefs reconciled: %v", prefs.Pretty()) + } + return changed +} + +// resolveExitNodeInPrefsLocked determines which exit node to use +// based on the specified prefs and netmap. It updates the exit node ID +// in the prefs if needed, and returns true if the exit node has changed. +// +// b.mu must be held. +func (b *LocalBackend) resolveExitNodeInPrefsLocked(prefs *ipn.Prefs) (changed bool) { + if b.resolveAutoExitNodeLocked(prefs) { + changed = true + } + if b.resolveExitNodeIPLocked(prefs) { + changed = true } - return newPrefs + return changed } // setNetMapLocked updates the LocalBackend state to reflect the newly @@ -5831,7 +6235,14 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { b.pauseOrResumeControlClientLocked() if nm != nil { - b.health.SetControlHealth(nm.ControlHealth) + messages := make(map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage) + for id, msg := range nm.DisplayMessages { + if msg.PrimaryAction != nil && !b.validPopBrowserURLLocked(msg.PrimaryAction.URL) { + msg.PrimaryAction = nil + } + messages[id] = msg + } + b.health.SetControlHealth(messages) } else { b.health.SetControlHealth(nil) } @@ -5887,6 +6298,7 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) { // the number of bytesRead. type responseBodyWrapper struct { io.ReadCloser + logVerbose bool bytesRx int64 bytesTx int64 log logger.Logf @@ -5908,8 +6320,22 @@ func (rbw *responseBodyWrapper) logAccess(err string) { // Some operating systems create and copy lots of 0 length hidden files for // tracking various states. Omit these to keep logs from being too verbose. - if rbw.contentLength > 0 { - rbw.log("taildrive: access: %s from %s to %s: status-code=%d ext=%q content-type=%q content-length=%.f tx=%.f rx=%.f err=%q", rbw.method, rbw.selfNodeKey, rbw.shareNodeKey, rbw.statusCode, rbw.fileExtension, rbw.contentType, roundTraffic(rbw.contentLength), roundTraffic(rbw.bytesTx), roundTraffic(rbw.bytesRx), err) + if rbw.logVerbose || rbw.contentLength > 0 { + levelPrefix := "" + if rbw.logVerbose { + levelPrefix = "[v1] " + } + rbw.log( + "%staildrive: access: %s from %s to %s: status-code=%d ext=%q content-type=%q content-length=%.f tx=%.f rx=%.f err=%q", + levelPrefix, + rbw.method, + rbw.selfNodeKey, + rbw.shareNodeKey, + rbw.statusCode, + rbw.fileExtension, + rbw.contentType, + roundTraffic(rbw.contentLength), + roundTraffic(rbw.bytesTx), roundTraffic(rbw.bytesRx), err) } } @@ -5964,17 +6390,8 @@ func (dt *driveTransport) RoundTrip(req *http.Request) (resp *http.Response, err defer func() { contentType := "unknown" - switch req.Method { - case httpm.PUT: - if ct := req.Header.Get("Content-Type"); ct != "" { - contentType = ct - } - case httpm.GET: - if ct := resp.Header.Get("Content-Type"); ct != "" { - contentType = ct - } - default: - return + if ct := req.Header.Get("Content-Type"); ct != "" { + contentType = ct } dt.b.mu.Lock() @@ -5988,6 +6405,7 @@ func (dt *driveTransport) RoundTrip(req *http.Request) (resp *http.Response, err rbw := responseBodyWrapper{ log: dt.b.logf, + logVerbose: req.Method != httpm.GET && req.Method != httpm.PUT, // other requests like PROPFIND are quite chatty, so we log those at verbose level method: req.Method, bytesTx: int64(bw.bytesRead), selfNodeKey: selfNodeKey, @@ -6438,6 +6856,15 @@ func (b *LocalBackend) OfferingAppConnector() bool { return b.appConnector != nil } +// AppConnector returns the current AppConnector, or nil if not configured. +// +// TODO(nickkhyl): move app connectors to [nodeBackend], or perhaps a feature package? +func (b *LocalBackend) AppConnector() *appc.AppConnector { + b.mu.Lock() + defer b.mu.Unlock() + return b.appConnector +} + // allowExitNodeDNSProxyToServeName reports whether the Exit Node DNS // proxy is allowed to serve responses for the provided DNS name. func (b *LocalBackend) allowExitNodeDNSProxyToServeName(name string) bool { @@ -6568,6 +6995,10 @@ func (b *LocalBackend) DebugReSTUN() error { return nil } +func (b *LocalBackend) DebugPeerRelayServers() set.Set[netip.Addr] { + return b.MagicConn().PeerRelays() +} + // ControlKnobs returns the node's control knobs. func (b *LocalBackend) ControlKnobs() *controlknobs.Knobs { return b.sys.ControlKnobs() @@ -6932,7 +7363,11 @@ func (b *LocalBackend) resetForProfileChangeLockedOnEntry(unlock unlockOnce) err // down, so no need to do any work. return nil } - b.currentNodeAtomic.Store(newNodeBackend()) + newNode := newNodeBackend(b.ctx, b.sys.Bus.Get()) + if oldNode := b.currentNodeAtomic.Swap(newNode); oldNode != nil { + oldNode.shutdown(errNodeContextChanged) + } + defer newNode.ready() b.setNetMapLocked(nil) // Reset netmap. b.updateFilterLocked(ipn.PrefsView{}) // Reset the NetworkMap in the engine @@ -6948,6 +7383,7 @@ func (b *LocalBackend) resetForProfileChangeLockedOnEntry(unlock unlockOnce) err b.serveConfig = ipn.ServeConfigView{} b.lastSuggestedExitNode = "" b.keyExpired = false + b.overrideExitNodePolicy = false b.resetAlwaysOnOverrideLocked() b.extHost.NotifyProfileChange(b.pm.CurrentProfile(), b.pm.CurrentPrefs(), false) b.setAtomicValuesFromPrefsLocked(b.pm.CurrentPrefs()) @@ -7282,27 +7718,17 @@ func allowedAutoRoute(ipp netip.Prefix) bool { var ErrNoPreferredDERP = errors.New("no preferred DERP, try again later") -// suggestExitNodeLocked computes a suggestion based on the current netmap and last netcheck report. If -// there are multiple equally good options, one is selected at random, so the result is not stable. To be -// eligible for consideration, the peer must have NodeAttrSuggestExitNode in its CapMap. -// -// Currently, peers with a DERP home are preferred over those without (typically this means Mullvad). -// Peers are selected based on having a DERP home that is the lowest latency to this device. For peers -// without a DERP home, we look for geographic proximity to this device's DERP home. -// -// netMap is an optional netmap to use that overrides b.netMap (needed for SetControlClientStatus before b.netMap is updated). -// If netMap is nil, then b.netMap is used. +// suggestExitNodeLocked computes a suggestion based on the current netmap and +// other optional factors. If there are multiple equally good options, one may +// be selected at random, so the result is not stable. To be eligible for +// consideration, the peer must have NodeAttrSuggestExitNode in its CapMap. // // b.mu.lock() must be held. -func (b *LocalBackend) suggestExitNodeLocked(netMap *netmap.NetworkMap) (response apitype.ExitNodeSuggestionResponse, err error) { - // netMap is an optional netmap to use that overrides b.netMap (needed for SetControlClientStatus before b.netMap is updated). If netMap is nil, then b.netMap is used. - if netMap == nil { - netMap = b.NetMap() - } +func (b *LocalBackend) suggestExitNodeLocked() (response apitype.ExitNodeSuggestionResponse, err error) { lastReport := b.MagicConn().GetLastNetcheckReport(b.ctx) prevSuggestion := b.lastSuggestedExitNode - res, err := suggestExitNode(lastReport, netMap, prevSuggestion, randomRegion, randomNode, b.getAllowedSuggestions()) + res, err := suggestExitNode(lastReport, b.currentNode(), prevSuggestion, randomRegion, randomNode, b.getAllowedSuggestions()) if err != nil { return res, err } @@ -7313,7 +7739,7 @@ func (b *LocalBackend) suggestExitNodeLocked(netMap *netmap.NetworkMap) (respons func (b *LocalBackend) SuggestExitNode() (response apitype.ExitNodeSuggestionResponse, err error) { b.mu.Lock() defer b.mu.Unlock() - return b.suggestExitNodeLocked(nil) + return b.suggestExitNodeLocked() } // getAllowedSuggestions returns a set of exit nodes permitted by the most recent @@ -7357,22 +7783,48 @@ func fillAllowedSuggestions() set.Set[tailcfg.StableNodeID] { return s } -func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSuggestion tailcfg.StableNodeID, selectRegion selectRegionFunc, selectNode selectNodeFunc, allowList set.Set[tailcfg.StableNodeID]) (res apitype.ExitNodeSuggestionResponse, err error) { +// suggestExitNode returns a suggestion for reasonably good exit node based on +// the current netmap and the previous suggestion. +func suggestExitNode(report *netcheck.Report, nb *nodeBackend, prevSuggestion tailcfg.StableNodeID, selectRegion selectRegionFunc, selectNode selectNodeFunc, allowList set.Set[tailcfg.StableNodeID]) (res apitype.ExitNodeSuggestionResponse, err error) { + switch { + case nb.SelfHasCap(tailcfg.NodeAttrTrafficSteering): + // The traffic-steering feature flag is enabled on this tailnet. + return suggestExitNodeUsingTrafficSteering(nb, allowList) + default: + return suggestExitNodeUsingDERP(report, nb, prevSuggestion, selectRegion, selectNode, allowList) + } +} + +// suggestExitNodeUsingDERP is the classic algorithm used to suggest exit nodes, +// before traffic steering was implemented. This handles the plain failover +// case, in addition to the optional Regional Routing. +// +// It computes a suggestion based on the current netmap and last netcheck +// report. If there are multiple equally good options, one is selected at +// random, so the result is not stable. To be eligible for consideration, the +// peer must have NodeAttrSuggestExitNode in its CapMap. +// +// Currently, peers with a DERP home are preferred over those without (typically +// this means Mullvad). Peers are selected based on having a DERP home that is +// the lowest latency to this device. For peers without a DERP home, we look for +// geographic proximity to this device's DERP home. +func suggestExitNodeUsingDERP(report *netcheck.Report, nb *nodeBackend, prevSuggestion tailcfg.StableNodeID, selectRegion selectRegionFunc, selectNode selectNodeFunc, allowList set.Set[tailcfg.StableNodeID]) (res apitype.ExitNodeSuggestionResponse, err error) { + netMap := nb.NetMap() if report == nil || report.PreferredDERP == 0 || netMap == nil || netMap.DERPMap == nil { return res, ErrNoPreferredDERP } - candidates := make([]tailcfg.NodeView, 0, len(netMap.Peers)) - for _, peer := range netMap.Peers { - if !peer.Valid() { - continue + // Use [nodeBackend.AppendMatchingPeers] instead of the netmap directly, + // since the netmap doesn't include delta updates (e.g., home DERP or Online + // status changes) from the control plane since the last full update. + candidates := nb.AppendMatchingPeers(nil, func(peer tailcfg.NodeView) bool { + if !peer.Valid() || !peer.Online().Get() { + return false } if allowList != nil && !allowList.Contains(peer.StableID()) { - continue - } - if peer.CapMap().Contains(tailcfg.NodeAttrSuggestExitNode) && tsaddr.ContainsExitRoutes(peer.AllowedIPs()) { - candidates = append(candidates, peer) + return false } - } + return peer.CapMap().Contains(tailcfg.NodeAttrSuggestExitNode) && tsaddr.ContainsExitRoutes(peer.AllowedIPs()) + }) if len(candidates) == 0 { return res, nil } @@ -7477,6 +7929,103 @@ func suggestExitNode(report *netcheck.Report, netMap *netmap.NetworkMap, prevSug return res, nil } +var ErrNoNetMap = errors.New("no network map, try again later") + +// suggestExitNodeUsingTrafficSteering uses traffic steering priority scores to +// pick one of the best exit nodes. These priorities are provided by Control in +// the node’s [tailcfg.Location]. To be eligible for consideration, the node +// must have NodeAttrSuggestExitNode in its CapMap. +func suggestExitNodeUsingTrafficSteering(nb *nodeBackend, allowed set.Set[tailcfg.StableNodeID]) (apitype.ExitNodeSuggestionResponse, error) { + nm := nb.NetMap() + if nm == nil { + return apitype.ExitNodeSuggestionResponse{}, ErrNoNetMap + } + + self := nb.Self() + if !self.Valid() { + return apitype.ExitNodeSuggestionResponse{}, ErrNoNetMap + } + + if !nb.SelfHasCap(tailcfg.NodeAttrTrafficSteering) { + panic("missing traffic-steering capability") + } + + var force tailcfg.NodeView + nodes := nb.AppendMatchingPeers(nil, func(p tailcfg.NodeView) bool { + if !p.Valid() { + return false + } + if !p.Online().Get() { + return false + } + if allowed != nil && !allowed.Contains(p.StableID()) { + return false + } + if !p.CapMap().Contains(tailcfg.NodeAttrSuggestExitNode) { + return false + } + if !tsaddr.ContainsExitRoutes(p.AllowedIPs()) { + return false + } + return true + }) + if force.Valid() { + nodes = append(nodes[:0], force) + } + + scores := make(map[tailcfg.NodeID]int, len(nodes)) + score := func(n tailcfg.NodeView) int { + id := n.ID() + s, ok := scores[id] + if !ok { + s = 0 // score of zero means incomparable + if hi := n.Hostinfo(); hi.Valid() { + if loc := hi.Location(); loc.Valid() { + s = loc.Priority() + } + } + scores[id] = s + } + return s + } + rdvHash := makeRendezvousHasher(self.ID()) + + var pick tailcfg.NodeView + if len(nodes) == 1 { + pick = nodes[0] + } + if len(nodes) > 1 { + // Find the highest scoring exit nodes. + slices.SortFunc(nodes, func(a, b tailcfg.NodeView) int { + c := cmp.Compare(score(b), score(a)) // Highest score first. + if c == 0 { + // Rendezvous hashing for reliably picking the + // same node from a list: tailscale/tailscale#16551. + return cmp.Compare(rdvHash(b.ID()), rdvHash(a.ID())) + } + return c + }) + + // TODO(sfllaw): add a temperature knob so that this client has + // a chance of picking the next best option. + pick = nodes[0] + } + + if !pick.Valid() { + return apitype.ExitNodeSuggestionResponse{}, nil + } + res := apitype.ExitNodeSuggestionResponse{ + ID: pick.StableID(), + Name: pick.Name(), + } + if hi := pick.Hostinfo(); hi.Valid() { + if loc := hi.Location(); loc.Valid() { + res.Location = loc + } + } + return res, nil +} + // pickWeighted chooses the node with highest priority given a list of mullvad nodes. func pickWeighted(candidates []tailcfg.NodeView) []tailcfg.NodeView { maxWeight := 0 @@ -7560,10 +8109,39 @@ func longLatDistance(fromLat, fromLong, toLat, toLong float64) float64 { return earthRadiusMeters * c } -// shouldAutoExitNode checks for the auto exit node MDM policy. -func shouldAutoExitNode() bool { - exitNodeIDStr, _ := syspolicy.GetString(syspolicy.ExitNodeID, "") - return exitNodeIDStr == "auto:any" +// makeRendezvousHasher returns a function that hashes a node ID to a uint64. +// https://en.wikipedia.org/wiki/Rendezvous_hashing +func makeRendezvousHasher(seed tailcfg.NodeID) func(tailcfg.NodeID) uint64 { + en := binary.BigEndian + return func(n tailcfg.NodeID) uint64 { + var b [16]byte + en.PutUint64(b[:], uint64(seed)) + en.PutUint64(b[8:], uint64(n)) + v := sha256.Sum256(b[:]) + return en.Uint64(v[:]) + } +} + +const ( + // unresolvedExitNodeID is a special [tailcfg.StableNodeID] value + // used as an exit node ID to install a blackhole route, preventing + // accidental non-exit-node usage until the [ipn.ExitNodeExpression] + // is evaluated and an actual exit node is selected. + // + // We use "auto:any" for compatibility with older, pre-[ipn.ExitNodeExpression] + // clients that have been using "auto:any" for this purpose for a long time. + unresolvedExitNodeID tailcfg.StableNodeID = "auto:any" +) + +func isAllowedAutoExitNodeID(exitNodeID tailcfg.StableNodeID) bool { + if exitNodeID == "" { + return false // an exit node is required + } + if nodes, _ := syspolicy.GetStringArray(syspolicy.AllowedSuggestedExitNodes, nil); nodes != nil { + return slices.Contains(nodes, string(exitNodeID)) + + } + return true // no policy configured; allow all exit nodes } // startAutoUpdate triggers an auto-update attempt. The actual update happens @@ -7634,7 +8212,7 @@ func (b *LocalBackend) srcIPHasCapForFilter(srcIP netip.Addr, cap tailcfg.NodeCa if !ok { return false } - n, ok := cn.PeerByID(nodeID) + n, ok := cn.NodeByID(nodeID) if !ok { return false } @@ -7708,3 +8286,29 @@ func (b *LocalBackend) vipServicesFromPrefsLocked(prefs ipn.PrefsView) []*tailcf var ( metricCurrentWatchIPNBus = clientmetric.NewGauge("localbackend_current_watch_ipn_bus") ) + +func (b *LocalBackend) stateEncrypted() opt.Bool { + switch runtime.GOOS { + case "android", "ios": + return opt.NewBool(true) + case "darwin": + switch { + case version.IsMacAppStore(): + return opt.NewBool(true) + case version.IsMacSysExt(): + // MacSys still stores its state in plaintext on disk in addition to + // the Keychain. A future release will clean up the on-disk state + // files. + // TODO(#15830): always return true here once MacSys is fully migrated. + sp, _ := syspolicy.GetBoolean(syspolicy.EncryptState, false) + return opt.NewBool(sp) + default: + // Probably self-compiled tailscaled, we don't use the Keychain + // there. + return opt.NewBool(false) + } + default: + _, ok := b.store.(ipn.EncryptedStateStore) + return opt.NewBool(ok) + } +} diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 1def6d98a9552..dd2837022f064 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -5,9 +5,11 @@ package ipnlocal import ( "context" + "encoding/binary" "encoding/json" "errors" "fmt" + "maps" "math" "net" "net/http" @@ -22,6 +24,8 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + memro "go4.org/mem" "go4.org/netipx" "golang.org/x/net/dns/dnsmessage" "tailscale.com/appc" @@ -53,6 +57,7 @@ import ( "tailscale.com/types/ptr" "tailscale.com/types/views" "tailscale.com/util/dnsname" + "tailscale.com/util/eventbus" "tailscale.com/util/mak" "tailscale.com/util/must" "tailscale.com/util/set" @@ -76,6 +81,18 @@ func inRemove(ip netip.Addr) bool { return false } +func makeNodeKeyFromID(nodeID tailcfg.NodeID) key.NodePublic { + raw := make([]byte, 32) + binary.BigEndian.PutUint64(raw[24:], uint64(nodeID)) + return key.NodePublicFromRaw32(memro.B(raw)) +} + +func makeDiscoKeyFromID(nodeID tailcfg.NodeID) (ret key.DiscoPublic) { + raw := make([]byte, 32) + binary.BigEndian.PutUint64(raw[24:], uint64(nodeID)) + return key.DiscoPublicFromRaw32(memro.B(raw)) +} + func TestShrinkDefaultRoute(t *testing.T) { tests := []struct { route string @@ -484,29 +501,30 @@ func TestLazyMachineKeyGeneration(t *testing.T) { func TestZeroExitNodeViaLocalAPI(t *testing.T) { lb := newTestLocalBackend(t) + user := &ipnauth.TestActor{} // Give it an initial exit node in use. - if _, err := lb.EditPrefs(&ipn.MaskedPrefs{ + if _, err := lb.EditPrefsAs(&ipn.MaskedPrefs{ ExitNodeIDSet: true, Prefs: ipn.Prefs{ ExitNodeID: "foo", }, - }); err != nil { + }, user); err != nil { t.Fatalf("enabling first exit node: %v", err) } // SetUseExitNodeEnabled(false) "remembers" the prior exit node. - if _, err := lb.SetUseExitNodeEnabled(false); err != nil { + if _, err := lb.SetUseExitNodeEnabled(user, false); err != nil { t.Fatal("expected failure") } // Zero the exit node - pv, err := lb.EditPrefs(&ipn.MaskedPrefs{ + pv, err := lb.EditPrefsAs(&ipn.MaskedPrefs{ ExitNodeIDSet: true, Prefs: ipn.Prefs{ ExitNodeID: "", }, - }) + }, user) if err != nil { t.Fatalf("enabling first exit node: %v", err) @@ -520,58 +538,962 @@ func TestZeroExitNodeViaLocalAPI(t *testing.T) { } -func TestSetUseExitNodeEnabled(t *testing.T) { - lb := newTestLocalBackend(t) +func TestSetUseExitNodeEnabled(t *testing.T) { + lb := newTestLocalBackend(t) + user := &ipnauth.TestActor{} + + // Can't turn it on if it never had an old value. + if _, err := lb.SetUseExitNodeEnabled(user, true); err == nil { + t.Fatal("expected success") + } + + // But we can turn it off when it's already off. + if _, err := lb.SetUseExitNodeEnabled(user, false); err != nil { + t.Fatal("expected failure") + } + + // Give it an initial exit node in use. + if _, err := lb.EditPrefsAs(&ipn.MaskedPrefs{ + ExitNodeIDSet: true, + Prefs: ipn.Prefs{ + ExitNodeID: "foo", + }, + }, user); err != nil { + t.Fatalf("enabling first exit node: %v", err) + } + + // Now turn off that exit node. + if prefs, err := lb.SetUseExitNodeEnabled(user, false); err != nil { + t.Fatal("expected failure") + } else { + if g, w := prefs.ExitNodeID(), tailcfg.StableNodeID(""); g != w { + t.Fatalf("unexpected exit node ID %q; want %q", g, w) + } + if g, w := prefs.InternalExitNodePrior(), tailcfg.StableNodeID("foo"); g != w { + t.Fatalf("unexpected exit node prior %q; want %q", g, w) + } + } + + // And turn it back on. + if prefs, err := lb.SetUseExitNodeEnabled(user, true); err != nil { + t.Fatal("expected failure") + } else { + if g, w := prefs.ExitNodeID(), tailcfg.StableNodeID("foo"); g != w { + t.Fatalf("unexpected exit node ID %q; want %q", g, w) + } + if g, w := prefs.InternalExitNodePrior(), tailcfg.StableNodeID("foo"); g != w { + t.Fatalf("unexpected exit node prior %q; want %q", g, w) + } + } + + // Verify we block setting an Internal field. + if _, err := lb.EditPrefsAs(&ipn.MaskedPrefs{ + InternalExitNodePriorSet: true, + }, user); err == nil { + t.Fatalf("unexpected success; want an error trying to set an internal field") + } +} + +func makeExitNode(id tailcfg.NodeID, opts ...peerOptFunc) tailcfg.NodeView { + return makePeer(id, append([]peerOptFunc{withCap(26), withSuggest(), withExitRoutes()}, opts...)...) +} + +func TestConfigureExitNode(t *testing.T) { + controlURL := "https://localhost:1/" + exitNode1 := makeExitNode(1, withName("node-1"), withDERP(1), withAddresses(netip.MustParsePrefix("100.64.1.1/32"))) + exitNode2 := makeExitNode(2, withName("node-2"), withDERP(2), withAddresses(netip.MustParsePrefix("100.64.1.2/32"))) + selfNode := makeExitNode(3, withName("node-3"), withDERP(1), withAddresses(netip.MustParsePrefix("100.64.1.3/32"))) + clientNetmap := buildNetmapWithPeers(selfNode, exitNode1, exitNode2) + + report := &netcheck.Report{ + RegionLatency: map[int]time.Duration{ + 1: 5 * time.Millisecond, + 2: 10 * time.Millisecond, + }, + PreferredDERP: 1, + } + + tests := []struct { + name string + prefs ipn.Prefs + netMap *netmap.NetworkMap + report *netcheck.Report + changePrefs *ipn.MaskedPrefs + useExitNodeEnabled *bool + exitNodeIDPolicy *tailcfg.StableNodeID + exitNodeIPPolicy *netip.Addr + exitNodeAllowedIDs []tailcfg.StableNodeID // nil if all IDs are allowed for auto exit nodes + exitNodeAllowOverride bool // whether [syspolicy.AllowExitNodeOverride] should be set to true + wantChangePrefsErr error // if non-nil, the error we expect from [LocalBackend.EditPrefsAs] + wantPrefs ipn.Prefs + wantExitNodeToggleErr error // if non-nil, the error we expect from [LocalBackend.SetUseExitNodeEnabled] + wantHostinfoExitNodeID tailcfg.StableNodeID + }{ + { + name: "exit-node-id-via-prefs", // set exit node ID via prefs + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ExitNodeID: exitNode1.StableID()}, + ExitNodeIDSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "exit-node-ip-via-prefs", // set exit node IP via prefs (should be resolved to an ID) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ExitNodeIP: exitNode1.Addresses().At(0).Addr()}, + ExitNodeIPSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "auto-exit-node-via-prefs/any", // set auto exit node via prefs + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "any"}, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "auto-exit-node-via-prefs/set-exit-node-id-via-prefs", // setting exit node ID explicitly should disable auto exit node + prefs: ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: "any", + ExitNodeID: exitNode1.StableID(), + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ExitNodeID: exitNode2.StableID()}, + ExitNodeIDSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), + AutoExitNode: "", // should be unset + }, + wantHostinfoExitNodeID: exitNode2.StableID(), + }, + { + name: "auto-exit-node-via-prefs/any/no-report", // set auto exit node via prefs, but no report means we can't resolve the exit node ID + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "any"}, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: unresolvedExitNodeID, // cannot resolve; traffic will be dropped + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: "", + }, + { + name: "auto-exit-node-via-prefs/any/no-netmap", // similarly, but without a netmap (no exit node should be selected) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "any"}, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: unresolvedExitNodeID, // cannot resolve; traffic will be dropped + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: "", + }, + { + name: "auto-exit-node-via-prefs/foo", // set auto exit node via prefs with an unknown/unsupported expression + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "foo"}, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" + AutoExitNode: "foo", + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "auto-exit-node-via-prefs/off", // toggle the exit node off after it was set to "any" + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "any"}, + AutoExitNodeSet: true, + }, + useExitNodeEnabled: ptr.To(false), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: "", + AutoExitNode: "", + InternalExitNodePrior: "auto:any", + }, + wantHostinfoExitNodeID: "", + }, + { + name: "auto-exit-node-via-prefs/on", // toggle the exit node on + prefs: ipn.Prefs{ + ControlURL: controlURL, + InternalExitNodePrior: "auto:any", + }, + netMap: clientNetmap, + report: report, + useExitNodeEnabled: ptr.To(true), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + AutoExitNode: "any", + InternalExitNodePrior: "auto:any", + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "id-via-policy", // set exit node ID via syspolicy + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + exitNodeIDPolicy: ptr.To(exitNode1.StableID()), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "id-via-policy/cannot-override-via-prefs/by-id", // syspolicy should take precedence over prefs + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + exitNodeIDPolicy: ptr.To(exitNode1.StableID()), + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + ExitNodeID: exitNode2.StableID(), // this should be ignored + }, + ExitNodeIDSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + wantChangePrefsErr: errManagedByPolicy, + }, + { + name: "id-via-policy/cannot-override-via-prefs/by-ip", // syspolicy should take precedence over prefs + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + exitNodeIDPolicy: ptr.To(exitNode1.StableID()), + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + ExitNodeIP: exitNode2.Addresses().At(0).Addr(), // this should be ignored + }, + ExitNodeIPSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + wantChangePrefsErr: errManagedByPolicy, + }, + { + name: "id-via-policy/cannot-override-via-prefs/by-auto-expr", // syspolicy should take precedence over prefs + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + exitNodeIDPolicy: ptr.To(exitNode1.StableID()), + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + AutoExitNode: "any", // this should be ignored + }, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + wantChangePrefsErr: errManagedByPolicy, + }, + { + name: "ip-via-policy", // set exit node IP via syspolicy (should be resolved to an ID) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + exitNodeIPPolicy: ptr.To(exitNode2.Addresses().At(0).Addr()), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), + }, + wantHostinfoExitNodeID: exitNode2.StableID(), + }, + { + name: "auto-any-via-policy", // set auto exit node via syspolicy (an exit node should be selected) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "auto-any-via-policy/no-report", // set auto exit node via syspolicy without a netcheck report (no exit node should be selected) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: nil, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: unresolvedExitNodeID, + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: "", + }, + { + name: "auto-any-via-policy/no-netmap", // similarly, but without a netmap (no exit node should be selected) + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: nil, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: unresolvedExitNodeID, + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: "", + }, + { + name: "auto-any-via-policy/no-netmap/with-existing", // set auto exit node via syspolicy without a netmap, but with a previously set exit node ID + prefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), // should be retained + }, + netMap: nil, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowedIDs: nil, // not configured, so all exit node IDs are implicitly allowed + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: exitNode2.StableID(), + }, + { + name: "auto-any-via-policy/no-netmap/with-allowed-existing", // same, but now with a syspolicy setting that explicitly allows the existing exit node ID + prefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), // should be retained + }, + netMap: nil, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowedIDs: []tailcfg.StableNodeID{ + exitNode2.StableID(), // the current exit node ID is allowed + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: exitNode2.StableID(), + }, + { + name: "auto-any-via-policy/no-netmap/with-disallowed-existing", // same, but now with a syspolicy setting that does not allow the existing exit node ID + prefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), // not allowed by [syspolicy.AllowedSuggestedExitNodes] + }, + netMap: nil, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowedIDs: []tailcfg.StableNodeID{ + exitNode1.StableID(), // a different exit node ID; the current one is not allowed + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: unresolvedExitNodeID, // we don't have a netmap yet, and the current exit node ID is not allowed; block traffic + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: "", + }, + { + name: "auto-any-via-policy/with-netmap/with-allowed-existing", // same, but now with a syspolicy setting that does not allow the existing exit node ID + prefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // not allowed by [syspolicy.AllowedSuggestedExitNodes] + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowedIDs: []tailcfg.StableNodeID{ + exitNode2.StableID(), // a different exit node ID; the current one is not allowed + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), // we have a netmap; switch to the best allowed exit node + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: exitNode2.StableID(), + }, + { + name: "auto-any-via-policy/with-netmap/switch-to-better", // if all exit nodes are allowed, switch to the best one once we have a netmap + prefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // switch to the best exit node + AutoExitNode: "any", + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "auto-foo-via-policy", // set auto exit node via syspolicy with an unknown/unsupported expression + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:foo")), + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" + AutoExitNode: "foo", + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "auto-foo-via-edit-prefs", // set auto exit node via EditPrefs with an unknown/unsupported expression + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{AutoExitNode: "foo"}, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // unknown exit node expressions should work as "any" + AutoExitNode: "foo", + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "auto-any-via-policy/toggle-off", // cannot toggle off the exit node if it was set via syspolicy + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + useExitNodeEnabled: ptr.To(false), // should fail with an error + wantExitNodeToggleErr: errManagedByPolicy, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // still enforced by the policy setting + AutoExitNode: "any", + InternalExitNodePrior: "", + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "auto-any-via-policy/allow-override/change", // changing the exit node is allowed by [syspolicy.AllowExitNodeOverride] + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowOverride: true, // allow changing the exit node + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + ExitNodeID: exitNode2.StableID(), // change the exit node ID + }, + ExitNodeIDSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode2.StableID(), // overridden by user + AutoExitNode: "", // cleared, as we are setting the exit node ID explicitly + }, + wantHostinfoExitNodeID: exitNode2.StableID(), + }, + { + name: "auto-any-via-policy/allow-override/clear", // clearing the exit node ID is not allowed by [syspolicy.AllowExitNodeOverride] + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowOverride: true, // allow changing, but not disabling, the exit node + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + ExitNodeID: "", // clearing the exit node ID disables the exit node and should not be allowed + }, + ExitNodeIDSet: true, + }, + wantChangePrefsErr: errManagedByPolicy, // edit prefs should fail with an error + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // still enforced by the policy setting + AutoExitNode: "any", + InternalExitNodePrior: "", + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "auto-any-via-policy/allow-override/toggle-off", // similarly, toggling off the exit node is not allowed even with [syspolicy.AllowExitNodeOverride] + prefs: ipn.Prefs{ + ControlURL: controlURL, + }, + netMap: clientNetmap, + report: report, + exitNodeIDPolicy: ptr.To(tailcfg.StableNodeID("auto:any")), + exitNodeAllowOverride: true, // allow changing, but not disabling, the exit node + useExitNodeEnabled: ptr.To(false), // should fail with an error + wantExitNodeToggleErr: errManagedByPolicy, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + ExitNodeID: exitNode1.StableID(), // still enforced by the policy setting + AutoExitNode: "any", + InternalExitNodePrior: "", + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + { + name: "auto-any-via-initial-prefs/no-netmap/clear-auto-exit-node", + prefs: ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: ipn.AnyExitNode, + }, + netMap: nil, // no netmap; exit node cannot be resolved + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + AutoExitNode: "", // clear the auto exit node + }, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: "", // cleared + ExitNodeID: "", // has never been resolved, so it should be cleared as well + }, + wantHostinfoExitNodeID: "", + }, + { + name: "auto-any-via-initial-prefs/with-netmap/clear-auto-exit-node", + prefs: ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: ipn.AnyExitNode, + }, + netMap: clientNetmap, // has a netmap; exit node will be resolved + report: report, + changePrefs: &ipn.MaskedPrefs{ + Prefs: ipn.Prefs{ + AutoExitNode: "", // clear the auto exit node + }, + AutoExitNodeSet: true, + }, + wantPrefs: ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: "", // cleared + ExitNodeID: exitNode1.StableID(), // a resolved exit node ID should be retained + }, + wantHostinfoExitNodeID: exitNode1.StableID(), + }, + } + syspolicy.RegisterWellKnownSettingsForTest(t) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Configure policy settings, if any. + store := source.NewTestStore(t) + if tt.exitNodeIDPolicy != nil { + store.SetStrings(source.TestSettingOf(syspolicy.ExitNodeID, string(*tt.exitNodeIDPolicy))) + } + if tt.exitNodeIPPolicy != nil { + store.SetStrings(source.TestSettingOf(syspolicy.ExitNodeIP, tt.exitNodeIPPolicy.String())) + } + if tt.exitNodeAllowedIDs != nil { + store.SetStringLists(source.TestSettingOf(syspolicy.AllowedSuggestedExitNodes, toStrings(tt.exitNodeAllowedIDs))) + } + if tt.exitNodeAllowOverride { + store.SetBooleans(source.TestSettingOf(syspolicy.AllowExitNodeOverride, true)) + } + if store.IsEmpty() { + // No syspolicy settings, so don't register a store. + // This allows the test to run in parallel with other tests. + t.Parallel() + } else { + // Register the store for syspolicy settings to make them available to the LocalBackend. + syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, store) + } + + // Create a new LocalBackend with the given prefs. + // Any syspolicy settings will be applied to the initial prefs. + lb := newTestLocalBackend(t) + lb.SetPrefsForTest(tt.prefs.Clone()) + // Then set the netcheck report and netmap, if any. + if tt.report != nil { + lb.MagicConn().SetLastNetcheckReportForTest(t.Context(), tt.report) + } + if tt.netMap != nil { + lb.SetControlClientStatus(lb.cc, controlclient.Status{NetMap: tt.netMap}) + } + + user := &ipnauth.TestActor{} + // If we have a changePrefs, apply it. + if tt.changePrefs != nil { + _, err := lb.EditPrefsAs(tt.changePrefs, user) + checkError(t, err, tt.wantChangePrefsErr, true) + } + + // If we need to flip exit node toggle on or off, do it. + if tt.useExitNodeEnabled != nil { + _, err := lb.SetUseExitNodeEnabled(user, *tt.useExitNodeEnabled) + checkError(t, err, tt.wantExitNodeToggleErr, true) + } + + // Now check the prefs. + opts := []cmp.Option{ + cmpopts.EquateComparable(netip.Addr{}, netip.Prefix{}), + } + if diff := cmp.Diff(&tt.wantPrefs, lb.Prefs().AsStruct(), opts...); diff != "" { + t.Errorf("Prefs(+got -want): %v", diff) + } + + // And check Hostinfo. + if got := lb.hostinfo.ExitNodeID; got != tt.wantHostinfoExitNodeID { + t.Errorf("Hostinfo.ExitNodeID got %s, want %s", got, tt.wantHostinfoExitNodeID) + } + }) + } +} + +func TestPrefsChangeDisablesExitNode(t *testing.T) { + tests := []struct { + name string + netMap *netmap.NetworkMap + prefs ipn.Prefs + change ipn.MaskedPrefs + wantDisablesExitNode bool + }{ + { + name: "has-exit-node-id/no-change", + prefs: ipn.Prefs{ + ExitNodeID: "test-exit-node", + }, + change: ipn.MaskedPrefs{}, + wantDisablesExitNode: false, + }, + { + name: "has-exit-node-ip/no-change", + prefs: ipn.Prefs{ + ExitNodeIP: netip.MustParseAddr("100.100.1.1"), + }, + change: ipn.MaskedPrefs{}, + wantDisablesExitNode: false, + }, + { + name: "has-auto-exit-node/no-change", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{}, + wantDisablesExitNode: false, + }, + { + name: "has-exit-node-id/non-exit-node-change", + prefs: ipn.Prefs{ + ExitNodeID: "test-exit-node", + }, + change: ipn.MaskedPrefs{ + WantRunningSet: true, + HostnameSet: true, + ExitNodeAllowLANAccessSet: true, + Prefs: ipn.Prefs{ + WantRunning: true, + Hostname: "test-hostname", + ExitNodeAllowLANAccess: true, + }, + }, + wantDisablesExitNode: false, + }, + { + name: "has-exit-node-ip/non-exit-node-change", + prefs: ipn.Prefs{ + ExitNodeIP: netip.MustParseAddr("100.100.1.1"), + }, + change: ipn.MaskedPrefs{ + WantRunningSet: true, + RouteAllSet: true, + ShieldsUpSet: true, + Prefs: ipn.Prefs{ + WantRunning: false, + RouteAll: false, + ShieldsUp: true, + }, + }, + wantDisablesExitNode: false, + }, + { + name: "has-auto-exit-node/non-exit-node-change", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{ + CorpDNSSet: true, + RouteAllSet: true, + ExitNodeAllowLANAccessSet: true, + Prefs: ipn.Prefs{ + CorpDNS: true, + RouteAll: false, + ExitNodeAllowLANAccess: true, + }, + }, + wantDisablesExitNode: false, + }, + { + name: "has-exit-node-id/change-exit-node-id", + prefs: ipn.Prefs{ + ExitNodeID: "exit-node-1", + }, + change: ipn.MaskedPrefs{ + ExitNodeIDSet: true, + Prefs: ipn.Prefs{ + ExitNodeID: "exit-node-2", + }, + }, + wantDisablesExitNode: false, // changing the exit node ID does not disable it + }, + { + name: "has-exit-node-id/enable-auto-exit-node", + prefs: ipn.Prefs{ + ExitNodeID: "exit-node-1", + }, + change: ipn.MaskedPrefs{ + AutoExitNodeSet: true, + Prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + }, + wantDisablesExitNode: false, // changing the exit node ID does not disable it + }, + { + name: "has-exit-node-id/clear-exit-node-id", + prefs: ipn.Prefs{ + ExitNodeID: "exit-node-1", + }, + change: ipn.MaskedPrefs{ + ExitNodeIDSet: true, + Prefs: ipn.Prefs{ + ExitNodeID: "", + }, + }, + wantDisablesExitNode: true, // clearing the exit node ID disables it + }, + { + name: "has-auto-exit-node/clear-exit-node-id", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{ + ExitNodeIDSet: true, + Prefs: ipn.Prefs{ + ExitNodeID: "", + }, + }, + wantDisablesExitNode: true, // clearing the exit node ID disables auto exit node as well... + }, + { + name: "has-auto-exit-node/clear-exit-node-id/but-keep-auto-exit-node", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{ + ExitNodeIDSet: true, + AutoExitNodeSet: true, + Prefs: ipn.Prefs{ + ExitNodeID: "", + AutoExitNode: ipn.AnyExitNode, + }, + }, + wantDisablesExitNode: false, // ... unless we explicitly keep the auto exit node enabled + }, + { + name: "has-auto-exit-node/clear-exit-node-ip", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{ + ExitNodeIPSet: true, + Prefs: ipn.Prefs{ + ExitNodeIP: netip.Addr{}, + }, + }, + wantDisablesExitNode: false, // auto exit node is still enabled + }, + { + name: "has-auto-exit-node/clear-auto-exit-node", + prefs: ipn.Prefs{ + AutoExitNode: ipn.AnyExitNode, + }, + change: ipn.MaskedPrefs{ + AutoExitNodeSet: true, + Prefs: ipn.Prefs{ + AutoExitNode: "", + }, + }, + wantDisablesExitNode: true, // clearing the auto exit while the exit node ID is unresolved disables exit node usage + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lb := newTestLocalBackend(t) + if tt.netMap != nil { + lb.SetControlClientStatus(lb.cc, controlclient.Status{NetMap: tt.netMap}) + } + // Set the initial prefs via SetPrefsForTest + // to apply necessary adjustments. + lb.SetPrefsForTest(tt.prefs.Clone()) + initialPrefs := lb.Prefs() + + // Check whether changeDisablesExitNodeLocked correctly identifies the change. + if got := lb.changeDisablesExitNodeLocked(initialPrefs, &tt.change); got != tt.wantDisablesExitNode { + t.Errorf("disablesExitNode: got %v; want %v", got, tt.wantDisablesExitNode) + } + + // Apply the change and check if it the actual behavior matches the expectation. + gotPrefs, err := lb.EditPrefsAs(&tt.change, &ipnauth.TestActor{}) + if err != nil { + t.Fatalf("EditPrefsAs failed: %v", err) + } + gotDisabledExitNode := initialPrefs.ExitNodeID() != "" && gotPrefs.ExitNodeID() == "" + if gotDisabledExitNode != tt.wantDisablesExitNode { + t.Errorf("disabledExitNode: got %v; want %v", gotDisabledExitNode, tt.wantDisablesExitNode) + } + }) + } +} + +func TestExitNodeNotifyOrder(t *testing.T) { + const controlURL = "https://localhost:1/" - // Can't turn it on if it never had an old value. - if _, err := lb.SetUseExitNodeEnabled(true); err == nil { - t.Fatal("expected success") + report := &netcheck.Report{ + RegionLatency: map[int]time.Duration{ + 1: 5 * time.Millisecond, + 2: 10 * time.Millisecond, + }, + PreferredDERP: 1, } - // But we can turn it off when it's already off. - if _, err := lb.SetUseExitNodeEnabled(false); err != nil { - t.Fatal("expected failure") - } + exitNode1 := makeExitNode(1, withName("node-1"), withDERP(1), withAddresses(netip.MustParsePrefix("100.64.1.1/32"))) + exitNode2 := makeExitNode(2, withName("node-2"), withDERP(2), withAddresses(netip.MustParsePrefix("100.64.1.2/32"))) + selfNode := makeExitNode(3, withName("node-3"), withDERP(1), withAddresses(netip.MustParsePrefix("100.64.1.3/32"))) + clientNetmap := buildNetmapWithPeers(selfNode, exitNode1, exitNode2) - // Give it an initial exit node in use. - if _, err := lb.EditPrefs(&ipn.MaskedPrefs{ - ExitNodeIDSet: true, - Prefs: ipn.Prefs{ - ExitNodeID: "foo", - }, - }); err != nil { - t.Fatalf("enabling first exit node: %v", err) - } + lb := newTestLocalBackend(t) + lb.sys.MagicSock.Get().SetLastNetcheckReportForTest(lb.ctx, report) + lb.SetPrefsForTest(&ipn.Prefs{ + ControlURL: controlURL, + AutoExitNode: ipn.AnyExitNode, + }) - // Now turn off that exit node. - if prefs, err := lb.SetUseExitNodeEnabled(false); err != nil { - t.Fatal("expected failure") - } else { - if g, w := prefs.ExitNodeID(), tailcfg.StableNodeID(""); g != w { - t.Fatalf("unexpected exit node ID %q; want %q", g, w) - } - if g, w := prefs.InternalExitNodePrior(), tailcfg.StableNodeID("foo"); g != w { - t.Fatalf("unexpected exit node prior %q; want %q", g, w) - } - } + nw := newNotificationWatcher(t, lb, ipnauth.Self) - // And turn it back on. - if prefs, err := lb.SetUseExitNodeEnabled(true); err != nil { - t.Fatal("expected failure") - } else { - if g, w := prefs.ExitNodeID(), tailcfg.StableNodeID("foo"); g != w { - t.Fatalf("unexpected exit node ID %q; want %q", g, w) - } - if g, w := prefs.InternalExitNodePrior(), tailcfg.StableNodeID("foo"); g != w { - t.Fatalf("unexpected exit node prior %q; want %q", g, w) - } + // Updating the netmap should trigger both a netmap notification + // and an exit node ID notification (since an exit node is selected). + // The netmap notification should be sent first. + nw.watch(0, []wantedNotification{ + wantNetmapNotify(clientNetmap), + wantExitNodeIDNotify(exitNode1.StableID()), + }) + lb.SetControlClientStatus(lb.cc, controlclient.Status{NetMap: clientNetmap}) + nw.check() +} + +func wantNetmapNotify(want *netmap.NetworkMap) wantedNotification { + return wantedNotification{ + name: "Netmap", + cond: func(t testing.TB, _ ipnauth.Actor, n *ipn.Notify) bool { + return n.NetMap == want + }, } +} - // Verify we block setting an Internal field. - if _, err := lb.EditPrefs(&ipn.MaskedPrefs{ - InternalExitNodePriorSet: true, - }); err == nil { - t.Fatalf("unexpected success; want an error trying to set an internal field") +func wantExitNodeIDNotify(want tailcfg.StableNodeID) wantedNotification { + return wantedNotification{ + name: fmt.Sprintf("ExitNodeID-%s", want), + cond: func(_ testing.TB, _ ipnauth.Actor, n *ipn.Notify) bool { + return n.Prefs != nil && n.Prefs.Valid() && n.Prefs.ExitNodeID() == want + }, } } @@ -793,6 +1715,7 @@ func TestStatusPeerCapabilities(t *testing.T) { (&tailcfg.Node{ ID: 1, StableID: "foo", + Key: makeNodeKeyFromID(1), IsWireGuardOnly: true, Hostinfo: (&tailcfg.Hostinfo{}).View(), Capabilities: []tailcfg.NodeCapability{tailcfg.CapabilitySSH}, @@ -803,16 +1726,28 @@ func TestStatusPeerCapabilities(t *testing.T) { (&tailcfg.Node{ ID: 2, StableID: "bar", + Key: makeNodeKeyFromID(2), Hostinfo: (&tailcfg.Hostinfo{}).View(), Capabilities: []tailcfg.NodeCapability{tailcfg.CapabilityAdmin}, CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{ tailcfg.CapabilityAdmin: {`{"test": "true}`}, }), }).View(), + (&tailcfg.Node{ + ID: 3, + StableID: "baz", + Key: makeNodeKeyFromID(3), + Hostinfo: (&tailcfg.Hostinfo{}).View(), + Capabilities: []tailcfg.NodeCapability{tailcfg.CapabilityOwner}, + CapMap: (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{ + tailcfg.CapabilityOwner: nil, + }), + }).View(), }, expectedPeerCapabilities: map[tailcfg.StableNodeID][]tailcfg.NodeCapability{ tailcfg.StableNodeID("foo"): {tailcfg.CapabilitySSH}, tailcfg.StableNodeID("bar"): {tailcfg.CapabilityAdmin}, + tailcfg.StableNodeID("baz"): {tailcfg.CapabilityOwner}, }, expectedPeerCapMap: map[tailcfg.StableNodeID]tailcfg.NodeCapMap{ tailcfg.StableNodeID("foo"): (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{ @@ -821,6 +1756,9 @@ func TestStatusPeerCapabilities(t *testing.T) { tailcfg.StableNodeID("bar"): (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{ tailcfg.CapabilityAdmin: {`{"test": "true}`}, }), + tailcfg.StableNodeID("baz"): (tailcfg.NodeCapMap)(map[tailcfg.NodeCapability][]tailcfg.RawMessage{ + tailcfg.CapabilityOwner: nil, + }), }, }, { @@ -829,12 +1767,14 @@ func TestStatusPeerCapabilities(t *testing.T) { (&tailcfg.Node{ ID: 1, StableID: "foo", + Key: makeNodeKeyFromID(1), IsWireGuardOnly: true, Hostinfo: (&tailcfg.Hostinfo{}).View(), }).View(), (&tailcfg.Node{ ID: 2, StableID: "bar", + Key: makeNodeKeyFromID(2), Hostinfo: (&tailcfg.Hostinfo{}).View(), }).View(), }, @@ -926,7 +1866,11 @@ func TestUpdateNetmapDelta(t *testing.T) { nm := &netmap.NetworkMap{} for i := range 5 { - nm.Peers = append(nm.Peers, (&tailcfg.Node{ID: (tailcfg.NodeID(i) + 1)}).View()) + id := tailcfg.NodeID(i + 1) + nm.Peers = append(nm.Peers, (&tailcfg.Node{ + ID: id, + Key: makeNodeKeyFromID(id), + }).View()) } b.currentNode().SetNetMap(nm) @@ -962,23 +1906,27 @@ func TestUpdateNetmapDelta(t *testing.T) { wants := []*tailcfg.Node{ { ID: 1, + Key: makeNodeKeyFromID(1), HomeDERP: 1, }, { ID: 2, + Key: makeNodeKeyFromID(2), Online: ptr.To(true), }, { ID: 3, + Key: makeNodeKeyFromID(3), Online: ptr.To(false), }, { ID: 4, + Key: makeNodeKeyFromID(4), LastSeen: ptr.To(someTime), }, } for _, want := range wants { - gotv, ok := b.currentNode().PeerByID(want.ID) + gotv, ok := b.currentNode().NodeByID(want.ID) if !ok { t.Errorf("netmap.Peer %v missing from b.profile.Peers", want.ID) continue @@ -997,12 +1945,14 @@ func TestWhoIs(t *testing.T) { SelfNode: (&tailcfg.Node{ ID: 1, User: 10, + Key: makeNodeKeyFromID(1), Addresses: []netip.Prefix{netip.MustParsePrefix("100.101.102.103/32")}, }).View(), Peers: []tailcfg.NodeView{ (&tailcfg.Node{ ID: 2, User: 20, + Key: makeNodeKeyFromID(2), Addresses: []netip.Prefix{netip.MustParsePrefix("100.200.200.200/32")}, }).View(), }, @@ -1592,6 +2542,7 @@ func dnsResponse(domain, address string) []byte { } func TestSetExitNodeIDPolicy(t *testing.T) { + zeroValHostinfoView := new(tailcfg.Hostinfo).View() pfx := netip.MustParsePrefix tests := []struct { name string @@ -1602,6 +2553,7 @@ func TestSetExitNodeIDPolicy(t *testing.T) { prefs *ipn.Prefs exitNodeIPWant string exitNodeIDWant string + autoExitNodeWant ipn.ExitNodeExpression prefsChanged bool nm *netmap.NetworkMap lastSuggestedExitNode tailcfg.StableNodeID @@ -1668,14 +2620,18 @@ func TestSetExitNodeIDPolicy(t *testing.T) { }).View(), Peers: []tailcfg.NodeView{ (&tailcfg.Node{ + ID: 201, Name: "a.tailnet", + Key: makeNodeKeyFromID(201), Addresses: []netip.Prefix{ pfx("100.0.0.201/32"), pfx("100::201/128"), }, }).View(), (&tailcfg.Node{ + ID: 202, Name: "b.tailnet", + Key: makeNodeKeyFromID(202), Addresses: []netip.Prefix{ pfx("100::202/128"), }, @@ -1701,18 +2657,24 @@ func TestSetExitNodeIDPolicy(t *testing.T) { }).View(), Peers: []tailcfg.NodeView{ (&tailcfg.Node{ + ID: 123, Name: "a.tailnet", StableID: tailcfg.StableNodeID("123"), + Key: makeNodeKeyFromID(123), Addresses: []netip.Prefix{ pfx("127.0.0.1/32"), pfx("100::201/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), (&tailcfg.Node{ + ID: 202, Name: "b.tailnet", + Key: makeNodeKeyFromID(202), Addresses: []netip.Prefix{ pfx("100::202/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), }, }, @@ -1733,18 +2695,24 @@ func TestSetExitNodeIDPolicy(t *testing.T) { }).View(), Peers: []tailcfg.NodeView{ (&tailcfg.Node{ + ID: 123, Name: "a.tailnet", StableID: tailcfg.StableNodeID("123"), + Key: makeNodeKeyFromID(123), Addresses: []netip.Prefix{ pfx("127.0.0.1/32"), pfx("100::201/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), (&tailcfg.Node{ + ID: 202, Name: "b.tailnet", + Key: makeNodeKeyFromID(202), Addresses: []netip.Prefix{ pfx("100::202/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), }, }, @@ -1767,36 +2735,61 @@ func TestSetExitNodeIDPolicy(t *testing.T) { }).View(), Peers: []tailcfg.NodeView{ (&tailcfg.Node{ + ID: 123, Name: "a.tailnet", StableID: tailcfg.StableNodeID("123"), + Key: makeNodeKeyFromID(123), Addresses: []netip.Prefix{ pfx("100.64.5.6/32"), pfx("100::201/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), (&tailcfg.Node{ + ID: 202, Name: "b.tailnet", + Key: makeNodeKeyFromID(202), Addresses: []netip.Prefix{ pfx("100::202/128"), }, + Hostinfo: zeroValHostinfoView, }).View(), }, }, }, { - name: "ExitNodeID key is set to auto and last suggested exit node is populated", + name: "ExitNodeID key is set to auto:any and last suggested exit node is populated", exitNodeIDKey: true, exitNodeID: "auto:any", lastSuggestedExitNode: "123", exitNodeIDWant: "123", + autoExitNodeWant: "any", prefsChanged: true, }, { - name: "ExitNodeID key is set to auto and last suggested exit node is not populated", - exitNodeIDKey: true, - exitNodeID: "auto:any", - prefsChanged: true, - exitNodeIDWant: "auto:any", + name: "ExitNodeID key is set to auto:any and last suggested exit node is not populated", + exitNodeIDKey: true, + exitNodeID: "auto:any", + exitNodeIDWant: "auto:any", + autoExitNodeWant: "any", + prefsChanged: true, + }, + { + name: "ExitNodeID key is set to auto:foo and last suggested exit node is populated", + exitNodeIDKey: true, + exitNodeID: "auto:foo", + lastSuggestedExitNode: "123", + exitNodeIDWant: "123", + autoExitNodeWant: "foo", + prefsChanged: true, + }, + { + name: "ExitNodeID key is set to auto:foo and last suggested exit node is not populated", + exitNodeIDKey: true, + exitNodeID: "auto:foo", + exitNodeIDWant: "auto:any", // should be "auto:any" for compatibility with existing clients + autoExitNodeWant: "foo", + prefsChanged: true, }, } @@ -1826,9 +2819,8 @@ func TestSetExitNodeIDPolicy(t *testing.T) { b.currentNode().SetNetMap(test.nm) b.pm = pm b.lastSuggestedExitNode = test.lastSuggestedExitNode - prefs := b.pm.prefs.AsStruct() - if changed := applySysPolicy(prefs, test.lastSuggestedExitNode, false) || setExitNodeID(prefs, test.nm); changed != test.prefsChanged { + if changed := b.reconcilePrefsLocked(prefs); changed != test.prefsChanged { t.Errorf("wanted prefs changed %v, got prefs changed %v", test.prefsChanged, changed) } @@ -1838,23 +2830,26 @@ func TestSetExitNodeIDPolicy(t *testing.T) { // preferences to change. b.SetPrefsForTest(pm.CurrentPrefs().AsStruct()) - if got := b.pm.prefs.ExitNodeID(); got != tailcfg.StableNodeID(test.exitNodeIDWant) { - t.Errorf("got %v want %v", got, test.exitNodeIDWant) + if got := b.Prefs().ExitNodeID(); got != tailcfg.StableNodeID(test.exitNodeIDWant) { + t.Errorf("ExitNodeID: got %q; want %q", got, test.exitNodeIDWant) } - if got := b.pm.prefs.ExitNodeIP(); test.exitNodeIPWant == "" { + if got := b.Prefs().ExitNodeIP(); test.exitNodeIPWant == "" { if got.String() != "invalid IP" { - t.Errorf("got %v want invalid IP", got) + t.Errorf("ExitNodeIP: got %v want invalid IP", got) } } else if got.String() != test.exitNodeIPWant { - t.Errorf("got %v want %v", got, test.exitNodeIPWant) + t.Errorf("ExitNodeIP: got %q; want %q", got, test.exitNodeIPWant) + } + if got := b.Prefs().AutoExitNode(); got != test.autoExitNodeWant { + t.Errorf("AutoExitNode: got %q; want %q", got, test.autoExitNodeWant) } }) } } func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { - peer1 := makePeer(1, withCap(26), withSuggest(), withExitRoutes()) - peer2 := makePeer(2, withCap(26), withSuggest(), withExitRoutes()) + peer1 := makePeer(1, withCap(26), withSuggest(), withOnline(true), withExitRoutes()) + peer2 := makePeer(2, withCap(26), withSuggest(), withOnline(true), withExitRoutes()) derpMap := &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ 1: { @@ -1893,8 +2888,10 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { }{ { // selected auto exit node goes offline - name: "exit-node-goes-offline", - lastSuggestedExitNode: peer1.StableID(), + name: "exit-node-goes-offline", + // PreferredDERP is 2, and it's also the region with the lowest latency. + // So, peer2 should be selected as the exit node. + lastSuggestedExitNode: peer2.StableID(), netmap: &netmap.NetworkMap{ Peers: []tailcfg.NodeView{ peer1, @@ -1905,14 +2902,14 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { muts: []*tailcfg.PeerChange{ { NodeID: 1, - Online: ptr.To(false), + Online: ptr.To(true), }, { NodeID: 2, - Online: ptr.To(true), + Online: ptr.To(false), // the selected exit node goes offline }, }, - exitNodeIDWant: peer2.StableID(), + exitNodeIDWant: peer1.StableID(), report: report, }, { @@ -1929,7 +2926,7 @@ func TestUpdateNetmapDeltaAutoExitNode(t *testing.T) { muts: []*tailcfg.PeerChange{ { NodeID: 1, - Online: ptr.To(false), + Online: ptr.To(false), // a different exit node goes offline }, { NodeID: 2, @@ -2097,8 +3094,8 @@ func TestAutoExitNodeSetNetInfoCallback(t *testing.T) { } func TestSetControlClientStatusAutoExitNode(t *testing.T) { - peer1 := makePeer(1, withCap(26), withSuggest(), withExitRoutes(), withNodeKey()) - peer2 := makePeer(2, withCap(26), withSuggest(), withExitRoutes(), withNodeKey()) + peer1 := makePeer(1, withCap(26), withSuggest(), withExitRoutes(), withOnline(true), withNodeKey()) + peer2 := makePeer(2, withCap(26), withSuggest(), withExitRoutes(), withOnline(true), withNodeKey()) derpMap := &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ 1: { @@ -2141,22 +3138,25 @@ func TestSetControlClientStatusAutoExitNode(t *testing.T) { )) syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore) b.currentNode().SetNetMap(nm) - b.lastSuggestedExitNode = peer1.StableID() + // Peer 2 should be the initial exit node, as it's better than peer 1 + // in terms of latency and DERP region. + b.lastSuggestedExitNode = peer2.StableID() b.sys.MagicSock.Get().SetLastNetcheckReportForTest(b.ctx, report) b.SetPrefsForTest(b.pm.CurrentPrefs().AsStruct()) - firstExitNode := b.Prefs().ExitNodeID() - newPeer1 := makePeer(1, withCap(26), withSuggest(), withExitRoutes(), withOnline(false), withNodeKey()) + offlinePeer2 := makePeer(2, withCap(26), withSuggest(), withExitRoutes(), withOnline(false), withNodeKey()) updatedNetmap := &netmap.NetworkMap{ Peers: []tailcfg.NodeView{ - newPeer1, - peer2, + peer1, + offlinePeer2, }, DERPMap: derpMap, } b.SetControlClientStatus(b.cc, controlclient.Status{NetMap: updatedNetmap}) - lastExitNode := b.Prefs().ExitNodeID() - if firstExitNode == lastExitNode { - t.Errorf("did not switch exit nodes despite auto exit node going offline") + // But now that peer 2 is offline, we should switch to peer 1. + wantExitNode := peer1.StableID() + gotExitNode := b.Prefs().ExitNodeID() + if gotExitNode != wantExitNode { + t.Errorf("did not switch exit nodes despite auto exit node going offline: got %q; want %q", gotExitNode, wantExitNode) } } @@ -2387,7 +3387,8 @@ func TestApplySysPolicy(t *testing.T) { t.Run("unit", func(t *testing.T) { prefs := tt.prefs.Clone() - gotAnyChange := applySysPolicy(prefs, "", false) + lb := newTestLocalBackend(t) + gotAnyChange := lb.applySysPolicyLocked(prefs) if gotAnyChange && prefs.Equals(&tt.prefs) { t.Errorf("anyChange but prefs is unchanged: %v", prefs.Pretty()) @@ -2535,7 +3536,8 @@ func TestPreferencePolicyInfo(t *testing.T) { prefs := defaultPrefs.AsStruct() pp.set(prefs, tt.initialValue) - gotAnyChange := applySysPolicy(prefs, "", false) + lb := newTestLocalBackend(t) + gotAnyChange := lb.applySysPolicyLocked(prefs) if gotAnyChange != tt.wantChange { t.Errorf("anyChange=%v, want %v", gotAnyChange, tt.wantChange) @@ -3216,10 +4218,14 @@ type peerOptFunc func(*tailcfg.Node) func makePeer(id tailcfg.NodeID, opts ...peerOptFunc) tailcfg.NodeView { node := &tailcfg.Node{ - ID: id, - StableID: tailcfg.StableNodeID(fmt.Sprintf("stable%d", id)), - Name: fmt.Sprintf("peer%d", id), - HomeDERP: int(id), + ID: id, + Key: makeNodeKeyFromID(id), + DiscoKey: makeDiscoKeyFromID(id), + StableID: tailcfg.StableNodeID(fmt.Sprintf("stable%d", id)), + Name: fmt.Sprintf("peer%d", id), + Online: ptr.To(true), + MachineAuthorized: true, + HomeDERP: int(id), } for _, opt := range opts { opt(node) @@ -3259,6 +4265,23 @@ func withLocation(loc tailcfg.LocationView) peerOptFunc { } } +func withLocationPriority(pri int) peerOptFunc { + return func(n *tailcfg.Node) { + var hi *tailcfg.Hostinfo + if n.Hostinfo.Valid() { + hi = n.Hostinfo.AsStruct() + } else { + hi = new(tailcfg.Hostinfo) + } + if hi.Location == nil { + hi.Location = new(tailcfg.Location) + } + hi.Location.Priority = pri + + n.Hostinfo = hi.View() + } +} + func withExitRoutes() peerOptFunc { return func(n *tailcfg.Node) { n.AllowedIPs = append(n.AllowedIPs, tsaddr.ExitRoutes()...) @@ -3289,6 +4312,12 @@ func withNodeKey() peerOptFunc { } } +func withAddresses(addresses ...netip.Prefix) peerOptFunc { + return func(n *tailcfg.Node) { + n.Addresses = append(n.Addresses, addresses...) + } +} + func deterministicRegionForTest(t testing.TB, want views.Slice[int], use int) selectRegionFunc { t.Helper() @@ -3787,7 +4816,11 @@ func TestSuggestExitNode(t *testing.T) { allowList = set.SetOf(tt.allowPolicy) } - got, err := suggestExitNode(tt.lastReport, tt.netMap, tt.lastSuggestion, selectRegion, selectNode, allowList) + nb := newNodeBackend(t.Context(), eventbus.New()) + defer nb.shutdown(errShutdown) + nb.SetNetMap(tt.netMap) + + got, err := suggestExitNode(tt.lastReport, nb, tt.lastSuggestion, selectRegion, selectNode, allowList) if got.Name != tt.wantName { t.Errorf("name=%v, want %v", got.Name, tt.wantName) } @@ -3821,95 +4854,450 @@ func TestSuggestExitNodePickWeighted(t *testing.T) { wantIDs []tailcfg.StableNodeID }{ { - name: "different priorities", - candidates: []tailcfg.NodeView{ - makePeer(2, withExitRoutes(), withLocation(location20.View())), - makePeer(3, withExitRoutes(), withLocation(location10.View())), + name: "different priorities", + candidates: []tailcfg.NodeView{ + makePeer(2, withExitRoutes(), withLocation(location20.View())), + makePeer(3, withExitRoutes(), withLocation(location10.View())), + }, + wantIDs: []tailcfg.StableNodeID{"stable2"}, + }, + { + name: "same priorities", + candidates: []tailcfg.NodeView{ + makePeer(2, withExitRoutes(), withLocation(location10.View())), + makePeer(3, withExitRoutes(), withLocation(location10.View())), + }, + wantIDs: []tailcfg.StableNodeID{"stable2", "stable3"}, + }, + { + name: "<1 candidates", + candidates: []tailcfg.NodeView{}, + }, + { + name: "1 candidate", + candidates: []tailcfg.NodeView{ + makePeer(2, withExitRoutes(), withLocation(location20.View())), + }, + wantIDs: []tailcfg.StableNodeID{"stable2"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := pickWeighted(tt.candidates) + gotIDs := make([]tailcfg.StableNodeID, 0, len(got)) + for _, n := range got { + if !n.Valid() { + gotIDs = append(gotIDs, "") + continue + } + gotIDs = append(gotIDs, n.StableID()) + } + if !views.SliceEqualAnyOrder(views.SliceOf(gotIDs), views.SliceOf(tt.wantIDs)) { + t.Errorf("node IDs = %v, want %v", gotIDs, tt.wantIDs) + } + }) + } +} + +func TestSuggestExitNodeLongLatDistance(t *testing.T) { + tests := []struct { + name string + fromLat float64 + fromLong float64 + toLat float64 + toLong float64 + want float64 + }{ + { + name: "zero values", + fromLat: 0, + fromLong: 0, + toLat: 0, + toLong: 0, + want: 0, + }, + { + name: "valid values", + fromLat: 40.73061, + fromLong: -73.935242, + toLat: 37.3382082, + toLong: -121.8863286, + want: 4117266.873301274, + }, + { + name: "valid values, locations in north and south of equator", + fromLat: 40.73061, + fromLong: -73.935242, + toLat: -33.861481, + toLong: 151.205475, + want: 15994089.144368416, + }, + } + // The wanted values are computed using a more precise algorithm using the WGS84 model but + // longLatDistance uses a spherical approximation for simplicity. To account for this, we allow for + // 10km of error. + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := longLatDistance(tt.fromLat, tt.fromLong, tt.toLat, tt.toLong) + const maxError = 10000 // 10km + if math.Abs(got-tt.want) > maxError { + t.Errorf("distance=%vm, want within %vm of %vm", got, maxError, tt.want) + } + }) + } +} + +func TestSuggestExitNodeTrafficSteering(t *testing.T) { + city := &tailcfg.Location{ + Country: "Canada", + CountryCode: "CA", + City: "Montreal", + CityCode: "MTR", + Latitude: 45.5053, + Longitude: -73.5525, + } + noLatLng := &tailcfg.Location{ + Country: "Canada", + CountryCode: "CA", + City: "Montreal", + CityCode: "MTR", + } + + selfNode := tailcfg.Node{ + ID: 0, // randomness is seeded off NetMap.SelfNode.ID + Addresses: []netip.Prefix{ + netip.MustParsePrefix("100.64.1.1/32"), + netip.MustParsePrefix("fe70::1/128"), + }, + CapMap: tailcfg.NodeCapMap{ + tailcfg.NodeAttrTrafficSteering: []tailcfg.RawMessage{}, + }, + } + + for _, tt := range []struct { + name string + + netMap *netmap.NetworkMap + lastExit tailcfg.StableNodeID + allowPolicy []tailcfg.StableNodeID + + wantID tailcfg.StableNodeID + wantName string + wantLoc *tailcfg.Location + wantPri int + + wantErr error + }{ + { + name: "no-netmap", + netMap: nil, + wantErr: ErrNoNetMap, + }, + { + name: "no-nodes", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{}, + }, + wantID: "", + }, + { + name: "no-exit-nodes", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1), + }, + }, + wantID: "", + }, + { + name: "exit-node-without-suggestion", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes()), + }, + }, + wantID: "", + }, + { + name: "suggested-exit-node-without-routes", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withSuggest()), + }, + }, + wantID: "", + }, + { + name: "suggested-exit-node", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest()), + }, + }, + wantID: "stable1", + wantName: "peer1", + }, + { + name: "suggest-exit-node-stable-pick", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest()), + makePeer(2, + withExitRoutes(), + withSuggest()), + makePeer(3, + withExitRoutes(), + withSuggest()), + makePeer(4, + withExitRoutes(), + withSuggest()), + }, + }, + // Change this, if the hashing function changes. + wantID: "stable3", + wantName: "peer3", + }, + { + name: "exit-nodes-with-and-without-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocationPriority(1)), + makePeer(2, + withExitRoutes(), + withSuggest()), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantPri: 1, + }, + { + name: "exit-nodes-without-and-with-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest()), + makePeer(2, + withExitRoutes(), + withSuggest(), + withLocationPriority(1)), + }, }, - wantIDs: []tailcfg.StableNodeID{"stable2"}, + wantID: "stable2", + wantName: "peer2", + wantPri: 1, }, { - name: "same priorities", - candidates: []tailcfg.NodeView{ - makePeer(2, withExitRoutes(), withLocation(location10.View())), - makePeer(3, withExitRoutes(), withLocation(location10.View())), + name: "exit-nodes-with-negative-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocationPriority(-1)), + makePeer(2, + withExitRoutes(), + withSuggest(), + withLocationPriority(-2)), + makePeer(3, + withExitRoutes(), + withSuggest(), + withLocationPriority(-3)), + makePeer(4, + withExitRoutes(), + withSuggest(), + withLocationPriority(-4)), + }, }, - wantIDs: []tailcfg.StableNodeID{"stable2", "stable3"}, + wantID: "stable1", + wantName: "peer1", + wantPri: -1, }, { - name: "<1 candidates", - candidates: []tailcfg.NodeView{}, + name: "exit-nodes-no-priority-beats-negative-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocationPriority(-1)), + makePeer(2, + withExitRoutes(), + withSuggest(), + withLocationPriority(-2)), + makePeer(3, + withExitRoutes(), + withSuggest()), + }, + }, + wantID: "stable3", + wantName: "peer3", }, { - name: "1 candidate", - candidates: []tailcfg.NodeView{ - makePeer(2, withExitRoutes(), withLocation(location20.View())), + name: "exit-nodes-same-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocationPriority(1)), + makePeer(2, + withExitRoutes(), + withSuggest(), + withLocationPriority(2)), // top + makePeer(3, + withExitRoutes(), + withSuggest(), + withLocationPriority(1)), + makePeer(4, + withExitRoutes(), + withSuggest(), + withLocationPriority(2)), // top + makePeer(5, + withExitRoutes(), + withSuggest(), + withLocationPriority(2)), // top + makePeer(6, + withExitRoutes(), + withSuggest()), + makePeer(7, + withExitRoutes(), + withSuggest(), + withLocationPriority(2)), // top + }, }, - wantIDs: []tailcfg.StableNodeID{"stable2"}, + wantID: "stable5", + wantName: "peer5", + wantPri: 2, }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := pickWeighted(tt.candidates) - gotIDs := make([]tailcfg.StableNodeID, 0, len(got)) - for _, n := range got { - if !n.Valid() { - gotIDs = append(gotIDs, "") - continue - } - gotIDs = append(gotIDs, n.StableID()) - } - if !views.SliceEqualAnyOrder(views.SliceOf(gotIDs), views.SliceOf(tt.wantIDs)) { - t.Errorf("node IDs = %v, want %v", gotIDs, tt.wantIDs) - } - }) - } -} - -func TestSuggestExitNodeLongLatDistance(t *testing.T) { - tests := []struct { - name string - fromLat float64 - fromLong float64 - toLat float64 - toLong float64 - want float64 - }{ { - name: "zero values", - fromLat: 0, - fromLong: 0, - toLat: 0, - toLong: 0, - want: 0, + name: "suggested-exit-node-with-city", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocation(city.View())), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantLoc: city, }, { - name: "valid values", - fromLat: 40.73061, - fromLong: -73.935242, - toLat: 37.3382082, - toLong: -121.8863286, - want: 4117266.873301274, + name: "suggested-exit-node-with-city-and-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocation(city.View()), + withLocationPriority(1)), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantLoc: city, + wantPri: 1, }, { - name: "valid values, locations in north and south of equator", - fromLat: 40.73061, - fromLong: -73.935242, - toLat: -33.861481, - toLong: 151.205475, - want: 15994089.144368416, + name: "suggested-exit-node-without-latlng", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocation(noLatLng.View())), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantLoc: noLatLng, }, - } - // The wanted values are computed using a more precise algorithm using the WGS84 model but - // longLatDistance uses a spherical approximation for simplicity. To account for this, we allow for - // 10km of error. - for _, tt := range tests { + { + name: "suggested-exit-node-without-latlng-with-priority", + netMap: &netmap.NetworkMap{ + SelfNode: selfNode.View(), + Peers: []tailcfg.NodeView{ + makePeer(1, + withExitRoutes(), + withSuggest(), + withLocation(noLatLng.View()), + withLocationPriority(1)), + }, + }, + wantID: "stable1", + wantName: "peer1", + wantLoc: noLatLng, + wantPri: 1, + }, + } { t.Run(tt.name, func(t *testing.T) { - got := longLatDistance(tt.fromLat, tt.fromLong, tt.toLat, tt.toLong) - const maxError = 10000 // 10km - if math.Abs(got-tt.want) > maxError { - t.Errorf("distance=%vm, want within %vm of %vm", got, maxError, tt.want) + var allowList set.Set[tailcfg.StableNodeID] + if tt.allowPolicy != nil { + allowList = set.SetOf(tt.allowPolicy) + } + + // HACK: NetMap.AllCaps is populated by Control: + if tt.netMap != nil { + caps := maps.Keys(tt.netMap.SelfNode.CapMap().AsMap()) + tt.netMap.AllCaps = set.SetOf(slices.Collect(caps)) + } + + nb := newNodeBackend(t.Context(), eventbus.New()) + defer nb.shutdown(errShutdown) + nb.SetNetMap(tt.netMap) + + got, err := suggestExitNodeUsingTrafficSteering(nb, allowList) + if tt.wantErr == nil && err != nil { + t.Fatalf("err=%v, want nil", err) + } + if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { + t.Fatalf("err=%v, want %v", err, tt.wantErr) + } + + if got.Name != tt.wantName { + t.Errorf("name=%q, want %q", got.Name, tt.wantName) + } + + if got.ID != tt.wantID { + t.Errorf("ID=%q, want %q", got.ID, tt.wantID) + } + + wantLoc := tt.wantLoc + if tt.wantPri != 0 { + if wantLoc == nil { + wantLoc = new(tailcfg.Location) + } + wantLoc.Priority = tt.wantPri + } + if diff := cmp.Diff(got.Location.AsStruct(), wantLoc); diff != "" { + t.Errorf("location mismatch (+want -got)\n%s", diff) } }) } @@ -3964,56 +5352,6 @@ func TestMinLatencyDERPregion(t *testing.T) { } } -func TestShouldAutoExitNode(t *testing.T) { - tests := []struct { - name string - exitNodeIDPolicyValue string - expectedBool bool - }{ - { - name: "auto:any", - exitNodeIDPolicyValue: "auto:any", - expectedBool: true, - }, - { - name: "no auto prefix", - exitNodeIDPolicyValue: "foo", - expectedBool: false, - }, - { - name: "auto prefix but empty suffix", - exitNodeIDPolicyValue: "auto:", - expectedBool: false, - }, - { - name: "auto prefix no colon", - exitNodeIDPolicyValue: "auto", - expectedBool: false, - }, - { - name: "auto prefix invalid suffix", - exitNodeIDPolicyValue: "auto:foo", - expectedBool: false, - }, - } - - syspolicy.RegisterWellKnownSettingsForTest(t) - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - policyStore := source.NewTestStoreOf(t, source.TestSettingOf( - syspolicy.ExitNodeID, tt.exitNodeIDPolicyValue, - )) - syspolicy.MustRegisterStoreForTest(t, "TestStore", setting.DeviceScope, policyStore) - - got := shouldAutoExitNode() - if got != tt.expectedBool { - t.Fatalf("expected %v got %v for %v policy value", tt.expectedBool, got, tt.exitNodeIDPolicyValue) - } - }) - } -} - func TestEnableAutoUpdates(t *testing.T) { lb := newTestLocalBackend(t) @@ -5402,3 +6740,200 @@ func TestSrcCapPacketFilter(t *testing.T) { t.Error("IsDrop() for node without cap = false, want true") } } + +func TestDisplayMessages(t *testing.T) { + b := newTestLocalBackend(t) + + // Pretend we're in a map poll so health updates get processed + ht := b.HealthTracker() + ht.SetIPNState("NeedsLogin", true) + ht.GotStreamedMapResponse() + + b.mu.Lock() + defer b.mu.Unlock() + b.setNetMapLocked(&netmap.NetworkMap{ + DisplayMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-message": { + Title: "Testing", + }, + }, + }) + + state := ht.CurrentState() + wantID := health.WarnableCode("control-health.test-message") + _, ok := state.Warnings[wantID] + + if !ok { + t.Errorf("no warning found with id %q", wantID) + } +} + +// TestDisplayMessagesURLFilter tests that we filter out any URLs that are not +// valid as a pop browser URL (see [LocalBackend.validPopBrowserURL]). +func TestDisplayMessagesURLFilter(t *testing.T) { + b := newTestLocalBackend(t) + + // Pretend we're in a map poll so health updates get processed + ht := b.HealthTracker() + ht.SetIPNState("NeedsLogin", true) + ht.GotStreamedMapResponse() + + b.mu.Lock() + defer b.mu.Unlock() + b.setNetMapLocked(&netmap.NetworkMap{ + DisplayMessages: map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-message": { + Title: "Testing", + Severity: tailcfg.SeverityHigh, + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "https://www.evil.com", + Label: "Phishing Link", + }, + }, + }, + }) + + state := ht.CurrentState() + wantID := health.WarnableCode("control-health.test-message") + got, ok := state.Warnings[wantID] + + if !ok { + t.Fatalf("no warning found with id %q", wantID) + } + + want := health.UnhealthyState{ + WarnableCode: wantID, + Title: "Testing", + Severity: health.SeverityHigh, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Unexpected message content (-want/+got):\n%s", diff) + } +} + +// TestDisplayMessageIPNBus checks that we send health messages appropriately +// based on whether the watcher has sent the [ipn.NotifyHealthActions] watch +// option or not. +func TestDisplayMessageIPNBus(t *testing.T) { + type test struct { + name string + mask ipn.NotifyWatchOpt + wantWarning health.UnhealthyState + } + + msgs := map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage{ + "test-message": { + Title: "Message title", + Text: "Message text.", + Severity: tailcfg.SeverityMedium, + PrimaryAction: &tailcfg.DisplayMessageAction{ + URL: "https://example.com", + Label: "Learn more", + }, + }, + } + + wantID := health.WarnableCode("control-health.test-message") + + for _, tt := range []test{ + { + name: "older-client-no-actions", + mask: 0, + wantWarning: health.UnhealthyState{ + WarnableCode: wantID, + Severity: health.SeverityMedium, + Title: "Message title", + Text: "Message text. Learn more: https://example.com", // PrimaryAction appended to text + PrimaryAction: nil, // PrimaryAction not included + }, + }, + { + name: "new-client-with-actions", + mask: ipn.NotifyHealthActions, + wantWarning: health.UnhealthyState{ + WarnableCode: wantID, + Severity: health.SeverityMedium, + Title: "Message title", + Text: "Message text.", + PrimaryAction: &health.UnhealthyStateAction{ + URL: "https://example.com", + Label: "Learn more", + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + lb := newLocalBackendWithTestControl(t, false, func(tb testing.TB, opts controlclient.Options) controlclient.Client { + return newClient(tb, opts) + }) + + ipnWatcher := newNotificationWatcher(t, lb, nil) + ipnWatcher.watch(tt.mask, []wantedNotification{{ + name: fmt.Sprintf("warning with ID %q", wantID), + cond: func(_ testing.TB, _ ipnauth.Actor, n *ipn.Notify) bool { + if n.Health == nil { + return false + } + got, ok := n.Health.Warnings[wantID] + if ok { + if diff := cmp.Diff(tt.wantWarning, got); diff != "" { + t.Errorf("unexpected warning details (-want/+got):\n%s", diff) + return true // we failed the test so tell the watcher we've seen what we need to to stop it waiting + } + } else { + got := slices.Collect(maps.Keys(n.Health.Warnings)) + t.Logf("saw warnings: %v", got) + } + return ok + }, + }}) + + lb.SetPrefsForTest(&ipn.Prefs{ + ControlURL: "https://localhost:1/", + WantRunning: true, + LoggedOut: false, + }) + if err := lb.Start(ipn.Options{}); err != nil { + t.Fatalf("(*LocalBackend).Start(): %v", err) + } + + cc := lb.cc.(*mockControl) + + // Assert that we are logged in and authorized, and also send our DisplayMessages + cc.send(nil, "", true, &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{MachineAuthorized: true}).View(), + DisplayMessages: msgs, + }) + + // Tell the health tracker that we are in a map poll because + // mockControl doesn't tell it + lb.HealthTracker().GotStreamedMapResponse() + + // Assert that we got the expected notification + ipnWatcher.check() + }) + } +} + +func checkError(tb testing.TB, got, want error, fatal bool) { + tb.Helper() + f := tb.Errorf + if fatal { + f = tb.Fatalf + } + if (want == nil) != (got == nil) || + (want != nil && got != nil && want.Error() != got.Error() && !errors.Is(got, want)) { + f("gotErr: %v; wantErr: %v", got, want) + } +} + +func toStrings[T ~string](in []T) []string { + out := make([]string, len(in)) + for i, v := range in { + out[i] = string(v) + } + return out +} diff --git a/ipn/ipnlocal/network-lock.go b/ipn/ipnlocal/network-lock.go index 36d39a465a654..10f0cc8278109 100644 --- a/ipn/ipnlocal/network-lock.go +++ b/ipn/ipnlocal/network-lock.go @@ -600,18 +600,14 @@ func (b *LocalBackend) NetworkLockInit(keys []tka.Key, disablementValues [][]byt var ourNodeKey key.NodePublic var nlPriv key.NLPrivate - b.mu.Lock() - - if !b.capTailnetLock { - b.mu.Unlock() - return errors.New("not permitted to enable tailnet lock") - } + b.mu.Lock() if p := b.pm.CurrentPrefs(); p.Valid() && p.Persist().Valid() && !p.Persist().PrivateNodeKey().IsZero() { ourNodeKey = p.Persist().PublicNodeKey() nlPriv = p.Persist().NetworkLockKey() } b.mu.Unlock() + if ourNodeKey.IsZero() || nlPriv.IsZero() { return errors.New("no node-key: is tailscale logged in?") } @@ -671,6 +667,13 @@ func (b *LocalBackend) NetworkLockInit(keys []tka.Key, disablementValues [][]byt return err } +// NetworkLockAllowed reports whether the node is allowed to use Tailnet Lock. +func (b *LocalBackend) NetworkLockAllowed() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.capTailnetLock +} + // Only use is in tests. func (b *LocalBackend) NetworkLockVerifySignatureForTest(nks tkatype.MarshaledSignature, nodeKey key.NodePublic) error { b.mu.Lock() diff --git a/ipn/ipnlocal/node_backend.go b/ipn/ipnlocal/node_backend.go index fb77f38ebc87a..ec503f1300ca5 100644 --- a/ipn/ipnlocal/node_backend.go +++ b/ipn/ipnlocal/node_backend.go @@ -5,6 +5,7 @@ package ipnlocal import ( "cmp" + "context" "net/netip" "slices" "sync" @@ -22,9 +23,11 @@ import ( "tailscale.com/types/ptr" "tailscale.com/types/views" "tailscale.com/util/dnsname" + "tailscale.com/util/eventbus" "tailscale.com/util/mak" "tailscale.com/util/slicesx" "tailscale.com/wgengine/filter" + "tailscale.com/wgengine/magicsock" ) // nodeBackend is node-specific [LocalBackend] state. It is usually the current node. @@ -39,7 +42,7 @@ import ( // Two pointers to different [nodeBackend] instances represent different local nodes. // However, there's currently a bug where a new [nodeBackend] might not be created // during an implicit node switch (see tailscale/corp#28014). - +// // In the future, we might want to include at least the following in this struct (in addition to the current fields). // However, not everything should be exported or otherwise made available to the outside world (e.g. [ipnext] extensions, // peer API handlers, etc.). @@ -61,13 +64,25 @@ import ( // Even if they're tied to the local node, instead of moving them here, we should extract the entire feature // into a separate package and have it install proper hooks. type nodeBackend struct { + ctx context.Context // canceled by [nodeBackend.shutdown] + ctxCancel context.CancelCauseFunc // cancels ctx + // filterAtomic is a stateful packet filter. Immutable once created, but can be // replaced with a new one. filterAtomic atomic.Pointer[filter.Filter] + // initialized once and immutable + eventClient *eventbus.Client + filterPub *eventbus.Publisher[magicsock.FilterUpdate] + nodeViewsPub *eventbus.Publisher[magicsock.NodeViewsUpdate] + nodeMutsPub *eventbus.Publisher[magicsock.NodeMutationsUpdate] + // TODO(nickkhyl): maybe use sync.RWMutex? mu sync.Mutex // protects the following fields + shutdownOnce sync.Once // guards calling [nodeBackend.shutdown] + readyCh chan struct{} // closed by [nodeBackend.ready]; nil after shutdown + // NetMap is the most recently set full netmap from the controlclient. // It can't be mutated in place once set. Because it can't be mutated in place, // delta updates from the control server don't apply to it. Instead, use @@ -88,12 +103,29 @@ type nodeBackend struct { nodeByAddr map[netip.Addr]tailcfg.NodeID } -func newNodeBackend() *nodeBackend { - cn := &nodeBackend{} +func newNodeBackend(ctx context.Context, bus *eventbus.Bus) *nodeBackend { + ctx, ctxCancel := context.WithCancelCause(ctx) + nb := &nodeBackend{ + ctx: ctx, + ctxCancel: ctxCancel, + eventClient: bus.Client("ipnlocal.nodeBackend"), + readyCh: make(chan struct{}), + } // Default filter blocks everything and logs nothing. noneFilter := filter.NewAllowNone(logger.Discard, &netipx.IPSet{}) - cn.filterAtomic.Store(noneFilter) - return cn + nb.filterAtomic.Store(noneFilter) + nb.filterPub = eventbus.Publish[magicsock.FilterUpdate](nb.eventClient) + nb.nodeViewsPub = eventbus.Publish[magicsock.NodeViewsUpdate](nb.eventClient) + nb.nodeMutsPub = eventbus.Publish[magicsock.NodeMutationsUpdate](nb.eventClient) + nb.filterPub.Publish(magicsock.FilterUpdate{Filter: nb.filterAtomic.Load()}) + return nb +} + +// Context returns a context that is canceled when the [nodeBackend] shuts down, +// either because [LocalBackend] is switching to a different [nodeBackend] +// or is shutting down itself. +func (nb *nodeBackend) Context() context.Context { + return nb.ctx } func (nb *nodeBackend) Self() tailcfg.NodeView { @@ -174,9 +206,14 @@ func (nb *nodeBackend) NodeByKey(k key.NodePublic) (_ tailcfg.NodeID, ok bool) { return 0, false } -func (nb *nodeBackend) PeerByID(id tailcfg.NodeID) (_ tailcfg.NodeView, ok bool) { +func (nb *nodeBackend) NodeByID(id tailcfg.NodeID) (_ tailcfg.NodeView, ok bool) { nb.mu.Lock() defer nb.mu.Unlock() + if nb.netMap != nil { + if self := nb.netMap.SelfNode; self.Valid() && self.ID() == id { + return self, true + } + } n, ok := nb.peers[id] return n, ok } @@ -349,6 +386,12 @@ func (nb *nodeBackend) SetNetMap(nm *netmap.NetworkMap) { nb.netMap = nm nb.updateNodeByAddrLocked() nb.updatePeersLocked() + nv := magicsock.NodeViewsUpdate{} + if nm != nil { + nv.SelfNode = nm.SelfNode + nv.Peers = nm.Peers + } + nb.nodeViewsPub.Publish(nv) } func (nb *nodeBackend) updateNodeByAddrLocked() { @@ -424,6 +467,9 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo // call (e.g. its endpoints + online status both change) var mutableNodes map[tailcfg.NodeID]*tailcfg.Node + update := magicsock.NodeMutationsUpdate{ + Mutations: make([]netmap.NodeMutation, 0, len(muts)), + } for _, m := range muts { n, ok := mutableNodes[m.NodeIDBeingMutated()] if !ok { @@ -434,12 +480,14 @@ func (nb *nodeBackend) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bo } n = nv.AsStruct() mak.Set(&mutableNodes, nv.ID(), n) + update.Mutations = append(update.Mutations, m) } m.Apply(n) } for nid, n := range mutableNodes { nb.peers[nid] = n.View() } + nb.nodeMutsPub.Publish(update) return true } @@ -461,6 +509,7 @@ func (nb *nodeBackend) filter() *filter.Filter { func (nb *nodeBackend) setFilter(f *filter.Filter) { nb.filterAtomic.Store(f) + nb.filterPub.Publish(magicsock.FilterUpdate{Filter: f}) } func (nb *nodeBackend) dnsConfigForNetmap(prefs ipn.PrefsView, selfExpired bool, logf logger.Logf, versionOS string) *dns.Config { @@ -475,6 +524,60 @@ func (nb *nodeBackend) exitNodeCanProxyDNS(exitNodeID tailcfg.StableNodeID) (doh return exitNodeCanProxyDNS(nb.netMap, nb.peers, exitNodeID) } +// ready signals that [LocalBackend] has completed the switch to this [nodeBackend] +// and any pending calls to [nodeBackend.Wait] must be unblocked. +func (nb *nodeBackend) ready() { + nb.mu.Lock() + defer nb.mu.Unlock() + if nb.readyCh != nil { + close(nb.readyCh) + } +} + +// Wait blocks until [LocalBackend] completes the switch to this [nodeBackend] +// and calls [nodeBackend.ready]. It returns an error if the provided context +// is canceled or if the [nodeBackend] shuts down or is already shut down. +// +// It must not be called with the [LocalBackend]'s internal mutex held as [LocalBackend] +// may need to acquire it to complete the switch. +// +// TODO(nickkhyl): Relax this restriction once [LocalBackend]'s state machine +// runs in its own goroutine, or if we decide that waiting for the state machine +// restart to finish isn't necessary for [LocalBackend] to consider the switch complete. +// We mostly need this because of [LocalBackend.Start] acquiring b.mu and the fact that +// methods like [LocalBackend.SwitchProfile] must report any errors returned by it. +// Perhaps we could report those errors asynchronously as [health.Warnable]s? +func (nb *nodeBackend) Wait(ctx context.Context) error { + nb.mu.Lock() + readyCh := nb.readyCh + nb.mu.Unlock() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-nb.ctx.Done(): + return context.Cause(nb.ctx) + case <-readyCh: + return nil + } +} + +// shutdown shuts down the [nodeBackend] and cancels its context +// with the provided cause. +func (nb *nodeBackend) shutdown(cause error) { + nb.shutdownOnce.Do(func() { + nb.doShutdown(cause) + }) +} + +func (nb *nodeBackend) doShutdown(cause error) { + nb.mu.Lock() + defer nb.mu.Unlock() + nb.ctxCancel(cause) + nb.readyCh = nil + nb.eventClient.Close() +} + // dnsConfigForNetmap returns a *dns.Config for the given netmap, // prefs, client OS version, and cloud hosting environment. // diff --git a/ipn/ipnlocal/node_backend_test.go b/ipn/ipnlocal/node_backend_test.go new file mode 100644 index 0000000000000..dc67d327c8041 --- /dev/null +++ b/ipn/ipnlocal/node_backend_test.go @@ -0,0 +1,123 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package ipnlocal + +import ( + "context" + "errors" + "testing" + "time" + + "tailscale.com/util/eventbus" +) + +func TestNodeBackendReadiness(t *testing.T) { + nb := newNodeBackend(t.Context(), eventbus.New()) + + // The node backend is not ready until [nodeBackend.ready] is called, + // and [nodeBackend.Wait] should fail with [context.DeadlineExceeded]. + ctx, cancelCtx := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancelCtx() + if err := nb.Wait(ctx); err != ctx.Err() { + t.Fatalf("Wait: got %v; want %v", err, ctx.Err()) + } + + // Start a goroutine to wait for the node backend to become ready. + waitDone := make(chan struct{}) + go func() { + if err := nb.Wait(context.Background()); err != nil { + t.Errorf("Wait: got %v; want nil", err) + } + close(waitDone) + }() + + // Call [nodeBackend.ready] to indicate that the node backend is now ready. + go nb.ready() + + // Once the backend is called, [nodeBackend.Wait] should return immediately without error. + if err := nb.Wait(context.Background()); err != nil { + t.Fatalf("Wait: got %v; want nil", err) + } + // And any pending waiters should also be unblocked. + <-waitDone +} + +func TestNodeBackendShutdown(t *testing.T) { + nb := newNodeBackend(t.Context(), eventbus.New()) + + shutdownCause := errors.New("test shutdown") + + // Start a goroutine to wait for the node backend to become ready. + // This test expects it to block until the node backend shuts down + // and then return the specified shutdown cause. + waitDone := make(chan struct{}) + go func() { + if err := nb.Wait(context.Background()); err != shutdownCause { + t.Errorf("Wait: got %v; want %v", err, shutdownCause) + } + close(waitDone) + }() + + // Call [nodeBackend.shutdown] to indicate that the node backend is shutting down. + nb.shutdown(shutdownCause) + + // Calling it again is fine, but should not change the shutdown cause. + nb.shutdown(errors.New("test shutdown again")) + + // After shutdown, [nodeBackend.Wait] should return with the specified shutdown cause. + if err := nb.Wait(context.Background()); err != shutdownCause { + t.Fatalf("Wait: got %v; want %v", err, shutdownCause) + } + // The context associated with the node backend should also be cancelled + // and its cancellation cause should match the shutdown cause. + if err := nb.Context().Err(); !errors.Is(err, context.Canceled) { + t.Fatalf("Context.Err: got %v; want %v", err, context.Canceled) + } + if cause := context.Cause(nb.Context()); cause != shutdownCause { + t.Fatalf("Cause: got %v; want %v", cause, shutdownCause) + } + // And any pending waiters should also be unblocked. + <-waitDone +} + +func TestNodeBackendReadyAfterShutdown(t *testing.T) { + nb := newNodeBackend(t.Context(), eventbus.New()) + + shutdownCause := errors.New("test shutdown") + nb.shutdown(shutdownCause) + nb.ready() // Calling ready after shutdown is a no-op, but should not panic, etc. + if err := nb.Wait(context.Background()); err != shutdownCause { + t.Fatalf("Wait: got %v; want %v", err, shutdownCause) + } +} + +func TestNodeBackendParentContextCancellation(t *testing.T) { + ctx, cancelCtx := context.WithCancel(context.Background()) + nb := newNodeBackend(ctx, eventbus.New()) + + cancelCtx() + + // Cancelling the parent context should cause [nodeBackend.Wait] + // to return with [context.Canceled]. + if err := nb.Wait(context.Background()); !errors.Is(err, context.Canceled) { + t.Fatalf("Wait: got %v; want %v", err, context.Canceled) + } + + // And the node backend's context should also be cancelled. + if err := nb.Context().Err(); !errors.Is(err, context.Canceled) { + t.Fatalf("Context.Err: got %v; want %v", err, context.Canceled) + } +} + +func TestNodeBackendConcurrentReadyAndShutdown(t *testing.T) { + nb := newNodeBackend(t.Context(), eventbus.New()) + + // Calling [nodeBackend.ready] and [nodeBackend.shutdown] concurrently + // should not cause issues, and [nodeBackend.Wait] should unblock, + // but the result of [nodeBackend.Wait] is intentionally undefined. + go nb.ready() + go nb.shutdown(errors.New("test shutdown")) + + nb.Wait(context.Background()) +} diff --git a/ipn/ipnlocal/peerapi.go b/ipn/ipnlocal/peerapi.go index 84aaecf7efb78..89554f0ff9eb1 100644 --- a/ipn/ipnlocal/peerapi.go +++ b/ipn/ipnlocal/peerapi.go @@ -247,6 +247,10 @@ func (h *peerAPIHandler) logf(format string, a ...any) { h.ps.b.logf("peerapi: "+format, a...) } +func (h *peerAPIHandler) logfv1(format string, a ...any) { + h.ps.b.logf("[v1] peerapi: "+format, a...) +} + // isAddressValid reports whether addr is a valid destination address for this // node originating from the peer. func (h *peerAPIHandler) isAddressValid(addr netip.Addr) bool { @@ -859,7 +863,7 @@ func dohQuery(r *http.Request) (dnsQuery []byte, publicErr string) { case "GET": q64 := r.FormValue("dns") if q64 == "" { - return nil, "missing 'dns' parameter" + return nil, "missing ‘dns’ parameter; try '?dns=' (DoH standard) or use '?q=' for JSON debug mode" } if base64.RawURLEncoding.DecodedLen(len(q64)) > maxQueryLen { return nil, "query too large" @@ -1015,6 +1019,7 @@ func (rbw *requestBodyWrapper) Read(b []byte) (int, error) { } func (h *peerAPIHandler) handleServeDrive(w http.ResponseWriter, r *http.Request) { + h.logfv1("taildrive: got %s request from %s", r.Method, h.peerNode.Key().ShortString()) if !h.ps.b.DriveSharingEnabled() { h.logf("taildrive: not enabled") http.Error(w, "taildrive not enabled", http.StatusNotFound) @@ -1055,21 +1060,23 @@ func (h *peerAPIHandler) handleServeDrive(w http.ResponseWriter, r *http.Request } r.Body = bw - if r.Method == httpm.PUT || r.Method == httpm.GET { - defer func() { - switch wr.statusCode { - case 304: - // 304s are particularly chatty so skip logging. - default: - contentType := "unknown" - if ct := wr.Header().Get("Content-Type"); ct != "" { - contentType = ct - } - - h.logf("taildrive: share: %s from %s to %s: status-code=%d ext=%q content-type=%q tx=%.f rx=%.f", r.Method, h.peerNode.Key().ShortString(), h.selfNode.Key().ShortString(), wr.statusCode, parseDriveFileExtensionForLog(r.URL.Path), contentType, roundTraffic(wr.contentLength), roundTraffic(bw.bytesRead)) + defer func() { + switch wr.statusCode { + case 304: + // 304s are particularly chatty so skip logging. + default: + log := h.logf + if r.Method != httpm.PUT && r.Method != httpm.GET { + log = h.logfv1 } - }() - } + contentType := "unknown" + if ct := wr.Header().Get("Content-Type"); ct != "" { + contentType = ct + } + + log("taildrive: share: %s from %s to %s: status-code=%d ext=%q content-type=%q tx=%.f rx=%.f", r.Method, h.peerNode.Key().ShortString(), h.selfNode.Key().ShortString(), wr.statusCode, parseDriveFileExtensionForLog(r.URL.Path), contentType, roundTraffic(wr.contentLength), roundTraffic(bw.bytesRead)) + } + }() r.URL.Path = strings.TrimPrefix(r.URL.Path, taildrivePrefix) fs.ServeHTTPWithPerms(p, wr, r) diff --git a/ipn/ipnlocal/serve.go b/ipn/ipnlocal/serve.go index 44d63fe54a902..36738b88119f5 100644 --- a/ipn/ipnlocal/serve.go +++ b/ipn/ipnlocal/serve.go @@ -1007,8 +1007,6 @@ func allNumeric(s string) bool { } func (b *LocalBackend) webServerConfig(hostname string, forVIPService tailcfg.ServiceName, port uint16) (c ipn.WebServerConfigView, ok bool) { - key := ipn.HostPort(fmt.Sprintf("%s:%v", hostname, port)) - b.mu.Lock() defer b.mu.Unlock() @@ -1016,8 +1014,12 @@ func (b *LocalBackend) webServerConfig(hostname string, forVIPService tailcfg.Se return c, false } if forVIPService != "" { + magicDNSSuffix := b.currentNode().NetMap().MagicDNSSuffix() + fqdn := strings.Join([]string{forVIPService.WithoutPrefix(), magicDNSSuffix}, ".") + key := ipn.HostPort(net.JoinHostPort(fqdn, fmt.Sprintf("%d", port))) return b.serveConfig.FindServiceWeb(forVIPService, key) } + key := ipn.HostPort(net.JoinHostPort(hostname, fmt.Sprintf("%d", port))) return b.serveConfig.FindWeb(key) } diff --git a/ipn/ipnlocal/serve_test.go b/ipn/ipnlocal/serve_test.go index b9370f8778e6b..57d1a4745a4a3 100644 --- a/ipn/ipnlocal/serve_test.go +++ b/ipn/ipnlocal/serve_test.go @@ -918,6 +918,7 @@ func newTestBackend(t *testing.T) *LocalBackend { ID: 152, ComputedName: "some-peer", User: tailcfg.UserID(1), + Key: makeNodeKeyFromID(152), Addresses: []netip.Prefix{ netip.MustParsePrefix("100.150.151.152/32"), }, @@ -927,6 +928,7 @@ func newTestBackend(t *testing.T) *LocalBackend { ComputedName: "some-tagged-peer", Tags: []string{"tag:server", "tag:test"}, User: tailcfg.UserID(1), + Key: makeNodeKeyFromID(153), Addresses: []netip.Prefix{ netip.MustParsePrefix("100.150.151.153/32"), }, diff --git a/ipn/ipnlocal/state_test.go b/ipn/ipnlocal/state_test.go index 5d9e8b169f0a5..c29589acc698c 100644 --- a/ipn/ipnlocal/state_test.go +++ b/ipn/ipnlocal/state_test.go @@ -6,6 +6,7 @@ package ipnlocal import ( "context" "errors" + "fmt" "net/netip" "strings" "sync" @@ -20,6 +21,7 @@ import ( "tailscale.com/control/controlclient" "tailscale.com/envknob" "tailscale.com/ipn" + "tailscale.com/ipn/ipnauth" "tailscale.com/ipn/ipnstate" "tailscale.com/ipn/store/mem" "tailscale.com/net/dns" @@ -606,7 +608,7 @@ func TestStateMachine(t *testing.T) { store.awaitWrite() t.Logf("\n\nLogout") notifies.expect(5) - b.Logout(context.Background()) + b.Logout(context.Background(), ipnauth.Self) { nn := notifies.drain(5) previousCC.assertCalls("pause", "Logout", "unpause", "Shutdown") @@ -636,7 +638,7 @@ func TestStateMachine(t *testing.T) { // A second logout should be a no-op as we are in the NeedsLogin state. t.Logf("\n\nLogout2") notifies.expect(0) - b.Logout(context.Background()) + b.Logout(context.Background(), ipnauth.Self) { notifies.drain(0) cc.assertCalls() @@ -649,7 +651,7 @@ func TestStateMachine(t *testing.T) { // AuthCantContinue state. t.Logf("\n\nLogout3") notifies.expect(3) - b.Logout(context.Background()) + b.Logout(context.Background(), ipnauth.Self) { notifies.drain(0) cc.assertCalls() @@ -1013,17 +1015,13 @@ func TestEditPrefsHasNoKeys(t *testing.T) { } type testStateStorage struct { - mem mem.Store + mem.Store written atomic.Bool } -func (s *testStateStorage) ReadState(id ipn.StateKey) ([]byte, error) { - return s.mem.ReadState(id) -} - func (s *testStateStorage) WriteState(id ipn.StateKey, bs []byte) error { s.written.Store(true) - return s.mem.WriteState(id, bs) + return s.Store.WriteState(id, bs) } // awaitWrite clears the "I've seen writes" bit, in prep for a future @@ -1112,8 +1110,17 @@ func TestEngineReconfigOnStateChange(t *testing.T) { enableLogging := false connect := &ipn.MaskedPrefs{Prefs: ipn.Prefs{WantRunning: true}, WantRunningSet: true} disconnect := &ipn.MaskedPrefs{Prefs: ipn.Prefs{WantRunning: false}, WantRunningSet: true} - node1 := testNetmapForNode(1, "node-1", []netip.Prefix{netip.MustParsePrefix("100.64.1.1/32")}) - node2 := testNetmapForNode(2, "node-2", []netip.Prefix{netip.MustParsePrefix("100.64.1.2/32")}) + node1 := buildNetmapWithPeers( + makePeer(1, withName("node-1"), withAddresses(netip.MustParsePrefix("100.64.1.1/32"))), + ) + node2 := buildNetmapWithPeers( + makePeer(2, withName("node-2"), withAddresses(netip.MustParsePrefix("100.64.1.2/32"))), + ) + node3 := buildNetmapWithPeers( + makePeer(3, withName("node-3"), withAddresses(netip.MustParsePrefix("100.64.1.3/32"))), + node1.SelfNode, + node2.SelfNode, + ) routesWithQuad100 := func(extra ...netip.Prefix) []netip.Prefix { return append(extra, netip.MustParsePrefix("100.100.100.100/32")) } @@ -1308,6 +1315,40 @@ func TestEngineReconfigOnStateChange(t *testing.T) { Hosts: hostsFor(node1), }, }, + { + name: "Start/Connect/Login/WithPeers", + steps: func(t *testing.T, lb *LocalBackend, cc func() *mockControl) { + mustDo(t)(lb.Start(ipn.Options{})) + mustDo2(t)(lb.EditPrefs(connect)) + cc().authenticated(node3) + }, + wantState: ipn.Starting, + wantCfg: &wgcfg.Config{ + Name: "tailscale", + NodeID: node3.SelfNode.StableID(), + Peers: []wgcfg.Peer{ + { + PublicKey: node1.SelfNode.Key(), + DiscoKey: node1.SelfNode.DiscoKey(), + }, + { + PublicKey: node2.SelfNode.Key(), + DiscoKey: node2.SelfNode.DiscoKey(), + }, + }, + Addresses: node3.SelfNode.Addresses().AsSlice(), + }, + wantRouterCfg: &router.Config{ + SNATSubnetRoutes: true, + NetfilterMode: preftype.NetfilterOn, + LocalAddrs: node3.SelfNode.Addresses().AsSlice(), + Routes: routesWithQuad100(), + }, + wantDNSCfg: &dns.Config{ + Routes: map[dnsname.FQDN][]*dnstype.Resolver{}, + Hosts: hostsFor(node3), + }, + }, } for _, tt := range tests { @@ -1322,8 +1363,18 @@ func TestEngineReconfigOnStateChange(t *testing.T) { t.Errorf("State: got %v; want %v", gotState, tt.wantState) } + if engine.Config() != nil { + for _, p := range engine.Config().Peers { + pKey := p.PublicKey.UntypedHexString() + _, err := lb.MagicConn().ParseEndpoint(pKey) + if err != nil { + t.Errorf("ParseEndpoint(%q) failed: %v", pKey, err) + } + } + } + opts := []cmp.Option{ - cmpopts.EquateComparable(key.NodePublic{}, netip.Addr{}, netip.Prefix{}), + cmpopts.EquateComparable(key.NodePublic{}, key.DiscoPublic{}, netip.Addr{}, netip.Prefix{}), } if diff := cmp.Diff(tt.wantCfg, engine.Config(), opts...); diff != "" { t.Errorf("wgcfg.Config(+got -want): %v", diff) @@ -1338,31 +1389,75 @@ func TestEngineReconfigOnStateChange(t *testing.T) { } } -func testNetmapForNode(userID tailcfg.UserID, name string, addresses []netip.Prefix) *netmap.NetworkMap { +func buildNetmapWithPeers(self tailcfg.NodeView, peers ...tailcfg.NodeView) *netmap.NetworkMap { const ( - domain = "example.com" - magicDNSSuffix = ".test.ts.net" + firstAutoUserID = tailcfg.UserID(10000) + domain = "example.com" + magicDNSSuffix = ".test.ts.net" ) - user := &tailcfg.UserProfile{ - ID: userID, - DisplayName: name, - LoginName: strings.Join([]string{name, domain}, "@"), - } - self := &tailcfg.Node{ - ID: tailcfg.NodeID(1000 + userID), - StableID: tailcfg.StableNodeID("stable-" + name), - User: user.ID, - Name: name + magicDNSSuffix, - Addresses: addresses, - MachineAuthorized: true, + + users := make(map[tailcfg.UserID]tailcfg.UserProfileView) + makeUserForNode := func(n *tailcfg.Node) { + var user *tailcfg.UserProfile + if n.User == 0 { + n.User = firstAutoUserID + tailcfg.UserID(n.ID) + user = &tailcfg.UserProfile{ + DisplayName: n.Name, + LoginName: n.Name, + } + } else if _, ok := users[n.User]; !ok { + user = &tailcfg.UserProfile{ + DisplayName: fmt.Sprintf("User %d", n.User), + LoginName: fmt.Sprintf("user-%d", n.User), + } + } + if user != nil { + user.ID = n.User + user.LoginName = strings.Join([]string{user.LoginName, domain}, "@") + users[n.User] = user.View() + } + } + + derpmap := &tailcfg.DERPMap{ + Regions: make(map[int]*tailcfg.DERPRegion), } + makeDERPRegionForNode := func(n *tailcfg.Node) { + if n.HomeDERP == 0 { + return // no DERP region + } + if _, ok := derpmap.Regions[n.HomeDERP]; !ok { + r := &tailcfg.DERPRegion{ + RegionID: n.HomeDERP, + RegionName: fmt.Sprintf("Region %d", n.HomeDERP), + } + r.Nodes = append(r.Nodes, &tailcfg.DERPNode{ + Name: fmt.Sprintf("%da", n.HomeDERP), + RegionID: n.HomeDERP, + }) + derpmap.Regions[n.HomeDERP] = r + } + } + + updateNode := func(n tailcfg.NodeView) tailcfg.NodeView { + mut := n.AsStruct() + makeUserForNode(mut) + makeDERPRegionForNode(mut) + mut.Name = mut.Name + magicDNSSuffix + return mut.View() + } + + self = updateNode(self) + for i := range peers { + peers[i] = updateNode(peers[i]) + } + return &netmap.NetworkMap{ - SelfNode: self.View(), - Name: self.Name, - Domain: domain, - UserProfiles: map[tailcfg.UserID]tailcfg.UserProfileView{ - user.ID: user.View(), - }, + SelfNode: self, + Name: self.Name(), + Domain: domain, + Peers: peers, + UserProfiles: users, + DERPMap: derpmap, } } @@ -1403,6 +1498,7 @@ func newLocalBackendWithMockEngineAndControl(t *testing.T, enableLogging bool) ( magicConn, err := magicsock.NewConn(magicsock.Options{ Logf: logf, + EventBus: sys.Bus.Get(), NetMon: dialer.NetMon(), Metrics: sys.UserMetricsRegistry(), HealthTracker: sys.HealthTracker(), diff --git a/ipn/ipnserver/actor.go b/ipn/ipnserver/actor.go index dd40924bbf542..9d86d2c825fda 100644 --- a/ipn/ipnserver/actor.go +++ b/ipn/ipnserver/actor.go @@ -144,7 +144,7 @@ func (a *actor) Username() (string, error) { } defer tok.Close() return tok.Username() - case "darwin", "linux", "illumos", "solaris": + case "darwin", "linux", "illumos", "solaris", "openbsd": uid, ok := a.ci.Creds().UserID() if !ok { return "", errors.New("missing user ID") diff --git a/ipn/ipnstate/ipnstate.go b/ipn/ipnstate/ipnstate.go index 89c6d7e24dbc5..e7ae2d62bd6b2 100644 --- a/ipn/ipnstate/ipnstate.go +++ b/ipn/ipnstate/ipnstate.go @@ -251,9 +251,10 @@ type PeerStatus struct { PrimaryRoutes *views.Slice[netip.Prefix] `json:",omitempty"` // Endpoints: - Addrs []string - CurAddr string // one of Addrs, or unique if roaming - Relay string // DERP region + Addrs []string + CurAddr string // one of Addrs, or unique if roaming + Relay string // DERP region + PeerRelay string // peer relay address (ip:port:vni) RxBytes int64 TxBytes int64 @@ -451,6 +452,9 @@ func (sb *StatusBuilder) AddPeer(peer key.NodePublic, st *PeerStatus) { if v := st.Relay; v != "" { e.Relay = v } + if v := st.PeerRelay; v != "" { + e.PeerRelay = v + } if v := st.UserID; v != 0 { e.UserID = v } @@ -697,10 +701,17 @@ type PingResult struct { Err string LatencySeconds float64 - // Endpoint is the ip:port if direct UDP was used. - // It is not currently set for TSMP pings. + // Endpoint is a string of the form "{ip}:{port}" if direct UDP was used. It + // is not currently set for TSMP. Endpoint string + // PeerRelay is a string of the form "{ip}:{port}:vni:{vni}" if a peer + // relay was used. It is not currently set for TSMP. Note that this field + // is not omitted during JSON encoding if it contains a zero value. This is + // done for consistency with the Endpoint field; this structure is exposed + // externally via localAPI, so we want to maintain the existing convention. + PeerRelay string + // DERPRegionID is non-zero DERP region ID if DERP was used. // It is not currently set for TSMP pings. DERPRegionID int @@ -735,6 +746,7 @@ func (pr *PingResult) ToPingResponse(pingType tailcfg.PingType) *tailcfg.PingRes Err: pr.Err, LatencySeconds: pr.LatencySeconds, Endpoint: pr.Endpoint, + PeerRelay: pr.PeerRelay, DERPRegionID: pr.DERPRegionID, DERPRegionCode: pr.DERPRegionCode, PeerAPIPort: pr.PeerAPIPort, diff --git a/ipn/localapi/localapi.go b/ipn/localapi/localapi.go index 99cb7c95b4dff..0acc5a65fca8a 100644 --- a/ipn/localapi/localapi.go +++ b/ipn/localapi/localapi.go @@ -20,6 +20,7 @@ import ( "net/url" "os" "path" + "reflect" "runtime" "slices" "strconv" @@ -32,6 +33,7 @@ import ( "tailscale.com/clientupdate" "tailscale.com/drive" "tailscale.com/envknob" + "tailscale.com/health/healthmsg" "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/ipn/ipnauth" @@ -82,71 +84,74 @@ var handler = map[string]LocalAPIHandler{ // The other /localapi/v0/NAME handlers are exact matches and contain only NAME // without a trailing slash: - "alpha-set-device-attrs": (*Handler).serveSetDeviceAttrs, // see tailscale/corp#24690 - "bugreport": (*Handler).serveBugReport, - "check-ip-forwarding": (*Handler).serveCheckIPForwarding, - "check-prefs": (*Handler).serveCheckPrefs, - "check-udp-gro-forwarding": (*Handler).serveCheckUDPGROForwarding, - "component-debug-logging": (*Handler).serveComponentDebugLogging, - "debug": (*Handler).serveDebug, - "debug-derp-region": (*Handler).serveDebugDERPRegion, - "debug-dial-types": (*Handler).serveDebugDialTypes, - "debug-log": (*Handler).serveDebugLog, - "debug-packet-filter-matches": (*Handler).serveDebugPacketFilterMatches, - "debug-packet-filter-rules": (*Handler).serveDebugPacketFilterRules, - "debug-peer-endpoint-changes": (*Handler).serveDebugPeerEndpointChanges, - "debug-portmap": (*Handler).serveDebugPortmap, - "derpmap": (*Handler).serveDERPMap, - "dev-set-state-store": (*Handler).serveDevSetStateStore, - "dial": (*Handler).serveDial, - "disconnect-control": (*Handler).disconnectControl, - "dns-osconfig": (*Handler).serveDNSOSConfig, - "dns-query": (*Handler).serveDNSQuery, - "drive/fileserver-address": (*Handler).serveDriveServerAddr, - "drive/shares": (*Handler).serveShares, - "goroutines": (*Handler).serveGoroutines, - "handle-push-message": (*Handler).serveHandlePushMessage, - "id-token": (*Handler).serveIDToken, - "login-interactive": (*Handler).serveLoginInteractive, - "logout": (*Handler).serveLogout, - "logtap": (*Handler).serveLogTap, - "metrics": (*Handler).serveMetrics, - "ping": (*Handler).servePing, - "pprof": (*Handler).servePprof, - "prefs": (*Handler).servePrefs, - "query-feature": (*Handler).serveQueryFeature, - "reload-config": (*Handler).reloadConfig, - "reset-auth": (*Handler).serveResetAuth, - "serve-config": (*Handler).serveServeConfig, - "set-dns": (*Handler).serveSetDNS, - "set-expiry-sooner": (*Handler).serveSetExpirySooner, - "set-gui-visible": (*Handler).serveSetGUIVisible, - "set-push-device-token": (*Handler).serveSetPushDeviceToken, - "set-udp-gro-forwarding": (*Handler).serveSetUDPGROForwarding, - "set-use-exit-node-enabled": (*Handler).serveSetUseExitNodeEnabled, - "start": (*Handler).serveStart, - "status": (*Handler).serveStatus, - "suggest-exit-node": (*Handler).serveSuggestExitNode, - "tka/affected-sigs": (*Handler).serveTKAAffectedSigs, - "tka/cosign-recovery-aum": (*Handler).serveTKACosignRecoveryAUM, - "tka/disable": (*Handler).serveTKADisable, - "tka/force-local-disable": (*Handler).serveTKALocalDisable, - "tka/generate-recovery-aum": (*Handler).serveTKAGenerateRecoveryAUM, - "tka/init": (*Handler).serveTKAInit, - "tka/log": (*Handler).serveTKALog, - "tka/modify": (*Handler).serveTKAModify, - "tka/sign": (*Handler).serveTKASign, - "tka/status": (*Handler).serveTKAStatus, - "tka/submit-recovery-aum": (*Handler).serveTKASubmitRecoveryAUM, - "tka/verify-deeplink": (*Handler).serveTKAVerifySigningDeeplink, - "tka/wrap-preauth-key": (*Handler).serveTKAWrapPreauthKey, - "update/check": (*Handler).serveUpdateCheck, - "update/install": (*Handler).serveUpdateInstall, - "update/progress": (*Handler).serveUpdateProgress, - "upload-client-metrics": (*Handler).serveUploadClientMetrics, - "usermetrics": (*Handler).serveUserMetrics, - "watch-ipn-bus": (*Handler).serveWatchIPNBus, - "whois": (*Handler).serveWhoIs, + "alpha-set-device-attrs": (*Handler).serveSetDeviceAttrs, // see tailscale/corp#24690 + "bugreport": (*Handler).serveBugReport, + "check-ip-forwarding": (*Handler).serveCheckIPForwarding, + "check-prefs": (*Handler).serveCheckPrefs, + "check-reverse-path-filtering": (*Handler).serveCheckReversePathFiltering, + "check-udp-gro-forwarding": (*Handler).serveCheckUDPGROForwarding, + "component-debug-logging": (*Handler).serveComponentDebugLogging, + "debug": (*Handler).serveDebug, + "debug-bus-events": (*Handler).serveDebugBusEvents, + "debug-bus-graph": (*Handler).serveEventBusGraph, + "debug-derp-region": (*Handler).serveDebugDERPRegion, + "debug-dial-types": (*Handler).serveDebugDialTypes, + "debug-log": (*Handler).serveDebugLog, + "debug-packet-filter-matches": (*Handler).serveDebugPacketFilterMatches, + "debug-packet-filter-rules": (*Handler).serveDebugPacketFilterRules, + "debug-peer-endpoint-changes": (*Handler).serveDebugPeerEndpointChanges, + "debug-portmap": (*Handler).serveDebugPortmap, + "derpmap": (*Handler).serveDERPMap, + "dev-set-state-store": (*Handler).serveDevSetStateStore, + "dial": (*Handler).serveDial, + "disconnect-control": (*Handler).disconnectControl, + "dns-osconfig": (*Handler).serveDNSOSConfig, + "dns-query": (*Handler).serveDNSQuery, + "drive/fileserver-address": (*Handler).serveDriveServerAddr, + "drive/shares": (*Handler).serveShares, + "goroutines": (*Handler).serveGoroutines, + "handle-push-message": (*Handler).serveHandlePushMessage, + "id-token": (*Handler).serveIDToken, + "login-interactive": (*Handler).serveLoginInteractive, + "logout": (*Handler).serveLogout, + "logtap": (*Handler).serveLogTap, + "metrics": (*Handler).serveMetrics, + "ping": (*Handler).servePing, + "pprof": (*Handler).servePprof, + "prefs": (*Handler).servePrefs, + "query-feature": (*Handler).serveQueryFeature, + "reload-config": (*Handler).reloadConfig, + "reset-auth": (*Handler).serveResetAuth, + "serve-config": (*Handler).serveServeConfig, + "set-dns": (*Handler).serveSetDNS, + "set-expiry-sooner": (*Handler).serveSetExpirySooner, + "set-gui-visible": (*Handler).serveSetGUIVisible, + "set-push-device-token": (*Handler).serveSetPushDeviceToken, + "set-udp-gro-forwarding": (*Handler).serveSetUDPGROForwarding, + "set-use-exit-node-enabled": (*Handler).serveSetUseExitNodeEnabled, + "start": (*Handler).serveStart, + "status": (*Handler).serveStatus, + "suggest-exit-node": (*Handler).serveSuggestExitNode, + "tka/affected-sigs": (*Handler).serveTKAAffectedSigs, + "tka/cosign-recovery-aum": (*Handler).serveTKACosignRecoveryAUM, + "tka/disable": (*Handler).serveTKADisable, + "tka/force-local-disable": (*Handler).serveTKALocalDisable, + "tka/generate-recovery-aum": (*Handler).serveTKAGenerateRecoveryAUM, + "tka/init": (*Handler).serveTKAInit, + "tka/log": (*Handler).serveTKALog, + "tka/modify": (*Handler).serveTKAModify, + "tka/sign": (*Handler).serveTKASign, + "tka/status": (*Handler).serveTKAStatus, + "tka/submit-recovery-aum": (*Handler).serveTKASubmitRecoveryAUM, + "tka/verify-deeplink": (*Handler).serveTKAVerifySigningDeeplink, + "tka/wrap-preauth-key": (*Handler).serveTKAWrapPreauthKey, + "update/check": (*Handler).serveUpdateCheck, + "update/install": (*Handler).serveUpdateInstall, + "update/progress": (*Handler).serveUpdateProgress, + "upload-client-metrics": (*Handler).serveUploadClientMetrics, + "usermetrics": (*Handler).serveUserMetrics, + "watch-ipn-bus": (*Handler).serveWatchIPNBus, + "whois": (*Handler).serveWhoIs, } // Register registers a new LocalAPI handler for the given name. @@ -330,7 +335,7 @@ func (h *Handler) serveIDToken(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - httpReq, err := http.NewRequest("POST", "https://unused/machine/id-token", bytes.NewReader(b)) + httpReq, err := http.NewRequest(httpm.POST, "https://unused/machine/id-token", bytes.NewReader(b)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -353,7 +358,7 @@ func (h *Handler) serveBugReport(w http.ResponseWriter, r *http.Request) { http.Error(w, "bugreport access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "only POST allowed", http.StatusMethodNotAllowed) return } @@ -480,7 +485,7 @@ func (h *Handler) serveSetDeviceAttrs(w http.ResponseWriter, r *http.Request) { http.Error(w, "set-device-attrs access denied", http.StatusForbidden) return } - if r.Method != "PATCH" { + if r.Method != httpm.PATCH { http.Error(w, "only PATCH allowed", http.StatusMethodNotAllowed) return } @@ -585,7 +590,7 @@ func (h *Handler) serveLogTap(w http.ResponseWriter, r *http.Request) { http.Error(w, "logtap access denied", http.StatusForbidden) return } - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "GET required", http.StatusMethodNotAllowed) return } @@ -637,7 +642,7 @@ func (h *Handler) serveDebug(w http.ResponseWriter, r *http.Request) { http.Error(w, "debug access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "POST required", http.StatusMethodNotAllowed) return } @@ -692,6 +697,15 @@ func (h *Handler) serveDebug(w http.ResponseWriter, r *http.Request) { break } h.b.DebugForcePreferDERP(n) + case "peer-relay-servers": + servers := h.b.DebugPeerRelayServers().Slice() + slices.SortFunc(servers, func(a, b netip.Addr) int { + return a.Compare(b) + }) + err = json.NewEncoder(w).Encode(servers) + if err == nil { + return + } case "": err = fmt.Errorf("missing parameter 'action'") default: @@ -710,7 +724,7 @@ func (h *Handler) serveDevSetStateStore(w http.ResponseWriter, r *http.Request) http.Error(w, "debug access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "POST required", http.StatusMethodNotAllowed) return } @@ -915,6 +929,131 @@ func (h *Handler) serveDebugPortmap(w http.ResponseWriter, r *http.Request) { } } +// EventError provides the JSON encoding of internal errors from event processing. +type EventError struct { + Error string +} + +// serveDebugBusEvents taps into the tailscaled/utils/eventbus and streams +// events to the client. +func (h *Handler) serveDebugBusEvents(w http.ResponseWriter, r *http.Request) { + // Require write access (~root) as the logs could contain something + // sensitive. + if !h.PermitWrite { + http.Error(w, "event bus access denied", http.StatusForbidden) + return + } + if r.Method != httpm.GET { + http.Error(w, "GET required", http.StatusMethodNotAllowed) + return + } + + bus, ok := h.LocalBackend().Sys().Bus.GetOK() + if !ok { + http.Error(w, "event bus not running", http.StatusNoContent) + return + } + + f, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", http.StatusInternalServerError) + return + } + + io.WriteString(w, `{"Event":"[event listener connected]\n"}`+"\n") + f.Flush() + + mon := bus.Debugger().WatchBus() + defer mon.Close() + + i := 0 + for { + select { + case <-r.Context().Done(): + fmt.Fprintf(w, `{"Event":"[event listener closed]\n"}`) + return + case <-mon.Done(): + return + case event := <-mon.Events(): + data := eventbus.DebugEvent{ + Count: i, + Type: reflect.TypeOf(event.Event).String(), + Event: event.Event, + From: event.From.Name(), + } + for _, client := range event.To { + data.To = append(data.To, client.Name()) + } + + if msg, err := json.Marshal(data); err != nil { + data.Event = EventError{Error: fmt.Sprintf( + "failed to marshal JSON for %T", event.Event, + )} + if errMsg, err := json.Marshal(data); err != nil { + fmt.Fprintf(w, + `{"Count": %d, "Event":"[ERROR] failed to marshal JSON for %T\n"}`, + i, event.Event) + } else { + w.Write(errMsg) + } + } else { + w.Write(msg) + } + f.Flush() + i++ + } + } +} + +// serveEventBusGraph taps into the event bus and dumps out the active graph of +// publishers and subscribers. It does not represent anything about the messages +// exchanged. +func (h *Handler) serveEventBusGraph(w http.ResponseWriter, r *http.Request) { + if r.Method != httpm.GET { + http.Error(w, "GET required", http.StatusMethodNotAllowed) + return + } + + bus, ok := h.LocalBackend().Sys().Bus.GetOK() + if !ok { + http.Error(w, "event bus not running", http.StatusPreconditionFailed) + return + } + + debugger := bus.Debugger() + clients := debugger.Clients() + + graph := map[string]eventbus.DebugTopic{} + + for _, client := range clients { + for _, pub := range debugger.PublishTypes(client) { + topic, ok := graph[pub.Name()] + if !ok { + topic = eventbus.DebugTopic{Name: pub.Name()} + } + topic.Publisher = client.Name() + graph[pub.Name()] = topic + } + for _, sub := range debugger.SubscribeTypes(client) { + topic, ok := graph[sub.Name()] + if !ok { + topic = eventbus.DebugTopic{Name: sub.Name()} + } + topic.Subscribers = append(topic.Subscribers, client.Name()) + graph[sub.Name()] = topic + } + } + + // The top level map is not really needed for the client, convert to a list. + topics := eventbus.DebugTopics{} + for _, v := range graph { + topics.Topics = append(topics.Topics, v) + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(topics) +} + func (h *Handler) serveComponentDebugLogging(w http.ResponseWriter, r *http.Request) { if !h.PermitWrite { http.Error(w, "debug access denied", http.StatusForbidden) @@ -1076,7 +1215,7 @@ func (h *Handler) serveResetAuth(w http.ResponseWriter, r *http.Request) { func (h *Handler) serveServeConfig(w http.ResponseWriter, r *http.Request) { switch r.Method { - case "GET": + case httpm.GET: if !h.PermitRead { http.Error(w, "serve config denied", http.StatusForbidden) return @@ -1092,7 +1231,7 @@ func (h *Handler) serveServeConfig(w http.ResponseWriter, r *http.Request) { w.Header().Set("Etag", etag) w.Header().Set("Content-Type", "application/json") w.Write(bts) - case "POST": + case httpm.POST: if !h.PermitWrite { http.Error(w, "serve config denied", http.StatusForbidden) return @@ -1155,7 +1294,6 @@ func authorizeServeConfigForGOOSAndUserContext(goos string, configIn *ipn.ServeC // should never happen. panic("unreachable") } - } func (h *Handler) serveCheckIPForwarding(w http.ResponseWriter, r *http.Request) { @@ -1175,6 +1313,32 @@ func (h *Handler) serveCheckIPForwarding(w http.ResponseWriter, r *http.Request) }) } +func (h *Handler) serveCheckReversePathFiltering(w http.ResponseWriter, r *http.Request) { + if !h.PermitRead { + http.Error(w, "reverse path filtering check access denied", http.StatusForbidden) + return + } + var warning string + + state := h.b.Sys().NetMon.Get().InterfaceState() + warn, err := netutil.CheckReversePathFiltering(state) + if err == nil && len(warn) > 0 { + var msg strings.Builder + msg.WriteString(healthmsg.WarnExitNodeUsage + ":\n") + for _, w := range warn { + msg.WriteString("- " + w + "\n") + } + msg.WriteString(healthmsg.DisableRPFilter) + warning = msg.String() + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(struct { + Warning string + }{ + Warning: warning, + }) +} + func (h *Handler) serveCheckUDPGROForwarding(w http.ResponseWriter, r *http.Request) { if !h.PermitRead { http.Error(w, "UDP GRO forwarding check access denied", http.StatusForbidden) @@ -1263,7 +1427,7 @@ func (h *Handler) serveDebugPeerEndpointChanges(w http.ResponseWriter, r *http.R // (in ipnserver.Server) provides the blocking until the connection is no longer // in use. func InUseOtherUserIPNStream(w http.ResponseWriter, r *http.Request, err error) (handled bool) { - if r.Method != "GET" || r.URL.Path != "/localapi/v0/watch-ipn-bus" { + if r.Method != httpm.GET || r.URL.Path != "/localapi/v0/watch-ipn-bus" { return false } js, err := json.Marshal(&ipn.Notify{ @@ -1328,7 +1492,7 @@ func (h *Handler) serveLoginInteractive(w http.ResponseWriter, r *http.Request) http.Error(w, "login access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "want POST", http.StatusBadRequest) return } @@ -1342,7 +1506,7 @@ func (h *Handler) serveStart(w http.ResponseWriter, r *http.Request) { http.Error(w, "access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "want POST", http.StatusBadRequest) return } @@ -1365,11 +1529,11 @@ func (h *Handler) serveLogout(w http.ResponseWriter, r *http.Request) { http.Error(w, "logout access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "want POST", http.StatusBadRequest) return } - err := h.b.Logout(r.Context()) + err := h.b.Logout(r.Context(), h.Actor) if err == nil { w.WriteHeader(http.StatusNoContent) return @@ -1384,7 +1548,7 @@ func (h *Handler) servePrefs(w http.ResponseWriter, r *http.Request) { } var prefs ipn.PrefsView switch r.Method { - case "PATCH": + case httpm.PATCH: if !h.PermitWrite { http.Error(w, "prefs write access denied", http.StatusForbidden) return @@ -1408,7 +1572,7 @@ func (h *Handler) servePrefs(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(resJSON{Error: err.Error()}) return } - case "GET", "HEAD": + case httpm.GET, httpm.HEAD: prefs = h.b.Prefs() default: http.Error(w, "unsupported method", http.StatusMethodNotAllowed) @@ -1448,9 +1612,9 @@ func (h *Handler) servePolicy(w http.ResponseWriter, r *http.Request) { var effectivePolicy *setting.Snapshot switch r.Method { - case "GET": + case httpm.GET: effectivePolicy = policy.Get() - case "POST": + case httpm.POST: effectivePolicy, err = policy.Reload() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1476,7 +1640,7 @@ func (h *Handler) serveCheckPrefs(w http.ResponseWriter, r *http.Request) { http.Error(w, "checkprefs access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "unsupported method", http.StatusMethodNotAllowed) return } @@ -1514,7 +1678,7 @@ func (h *Handler) serveSetDNS(w http.ResponseWriter, r *http.Request) { http.Error(w, "access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "want POST", http.StatusBadRequest) return } @@ -1529,7 +1693,7 @@ func (h *Handler) serveSetDNS(w http.ResponseWriter, r *http.Request) { } func (h *Handler) serveDERPMap(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "want GET", http.StatusBadRequest) return } @@ -1546,7 +1710,7 @@ func (h *Handler) serveSetExpirySooner(w http.ResponseWriter, r *http.Request) { http.Error(w, "access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "POST required", http.StatusMethodNotAllowed) return } @@ -1574,7 +1738,7 @@ func (h *Handler) serveSetExpirySooner(w http.ResponseWriter, r *http.Request) { func (h *Handler) servePing(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "want POST", http.StatusBadRequest) return } @@ -1620,7 +1784,7 @@ func (h *Handler) servePing(w http.ResponseWriter, r *http.Request) { } func (h *Handler) serveDial(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "POST required", http.StatusMethodNotAllowed) return } @@ -1683,7 +1847,7 @@ func (h *Handler) serveSetPushDeviceToken(w http.ResponseWriter, r *http.Request http.Error(w, "set push device token access denied", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "unsupported method", http.StatusMethodNotAllowed) return } @@ -1701,7 +1865,7 @@ func (h *Handler) serveHandlePushMessage(w http.ResponseWriter, r *http.Request) http.Error(w, "handle push message not allowed", http.StatusForbidden) return } - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "unsupported method", http.StatusMethodNotAllowed) return } @@ -1718,7 +1882,7 @@ func (h *Handler) serveHandlePushMessage(w http.ResponseWriter, r *http.Request) } func (h *Handler) serveUploadClientMetrics(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "unsupported method", http.StatusMethodNotAllowed) return } @@ -1819,7 +1983,7 @@ func (h *Handler) serveSetUseExitNodeEnabled(w http.ResponseWriter, r *http.Requ http.Error(w, "invalid 'enabled' parameter", http.StatusBadRequest) return } - prefs, err := h.b.SetUseExitNodeEnabled(v) + prefs, err := h.b.SetUseExitNodeEnabled(h.Actor, v) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -1879,6 +2043,11 @@ func (h *Handler) serveTKAInit(w http.ResponseWriter, r *http.Request) { return } + if !h.b.NetworkLockAllowed() { + http.Error(w, "Tailnet Lock is not supported on your pricing plan", http.StatusForbidden) + return + } + if err := h.b.NetworkLockInit(req.Keys, req.DisablementValues, req.SupportDisablement); err != nil { http.Error(w, "initialization failed: "+err.Error(), http.StatusInternalServerError) return @@ -2309,7 +2478,7 @@ func (h *Handler) serveQueryFeature(w http.ResponseWriter, r *http.Request) { } req, err := http.NewRequestWithContext(r.Context(), - "POST", "https://unused/machine/feature/query", bytes.NewReader(b)) + httpm.POST, "https://unused/machine/feature/query", bytes.NewReader(b)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -2388,7 +2557,7 @@ func (h *Handler) serveDebugLog(w http.ResponseWriter, r *http.Request) { // Effectively, it tells us whether serveUpdateInstall will be able to install // an update for us. func (h *Handler) serveUpdateCheck(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "only GET allowed", http.StatusMethodNotAllowed) return } @@ -2417,7 +2586,7 @@ func (h *Handler) serveUpdateCheck(w http.ResponseWriter, r *http.Request) { // serveUpdateProgress after pinging this endpoint to check how the update is // going. func (h *Handler) serveUpdateInstall(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { + if r.Method != httpm.POST { http.Error(w, "only POST allowed", http.StatusMethodNotAllowed) return } @@ -2432,7 +2601,7 @@ func (h *Handler) serveUpdateInstall(w http.ResponseWriter, r *http.Request) { // log messages in order from oldest to newest. If an update is not in progress, // the returned slice will be empty. func (h *Handler) serveUpdateProgress(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "only GET allowed", http.StatusMethodNotAllowed) return } @@ -2488,7 +2657,7 @@ func (h *Handler) serveDNSOSConfig(w http.ResponseWriter, r *http.Request) { // // The response if successful is a DNSQueryResponse JSON object. func (h *Handler) serveDNSQuery(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "only GET allowed", http.StatusMethodNotAllowed) return } @@ -2525,7 +2694,7 @@ func (h *Handler) serveDNSQuery(w http.ResponseWriter, r *http.Request) { // serveDriveServerAddr handles updates of the Taildrive file server address. func (h *Handler) serveDriveServerAddr(w http.ResponseWriter, r *http.Request) { - if r.Method != "PUT" { + if r.Method != httpm.PUT { http.Error(w, "only PUT allowed", http.StatusMethodNotAllowed) return } @@ -2552,7 +2721,7 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) { return } switch r.Method { - case "PUT": + case httpm.PUT: var share drive.Share err := json.NewDecoder(r.Body).Decode(&share) if err != nil { @@ -2588,7 +2757,7 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) { return } w.WriteHeader(http.StatusCreated) - case "DELETE": + case httpm.DELETE: b, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -2604,7 +2773,7 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) { return } w.WriteHeader(http.StatusNoContent) - case "POST": + case httpm.POST: var names [2]string err := json.NewDecoder(r.Body).Decode(&names) if err != nil { @@ -2629,7 +2798,7 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) { return } w.WriteHeader(http.StatusNoContent) - case "GET": + case httpm.GET: shares := h.b.DriveGetShares() err := json.NewEncoder(w).Encode(shares) if err != nil { @@ -2643,7 +2812,7 @@ func (h *Handler) serveShares(w http.ResponseWriter, r *http.Request) { // serveSuggestExitNode serves a POST endpoint for returning a suggested exit node. func (h *Handler) serveSuggestExitNode(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { + if r.Method != httpm.GET { http.Error(w, "only GET allowed", http.StatusMethodNotAllowed) return } diff --git a/ipn/prefs.go b/ipn/prefs.go index 01275a7e25bdc..71a80b1828760 100644 --- a/ipn/prefs.go +++ b/ipn/prefs.go @@ -94,6 +94,25 @@ type Prefs struct { ExitNodeID tailcfg.StableNodeID ExitNodeIP netip.Addr + // AutoExitNode is an optional expression that specifies whether and how + // tailscaled should pick an exit node automatically. + // + // If specified, tailscaled will use an exit node based on the expression, + // and will re-evaluate the selection periodically as network conditions, + // available exit nodes, or policy settings change. A blackhole route will + // be installed to prevent traffic from escaping to the local network until + // an exit node is selected. It takes precedence over ExitNodeID and ExitNodeIP. + // + // If empty, tailscaled will not automatically select an exit node. + // + // If the specified expression is invalid or unsupported by the client, + // it falls back to the behavior of [AnyExitNode]. + // + // As of 2025-07-02, the only supported value is [AnyExitNode]. + // It's a string rather than a boolean to allow future extensibility + // (e.g., AutoExitNode = "mullvad" or AutoExitNode = "geo:us"). + AutoExitNode ExitNodeExpression `json:",omitempty"` + // InternalExitNodePrior is the most recently used ExitNodeID in string form. It is set by // the backend on transition from exit node on to off and used by the // backend. @@ -325,6 +344,7 @@ type MaskedPrefs struct { RouteAllSet bool `json:",omitempty"` ExitNodeIDSet bool `json:",omitempty"` ExitNodeIPSet bool `json:",omitempty"` + AutoExitNodeSet bool `json:",omitempty"` InternalExitNodePriorSet bool `json:",omitempty"` // Internal; can't be set by LocalAPI clients ExitNodeAllowLANAccessSet bool `json:",omitempty"` CorpDNSSet bool `json:",omitempty"` @@ -533,6 +553,9 @@ func (p *Prefs) pretty(goos string) string { } else if !p.ExitNodeID.IsZero() { fmt.Fprintf(&sb, "exit=%v lan=%t ", p.ExitNodeID, p.ExitNodeAllowLANAccess) } + if p.AutoExitNode.IsSet() { + fmt.Fprintf(&sb, "auto=%v ", p.AutoExitNode) + } if len(p.AdvertiseRoutes) > 0 || goos == "linux" { fmt.Fprintf(&sb, "routes=%v ", p.AdvertiseRoutes) } @@ -609,6 +632,7 @@ func (p *Prefs) Equals(p2 *Prefs) bool { p.RouteAll == p2.RouteAll && p.ExitNodeID == p2.ExitNodeID && p.ExitNodeIP == p2.ExitNodeIP && + p.AutoExitNode == p2.AutoExitNode && p.InternalExitNodePrior == p2.InternalExitNodePrior && p.ExitNodeAllowLANAccess == p2.ExitNodeAllowLANAccess && p.CorpDNS == p2.CorpDNS && @@ -804,6 +828,7 @@ func isRemoteIP(st *ipnstate.Status, ip netip.Addr) bool { func (p *Prefs) ClearExitNode() { p.ExitNodeID = "" p.ExitNodeIP = netip.Addr{} + p.AutoExitNode = "" } // ExitNodeLocalIPError is returned when the requested IP address for an exit @@ -1043,3 +1068,45 @@ func (p *LoginProfile) Equals(p2 *LoginProfile) bool { p.LocalUserID == p2.LocalUserID && p.ControlURL == p2.ControlURL } + +// ExitNodeExpression is a string that specifies how an exit node +// should be selected. An empty string means that no exit node +// should be selected. +// +// As of 2025-07-02, the only supported value is [AnyExitNode]. +type ExitNodeExpression string + +// AnyExitNode indicates that the exit node should be automatically +// selected from the pool of available exit nodes, excluding any +// disallowed by policy (e.g., [syspolicy.AllowedSuggestedExitNodes]). +// The exact implementation is subject to change, but exit nodes +// offering the best performance will be preferred. +const AnyExitNode ExitNodeExpression = "any" + +// IsSet reports whether the expression is non-empty and can be used +// to select an exit node. +func (e ExitNodeExpression) IsSet() bool { + return e != "" +} + +const ( + // AutoExitNodePrefix is the prefix used in [syspolicy.ExitNodeID] values and CLI + // to indicate that the string following the prefix is an [ipn.ExitNodeExpression]. + AutoExitNodePrefix = "auto:" +) + +// ParseAutoExitNodeString attempts to parse the given string +// as an [ExitNodeExpression]. +// +// It returns the parsed expression and true on success, +// or an empty string and false if the input does not appear to be +// an [ExitNodeExpression] (i.e., it doesn't start with "auto:"). +// +// It is mainly used to parse the [syspolicy.ExitNodeID] value +// when it is set to "auto:" (e.g., auto:any). +func ParseAutoExitNodeString[T ~string](s T) (_ ExitNodeExpression, ok bool) { + if expr, ok := strings.CutPrefix(string(s), AutoExitNodePrefix); ok && expr != "" { + return ExitNodeExpression(expr), true + } + return "", false +} diff --git a/ipn/prefs_test.go b/ipn/prefs_test.go index d28d161db422e..43e360c6af0c2 100644 --- a/ipn/prefs_test.go +++ b/ipn/prefs_test.go @@ -40,6 +40,7 @@ func TestPrefsEqual(t *testing.T) { "RouteAll", "ExitNodeID", "ExitNodeIP", + "AutoExitNode", "InternalExitNodePrior", "ExitNodeAllowLANAccess", "CorpDNS", @@ -150,6 +151,17 @@ func TestPrefsEqual(t *testing.T) { true, }, + { + &Prefs{AutoExitNode: ""}, + &Prefs{AutoExitNode: "auto:any"}, + false, + }, + { + &Prefs{AutoExitNode: "auto:any"}, + &Prefs{AutoExitNode: "auto:any"}, + true, + }, + { &Prefs{}, &Prefs{ExitNodeAllowLANAccess: true}, @@ -1117,3 +1129,62 @@ func TestPrefsDowngrade(t *testing.T) { t.Fatal("AllowSingleHosts should be true") } } + +func TestParseAutoExitNodeString(t *testing.T) { + tests := []struct { + name string + exitNodeID string + wantOk bool + wantExpr ExitNodeExpression + }{ + { + name: "empty expr", + exitNodeID: "", + wantOk: false, + wantExpr: "", + }, + { + name: "no auto prefix", + exitNodeID: "foo", + wantOk: false, + wantExpr: "", + }, + { + name: "auto:any", + exitNodeID: "auto:any", + wantOk: true, + wantExpr: AnyExitNode, + }, + { + name: "auto:foo", + exitNodeID: "auto:foo", + wantOk: true, + wantExpr: "foo", + }, + { + name: "auto prefix but empty suffix", + exitNodeID: "auto:", + wantOk: false, + wantExpr: "", + }, + { + name: "auto prefix no colon", + exitNodeID: "auto", + wantOk: false, + wantExpr: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotExpr, gotOk := ParseAutoExitNodeString(tt.exitNodeID) + if gotOk != tt.wantOk || gotExpr != tt.wantExpr { + if tt.wantOk { + t.Fatalf("got %v (%q); want %v (%q)", gotOk, gotExpr, tt.wantOk, tt.wantExpr) + } else { + t.Fatalf("got %v (%q); want false", gotOk, gotExpr) + } + } + }) + } +} diff --git a/ipn/serve.go b/ipn/serve.go index ac92287bdc08f..a0f1334d7d150 100644 --- a/ipn/serve.go +++ b/ipn/serve.go @@ -166,26 +166,44 @@ type HTTPHandler struct { // WebHandlerExists reports whether if the ServeConfig Web handler exists for // the given host:port and mount point. -func (sc *ServeConfig) WebHandlerExists(hp HostPort, mount string) bool { - h := sc.GetWebHandler(hp, mount) +func (sc *ServeConfig) WebHandlerExists(svcName tailcfg.ServiceName, hp HostPort, mount string) bool { + h := sc.GetWebHandler(svcName, hp, mount) return h != nil } // GetWebHandler returns the HTTPHandler for the given host:port and mount point. // Returns nil if the handler does not exist. -func (sc *ServeConfig) GetWebHandler(hp HostPort, mount string) *HTTPHandler { - if sc == nil || sc.Web[hp] == nil { +func (sc *ServeConfig) GetWebHandler(svcName tailcfg.ServiceName, hp HostPort, mount string) *HTTPHandler { + if sc == nil { + return nil + } + if svcName != "" { + if svc, ok := sc.Services[svcName]; ok && svc.Web != nil { + if webCfg, ok := svc.Web[hp]; ok { + return webCfg.Handlers[mount] + } + } + return nil + } + if sc.Web[hp] == nil { return nil } return sc.Web[hp].Handlers[mount] } -// GetTCPPortHandler returns the TCPPortHandler for the given port. -// If the port is not configured, nil is returned. -func (sc *ServeConfig) GetTCPPortHandler(port uint16) *TCPPortHandler { +// GetTCPPortHandler returns the TCPPortHandler for the given port. If the port +// is not configured, nil is returned. Parameter svcName can be tailcfg.NoService +// for local serve or a service name for a service hosted on node. +func (sc *ServeConfig) GetTCPPortHandler(port uint16, svcName tailcfg.ServiceName) *TCPPortHandler { if sc == nil { return nil } + if svcName != "" { + if svc, ok := sc.Services[svcName]; ok && svc != nil { + return svc.TCP[port] + } + return nil + } return sc.TCP[port] } @@ -227,34 +245,78 @@ func (sc *ServeConfig) IsTCPForwardingAny() bool { return false } -// IsTCPForwardingOnPort reports whether if ServeConfig is currently forwarding -// in TCPForward mode on the given port. This is exclusive of Web/HTTPS serving. -func (sc *ServeConfig) IsTCPForwardingOnPort(port uint16) bool { - if sc == nil || sc.TCP[port] == nil { +// IsTCPForwardingOnPort reports whether ServeConfig is currently forwarding +// in TCPForward mode on the given port for local or a service. svcName will +// either be noService (empty string) for local serve or a serviceName for service +// hosted on node. Notice TCPForwarding is exclusive with Web/HTTPS serving. +func (sc *ServeConfig) IsTCPForwardingOnPort(port uint16, svcName tailcfg.ServiceName) bool { + if sc == nil { + return false + } + + if svcName != "" { + svc, ok := sc.Services[svcName] + if !ok || svc == nil { + return false + } + if svc.TCP[port] == nil { + return false + } + } else if sc.TCP[port] == nil { return false } - return !sc.IsServingWeb(port) + return !sc.IsServingWeb(port, svcName) } -// IsServingWeb reports whether if ServeConfig is currently serving Web -// (HTTP/HTTPS) on the given port. This is exclusive of TCPForwarding. -func (sc *ServeConfig) IsServingWeb(port uint16) bool { - return sc.IsServingHTTP(port) || sc.IsServingHTTPS(port) +// IsServingWeb reports whether ServeConfig is currently serving Web (HTTP/HTTPS) +// on the given port for local or a service. svcName will be either tailcfg.NoService, +// or a serviceName for service hosted on node. This is exclusive with TCPForwarding. +func (sc *ServeConfig) IsServingWeb(port uint16, svcName tailcfg.ServiceName) bool { + return sc.IsServingHTTP(port, svcName) || sc.IsServingHTTPS(port, svcName) } -// IsServingHTTPS reports whether if ServeConfig is currently serving HTTPS on -// the given port. This is exclusive of HTTP and TCPForwarding. -func (sc *ServeConfig) IsServingHTTPS(port uint16) bool { - if sc == nil || sc.TCP[port] == nil { +// IsServingHTTPS reports whether ServeConfig is currently serving HTTPS on +// the given port for local or a service. svcName will be either tailcfg.NoService +// for local serve, or a serviceName for service hosted on node. This is exclusive +// with HTTP and TCPForwarding. +func (sc *ServeConfig) IsServingHTTPS(port uint16, svcName tailcfg.ServiceName) bool { + if sc == nil { + return false + } + var tcpHandlers map[uint16]*TCPPortHandler + if svcName != "" { + if svc := sc.Services[svcName]; svc != nil { + tcpHandlers = svc.TCP + } + } else { + tcpHandlers = sc.TCP + } + + th := tcpHandlers[port] + if th == nil { return false } - return sc.TCP[port].HTTPS + return th.HTTPS } -// IsServingHTTP reports whether if ServeConfig is currently serving HTTP on the -// given port. This is exclusive of HTTPS and TCPForwarding. -func (sc *ServeConfig) IsServingHTTP(port uint16) bool { - if sc == nil || sc.TCP[port] == nil { +// IsServingHTTP reports whether ServeConfig is currently serving HTTP on the +// given port for local or a service. svcName will be either tailcfg.NoService for +// local serve, or a serviceName for service hosted on node. This is exclusive +// with HTTPS and TCPForwarding. +func (sc *ServeConfig) IsServingHTTP(port uint16, svcName tailcfg.ServiceName) bool { + if sc == nil { + return false + } + if svcName != "" { + if svc := sc.Services[svcName]; svc != nil { + if svc.TCP[port] != nil { + return svc.TCP[port].HTTP + } + } + return false + } + + if sc.TCP[port] == nil { return false } return sc.TCP[port].HTTP @@ -280,21 +342,38 @@ func (sc *ServeConfig) FindConfig(port uint16) (*ServeConfig, bool) { // SetWebHandler sets the given HTTPHandler at the specified host, port, // and mount in the serve config. sc.TCP is also updated to reflect web -// serving usage of the given port. -func (sc *ServeConfig) SetWebHandler(handler *HTTPHandler, host string, port uint16, mount string, useTLS bool) { +// serving usage of the given port. The st argument is needed when setting +// a web handler for a service, otherwise it can be nil. mds is the Magic DNS +// suffix, which is used to recreate serve's host. +func (sc *ServeConfig) SetWebHandler(handler *HTTPHandler, host string, port uint16, mount string, useTLS bool, mds string) { if sc == nil { sc = new(ServeConfig) } - mak.Set(&sc.TCP, port, &TCPPortHandler{HTTPS: useTLS, HTTP: !useTLS}) - hp := HostPort(net.JoinHostPort(host, strconv.Itoa(int(port)))) - if _, ok := sc.Web[hp]; !ok { - mak.Set(&sc.Web, hp, new(WebServerConfig)) + tcpMap := &sc.TCP + webServerMap := &sc.Web + hostName := host + if svcName := tailcfg.AsServiceName(host); svcName != "" { + hostName = strings.Join([]string{svcName.WithoutPrefix(), mds}, ".") + svc, ok := sc.Services[svcName] + if !ok { + svc = new(ServiceConfig) + mak.Set(&sc.Services, svcName, svc) + } + tcpMap = &svc.TCP + webServerMap = &svc.Web } - mak.Set(&sc.Web[hp].Handlers, mount, handler) + mak.Set(tcpMap, port, &TCPPortHandler{HTTPS: useTLS, HTTP: !useTLS}) + hp := HostPort(net.JoinHostPort(hostName, strconv.Itoa(int(port)))) + webCfg, ok := (*webServerMap)[hp] + if !ok { + webCfg = new(WebServerConfig) + mak.Set(webServerMap, hp, webCfg) + } + mak.Set(&webCfg.Handlers, mount, handler) // TODO(tylersmalley): handle multiple web handlers from foreground mode - for k, v := range sc.Web[hp].Handlers { + for k, v := range webCfg.Handlers { if v == handler { continue } @@ -305,7 +384,7 @@ func (sc *ServeConfig) SetWebHandler(handler *HTTPHandler, host string, port uin m1 := strings.TrimSuffix(mount, "/") m2 := strings.TrimSuffix(k, "/") if m1 == m2 { - delete(sc.Web[hp].Handlers, k) + delete(webCfg.Handlers, k) } } } @@ -318,9 +397,19 @@ func (sc *ServeConfig) SetTCPForwarding(port uint16, fwdAddr string, terminateTL if sc == nil { sc = new(ServeConfig) } - mak.Set(&sc.TCP, port, &TCPPortHandler{TCPForward: fwdAddr}) + tcpPortHandler := &sc.TCP + if svcName := tailcfg.AsServiceName(host); svcName != "" { + svcConfig, ok := sc.Services[svcName] + if !ok { + svcConfig = new(ServiceConfig) + mak.Set(&sc.Services, svcName, svcConfig) + } + tcpPortHandler = &svcConfig.TCP + } + mak.Set(tcpPortHandler, port, &TCPPortHandler{TCPForward: fwdAddr}) + if terminateTLS { - sc.TCP[port].TerminateTLS = host + (*tcpPortHandler)[port].TerminateTLS = host } } @@ -344,9 +433,9 @@ func (sc *ServeConfig) SetFunnel(host string, port uint16, setOn bool) { } } -// RemoveWebHandler deletes the web handlers at all of the given mount points -// for the provided host and port in the serve config. If cleanupFunnel is -// true, this also removes the funnel value for this port if no handlers remain. +// RemoveWebHandler deletes the web handlers at all of the given mount points for the +// provided host and port in the serve config for the node (as opposed to a service). +// If cleanupFunnel is true, this also removes the funnel value for this port if no handlers remain. func (sc *ServeConfig) RemoveWebHandler(host string, port uint16, mounts []string, cleanupFunnel bool) { hp := HostPort(net.JoinHostPort(host, strconv.Itoa(int(port)))) @@ -374,9 +463,50 @@ func (sc *ServeConfig) RemoveWebHandler(host string, port uint16, mounts []strin } } +// RemoveServiceWebHandler deletes the web handlers at all of the given mount points +// for the provided host and port in the serve config for the given service. +func (sc *ServeConfig) RemoveServiceWebHandler(svcName tailcfg.ServiceName, hostName string, port uint16, mounts []string) { + hp := HostPort(net.JoinHostPort(hostName, strconv.Itoa(int(port)))) + + svc, ok := sc.Services[svcName] + if !ok || svc == nil { + return + } + + // Delete existing handler, then cascade delete if empty. + for _, m := range mounts { + delete(svc.Web[hp].Handlers, m) + } + if len(svc.Web[hp].Handlers) == 0 { + delete(svc.Web, hp) + delete(svc.TCP, port) + } + if len(svc.Web) == 0 && len(svc.TCP) == 0 { + delete(sc.Services, svcName) + } + if len(sc.Services) == 0 { + sc.Services = nil + } +} + // RemoveTCPForwarding deletes the TCP forwarding configuration for the given // port from the serve config. -func (sc *ServeConfig) RemoveTCPForwarding(port uint16) { +func (sc *ServeConfig) RemoveTCPForwarding(svcName tailcfg.ServiceName, port uint16) { + if svcName != "" { + if svc := sc.Services[svcName]; svc != nil { + delete(svc.TCP, port) + if len(svc.TCP) == 0 { + svc.TCP = nil + } + if len(svc.Web) == 0 && len(svc.TCP) == 0 { + delete(sc.Services, svcName) + } + if len(sc.Services) == 0 { + sc.Services = nil + } + } + return + } delete(sc.TCP, port) if len(sc.TCP) == 0 { sc.TCP = nil diff --git a/ipn/serve_test.go b/ipn/serve_test.go index ae1d56eef6b09..7028c1e17cd71 100644 --- a/ipn/serve_test.go +++ b/ipn/serve_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package ipn import ( @@ -127,6 +128,121 @@ func TestHasPathHandler(t *testing.T) { } } +func TestIsTCPForwardingOnPort(t *testing.T) { + tests := []struct { + name string + cfg ServeConfig + svcName tailcfg.ServiceName + port uint16 + want bool + }{ + { + name: "empty-config", + cfg: ServeConfig{}, + svcName: "", + port: 80, + want: false, + }, + { + name: "node-tcp-config-match", + cfg: ServeConfig{ + TCP: map[uint16]*TCPPortHandler{80: {TCPForward: "10.0.0.123:3000"}}, + }, + svcName: "", + port: 80, + want: true, + }, + { + name: "node-tcp-config-no-match", + cfg: ServeConfig{ + TCP: map[uint16]*TCPPortHandler{80: {TCPForward: "10.0.0.123:3000"}}, + }, + svcName: "", + port: 443, + want: false, + }, + { + name: "node-tcp-config-no-match-with-service", + cfg: ServeConfig{ + TCP: map[uint16]*TCPPortHandler{80: {TCPForward: "10.0.0.123:3000"}}, + }, + svcName: "svc:bar", + port: 80, + want: false, + }, + { + name: "node-web-config-no-match", + cfg: ServeConfig{ + TCP: map[uint16]*TCPPortHandler{80: {HTTPS: true}}, + Web: map[HostPort]*WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*HTTPHandler{ + "/": {Text: "Hello, world!"}, + }, + }, + }, + }, + svcName: "", + port: 80, + want: false, + }, + { + name: "service-tcp-config-match", + cfg: ServeConfig{ + Services: map[tailcfg.ServiceName]*ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*TCPPortHandler{80: {TCPForward: "10.0.0.123:3000"}}, + }, + }, + }, + svcName: "svc:foo", + port: 80, + want: true, + }, + { + name: "service-tcp-config-no-match", + cfg: ServeConfig{ + Services: map[tailcfg.ServiceName]*ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*TCPPortHandler{80: {TCPForward: "10.0.0.123:3000"}}, + }, + }, + }, + svcName: "svc:bar", + port: 80, + want: false, + }, + { + name: "service-web-config-no-match", + cfg: ServeConfig{ + Services: map[tailcfg.ServiceName]*ServiceConfig{ + "svc:foo": { + TCP: map[uint16]*TCPPortHandler{80: {HTTPS: true}}, + Web: map[HostPort]*WebServerConfig{ + "foo.test.ts.net:80": { + Handlers: map[string]*HTTPHandler{ + "/": {Text: "Hello, world!"}, + }, + }, + }, + }, + }, + }, + svcName: "svc:foo", + port: 80, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.cfg.IsTCPForwardingOnPort(tt.port, tt.svcName) + if tt.want != got { + t.Errorf("IsTCPForwardingOnPort() = %v, want %v", got, tt.want) + } + }) + } +} + func TestExpandProxyTargetDev(t *testing.T) { tests := []struct { name string diff --git a/ipn/store.go b/ipn/store.go index 550aa8cba819a..9da5288c0d371 100644 --- a/ipn/store.go +++ b/ipn/store.go @@ -113,3 +113,9 @@ func ReadStoreInt(store StateStore, id StateKey) (int64, error) { func PutStoreInt(store StateStore, id StateKey, val int64) error { return WriteState(store, id, fmt.Appendf(nil, "%d", val)) } + +// EncryptedStateStore is a marker interface implemented by StateStores that +// encrypt data at rest. +type EncryptedStateStore interface { + stateStoreIsEncrypted() +} diff --git a/ipn/store/kubestore/store_kube.go b/ipn/store/kubestore/store_kube.go index 14025bbb4150a..a9ad514e755b2 100644 --- a/ipn/store/kubestore/store_kube.go +++ b/ipn/store/kubestore/store_kube.go @@ -394,8 +394,8 @@ func (s *Store) canPatchSecret(secret string) bool { // certSecretSelector returns a label selector that can be used to list all // Secrets that aren't Tailscale state Secrets and contain TLS certificates for // HTTPS endpoints that this node serves. -// Currently (3/2025) this only applies to the Kubernetes Operator's ingress -// ProxyGroup. +// Currently (7/2025) this only applies to the Kubernetes Operator's ProxyGroup +// when spec.Type is "ingress" or "kube-apiserver". func (s *Store) certSecretSelector() map[string]string { if s.podName == "" { return map[string]string{} @@ -406,7 +406,7 @@ func (s *Store) certSecretSelector() map[string]string { } pgName := s.podName[:p] return map[string]string{ - kubetypes.LabelSecretType: "certs", + kubetypes.LabelSecretType: kubetypes.LabelSecretTypeCerts, kubetypes.LabelManaged: "true", "tailscale.com/proxy-group": pgName, } diff --git a/ipn/store/kubestore/store_kube_test.go b/ipn/store/kubestore/store_kube_test.go index 0d709264e5c08..9a49f30288840 100644 --- a/ipn/store/kubestore/store_kube_test.go +++ b/ipn/store/kubestore/store_kube_test.go @@ -17,6 +17,7 @@ import ( "tailscale.com/ipn/store/mem" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" + "tailscale.com/kube/kubetypes" ) func TestWriteState(t *testing.T) { @@ -516,7 +517,7 @@ func TestNewWithClient(t *testing.T) { ) certSecretsLabels := map[string]string{ - "tailscale.com/secret-type": "certs", + "tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts, "tailscale.com/managed": "true", "tailscale.com/proxy-group": "ingress-proxies", } @@ -582,7 +583,7 @@ func TestNewWithClient(t *testing.T) { makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"), makeSecret("some-other-secret", nil, "3"), makeSecret("app3.other-proxies.ts.net", map[string]string{ - "tailscale.com/secret-type": "certs", + "tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts, "tailscale.com/managed": "true", "tailscale.com/proxy-group": "some-other-proxygroup", }, "4"), @@ -606,7 +607,7 @@ func TestNewWithClient(t *testing.T) { makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"), makeSecret("some-other-secret", nil, "3"), makeSecret("app3.other-proxies.ts.net", map[string]string{ - "tailscale.com/secret-type": "certs", + "tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts, "tailscale.com/managed": "true", "tailscale.com/proxy-group": "some-other-proxygroup", }, "4"), diff --git a/ipn/store/stores.go b/ipn/store/stores.go index 1f98891bff248..bf175da41d8aa 100644 --- a/ipn/store/stores.go +++ b/ipn/store/stores.go @@ -7,10 +7,14 @@ package store import ( "bytes" "encoding/json" + "errors" "fmt" + "iter" + "maps" "os" "path/filepath" "runtime" + "slices" "strings" "sync" @@ -20,6 +24,7 @@ import ( "tailscale.com/paths" "tailscale.com/types/logger" "tailscale.com/util/mak" + "tailscale.com/util/testenv" ) // Provider returns a StateStore for the provided path. @@ -32,6 +37,9 @@ func init() { var knownStores map[string]Provider +// TPMPrefix is the path prefix used for TPM-encrypted StateStore. +const TPMPrefix = "tpmseal:" + // New returns a StateStore based on the provided arg // and registered stores. // The arg is of the form "prefix:rest", where prefix was previously @@ -45,18 +53,31 @@ var knownStores map[string]Provider // the suffix an AWS ARN for an SSM. // - (Linux-only) if the string begins with "kube:", // the suffix is a Kubernetes secret name +// - (Linux or Windows) if the string begins with "tpmseal:", the suffix is +// filepath that is sealed with the local TPM device. // - In all other cases, the path is treated as a filepath. func New(logf logger.Logf, path string) (ipn.StateStore, error) { for prefix, sf := range knownStores { if strings.HasPrefix(path, prefix) { // We can't strip the prefix here as some NewStoreFunc (like arn:) // expect the prefix. + if prefix == TPMPrefix { + if runtime.GOOS == "windows" { + path = TPMPrefix + TryWindowsAppDataMigration(logf, strings.TrimPrefix(path, TPMPrefix)) + } + if err := maybeMigrateLocalStateFile(logf, path); err != nil { + return nil, fmt.Errorf("failed to migrate existing state file to TPM-sealed format: %w", err) + } + } return sf(logf, path) } } if runtime.GOOS == "windows" { path = TryWindowsAppDataMigration(logf, path) } + if err := maybeMigrateLocalStateFile(logf, path); err != nil { + return nil, fmt.Errorf("failed to migrate existing TPM-sealed state file to plaintext format: %w", err) + } return NewFileStore(logf, path) } @@ -75,6 +96,29 @@ func Register(prefix string, fn Provider) { mak.Set(&knownStores, prefix, fn) } +// RegisterForTest registers a prefix to be used for NewStore in tests. An +// existing registered prefix will be replaced. +func RegisterForTest(t testenv.TB, prefix string, fn Provider) { + if len(prefix) == 0 { + panic("prefix is empty") + } + old := maps.Clone(knownStores) + t.Cleanup(func() { knownStores = old }) + + mak.Set(&knownStores, prefix, fn) +} + +// HasKnownProviderPrefix reports whether path uses one of the registered +// Provider prefixes. +func HasKnownProviderPrefix(path string) bool { + for prefix := range knownStores { + if strings.HasPrefix(path, prefix) { + return true + } + } + return false +} + // TryWindowsAppDataMigration attempts to copy the Windows state file // from its old location to the new location. (Issue 2856) // @@ -177,3 +221,123 @@ func (s *FileStore) WriteState(id ipn.StateKey, bs []byte) error { } return atomicfile.WriteFile(s.path, bs, 0600) } + +func (s *FileStore) All() iter.Seq2[ipn.StateKey, []byte] { + return func(yield func(ipn.StateKey, []byte) bool) { + s.mu.Lock() + defer s.mu.Unlock() + + for k, v := range s.cache { + if !yield(k, v) { + break + } + } + } +} + +// Ensure FileStore implements ExportableStore for migration to/from +// tpm.tpmStore. +var _ ExportableStore = (*FileStore)(nil) + +// ExportableStore is an ipn.StateStore that can export all of its contents. +// This interface is optional to implement, and used for migrating the state +// between different store implementations. +type ExportableStore interface { + ipn.StateStore + + // All returns an iterator over all store keys. Using ReadState or + // WriteState is not safe while iterating and can lead to a deadlock. The + // order of keys in the iterator is not specified and may change between + // runs. + All() iter.Seq2[ipn.StateKey, []byte] +} + +func maybeMigrateLocalStateFile(logf logger.Logf, path string) error { + path, toTPM := strings.CutPrefix(path, TPMPrefix) + + // Extract JSON keys from the file on disk and guess what kind it is. + bs, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + var content map[string]any + if err := json.Unmarshal(bs, &content); err != nil { + return fmt.Errorf("failed to unmarshal %q: %w", path, err) + } + keys := slices.Sorted(maps.Keys(content)) + tpmKeys := []string{"key", "nonce", "data"} + slices.Sort(tpmKeys) + // TPM-sealed files will have exactly these keys. + existingFileSealed := slices.Equal(keys, tpmKeys) + // Plaintext files for nodes that registered at least once will have this + // key, plus other dynamic ones. + _, existingFilePlaintext := content["_machinekey"] + isTPM := existingFileSealed && !existingFilePlaintext + + if isTPM == toTPM { + // No migration needed. + return nil + } + + newTPMStore, ok := knownStores[TPMPrefix] + if !ok { + return errors.New("this build does not support TPM integration") + } + + // Open from (old format) and to (new format) stores for migration. The + // "to" store will be at tmpPath. + var from, to ipn.StateStore + tmpPath := path + ".tmp" + if toTPM { + // Migrate plaintext file to be TPM-sealed. + from, err = NewFileStore(logf, path) + if err != nil { + return fmt.Errorf("NewFileStore(%q): %w", path, err) + } + to, err = newTPMStore(logf, TPMPrefix+tmpPath) + if err != nil { + return fmt.Errorf("newTPMStore(%q): %w", tmpPath, err) + } + } else { + // Migrate TPM-selaed file to plaintext. + from, err = newTPMStore(logf, TPMPrefix+path) + if err != nil { + return fmt.Errorf("newTPMStore(%q): %w", path, err) + } + to, err = NewFileStore(logf, tmpPath) + if err != nil { + return fmt.Errorf("NewFileStore(%q): %w", tmpPath, err) + } + } + defer os.Remove(tmpPath) + + fromExp, ok := from.(ExportableStore) + if !ok { + return fmt.Errorf("%T does not implement the exportableStore interface", from) + } + + // Copy all the items. This is pretty inefficient, because both stores + // write the file to disk for each WriteState, but that's ok for a one-time + // migration. + for k, v := range fromExp.All() { + if err := to.WriteState(k, v); err != nil { + return err + } + } + + // Finally, overwrite the state file with the new one we created at + // tmpPath. + if err := atomicfile.Rename(tmpPath, path); err != nil { + return err + } + + if toTPM { + logf("migrated %q from plaintext to TPM-sealed format", path) + } else { + logf("migrated %q from TPM-sealed to plaintext format", path) + } + return nil +} diff --git a/ipn/store_test.go b/ipn/store_test.go index fcc082d8a8a87..4dd7321b9048d 100644 --- a/ipn/store_test.go +++ b/ipn/store_test.go @@ -5,6 +5,7 @@ package ipn import ( "bytes" + "iter" "sync" "testing" @@ -31,6 +32,19 @@ func (s *memStore) WriteState(k StateKey, v []byte) error { return nil } +func (s *memStore) All() iter.Seq2[StateKey, []byte] { + return func(yield func(StateKey, []byte) bool) { + s.mu.Lock() + defer s.mu.Unlock() + + for k, v := range s.m { + if !yield(k, v) { + break + } + } + } +} + func TestWriteState(t *testing.T) { var ss StateStore = new(memStore) WriteState(ss, "foo", []byte("bar")) diff --git a/k8s-operator/api-proxy/env.go b/k8s-operator/api-proxy/env.go deleted file mode 100644 index c0640ab1e16bf..0000000000000 --- a/k8s-operator/api-proxy/env.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -//go:build !plan9 - -package apiproxy - -import ( - "os" - - "tailscale.com/types/opt" -) - -func defaultBool(envName string, defVal bool) bool { - vs := os.Getenv(envName) - if vs == "" { - return defVal - } - v, _ := opt.Bool(vs).Get() - return v -} - -func defaultEnv(envName, defVal string) string { - v := os.Getenv(envName) - if v == "" { - return defVal - } - return v -} diff --git a/k8s-operator/api-proxy/proxy.go b/k8s-operator/api-proxy/proxy.go index 7c7260b94af39..a0f2f930b8067 100644 --- a/k8s-operator/api-proxy/proxy.go +++ b/k8s-operator/api-proxy/proxy.go @@ -6,22 +6,24 @@ package apiproxy import ( + "context" "crypto/tls" + "errors" "fmt" - "log" + "net" "net/http" "net/http/httputil" "net/netip" "net/url" - "os" "strings" + "time" - "github.com/pkg/errors" "go.uber.org/zap" "k8s.io/client-go/rest" "k8s.io/client-go/transport" "tailscale.com/client/local" "tailscale.com/client/tailscale/apitype" + "tailscale.com/k8s-operator/sessionrecording" ksr "tailscale.com/k8s-operator/sessionrecording" "tailscale.com/kube/kubetypes" "tailscale.com/tailcfg" @@ -37,123 +39,54 @@ var ( whoIsKey = ctxkey.New("", (*apitype.WhoIsResponse)(nil)) ) -type APIServerProxyMode int - -func (a APIServerProxyMode) String() string { - switch a { - case APIServerProxyModeDisabled: - return "disabled" - case APIServerProxyModeEnabled: - return "auth" - case APIServerProxyModeNoAuth: - return "noauth" - default: - return "unknown" - } -} - -const ( - APIServerProxyModeDisabled APIServerProxyMode = iota - APIServerProxyModeEnabled - APIServerProxyModeNoAuth -) - -func ParseAPIProxyMode() APIServerProxyMode { - haveAuthProxyEnv := os.Getenv("AUTH_PROXY") != "" - haveAPIProxyEnv := os.Getenv("APISERVER_PROXY") != "" - switch { - case haveAPIProxyEnv && haveAuthProxyEnv: - log.Fatal("AUTH_PROXY and APISERVER_PROXY are mutually exclusive") - case haveAuthProxyEnv: - var authProxyEnv = defaultBool("AUTH_PROXY", false) // deprecated - if authProxyEnv { - return APIServerProxyModeEnabled - } - return APIServerProxyModeDisabled - case haveAPIProxyEnv: - var apiProxyEnv = defaultEnv("APISERVER_PROXY", "") // true, false or "noauth" - switch apiProxyEnv { - case "true": - return APIServerProxyModeEnabled - case "false", "": - return APIServerProxyModeDisabled - case "noauth": - return APIServerProxyModeNoAuth - default: - panic(fmt.Sprintf("unknown APISERVER_PROXY value %q", apiProxyEnv)) - } - } - return APIServerProxyModeDisabled -} - -// maybeLaunchAPIServerProxy launches the auth proxy, which is a small HTTP server -// that authenticates requests using the Tailscale LocalAPI and then proxies -// them to the kube-apiserver. -func MaybeLaunchAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, s *tsnet.Server, mode APIServerProxyMode) { - if mode == APIServerProxyModeDisabled { - return - } - startlog := zlog.Named("launchAPIProxy") - if mode == APIServerProxyModeNoAuth { +// NewAPIServerProxy creates a new APIServerProxy that's ready to start once Run +// is called. No network traffic will flow until Run is called. +// +// authMode controls how the proxy behaves: +// - true: the proxy is started and requests are impersonated using the +// caller's Tailscale identity and the rules defined in the tailnet ACLs. +// - false: the proxy is started and requests are passed through to the +// Kubernetes API without any auth modifications. +func NewAPIServerProxy(zlog *zap.SugaredLogger, restConfig *rest.Config, ts *tsnet.Server, mode kubetypes.APIServerProxyMode, https bool) (*APIServerProxy, error) { + if mode == kubetypes.APIServerProxyModeNoAuth { restConfig = rest.AnonymousClientConfig(restConfig) } + cfg, err := restConfig.TransportConfig() if err != nil { - startlog.Fatalf("could not get rest.TransportConfig(): %v", err) + return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err) } - // Kubernetes uses SPDY for exec and port-forward, however SPDY is - // incompatible with HTTP/2; so disable HTTP/2 in the proxy. tr := http.DefaultTransport.(*http.Transport).Clone() tr.TLSClientConfig, err = transport.TLSConfigFor(cfg) if err != nil { - startlog.Fatalf("could not get transport.TLSConfigFor(): %v", err) + return nil, fmt.Errorf("could not get transport.TLSConfigFor(): %w", err) } tr.TLSNextProto = make(map[string]func(authority string, c *tls.Conn) http.RoundTripper) rt, err := transport.HTTPWrappersForConfig(cfg, tr) if err != nil { - startlog.Fatalf("could not get rest.TransportConfig(): %v", err) + return nil, fmt.Errorf("could not get rest.TransportConfig(): %w", err) } - go runAPIServerProxy(s, rt, zlog.Named("apiserver-proxy"), mode, restConfig.Host) -} -// runAPIServerProxy runs an HTTP server that authenticates requests using the -// Tailscale LocalAPI and then proxies them to the Kubernetes API. -// It listens on :443 and uses the Tailscale HTTPS certificate. -// s will be started if it is not already running. -// rt is used to proxy requests to the Kubernetes API. -// -// mode controls how the proxy behaves: -// - apiserverProxyModeDisabled: the proxy is not started. -// - apiserverProxyModeEnabled: the proxy is started and requests are impersonated using the -// caller's identity from the Tailscale LocalAPI. -// - apiserverProxyModeNoAuth: the proxy is started and requests are not impersonated and -// are passed through to the Kubernetes API. -// -// It never returns. -func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredLogger, mode APIServerProxyMode, host string) { - if mode == APIServerProxyModeDisabled { - return - } - ln, err := ts.Listen("tcp", ":443") + u, err := url.Parse(restConfig.Host) if err != nil { - log.Fatalf("could not listen on :443: %v", err) + return nil, fmt.Errorf("failed to parse URL %w", err) } - u, err := url.Parse(host) - if err != nil { - log.Fatalf("runAPIServerProxy: failed to parse URL %v", err) + if u.Scheme == "" || u.Host == "" { + return nil, fmt.Errorf("the API server proxy requires host and scheme but got: %q", restConfig.Host) } lc, err := ts.LocalClient() if err != nil { - log.Fatalf("could not get local client: %v", err) + return nil, fmt.Errorf("could not get local client: %w", err) } - ap := &apiserverProxy{ - log: log, + ap := &APIServerProxy{ + log: zlog, lc: lc, - mode: mode, + authMode: mode == kubetypes.APIServerProxyModeAuth, + https: https, upstreamURL: u, ts: ts, } @@ -164,41 +97,96 @@ func runAPIServerProxy(ts *tsnet.Server, rt http.RoundTripper, log *zap.SugaredL Transport: rt, } + return ap, nil +} + +// Run starts the HTTP server that authenticates requests using the +// Tailscale LocalAPI and then proxies them to the Kubernetes API. +// It listens on :443 and uses the Tailscale HTTPS certificate. +// +// It return when ctx is cancelled or ServeTLS fails. +func (ap *APIServerProxy) Run(ctx context.Context) error { mux := http.NewServeMux() mux.HandleFunc("/", ap.serveDefault) mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecSPDY) mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/exec", ap.serveExecWS) + mux.HandleFunc("POST /api/v1/namespaces/{namespace}/pods/{pod}/attach", ap.serveAttachSPDY) + mux.HandleFunc("GET /api/v1/namespaces/{namespace}/pods/{pod}/attach", ap.serveAttachWS) + + ap.hs = &http.Server{ + Handler: mux, + ErrorLog: zap.NewStdLog(ap.log.Desugar()), + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), + } + + mode := "noauth" + if ap.authMode { + mode = "auth" + } + var proxyLn net.Listener + var serve func(ln net.Listener) error + if ap.https { + var err error + proxyLn, err = ap.ts.Listen("tcp", ":443") + if err != nil { + return fmt.Errorf("could not listen on :443: %w", err) + } + serve = func(ln net.Listener) error { + return ap.hs.ServeTLS(ln, "", "") + } - hs := &http.Server{ // Kubernetes uses SPDY for exec and port-forward, however SPDY is // incompatible with HTTP/2; so disable HTTP/2 in the proxy. - TLSConfig: &tls.Config{ - GetCertificate: lc.GetCertificate, + ap.hs.TLSConfig = &tls.Config{ + GetCertificate: ap.lc.GetCertificate, NextProtos: []string{"http/1.1"}, - }, - TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), - Handler: mux, + } + } else { + var err error + proxyLn, err = net.Listen("tcp", "localhost:80") + if err != nil { + return fmt.Errorf("could not listen on :80: %w", err) + } + serve = ap.hs.Serve } - log.Infof("API server proxy in %q mode is listening on %s", mode, ln.Addr()) - if err := hs.ServeTLS(ln, "", ""); err != nil { - log.Fatalf("runAPIServerProxy: failed to serve %v", err) + + errs := make(chan error) + go func() { + ap.log.Infof("API server proxy in %s mode is listening on %s", mode, proxyLn.Addr()) + if err := serve(proxyLn); err != nil && err != http.ErrServerClosed { + errs <- fmt.Errorf("error serving: %w", err) + } + }() + + select { + case <-ctx.Done(): + case err := <-errs: + ap.hs.Close() + return err } + + // Graceful shutdown with a timeout of 10s. + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return ap.hs.Shutdown(shutdownCtx) } -// apiserverProxy is an [net/http.Handler] that authenticates requests using the Tailscale +// APIServerProxy is an [net/http.Handler] that authenticates requests using the Tailscale // LocalAPI and then proxies them to the Kubernetes API. -type apiserverProxy struct { +type APIServerProxy struct { log *zap.SugaredLogger lc *local.Client rp *httputil.ReverseProxy - mode APIServerProxyMode + authMode bool // Whether to run with impersonation using caller's tailnet identity. + https bool // Whether to serve on https for the device hostname; true for k8s-operator, false (and localhost) for k8s-proxy. ts *tsnet.Server + hs *http.Server upstreamURL *url.URL } // serveDefault is the default handler for Kubernetes API server requests. -func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) { +func (ap *APIServerProxy) serveDefault(w http.ResponseWriter, r *http.Request) { who, err := ap.whoIs(r) if err != nil { ap.authError(w, err) @@ -208,19 +196,31 @@ func (ap *apiserverProxy) serveDefault(w http.ResponseWriter, r *http.Request) { ap.rp.ServeHTTP(w, r.WithContext(whoIsKey.WithValue(r.Context(), who))) } -// serveExecSPDY serves 'kubectl exec' requests for sessions streamed over SPDY, +// serveExecSPDY serves '/exec' requests for sessions streamed over SPDY, +// optionally configuring the kubectl exec sessions to be recorded. +func (ap *APIServerProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) { + ap.sessionForProto(w, r, ksr.ExecSessionType, ksr.SPDYProtocol) +} + +// serveExecWS serves '/exec' requests for sessions streamed over WebSocket, +// optionally configuring the kubectl exec sessions to be recorded. +func (ap *APIServerProxy) serveExecWS(w http.ResponseWriter, r *http.Request) { + ap.sessionForProto(w, r, ksr.ExecSessionType, ksr.WSProtocol) +} + +// serveAttachSPDY serves '/attach' requests for sessions streamed over SPDY, // optionally configuring the kubectl exec sessions to be recorded. -func (ap *apiserverProxy) serveExecSPDY(w http.ResponseWriter, r *http.Request) { - ap.execForProto(w, r, ksr.SPDYProtocol) +func (ap *APIServerProxy) serveAttachSPDY(w http.ResponseWriter, r *http.Request) { + ap.sessionForProto(w, r, ksr.AttachSessionType, ksr.SPDYProtocol) } -// serveExecWS serves 'kubectl exec' requests for sessions streamed over WebSocket, +// serveAttachWS serves '/attach' requests for sessions streamed over WebSocket, // optionally configuring the kubectl exec sessions to be recorded. -func (ap *apiserverProxy) serveExecWS(w http.ResponseWriter, r *http.Request) { - ap.execForProto(w, r, ksr.WSProtocol) +func (ap *APIServerProxy) serveAttachWS(w http.ResponseWriter, r *http.Request) { + ap.sessionForProto(w, r, ksr.AttachSessionType, ksr.WSProtocol) } -func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, proto ksr.Protocol) { +func (ap *APIServerProxy) sessionForProto(w http.ResponseWriter, r *http.Request, sessionType sessionrecording.SessionType, proto ksr.Protocol) { const ( podNameKey = "pod" namespaceNameKey = "namespace" @@ -235,7 +235,7 @@ func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, p counterNumRequestsProxied.Add(1) failOpen, addrs, err := determineRecorderConfig(who) if err != nil { - ap.log.Errorf("error trying to determine whether the 'kubectl exec' session needs to be recorded: %v", err) + ap.log.Errorf("error trying to determine whether the 'kubectl %s' session needs to be recorded: %v", sessionType, err) return } if failOpen && len(addrs) == 0 { // will not record @@ -244,7 +244,7 @@ func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, p } ksr.CounterSessionRecordingsAttempted.Add(1) // at this point we know that users intended for this session to be recorded if !failOpen && len(addrs) == 0 { - msg := "forbidden: 'kubectl exec' session must be recorded, but no recorders are available." + msg := fmt.Sprintf("forbidden: 'kubectl %s' session must be recorded, but no recorders are available.", sessionType) ap.log.Error(msg) http.Error(w, msg, http.StatusForbidden) return @@ -266,26 +266,27 @@ func (ap *apiserverProxy) execForProto(w http.ResponseWriter, r *http.Request, p } opts := ksr.HijackerOpts{ - Req: r, - W: w, - Proto: proto, - TS: ap.ts, - Who: who, - Addrs: addrs, - FailOpen: failOpen, - Pod: r.PathValue(podNameKey), - Namespace: r.PathValue(namespaceNameKey), - Log: ap.log, - } - h := ksr.New(opts) + Req: r, + W: w, + Proto: proto, + SessionType: sessionType, + TS: ap.ts, + Who: who, + Addrs: addrs, + FailOpen: failOpen, + Pod: r.PathValue(podNameKey), + Namespace: r.PathValue(namespaceNameKey), + Log: ap.log, + } + h := ksr.NewHijacker(opts) ap.rp.ServeHTTP(h, r.WithContext(whoIsKey.WithValue(r.Context(), who))) } -func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) { - r.URL.Scheme = h.upstreamURL.Scheme - r.URL.Host = h.upstreamURL.Host - if h.mode == APIServerProxyModeNoAuth { +func (ap *APIServerProxy) addImpersonationHeadersAsRequired(r *http.Request) { + r.URL.Scheme = ap.upstreamURL.Scheme + r.URL.Host = ap.upstreamURL.Host + if !ap.authMode { // If we are not providing authentication, then we are just // proxying to the Kubernetes API, so we don't need to do // anything else. @@ -310,16 +311,32 @@ func (h *apiserverProxy) addImpersonationHeadersAsRequired(r *http.Request) { } // Now add the impersonation headers that we want. - if err := addImpersonationHeaders(r, h.log); err != nil { - log.Print("failed to add impersonation headers: ", err.Error()) + if err := addImpersonationHeaders(r, ap.log); err != nil { + ap.log.Errorf("failed to add impersonation headers: %v", err) } } -func (ap *apiserverProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) { - return ap.lc.WhoIs(r.Context(), r.RemoteAddr) +func (ap *APIServerProxy) whoIs(r *http.Request) (*apitype.WhoIsResponse, error) { + who, remoteErr := ap.lc.WhoIs(r.Context(), r.RemoteAddr) + if remoteErr == nil { + ap.log.Debugf("WhoIs from remote addr: %s", r.RemoteAddr) + return who, nil + } + + var fwdErr error + fwdFor := r.Header.Get("X-Forwarded-For") + if fwdFor != "" && !ap.https { + who, fwdErr = ap.lc.WhoIs(r.Context(), fwdFor) + if fwdErr == nil { + ap.log.Debugf("WhoIs from X-Forwarded-For header: %s", fwdFor) + return who, nil + } + } + + return nil, errors.Join(remoteErr, fwdErr) } -func (ap *apiserverProxy) authError(w http.ResponseWriter, err error) { +func (ap *APIServerProxy) authError(w http.ResponseWriter, err error) { ap.log.Errorf("failed to authenticate caller: %v", err) http.Error(w, "failed to authenticate caller", http.StatusInternalServerError) } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 03bb8989b9782..564c87f503a22 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -21,6 +21,21 @@ +#### APIServerProxyMode + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [auth noauth] +- Type: string + +_Appears in:_ +- [KubeAPIServerConfig](#kubeapiserverconfig) + + + #### AppConnector @@ -142,7 +157,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `env` _[Env](#env) array_ | List of environment variables to set in the container.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables
Note that environment variables provided here will take precedence
over Tailscale-specific environment variables set by the operator,
however running proxies with custom values for Tailscale environment
variables (i.e TS_USERSPACE) is not recommended and might break in
the future. | | | -| `image` _string_ | Container image name. By default images are pulled from
docker.io/tailscale/tailscale, but the official images are also
available at ghcr.io/tailscale/tailscale. Specifying image name here
will override any proxy image values specified via the Kubernetes
operator's Helm chart values or PROXY_IMAGE env var in the operator
Deployment.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | | +| `image` _string_ | Container image name. By default images are pulled from docker.io/tailscale,
but the official images are also available at ghcr.io/tailscale.
For all uses except on ProxyGroups of type "kube-apiserver", this image must
be either tailscale/tailscale, or an equivalent mirror of that image.
To apply to ProxyGroups of type "kube-apiserver", this image must be
tailscale/k8s-proxy or a mirror of that image.
For "tailscale/tailscale"-based proxies, specifying image name here will
override any proxy image values specified via the Kubernetes operator's
Helm chart values or PROXY_IMAGE env var in the operator Deployment.
For "tailscale/k8s-proxy"-based proxies, there is currently no way to
configure your own default, and this field is the only way to use a
custom image.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | | | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#pullpolicy-v1-core)_ | Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image | | Enum: [Always Never IfNotPresent]
| | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#resourcerequirements-v1-core)_ | Container resource requirements.
By default Tailscale Kubernetes operator does not apply any resource
requirements. The amount of resources required wil depend on the
amount of resources the operator needs to parse, usage patterns and
cluster size.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources | | | | `securityContext` _[SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#securitycontext-v1-core)_ | Container security context.
Security context specified here will override the security context set by the operator.
By default the operator sets the Tailscale container and the Tailscale init container to privileged
for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup.
You can reduce the permissions of the Tailscale container to cap NET_ADMIN by
installing device plugin in your cluster and configuring the proxies tun device to be created
by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context | | | @@ -313,6 +328,23 @@ _Appears in:_ +#### KubeAPIServerConfig + + + +KubeAPIServerConfig contains configuration specific to the kube-apiserver ProxyGroup type. + + + +_Appears in:_ +- [ProxyGroupSpec](#proxygroupspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `mode` _[APIServerProxyMode](#apiserverproxymode)_ | Mode to run the API server proxy in. Supported modes are auth and noauth.
In auth mode, requests from the tailnet proxied over to the Kubernetes
API server are additionally impersonated using the sender's tailnet identity.
If not specified, defaults to auth mode. | | Enum: [auth noauth]
Type: string
| +| `hostname` _string_ | Hostname is the hostname with which to expose the Kubernetes API server
proxies. Must be a valid DNS label no longer than 63 characters. If not
specified, the name of the ProxyGroup is used as the hostname. Must be
unique across the whole tailnet. | | Pattern: `^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`
Type: string
| + + #### LabelValue _Underlying type:_ _string_ @@ -390,6 +422,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `image` _[NameserverImage](#nameserverimage)_ | Nameserver image. Defaults to tailscale/k8s-nameserver:unstable. | | | +| `service` _[NameserverService](#nameserverservice)_ | Service configuration. | | | #### NameserverImage @@ -409,6 +442,22 @@ _Appears in:_ | `tag` _string_ | Tag defaults to unstable. | | | +#### NameserverService + + + + + + + +_Appears in:_ +- [Nameserver](#nameserver) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `clusterIP` _string_ | ClusterIP sets the static IP of the service used by the nameserver. | | | + + #### NameserverStatus @@ -422,7 +471,24 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
Currently you must manually update your cluster DNS config to add
this address as a stub nameserver for ts.net for cluster workloads to be
able to resolve MagicDNS names associated with egress or Ingress
proxies.
The IP address will change if you delete and recreate the DNSConfig. | | | +| `ip` _string_ | IP is the ClusterIP of the Service fronting the deployed ts.net nameserver.
Currently, you must manually update your cluster DNS config to add
this address as a stub nameserver for ts.net for cluster workloads to be
able to resolve MagicDNS names associated with egress or Ingress
proxies.
The IP address will change if you delete and recreate the DNSConfig. | | | + + +#### NodePortConfig + + + + + + + +_Appears in:_ +- [StaticEndpointsConfig](#staticendpointsconfig) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `ports` _[PortRange](#portrange) array_ | The port ranges from which the operator will select NodePorts for the Services.
You must ensure that firewall rules allow UDP ingress traffic for these ports
to the node's external IPs.
The ports must be in the range of service node ports for the cluster (default `30000-32767`).
See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. | | MinItems: 1
| +| `selector` _object (keys:string, values:string)_ | A selector which will be used to select the node's that will have their `ExternalIP`'s advertised
by the ProxyGroup as Static Endpoints. | | | #### Pod @@ -442,7 +508,7 @@ _Appears in:_ | `annotations` _object (keys:string, values:string)_ | Annotations that will be added to the proxy Pod.
Any annotations specified here will be merged with the default
annotations applied to the Pod by the Tailscale Kubernetes operator.
Annotations must be valid Kubernetes annotations.
https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set | | | | `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#affinity-v1-core)_ | Proxy Pod's affinity rules.
By default, the Tailscale Kubernetes operator does not apply any affinity rules.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#affinity | | | | `tailscaleContainer` _[Container](#container)_ | Configuration for the proxy container running tailscale. | | | -| `tailscaleInitContainer` _[Container](#container)_ | Configuration for the proxy init container that enables forwarding. | | | +| `tailscaleInitContainer` _[Container](#container)_ | Configuration for the proxy init container that enables forwarding.
Not valid to apply to ProxyGroups of type "kube-apiserver". | | | | `securityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#podsecuritycontext-v1-core)_ | Proxy Pod's security context.
By default Tailscale Kubernetes operator does not apply any Pod
security context.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2 | | | | `imagePullSecrets` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#localobjectreference-v1-core) array_ | Proxy Pod's image pull Secrets.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec | | | | `nodeName` _string_ | Proxy Pod's node name.
https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling | | | @@ -451,6 +517,26 @@ _Appears in:_ | `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#topologyspreadconstraint-v1-core) array_ | Proxy Pod's topology spread constraints.
By default Tailscale Kubernetes operator does not apply any topology spread constraints.
https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ | | | +#### PortRange + + + + + + + +_Appears in:_ +- [NodePortConfig](#nodeportconfig) +- [PortRanges](#portranges) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `port` _integer_ | port represents a port selected to be used. This is a required field. | | | +| `endPort` _integer_ | endPort indicates that the range of ports from port to endPort if set, inclusive,
should be used. This field cannot be defined if the port field is not defined.
The endPort must be either unset, or equal or greater than port. | | | + + + + #### ProxyClass @@ -518,6 +604,7 @@ _Appears in:_ | `metrics` _[Metrics](#metrics)_ | Configuration for proxy metrics. Metrics are currently not supported
for egress proxies and for Ingress proxies that have been configured
with tailscale.com/experimental-forward-cluster-traffic-via-ingress
annotation. Note that the metrics are currently considered unstable
and will likely change in breaking ways in the future - we only
recommend that you use those for debugging purposes. | | | | `tailscale` _[TailscaleConfig](#tailscaleconfig)_ | TailscaleConfig contains options to configure the tailscale-specific
parameters of proxies. | | | | `useLetsEncryptStagingEnvironment` _boolean_ | Set UseLetsEncryptStagingEnvironment to true to issue TLS
certificates for any HTTPS endpoints exposed to the tailnet from
LetsEncrypt's staging environment.
https://letsencrypt.org/docs/staging-environment/
This setting only affects Tailscale Ingress resources.
By default Ingress TLS certificates are issued from LetsEncrypt's
production environment.
Changing this setting true -> false, will result in any
existing certs being re-issued from the production environment.
Changing this setting false (default) -> true, when certs have already
been provisioned from production environment will NOT result in certs
being re-issued from the staging environment before they need to be
renewed. | | | +| `staticEndpoints` _[StaticEndpointsConfig](#staticendpointsconfig)_ | Configuration for 'static endpoints' on proxies in order to facilitate
direct connections from other devices on the tailnet.
See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. | | | #### ProxyClassStatus @@ -541,15 +628,22 @@ _Appears in:_ ProxyGroup defines a set of Tailscale devices that will act as proxies. -Currently only egress ProxyGroups are supported. +Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver +proxies. In addition to running a highly available set of proxies, ingress +and egress ProxyGroups also allow for serving many annotated Services from a +single set of proxies to minimise resource consumption. + +For ingress and egress, use the tailscale.com/proxy-group annotation on a +Service to specify that the proxy should be implemented by a ProxyGroup +instead of a single dedicated proxy. -Use the tailscale.com/proxy-group annotation on a Service to specify that -the egress proxy should be implemented by a ProxyGroup instead of a single -dedicated proxy. In addition to running a highly available set of proxies, -ProxyGroup also allows for serving many annotated Services from a single -set of proxies to minimise resource consumption. +More info: +* https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +* https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress -More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +For kube-apiserver, the ProxyGroup is a standalone resource. Use the +spec.kubeAPIServer field to configure options specific to the kube-apiserver +ProxyGroup type. @@ -600,11 +694,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress and ingress.
Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress]
Type: string
| +| `type` _[ProxyGroupType](#proxygrouptype)_ | Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver.
Type is immutable once a ProxyGroup is created. | | Enum: [egress ingress kube-apiserver]
Type: string
| | `tags` _[Tags](#tags)_ | Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s].
If you specify custom tags here, make sure you also make the operator
an owner of these tags.
See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
Tags cannot be changed once a ProxyGroup device has been created.
Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$`
Type: string
| | `replicas` _integer_ | Replicas specifies how many replicas to create the StatefulSet with.
Defaults to 2. | | Minimum: 0
| | `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix is the hostname prefix to use for tailnet devices created
by the ProxyGroup. Each device will have the integer number from its
StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$`
Type: string
| | `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that contains
configuration options that should be applied to the resources created
for this ProxyGroup. If unset, and there is no default ProxyClass
configured, the operator will create resources with the default
configuration. | | | +| `kubeAPIServer` _[KubeAPIServerConfig](#kubeapiserverconfig)_ | KubeAPIServer contains configuration specific to the kube-apiserver
ProxyGroup type. This field is only used when Type is set to "kube-apiserver". | | | #### ProxyGroupStatus @@ -620,8 +715,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the ProxyGroup
resources. Known condition types are `ProxyGroupReady`. | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the ProxyGroup
resources. Known condition types include `ProxyGroupReady` and
`ProxyGroupAvailable`.
* `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and
all expected conditions are true.
* `ProxyGroupAvailable` indicates that at least one proxy is ready to
serve traffic.
For ProxyGroups of type kube-apiserver, there are two additional conditions:
* `KubeAPIServerProxyConfigured` indicates that at least one API server
proxy is configured and ready to serve traffic.
* `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is
valid. | | | | `devices` _[TailnetDevice](#tailnetdevice) array_ | List of tailnet devices associated with the ProxyGroup StatefulSet. | | | +| `url` _string_ | URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if
any. Only applies to ProxyGroups of type kube-apiserver. | | | #### ProxyGroupType @@ -631,7 +727,7 @@ _Underlying type:_ _string_ _Validation:_ -- Enum: [egress ingress] +- Enum: [egress ingress kube-apiserver] - Type: string _Appears in:_ @@ -935,6 +1031,22 @@ _Appears in:_ | `pod` _[Pod](#pod)_ | Configuration for the proxy Pod. | | | +#### StaticEndpointsConfig + + + + + + + +_Appears in:_ +- [ProxyClassSpec](#proxyclassspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `nodePort` _[NodePortConfig](#nodeportconfig)_ | The configuration for static endpoints using NodePort Services. | | | + + #### Storage @@ -1015,6 +1127,7 @@ _Appears in:_ | --- | --- | --- | --- | | `hostname` _string_ | Hostname is the fully qualified domain name of the device.
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
node. | | | | `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
assigned to the device. | | | +| `staticEndpoints` _string array_ | StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. | | | #### TailscaleConfig diff --git a/k8s-operator/apis/v1alpha1/types_connector.go b/k8s-operator/apis/v1alpha1/types_connector.go index b8b7a935e3a3f..ce6a1411b9ea8 100644 --- a/k8s-operator/apis/v1alpha1/types_connector.go +++ b/k8s-operator/apis/v1alpha1/types_connector.go @@ -205,11 +205,12 @@ type ConnectorStatus struct { type ConditionType string const ( - ConnectorReady ConditionType = `ConnectorReady` - ProxyClassReady ConditionType = `ProxyClassReady` - ProxyGroupReady ConditionType = `ProxyGroupReady` - ProxyReady ConditionType = `TailscaleProxyReady` // a Tailscale-specific condition type for corev1.Service - RecorderReady ConditionType = `RecorderReady` + ConnectorReady ConditionType = `ConnectorReady` + ProxyClassReady ConditionType = `ProxyClassReady` + ProxyGroupReady ConditionType = `ProxyGroupReady` // All proxy Pods running. + ProxyGroupAvailable ConditionType = `ProxyGroupAvailable` // At least one proxy Pod running. + ProxyReady ConditionType = `TailscaleProxyReady` // a Tailscale-specific condition type for corev1.Service + RecorderReady ConditionType = `RecorderReady` // EgressSvcValid gets set on a user configured ExternalName Service that defines a tailnet target to be exposed // on a ProxyGroup. // Set to true if the user provided configuration is valid. @@ -225,4 +226,7 @@ const ( IngressSvcValid ConditionType = `TailscaleIngressSvcValid` IngressSvcConfigured ConditionType = `TailscaleIngressSvcConfigured` + + KubeAPIServerProxyValid ConditionType = `KubeAPIServerProxyValid` // The kubeAPIServer config for the ProxyGroup is valid. + KubeAPIServerProxyConfigured ConditionType = `KubeAPIServerProxyConfigured` // At least one of the ProxyGroup's Pods is advertising the kube-apiserver proxy's hostname. ) diff --git a/k8s-operator/apis/v1alpha1/types_proxyclass.go b/k8s-operator/apis/v1alpha1/types_proxyclass.go index 899abf096bb86..6a4114bfa83da 100644 --- a/k8s-operator/apis/v1alpha1/types_proxyclass.go +++ b/k8s-operator/apis/v1alpha1/types_proxyclass.go @@ -6,6 +6,10 @@ package v1alpha1 import ( + "fmt" + "iter" + "strings" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -82,6 +86,124 @@ type ProxyClassSpec struct { // renewed. // +optional UseLetsEncryptStagingEnvironment bool `json:"useLetsEncryptStagingEnvironment,omitempty"` + // Configuration for 'static endpoints' on proxies in order to facilitate + // direct connections from other devices on the tailnet. + // See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. + // +optional + StaticEndpoints *StaticEndpointsConfig `json:"staticEndpoints,omitempty"` +} + +type StaticEndpointsConfig struct { + // The configuration for static endpoints using NodePort Services. + NodePort *NodePortConfig `json:"nodePort"` +} + +type NodePortConfig struct { + // The port ranges from which the operator will select NodePorts for the Services. + // You must ensure that firewall rules allow UDP ingress traffic for these ports + // to the node's external IPs. + // The ports must be in the range of service node ports for the cluster (default `30000-32767`). + // See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. + // +kubebuilder:validation:MinItems=1 + Ports []PortRange `json:"ports"` + // A selector which will be used to select the node's that will have their `ExternalIP`'s advertised + // by the ProxyGroup as Static Endpoints. + Selector map[string]string `json:"selector,omitempty"` +} + +// PortRanges is a list of PortRange(s) +type PortRanges []PortRange + +func (prs PortRanges) String() string { + var prStrings []string + + for _, pr := range prs { + prStrings = append(prStrings, pr.String()) + } + + return strings.Join(prStrings, ", ") +} + +// All allows us to iterate over all the ports in the PortRanges +func (prs PortRanges) All() iter.Seq[uint16] { + return func(yield func(uint16) bool) { + for _, pr := range prs { + end := pr.EndPort + if end == 0 { + end = pr.Port + } + + for port := pr.Port; port <= end; port++ { + if !yield(port) { + return + } + } + } + } +} + +// Contains reports whether port is in any of the PortRanges. +func (prs PortRanges) Contains(port uint16) bool { + for _, r := range prs { + if r.Contains(port) { + return true + } + } + + return false +} + +// ClashesWith reports whether the supplied PortRange clashes with any of the PortRanges. +func (prs PortRanges) ClashesWith(pr PortRange) bool { + for p := range prs.All() { + if pr.Contains(p) { + return true + } + } + + return false +} + +type PortRange struct { + // port represents a port selected to be used. This is a required field. + Port uint16 `json:"port"` + + // endPort indicates that the range of ports from port to endPort if set, inclusive, + // should be used. This field cannot be defined if the port field is not defined. + // The endPort must be either unset, or equal or greater than port. + // +optional + EndPort uint16 `json:"endPort,omitempty"` +} + +// Contains reports whether port is in pr. +func (pr PortRange) Contains(port uint16) bool { + switch pr.EndPort { + case 0: + return port == pr.Port + default: + return port >= pr.Port && port <= pr.EndPort + } +} + +// String returns the PortRange in a string form. +func (pr PortRange) String() string { + if pr.EndPort == 0 { + return fmt.Sprintf("%d", pr.Port) + } + + return fmt.Sprintf("%d-%d", pr.Port, pr.EndPort) +} + +// IsValid reports whether the port range is valid. +func (pr PortRange) IsValid() bool { + if pr.Port == 0 { + return false + } + if pr.EndPort == 0 { + return true + } + + return pr.Port <= pr.EndPort } type TailscaleConfig struct { @@ -142,6 +264,7 @@ type Pod struct { // +optional TailscaleContainer *Container `json:"tailscaleContainer,omitempty"` // Configuration for the proxy init container that enables forwarding. + // Not valid to apply to ProxyGroups of type "kube-apiserver". // +optional TailscaleInitContainer *Container `json:"tailscaleInitContainer,omitempty"` // Proxy Pod's security context. @@ -242,12 +365,21 @@ type Container struct { // the future. // +optional Env []Env `json:"env,omitempty"` - // Container image name. By default images are pulled from - // docker.io/tailscale/tailscale, but the official images are also - // available at ghcr.io/tailscale/tailscale. Specifying image name here - // will override any proxy image values specified via the Kubernetes - // operator's Helm chart values or PROXY_IMAGE env var in the operator - // Deployment. + // Container image name. By default images are pulled from docker.io/tailscale, + // but the official images are also available at ghcr.io/tailscale. + // + // For all uses except on ProxyGroups of type "kube-apiserver", this image must + // be either tailscale/tailscale, or an equivalent mirror of that image. + // To apply to ProxyGroups of type "kube-apiserver", this image must be + // tailscale/k8s-proxy or a mirror of that image. + // + // For "tailscale/tailscale"-based proxies, specifying image name here will + // override any proxy image values specified via the Kubernetes operator's + // Helm chart values or PROXY_IMAGE env var in the operator Deployment. + // For "tailscale/k8s-proxy"-based proxies, there is currently no way to + // configure your own default, and this field is the only way to use a + // custom image. + // // https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image // +optional Image string `json:"image,omitempty"` diff --git a/k8s-operator/apis/v1alpha1/types_proxygroup.go b/k8s-operator/apis/v1alpha1/types_proxygroup.go index ac87cc6caf892..28fd9e00973c5 100644 --- a/k8s-operator/apis/v1alpha1/types_proxygroup.go +++ b/k8s-operator/apis/v1alpha1/types_proxygroup.go @@ -13,19 +13,27 @@ import ( // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster,shortName=pg // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ProxyGroupReady")].reason`,description="Status of the deployed ProxyGroup resources." +// +kubebuilder:printcolumn:name="URL",type="string",JSONPath=`.status.url`,description="URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver." // +kubebuilder:printcolumn:name="Type",type="string",JSONPath=`.spec.type`,description="ProxyGroup type." // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // ProxyGroup defines a set of Tailscale devices that will act as proxies. -// Currently only egress ProxyGroups are supported. +// Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver +// proxies. In addition to running a highly available set of proxies, ingress +// and egress ProxyGroups also allow for serving many annotated Services from a +// single set of proxies to minimise resource consumption. // -// Use the tailscale.com/proxy-group annotation on a Service to specify that -// the egress proxy should be implemented by a ProxyGroup instead of a single -// dedicated proxy. In addition to running a highly available set of proxies, -// ProxyGroup also allows for serving many annotated Services from a single -// set of proxies to minimise resource consumption. +// For ingress and egress, use the tailscale.com/proxy-group annotation on a +// Service to specify that the proxy should be implemented by a ProxyGroup +// instead of a single dedicated proxy. // -// More info: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +// More info: +// * https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +// * https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress +// +// For kube-apiserver, the ProxyGroup is a standalone resource. Use the +// spec.kubeAPIServer field to configure options specific to the kube-apiserver +// ProxyGroup type. type ProxyGroup struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -49,7 +57,7 @@ type ProxyGroupList struct { } type ProxyGroupSpec struct { - // Type of the ProxyGroup proxies. Supported types are egress and ingress. + // Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. // Type is immutable once a ProxyGroup is created. // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup type is immutable" Type ProxyGroupType `json:"type"` @@ -84,11 +92,30 @@ type ProxyGroupSpec struct { // configuration. // +optional ProxyClass string `json:"proxyClass,omitempty"` + + // KubeAPIServer contains configuration specific to the kube-apiserver + // ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + // +optional + KubeAPIServer *KubeAPIServerConfig `json:"kubeAPIServer,omitempty"` } type ProxyGroupStatus struct { // List of status conditions to indicate the status of the ProxyGroup - // resources. Known condition types are `ProxyGroupReady`. + // resources. Known condition types include `ProxyGroupReady` and + // `ProxyGroupAvailable`. + // + // * `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and + // all expected conditions are true. + // * `ProxyGroupAvailable` indicates that at least one proxy is ready to + // serve traffic. + // + // For ProxyGroups of type kube-apiserver, there are two additional conditions: + // + // * `KubeAPIServerProxyConfigured` indicates that at least one API server + // proxy is configured and ready to serve traffic. + // * `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is + // valid. + // // +listType=map // +listMapKey=type // +optional @@ -99,6 +126,11 @@ type ProxyGroupStatus struct { // +listMapKey=hostname // +optional Devices []TailnetDevice `json:"devices,omitempty"` + + // URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if + // any. Only applies to ProxyGroups of type kube-apiserver. + // +optional + URL string `json:"url,omitempty"` } type TailnetDevice struct { @@ -111,17 +143,50 @@ type TailnetDevice struct { // assigned to the device. // +optional TailnetIPs []string `json:"tailnetIPs,omitempty"` + + // StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. + // +optional + StaticEndpoints []string `json:"staticEndpoints,omitempty"` } // +kubebuilder:validation:Type=string -// +kubebuilder:validation:Enum=egress;ingress +// +kubebuilder:validation:Enum=egress;ingress;kube-apiserver type ProxyGroupType string const ( - ProxyGroupTypeEgress ProxyGroupType = "egress" - ProxyGroupTypeIngress ProxyGroupType = "ingress" + ProxyGroupTypeEgress ProxyGroupType = "egress" + ProxyGroupTypeIngress ProxyGroupType = "ingress" + ProxyGroupTypeKubernetesAPIServer ProxyGroupType = "kube-apiserver" +) + +// +kubebuilder:validation:Type=string +// +kubebuilder:validation:Enum=auth;noauth +type APIServerProxyMode string + +const ( + APIServerProxyModeAuth APIServerProxyMode = "auth" + APIServerProxyModeNoAuth APIServerProxyMode = "noauth" ) // +kubebuilder:validation:Type=string // +kubebuilder:validation:Pattern=`^[a-z0-9][a-z0-9-]{0,61}$` type HostnamePrefix string + +// KubeAPIServerConfig contains configuration specific to the kube-apiserver ProxyGroup type. +type KubeAPIServerConfig struct { + // Mode to run the API server proxy in. Supported modes are auth and noauth. + // In auth mode, requests from the tailnet proxied over to the Kubernetes + // API server are additionally impersonated using the sender's tailnet identity. + // If not specified, defaults to auth mode. + // +optional + Mode *APIServerProxyMode `json:"mode,omitempty"` + + // Hostname is the hostname with which to expose the Kubernetes API server + // proxies. Must be a valid DNS label no longer than 63 characters. If not + // specified, the name of the ProxyGroup is used as the hostname. Must be + // unique across the whole tailnet. + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Pattern=`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$` + // +optional + Hostname string `json:"hostname,omitempty"` +} diff --git a/k8s-operator/apis/v1alpha1/types_tsdnsconfig.go b/k8s-operator/apis/v1alpha1/types_tsdnsconfig.go index 0178d60eab606..0e26ee6476d7a 100644 --- a/k8s-operator/apis/v1alpha1/types_tsdnsconfig.go +++ b/k8s-operator/apis/v1alpha1/types_tsdnsconfig.go @@ -82,6 +82,9 @@ type Nameserver struct { // Nameserver image. Defaults to tailscale/k8s-nameserver:unstable. // +optional Image *NameserverImage `json:"image,omitempty"` + // Service configuration. + // +optional + Service *NameserverService `json:"service,omitempty"` } type NameserverImage struct { @@ -93,6 +96,12 @@ type NameserverImage struct { Tag string `json:"tag,omitempty"` } +type NameserverService struct { + // ClusterIP sets the static IP of the service used by the nameserver. + // +optional + ClusterIP string `json:"clusterIP,omitempty"` +} + type DNSConfigStatus struct { // +listType=map // +listMapKey=type @@ -105,7 +114,7 @@ type DNSConfigStatus struct { type NameserverStatus struct { // IP is the ClusterIP of the Service fronting the deployed ts.net nameserver. - // Currently you must manually update your cluster DNS config to add + // Currently, you must manually update your cluster DNS config to add // this address as a stub nameserver for ts.net for cluster workloads to be // able to resolve MagicDNS names associated with egress or Ingress // proxies. diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index e091272075ce2..6586c13546f4f 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -316,6 +316,26 @@ func (in *Env) DeepCopy() *Env { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) { + *out = *in + if in.Mode != nil { + in, out := &in.Mode, &out.Mode + *out = new(APIServerProxyMode) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeAPIServerConfig. +func (in *KubeAPIServerConfig) DeepCopy() *KubeAPIServerConfig { + if in == nil { + return nil + } + out := new(KubeAPIServerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in Labels) DeepCopyInto(out *Labels) { { @@ -365,6 +385,11 @@ func (in *Nameserver) DeepCopyInto(out *Nameserver) { *out = new(NameserverImage) **out = **in } + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = new(NameserverService) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Nameserver. @@ -392,6 +417,21 @@ func (in *NameserverImage) DeepCopy() *NameserverImage { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NameserverService) DeepCopyInto(out *NameserverService) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NameserverService. +func (in *NameserverService) DeepCopy() *NameserverService { + if in == nil { + return nil + } + out := new(NameserverService) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NameserverStatus) DeepCopyInto(out *NameserverStatus) { *out = *in @@ -407,6 +447,33 @@ func (in *NameserverStatus) DeepCopy() *NameserverStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePortConfig) DeepCopyInto(out *NodePortConfig) { + *out = *in + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]PortRange, len(*in)) + copy(*out, *in) + } + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePortConfig. +func (in *NodePortConfig) DeepCopy() *NodePortConfig { + if in == nil { + return nil + } + out := new(NodePortConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pod) DeepCopyInto(out *Pod) { *out = *in @@ -482,6 +549,40 @@ func (in *Pod) DeepCopy() *Pod { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortRange) DeepCopyInto(out *PortRange) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRange. +func (in *PortRange) DeepCopy() *PortRange { + if in == nil { + return nil + } + out := new(PortRange) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in PortRanges) DeepCopyInto(out *PortRanges) { + { + in := &in + *out = make(PortRanges, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortRanges. +func (in PortRanges) DeepCopy() PortRanges { + if in == nil { + return nil + } + out := new(PortRanges) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxyClass) DeepCopyInto(out *ProxyClass) { *out = *in @@ -559,6 +660,11 @@ func (in *ProxyClassSpec) DeepCopyInto(out *ProxyClassSpec) { *out = new(TailscaleConfig) **out = **in } + if in.StaticEndpoints != nil { + in, out := &in.StaticEndpoints, &out.StaticEndpoints + *out = new(StaticEndpointsConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyClassSpec. @@ -665,6 +771,11 @@ func (in *ProxyGroupSpec) DeepCopyInto(out *ProxyGroupSpec) { *out = new(int32) **out = **in } + if in.KubeAPIServer != nil { + in, out := &in.KubeAPIServer, &out.KubeAPIServer + *out = new(KubeAPIServerConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyGroupSpec. @@ -1096,6 +1207,26 @@ func (in *StatefulSet) DeepCopy() *StatefulSet { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticEndpointsConfig) DeepCopyInto(out *StaticEndpointsConfig) { + *out = *in + if in.NodePort != nil { + in, out := &in.NodePort, &out.NodePort + *out = new(NodePortConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticEndpointsConfig. +func (in *StaticEndpointsConfig) DeepCopy() *StaticEndpointsConfig { + if in == nil { + return nil + } + out := new(StaticEndpointsConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Storage) DeepCopyInto(out *Storage) { *out = *in @@ -1163,6 +1294,11 @@ func (in *TailnetDevice) DeepCopyInto(out *TailnetDevice) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.StaticEndpoints != nil { + in, out := &in.StaticEndpoints, &out.StaticEndpoints + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetDevice. diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go index abe8f7f9cc6fa..ae465a728f0ff 100644 --- a/k8s-operator/conditions.go +++ b/k8s-operator/conditions.go @@ -137,14 +137,33 @@ func ProxyClassIsReady(pc *tsapi.ProxyClass) bool { } func ProxyGroupIsReady(pg *tsapi.ProxyGroup) bool { + cond := proxyGroupCondition(pg, tsapi.ProxyGroupReady) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation +} + +func ProxyGroupAvailable(pg *tsapi.ProxyGroup) bool { + cond := proxyGroupCondition(pg, tsapi.ProxyGroupAvailable) + return cond != nil && cond.Status == metav1.ConditionTrue +} + +func KubeAPIServerProxyValid(pg *tsapi.ProxyGroup) (valid bool, set bool) { + cond := proxyGroupCondition(pg, tsapi.KubeAPIServerProxyValid) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation, cond != nil +} + +func KubeAPIServerProxyConfigured(pg *tsapi.ProxyGroup) bool { + cond := proxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured) + return cond != nil && cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation +} + +func proxyGroupCondition(pg *tsapi.ProxyGroup, condType tsapi.ConditionType) *metav1.Condition { idx := xslices.IndexFunc(pg.Status.Conditions, func(cond metav1.Condition) bool { - return cond.Type == string(tsapi.ProxyGroupReady) + return cond.Type == string(condType) }) if idx == -1 { - return false + return nil } - cond := pg.Status.Conditions[idx] - return cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == pg.Generation + return &pg.Status.Conditions[idx] } func DNSCfgIsReady(cfg *tsapi.DNSConfig) bool { diff --git a/k8s-operator/sessionrecording/fakes/fakes.go b/k8s-operator/sessionrecording/fakes/fakes.go index 9eb1047e4242f..94853df195f7c 100644 --- a/k8s-operator/sessionrecording/fakes/fakes.go +++ b/k8s-operator/sessionrecording/fakes/fakes.go @@ -10,13 +10,13 @@ package fakes import ( "bytes" "encoding/json" + "fmt" + "math/rand" "net" "sync" "testing" "time" - "math/rand" - "tailscale.com/sessionrecording" "tailscale.com/tstime" ) @@ -107,7 +107,13 @@ func CastLine(t *testing.T, p []byte, clock tstime.Clock) []byte { return append(j, '\n') } -func AsciinemaResizeMsg(t *testing.T, width, height int) []byte { +func AsciinemaCastResizeMsg(t *testing.T, width, height int) []byte { + msg := fmt.Sprintf(`[0,"r","%dx%d"]`, height, width) + + return append([]byte(msg), '\n') +} + +func AsciinemaCastHeaderMsg(t *testing.T, width, height int) []byte { t.Helper() ch := sessionrecording.CastHeader{ Width: width, diff --git a/k8s-operator/sessionrecording/hijacker.go b/k8s-operator/sessionrecording/hijacker.go index a9ed658964787..789a9fdb9f6a3 100644 --- a/k8s-operator/sessionrecording/hijacker.go +++ b/k8s-operator/sessionrecording/hijacker.go @@ -4,7 +4,7 @@ //go:build !plan9 // Package sessionrecording contains functionality for recording Kubernetes API -// server proxy 'kubectl exec' sessions. +// server proxy 'kubectl exec/attach' sessions. package sessionrecording import ( @@ -35,14 +35,20 @@ import ( ) const ( - SPDYProtocol Protocol = "SPDY" - WSProtocol Protocol = "WebSocket" + SPDYProtocol Protocol = "SPDY" + WSProtocol Protocol = "WebSocket" + ExecSessionType SessionType = "exec" + AttachSessionType SessionType = "attach" ) // Protocol is the streaming protocol of the hijacked session. Supported // protocols are SPDY and WebSocket. type Protocol string +// SessionType is the type of session initiated with `kubectl` +// (`exec` or `attach`) +type SessionType string + var ( // CounterSessionRecordingsAttempted counts the number of session recording attempts. CounterSessionRecordingsAttempted = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_attempted") @@ -51,7 +57,7 @@ var ( counterSessionRecordingsUploaded = clientmetric.NewCounter("k8s_auth_proxy_session_recordings_uploaded") ) -func New(opts HijackerOpts) *Hijacker { +func NewHijacker(opts HijackerOpts) *Hijacker { return &Hijacker{ ts: opts.TS, req: opts.Req, @@ -63,25 +69,27 @@ func New(opts HijackerOpts) *Hijacker { failOpen: opts.FailOpen, proto: opts.Proto, log: opts.Log, + sessionType: opts.SessionType, connectToRecorder: sessionrecording.ConnectToRecorder, } } type HijackerOpts struct { - TS *tsnet.Server - Req *http.Request - W http.ResponseWriter - Who *apitype.WhoIsResponse - Addrs []netip.AddrPort - Log *zap.SugaredLogger - Pod string - Namespace string - FailOpen bool - Proto Protocol + TS *tsnet.Server + Req *http.Request + W http.ResponseWriter + Who *apitype.WhoIsResponse + Addrs []netip.AddrPort + Log *zap.SugaredLogger + Pod string + Namespace string + FailOpen bool + Proto Protocol + SessionType SessionType } // Hijacker implements [net/http.Hijacker] interface. -// It must be configured with an http request for a 'kubectl exec' session that +// It must be configured with an http request for a 'kubectl exec/attach' session that // needs to be recorded. It knows how to hijack the connection and configure for // the session contents to be sent to a tsrecorder instance. type Hijacker struct { @@ -90,12 +98,13 @@ type Hijacker struct { req *http.Request who *apitype.WhoIsResponse log *zap.SugaredLogger - pod string // pod being exec-d - ns string // namespace of the pod being exec-d + pod string // pod being exec/attach-d + ns string // namespace of the pod being exec/attach-d addrs []netip.AddrPort // tsrecorder addresses failOpen bool // whether to fail open if recording fails connectToRecorder RecorderDialFn - proto Protocol // streaming protocol + proto Protocol // streaming protocol + sessionType SessionType // subcommand, e.g., "exec, attach" } // RecorderDialFn dials the specified netip.AddrPorts that should be tsrecorder @@ -105,7 +114,7 @@ type Hijacker struct { // after having been established, an error is sent down the channel. type RecorderDialFn func(context.Context, []netip.AddrPort, netx.DialFunc) (io.WriteCloser, []*tailcfg.SSHRecordingAttempt, <-chan error, error) -// Hijack hijacks a 'kubectl exec' session and configures for the session +// Hijack hijacks a 'kubectl exec/attach' session and configures for the session // contents to be sent to a recorder. func (h *Hijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) { h.log.Infof("recorder addrs: %v, failOpen: %v", h.addrs, h.failOpen) @@ -114,7 +123,7 @@ func (h *Hijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) { return nil, nil, fmt.Errorf("error hijacking connection: %w", err) } - conn, err := h.setUpRecording(context.Background(), reqConn) + conn, err := h.setUpRecording(h.req.Context(), reqConn) if err != nil { return nil, nil, fmt.Errorf("error setting up session recording: %w", err) } @@ -138,7 +147,7 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, err error errChan <-chan error ) - h.log.Infof("kubectl exec session will be recorded, recorders: %v, fail open policy: %t", h.addrs, h.failOpen) + h.log.Infof("kubectl %s session will be recorded, recorders: %v, fail open policy: %t", h.sessionType, h.addrs, h.failOpen) qp := h.req.URL.Query() container := strings.Join(qp[containerKey], "") var recorderAddr net.Addr @@ -161,7 +170,7 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, } return nil, errors.New(msg) } else { - h.log.Infof("exec session to container %q in Pod %q namespace %q will be recorded, the recording will be sent to a tsrecorder instance at %q", container, h.pod, h.ns, recorderAddr) + h.log.Infof("%s session to container %q in Pod %q namespace %q will be recorded, the recording will be sent to a tsrecorder instance at %q", h.sessionType, container, h.pod, h.ns, recorderAddr) } cl := tstime.DefaultClock{} @@ -175,9 +184,10 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, SrcNode: strings.TrimSuffix(h.who.Node.Name, "."), SrcNodeID: h.who.Node.StableID, Kubernetes: &sessionrecording.Kubernetes{ - PodName: h.pod, - Namespace: h.ns, - Container: container, + PodName: h.pod, + Namespace: h.ns, + Container: container, + SessionType: string(h.sessionType), }, } if !h.who.Node.IsTagged() { @@ -190,9 +200,15 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, var lc net.Conn switch h.proto { case SPDYProtocol: - lc = spdy.New(conn, rec, ch, hasTerm, h.log) + lc, err = spdy.New(ctx, conn, rec, ch, hasTerm, h.log) + if err != nil { + return nil, fmt.Errorf("failed to initialize spdy connection: %w", err) + } case WSProtocol: - lc = ws.New(conn, rec, ch, hasTerm, h.log) + lc, err = ws.New(ctx, conn, rec, ch, hasTerm, h.log) + if err != nil { + return nil, fmt.Errorf("failed to initialize websocket connection: %w", err) + } default: return nil, fmt.Errorf("unknown protocol: %s", h.proto) } @@ -209,7 +225,7 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, h.log.Info("finished uploading the recording") return } - msg := fmt.Sprintf("connection to the session recorder errorred: %v;", err) + msg := fmt.Sprintf("connection to the session recorder errored: %v;", err) if h.failOpen { msg += msg + "; failure mode is 'fail open'; continuing session without recording." h.log.Info(msg) @@ -221,7 +237,6 @@ func (h *Hijacker) setUpRecording(ctx context.Context, conn net.Conn) (net.Conn, if err := lc.Close(); err != nil { h.log.Infof("error closing recorder connections: %v", err) } - return }() return lc, nil } diff --git a/k8s-operator/sessionrecording/hijacker_test.go b/k8s-operator/sessionrecording/hijacker_test.go index 880015b22c2d0..cac6f55c7c7d7 100644 --- a/k8s-operator/sessionrecording/hijacker_test.go +++ b/k8s-operator/sessionrecording/hijacker_test.go @@ -91,7 +91,7 @@ func Test_Hijacker(t *testing.T) { who: &apitype.WhoIsResponse{Node: &tailcfg.Node{}, UserProfile: &tailcfg.UserProfile{}}, log: zl.Sugar(), ts: &tsnet.Server{}, - req: &http.Request{URL: &url.URL{}}, + req: &http.Request{URL: &url.URL{RawQuery: "tty=true"}}, proto: tt.proto, } ctx := context.Background() diff --git a/k8s-operator/sessionrecording/spdy/conn.go b/k8s-operator/sessionrecording/spdy/conn.go index 455c2225ad921..9fefca11fc2b8 100644 --- a/k8s-operator/sessionrecording/spdy/conn.go +++ b/k8s-operator/sessionrecording/spdy/conn.go @@ -4,11 +4,12 @@ //go:build !plan9 // Package spdy contains functionality for parsing SPDY streaming sessions. This -// is used for 'kubectl exec' session recording. +// is used for 'kubectl exec/attach' session recording. package spdy import ( "bytes" + "context" "encoding/binary" "encoding/json" "fmt" @@ -24,29 +25,50 @@ import ( ) // New wraps the provided network connection and returns a connection whose reads and writes will get triggered as data is received on the hijacked connection. -// The connection must be a hijacked connection for a 'kubectl exec' session using SPDY. +// The connection must be a hijacked connection for a 'kubectl exec/attach' session using SPDY. // The hijacked connection is used to transmit SPDY streams between Kubernetes client ('kubectl') and the destination container. // Data read from the underlying network connection is data sent via one of the SPDY streams from the client to the container. // Data written to the underlying connection is data sent from the container to the client. // We parse the data and send everything for the stdout/stderr streams to the configured tsrecorder as an asciinema recording with the provided header. // https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/4006-transition-spdy-to-websockets#background-remotecommand-subprotocol -func New(nc net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, hasTerm bool, log *zap.SugaredLogger) net.Conn { - return &conn{ - Conn: nc, - rec: rec, - ch: ch, - log: log, - hasTerm: hasTerm, - initialTermSizeSet: make(chan struct{}), +func New(ctx context.Context, nc net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, hasTerm bool, log *zap.SugaredLogger) (net.Conn, error) { + lc := &conn{ + Conn: nc, + ctx: ctx, + rec: rec, + ch: ch, + log: log, + hasTerm: hasTerm, + initialCastHeaderSent: make(chan struct{}, 1), } + + // if there is no term, we don't need to wait for a resize message + if !hasTerm { + var err error + lc.writeCastHeaderOnce.Do(func() { + // If this is a session with a terminal attached, + // we must wait for the terminal width and + // height to be parsed from a resize message + // before sending CastHeader, else tsrecorder + // will not be able to play this recording. + err = lc.rec.WriteCastHeader(ch) + close(lc.initialCastHeaderSent) + }) + if err != nil { + return nil, fmt.Errorf("error writing CastHeader: %w", err) + } + } + + return lc, nil } // conn is a wrapper around net.Conn. It reads the bytestream for a 'kubectl -// exec' session streamed using SPDY protocol, sends session recording data to +// exec/attach' session streamed using SPDY protocol, sends session recording data to // the configured recorder and forwards the raw bytes to the original // destination. type conn struct { net.Conn + ctx context.Context // rec knows how to send data written to it to a tsrecorder instance. rec *tsrecorder.Client @@ -63,7 +85,7 @@ type conn struct { // CastHeader must be sent before any payload. If the session has a // terminal attached, the CastHeader must have '.Width' and '.Height' // fields set for the tsrecorder UI to be able to play the recording. - // For 'kubectl exec' sessions, terminal width and height are sent as a + // For 'kubectl exec/attach' sessions, terminal width and height are sent as a // resize message on resize stream from the client when the session // starts as well as at any time the client detects a terminal change. // We can intercept the resize message on Read calls. As there is no @@ -79,15 +101,10 @@ type conn struct { // writeCastHeaderOnce is used to ensure CastHeader gets sent to tsrecorder once. writeCastHeaderOnce sync.Once hasTerm bool // whether the session had TTY attached - // initialTermSizeSet channel gets sent a value once, when the Read has - // received a resize message and set the initial terminal size. It must - // be set to a buffered channel to prevent Reads being blocked on the - // first stdout/stderr write reading from the channel. - initialTermSizeSet chan struct{} - // sendInitialTermSizeSetOnce is used to ensure that a value is sent to - // initialTermSizeSet channel only once, when the initial resize message - // is received. - sendinitialTermSizeSetOnce sync.Once + // initialCastHeaderSent is a channel to ensure that the cast + // header is the first thing that is streamed to the session recorder. + // Otherwise the stream will fail. + initialCastHeaderSent chan struct{} zlibReqReader zlibReader // writeBuf is used to store data written to the connection that has not @@ -124,7 +141,7 @@ func (c *conn) Read(b []byte) (int, error) { } c.readBuf.Next(len(sf.Raw)) // advance buffer past the parsed frame - if !sf.Ctrl { // data frame + if !sf.Ctrl && c.hasTerm { // data frame switch sf.StreamID { case c.resizeStreamID.Load(): @@ -140,10 +157,19 @@ func (c *conn) Read(b []byte) (int, error) { // subsequent resize message, we need to send asciinema // resize message. var isInitialResize bool - c.sendinitialTermSizeSetOnce.Do(func() { + c.writeCastHeaderOnce.Do(func() { isInitialResize = true - close(c.initialTermSizeSet) // unblock sending of CastHeader + // If this is a session with a terminal attached, + // we must wait for the terminal width and + // height to be parsed from a resize message + // before sending CastHeader, else tsrecorder + // will not be able to play this recording. + err = c.rec.WriteCastHeader(c.ch) + close(c.initialCastHeaderSent) }) + if err != nil { + return 0, fmt.Errorf("error writing CastHeader: %w", err) + } if !isInitialResize { if err := c.rec.WriteResize(c.ch.Height, c.ch.Width); err != nil { return 0, fmt.Errorf("error writing resize message: %w", err) @@ -190,24 +216,14 @@ func (c *conn) Write(b []byte) (int, error) { if !sf.Ctrl { switch sf.StreamID { case c.stdoutStreamID.Load(), c.stderrStreamID.Load(): - var err error - c.writeCastHeaderOnce.Do(func() { - // If this is a session with a terminal attached, - // we must wait for the terminal width and - // height to be parsed from a resize message - // before sending CastHeader, else tsrecorder - // will not be able to play this recording. - if c.hasTerm { - c.log.Debugf("write: waiting for the initial terminal size to be set before proceeding with sending the first payload") - <-c.initialTermSizeSet + // we must wait for confirmation that the initial cast header was sent before proceeding with any more writes + select { + case <-c.ctx.Done(): + return 0, c.ctx.Err() + case <-c.initialCastHeaderSent: + if err := c.rec.WriteOutput(sf.Payload); err != nil { + return 0, fmt.Errorf("error sending payload to session recorder: %w", err) } - err = c.rec.WriteCastHeader(c.ch) - }) - if err != nil { - return 0, fmt.Errorf("error writing CastHeader: %w", err) - } - if err := c.rec.WriteOutput(sf.Payload); err != nil { - return 0, fmt.Errorf("error sending payload to session recorder: %w", err) } } } diff --git a/k8s-operator/sessionrecording/spdy/conn_test.go b/k8s-operator/sessionrecording/spdy/conn_test.go index 3485d61c4f454..3c1cb8427d822 100644 --- a/k8s-operator/sessionrecording/spdy/conn_test.go +++ b/k8s-operator/sessionrecording/spdy/conn_test.go @@ -6,10 +6,12 @@ package spdy import ( + "context" "encoding/json" "fmt" "reflect" "testing" + "time" "go.uber.org/zap" "tailscale.com/k8s-operator/sessionrecording/fakes" @@ -29,15 +31,11 @@ func Test_Writes(t *testing.T) { } cl := tstest.NewClock(tstest.ClockOpts{}) tests := []struct { - name string - inputs [][]byte - wantForwarded []byte - wantRecorded []byte - firstWrite bool - width int - height int - sendInitialResize bool - hasTerm bool + name string + inputs [][]byte + wantForwarded []byte + wantRecorded []byte + hasTerm bool }{ { name: "single_write_control_frame_with_payload", @@ -78,24 +76,17 @@ func Test_Writes(t *testing.T) { wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl), }, { - name: "single_first_write_stdout_data_frame_with_payload_sess_has_terminal", - inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}}, - wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}, - wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...), - width: 10, - height: 20, - hasTerm: true, - firstWrite: true, - sendInitialResize: true, + name: "single_first_write_stdout_data_frame_with_payload_sess_has_terminal", + inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}}, + wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}, + wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl), + hasTerm: true, }, { name: "single_first_write_stdout_data_frame_with_payload_sess_does_not_have_terminal", inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}}, wantForwarded: []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x5, 0x1, 0x2, 0x3, 0x4, 0x5}, - wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl)...), - width: 10, - height: 20, - firstWrite: true, + wantRecorded: fakes.CastLine(t, []byte{0x1, 0x2, 0x3, 0x4, 0x5}, cl), }, } for _, tt := range tests { @@ -104,29 +95,25 @@ func Test_Writes(t *testing.T) { sr := &fakes.TestSessionRecorder{} rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar()) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() c := &conn{ - Conn: tc, - log: zl.Sugar(), - rec: rec, - ch: sessionrecording.CastHeader{ - Width: tt.width, - Height: tt.height, - }, - initialTermSizeSet: make(chan struct{}), - hasTerm: tt.hasTerm, - } - if !tt.firstWrite { - // this test case does not intend to test that cast header gets written once - c.writeCastHeaderOnce.Do(func() {}) - } - if tt.sendInitialResize { - close(c.initialTermSizeSet) + ctx: ctx, + Conn: tc, + log: zl.Sugar(), + rec: rec, + ch: sessionrecording.CastHeader{}, + initialCastHeaderSent: make(chan struct{}), + hasTerm: tt.hasTerm, } + c.writeCastHeaderOnce.Do(func() { + close(c.initialCastHeaderSent) + }) + c.stdoutStreamID.Store(stdoutStreamID) c.stderrStreamID.Store(stderrStreamID) for i, input := range tt.inputs { - c.hasTerm = tt.hasTerm if _, err := c.Write(input); err != nil { t.Errorf("[%d] spdyRemoteConnRecorder.Write() unexpected error %v", i, err) } @@ -171,11 +158,25 @@ func Test_Reads(t *testing.T) { wantResizeStreamID uint32 wantWidth int wantHeight int + wantRecorded []byte resizeStreamIDBeforeRead uint32 }{ { name: "resize_data_frame_single_read", inputs: [][]byte{append([]byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg...)}, + wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), + resizeStreamIDBeforeRead: 1, + wantWidth: 10, + wantHeight: 20, + }, + { + name: "resize_data_frame_many", + inputs: [][]byte{ + append([]byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg...), + append([]byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg...), + }, + wantRecorded: append(fakes.AsciinemaCastHeaderMsg(t, 10, 20), fakes.AsciinemaCastResizeMsg(t, 10, 20)...), + resizeStreamIDBeforeRead: 1, wantWidth: 10, wantHeight: 20, @@ -183,6 +184,7 @@ func Test_Reads(t *testing.T) { { name: "resize_data_frame_two_reads", inputs: [][]byte{{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, uint8(len(resizeMsg))}, resizeMsg}, + wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), resizeStreamIDBeforeRead: 1, wantWidth: 10, wantHeight: 20, @@ -215,11 +217,15 @@ func Test_Reads(t *testing.T) { tc := &fakes.TestConn{} sr := &fakes.TestSessionRecorder{} rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar()) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() c := &conn{ - Conn: tc, - log: zl.Sugar(), - rec: rec, - initialTermSizeSet: make(chan struct{}), + ctx: ctx, + Conn: tc, + log: zl.Sugar(), + rec: rec, + initialCastHeaderSent: make(chan struct{}), + hasTerm: true, } c.resizeStreamID.Store(tt.resizeStreamIDBeforeRead) @@ -251,6 +257,12 @@ func Test_Reads(t *testing.T) { t.Errorf("want height: %v, got %v", tt.wantHeight, c.ch.Height) } } + + // Assert that the expected bytes have been forwarded to the session recorder. + gotRecorded := sr.Bytes() + if !reflect.DeepEqual(gotRecorded, tt.wantRecorded) { + t.Errorf("expected bytes not recorded, wants\n%v\ngot\n%v", tt.wantRecorded, gotRecorded) + } }) } } diff --git a/k8s-operator/sessionrecording/tsrecorder/tsrecorder.go b/k8s-operator/sessionrecording/tsrecorder/tsrecorder.go index af5fcb8da641a..a5bdf7ddddeeb 100644 --- a/k8s-operator/sessionrecording/tsrecorder/tsrecorder.go +++ b/k8s-operator/sessionrecording/tsrecorder/tsrecorder.go @@ -25,6 +25,7 @@ func New(conn io.WriteCloser, clock tstime.Clock, start time.Time, failOpen bool clock: clock, conn: conn, failOpen: failOpen, + logger: logger, } } diff --git a/k8s-operator/sessionrecording/ws/conn.go b/k8s-operator/sessionrecording/ws/conn.go index 86029f67b1f13..a34379658caa2 100644 --- a/k8s-operator/sessionrecording/ws/conn.go +++ b/k8s-operator/sessionrecording/ws/conn.go @@ -3,12 +3,13 @@ //go:build !plan9 -// package ws has functionality to parse 'kubectl exec' sessions streamed using +// package ws has functionality to parse 'kubectl exec/attach' sessions streamed using // WebSocket protocol. package ws import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -24,31 +25,53 @@ import ( ) // New wraps the provided network connection and returns a connection whose reads and writes will get triggered as data is received on the hijacked connection. -// The connection must be a hijacked connection for a 'kubectl exec' session using WebSocket protocol and a *.channel.k8s.io subprotocol. +// The connection must be a hijacked connection for a 'kubectl exec/attach' session using WebSocket protocol and a *.channel.k8s.io subprotocol. // The hijacked connection is used to transmit *.channel.k8s.io streams between Kubernetes client ('kubectl') and the destination proxy controlled by Kubernetes. // Data read from the underlying network connection is data sent via one of the streams from the client to the container. // Data written to the underlying connection is data sent from the container to the client. // We parse the data and send everything for the stdout/stderr streams to the configured tsrecorder as an asciinema recording with the provided header. // https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/4006-transition-spdy-to-websockets#proposal-new-remotecommand-sub-protocol-version---v5channelk8sio -func New(c net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, hasTerm bool, log *zap.SugaredLogger) net.Conn { - return &conn{ - Conn: c, - rec: rec, - ch: ch, - hasTerm: hasTerm, - log: log, - initialTermSizeSet: make(chan struct{}, 1), +func New(ctx context.Context, c net.Conn, rec *tsrecorder.Client, ch sessionrecording.CastHeader, hasTerm bool, log *zap.SugaredLogger) (net.Conn, error) { + lc := &conn{ + Conn: c, + ctx: ctx, + rec: rec, + ch: ch, + hasTerm: hasTerm, + log: log, + initialCastHeaderSent: make(chan struct{}, 1), } + + // if there is no term, we don't need to wait for a resize message + if !hasTerm { + var err error + lc.writeCastHeaderOnce.Do(func() { + // If this is a session with a terminal attached, + // we must wait for the terminal width and + // height to be parsed from a resize message + // before sending CastHeader, else tsrecorder + // will not be able to play this recording. + err = lc.rec.WriteCastHeader(ch) + close(lc.initialCastHeaderSent) + }) + if err != nil { + return nil, fmt.Errorf("error writing CastHeader: %w", err) + } + } + + return lc, nil } // conn is a wrapper around net.Conn. It reads the bytestream -// for a 'kubectl exec' session, sends session recording data to the configured +// for a 'kubectl exec/attach' session, sends session recording data to the configured // recorder and forwards the raw bytes to the original destination. // A new conn is created per session. -// conn only knows to how to read a 'kubectl exec' session that is streamed using WebSocket protocol. +// conn only knows to how to read a 'kubectl exec/attach' session that is streamed using WebSocket protocol. // https://www.rfc-editor.org/rfc/rfc6455 type conn struct { net.Conn + + ctx context.Context // rec knows how to send data to a tsrecorder instance. rec *tsrecorder.Client @@ -56,7 +79,7 @@ type conn struct { // CastHeader must be sent before any payload. If the session has a // terminal attached, the CastHeader must have '.Width' and '.Height' // fields set for the tsrecorder UI to be able to play the recording. - // For 'kubectl exec' sessions, terminal width and height are sent as a + // For 'kubectl exec/attach' sessions, terminal width and height are sent as a // resize message on resize stream from the client when the session // starts as well as at any time the client detects a terminal change. // We can intercept the resize message on Read calls. As there is no @@ -72,15 +95,10 @@ type conn struct { // writeCastHeaderOnce is used to ensure CastHeader gets sent to tsrecorder once. writeCastHeaderOnce sync.Once hasTerm bool // whether the session has TTY attached - // initialTermSizeSet channel gets sent a value once, when the Read has - // received a resize message and set the initial terminal size. It must - // be set to a buffered channel to prevent Reads being blocked on the - // first stdout/stderr write reading from the channel. - initialTermSizeSet chan struct{} - // sendInitialTermSizeSetOnce is used to ensure that a value is sent to - // initialTermSizeSet channel only once, when the initial resize message - // is received. - sendInitialTermSizeSetOnce sync.Once + // initialCastHeaderSent is a boolean that is set to ensure that the cast + // header is the first thing that is streamed to the session recorder. + // Otherwise the stream will fail. + initialCastHeaderSent chan struct{} log *zap.SugaredLogger @@ -130,6 +148,8 @@ func (c *conn) Read(b []byte) (int, error) { return 0, nil } + // TODO(tomhjp): If we get multiple frames in a single Read with different + // types, we may parse the second frame with the wrong type. typ := messageType(opcode(b)) if (typ == noOpcode && c.readMsgIsIncomplete()) || c.readBufHasIncompleteFragment() { // subsequent fragment if typ, err = c.curReadMsgType(); err != nil { @@ -139,6 +159,8 @@ func (c *conn) Read(b []byte) (int, error) { // A control message can not be fragmented and we are not interested in // these messages. Just return. + // TODO(tomhjp): If we get multiple frames in a single Read, we may skip + // some non-control messages. if isControlMessage(typ) { return n, nil } @@ -151,54 +173,65 @@ func (c *conn) Read(b []byte) (int, error) { return n, nil } - readMsg := &message{typ: typ} // start a new message... - // ... or pick up an already started one if the previous fragment was not final. - if c.readMsgIsIncomplete() || c.readBufHasIncompleteFragment() { - readMsg = c.currentReadMsg - } - if _, err := c.readBuf.Write(b[:n]); err != nil { return 0, fmt.Errorf("[unexpected] error writing message contents to read buffer: %w", err) } - ok, err := readMsg.Parse(c.readBuf.Bytes(), c.log) - if err != nil { - return 0, fmt.Errorf("error parsing message: %v", err) - } - if !ok { // incomplete fragment - return n, nil - } - c.readBuf.Next(len(readMsg.raw)) - - if readMsg.isFinalized && !c.readMsgIsIncomplete() { - // Stream IDs for websocket streams are static. - // https://github.com/kubernetes/client-go/blob/v0.30.0-rc.1/tools/remotecommand/websocket.go#L218 - if readMsg.streamID.Load() == remotecommand.StreamResize { - var msg tsrecorder.ResizeMsg - if err = json.Unmarshal(readMsg.payload, &msg); err != nil { - return 0, fmt.Errorf("error umarshalling resize message: %w", err) - } + for c.readBuf.Len() != 0 { + readMsg := &message{typ: typ} // start a new message... + // ... or pick up an already started one if the previous fragment was not final. + if c.readMsgIsIncomplete() { + readMsg = c.currentReadMsg + } - c.ch.Width = msg.Width - c.ch.Height = msg.Height - - // If this is initial resize message, the width and - // height will be sent in the CastHeader. If this is a - // subsequent resize message, we need to send asciinema - // resize message. - var isInitialResize bool - c.sendInitialTermSizeSetOnce.Do(func() { - isInitialResize = true - close(c.initialTermSizeSet) // unblock sending of CastHeader - }) - if !isInitialResize { - if err := c.rec.WriteResize(c.ch.Height, c.ch.Width); err != nil { - return 0, fmt.Errorf("error writing resize message: %w", err) + ok, err := readMsg.Parse(c.readBuf.Bytes(), c.log) + if err != nil { + return 0, fmt.Errorf("error parsing message: %v", err) + } + if !ok { // incomplete fragment + return n, nil + } + c.readBuf.Next(len(readMsg.raw)) + + if readMsg.isFinalized && !c.readMsgIsIncomplete() { + // we want to send stream resize messages for terminal sessions + // Stream IDs for websocket streams are static. + // https://github.com/kubernetes/client-go/blob/v0.30.0-rc.1/tools/remotecommand/websocket.go#L218 + if readMsg.streamID.Load() == remotecommand.StreamResize && c.hasTerm { + var msg tsrecorder.ResizeMsg + if err = json.Unmarshal(readMsg.payload, &msg); err != nil { + return 0, fmt.Errorf("error umarshalling resize message: %w", err) + } + + c.ch.Width = msg.Width + c.ch.Height = msg.Height + + var isInitialResize bool + c.writeCastHeaderOnce.Do(func() { + isInitialResize = true + // If this is a session with a terminal attached, + // we must wait for the terminal width and + // height to be parsed from a resize message + // before sending CastHeader, else tsrecorder + // will not be able to play this recording. + err = c.rec.WriteCastHeader(c.ch) + close(c.initialCastHeaderSent) + }) + if err != nil { + return 0, fmt.Errorf("error writing CastHeader: %w", err) + } + + if !isInitialResize { + if err := c.rec.WriteResize(msg.Height, msg.Width); err != nil { + return 0, fmt.Errorf("error writing resize message: %w", err) + } } } } + + c.currentReadMsg = readMsg } - c.currentReadMsg = readMsg + return n, nil } @@ -244,39 +277,33 @@ func (c *conn) Write(b []byte) (int, error) { c.log.Errorf("write: parsing a message errored: %v", err) return 0, fmt.Errorf("write: error parsing message: %v", err) } + c.currentWriteMsg = writeMsg if !ok { // incomplete fragment return len(b), nil } + c.writeBuf.Next(len(writeMsg.raw)) // advance frame if len(writeMsg.payload) != 0 && writeMsg.isFinalized { if writeMsg.streamID.Load() == remotecommand.StreamStdOut || writeMsg.streamID.Load() == remotecommand.StreamStdErr { - var err error - c.writeCastHeaderOnce.Do(func() { - // If this is a session with a terminal attached, - // we must wait for the terminal width and - // height to be parsed from a resize message - // before sending CastHeader, else tsrecorder - // will not be able to play this recording. - if c.hasTerm { - c.log.Debug("waiting for terminal size to be set before starting to send recorded data") - <-c.initialTermSizeSet + // we must wait for confirmation that the initial cast header was sent before proceeding with any more writes + select { + case <-c.ctx.Done(): + return 0, c.ctx.Err() + case <-c.initialCastHeaderSent: + if err := c.rec.WriteOutput(writeMsg.payload); err != nil { + return 0, fmt.Errorf("error writing message to recorder: %w", err) } - err = c.rec.WriteCastHeader(c.ch) - }) - if err != nil { - return 0, fmt.Errorf("error writing CastHeader: %w", err) - } - if err := c.rec.WriteOutput(writeMsg.payload); err != nil { - return 0, fmt.Errorf("error writing message to recorder: %v", err) } } } + _, err = c.Conn.Write(c.currentWriteMsg.raw) if err != nil { c.log.Errorf("write: error writing to conn: %v", err) } + return len(b), nil } @@ -321,6 +348,7 @@ func (c *conn) writeMsgIsIncomplete() bool { func (c *conn) readMsgIsIncomplete() bool { return c.currentReadMsg != nil && !c.currentReadMsg.isFinalized } + func (c *conn) curReadMsgType() (messageType, error) { if c.currentReadMsg != nil { return c.currentReadMsg.typ, nil diff --git a/k8s-operator/sessionrecording/ws/conn_test.go b/k8s-operator/sessionrecording/ws/conn_test.go index 11174480ba605..f2fd4ea55f554 100644 --- a/k8s-operator/sessionrecording/ws/conn_test.go +++ b/k8s-operator/sessionrecording/ws/conn_test.go @@ -6,9 +6,12 @@ package ws import ( + "context" "fmt" "reflect" + "runtime/debug" "testing" + "time" "go.uber.org/zap" "k8s.io/apimachinery/pkg/util/remotecommand" @@ -26,46 +29,93 @@ func Test_conn_Read(t *testing.T) { // Resize stream ID + {"width": 10, "height": 20} testResizeMsg := []byte{byte(remotecommand.StreamResize), 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x3a, 0x32, 0x30, 0x7d} lenResizeMsgPayload := byte(len(testResizeMsg)) - + cl := tstest.NewClock(tstest.ClockOpts{}) tests := []struct { - name string - inputs [][]byte - wantWidth int - wantHeight int + name string + inputs [][]byte + wantCastHeaderWidth int + wantCastHeaderHeight int + wantRecorded []byte }{ { name: "single_read_control_message", inputs: [][]byte{{0x88, 0x0}}, }, { - name: "single_read_resize_message", - inputs: [][]byte{append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...)}, - wantWidth: 10, - wantHeight: 20, + name: "single_read_resize_message", + inputs: [][]byte{append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...)}, + wantCastHeaderWidth: 10, + wantCastHeaderHeight: 20, + wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), + }, + { + name: "resize_data_frame_many", + inputs: [][]byte{ + append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...), + append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...), + }, + wantRecorded: append(fakes.AsciinemaCastHeaderMsg(t, 10, 20), fakes.AsciinemaCastResizeMsg(t, 10, 20)...), + wantCastHeaderWidth: 10, + wantCastHeaderHeight: 20, + }, + { + name: "resize_data_frame_two_in_one_read", + inputs: [][]byte{ + fmt.Appendf(nil, "%s%s", + append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...), + append([]byte{0x82, lenResizeMsgPayload}, testResizeMsg...), + ), + }, + wantRecorded: append(fakes.AsciinemaCastHeaderMsg(t, 10, 20), fakes.AsciinemaCastResizeMsg(t, 10, 20)...), + wantCastHeaderWidth: 10, + wantCastHeaderHeight: 20, }, { - name: "two_reads_resize_message", - inputs: [][]byte{{0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x3a, 0x32, 0x30, 0x7d}}, - wantWidth: 10, - wantHeight: 20, + name: "two_reads_resize_message", + inputs: [][]byte{ + // op, len, stream ID, `{"width` + {0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, + // op, len, stream ID, `:10,"height":20}` + {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x3a, 0x32, 0x30, 0x7d}, + }, + wantCastHeaderWidth: 10, + wantCastHeaderHeight: 20, + wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), }, { - name: "three_reads_resize_message_with_split_fragment", - inputs: [][]byte{{0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, {0x80, 0x11, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74}, {0x22, 0x3a, 0x32, 0x30, 0x7d}}, - wantWidth: 10, - wantHeight: 20, + name: "three_reads_resize_message_with_split_fragment", + inputs: [][]byte{ + // op, len, stream ID, `{"width"` + {0x2, 0x9, 0x4, 0x7b, 0x22, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22}, + // op, len, stream ID, `:10,"height` + {0x00, 0x0c, 0x4, 0x3a, 0x31, 0x30, 0x2c, 0x22, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74}, + // op, len, stream ID, `":20}` + {0x80, 0x06, 0x4, 0x22, 0x3a, 0x32, 0x30, 0x7d}, + }, + wantCastHeaderWidth: 10, + wantCastHeaderHeight: 20, + wantRecorded: fakes.AsciinemaCastHeaderMsg(t, 10, 20), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + l := zl.Sugar() tc := &fakes.TestConn{} + sr := &fakes.TestSessionRecorder{} + rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar()) tc.ResetReadBuf() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() c := &conn{ - Conn: tc, - log: zl.Sugar(), + ctx: ctx, + Conn: tc, + log: l, + hasTerm: true, + initialCastHeaderSent: make(chan struct{}), + rec: rec, } for i, input := range tt.inputs { - c.initialTermSizeSet = make(chan struct{}) if err := tc.WriteReadBufBytes(input); err != nil { t.Fatalf("writing bytes to test conn: %v", err) } @@ -75,14 +125,20 @@ func Test_conn_Read(t *testing.T) { return } } - if tt.wantHeight != 0 || tt.wantWidth != 0 { - if tt.wantWidth != c.ch.Width { - t.Errorf("wants width: %v, got %v", tt.wantWidth, c.ch.Width) + + if tt.wantCastHeaderHeight != 0 || tt.wantCastHeaderWidth != 0 { + if tt.wantCastHeaderWidth != c.ch.Width { + t.Errorf("wants width: %v, got %v", tt.wantCastHeaderWidth, c.ch.Width) } - if tt.wantHeight != c.ch.Height { - t.Errorf("want height: %v, got %v", tt.wantHeight, c.ch.Height) + if tt.wantCastHeaderHeight != c.ch.Height { + t.Errorf("want height: %v, got %v", tt.wantCastHeaderHeight, c.ch.Height) } } + + gotRecorded := sr.Bytes() + if !reflect.DeepEqual(gotRecorded, tt.wantRecorded) { + t.Errorf("expected bytes not recorded, wants\n%v\ngot\n%v", string(tt.wantRecorded), string(gotRecorded)) + } }) } } @@ -94,15 +150,11 @@ func Test_conn_Write(t *testing.T) { } cl := tstest.NewClock(tstest.ClockOpts{}) tests := []struct { - name string - inputs [][]byte - wantForwarded []byte - wantRecorded []byte - firstWrite bool - width int - height int - hasTerm bool - sendInitialResize bool + name string + inputs [][]byte + wantForwarded []byte + wantRecorded []byte + hasTerm bool }{ { name: "single_write_control_frame", @@ -130,10 +182,7 @@ func Test_conn_Write(t *testing.T) { name: "single_write_stdout_data_message_with_cast_header", inputs: [][]byte{{0x82, 0x3, 0x1, 0x7, 0x8}}, wantForwarded: []byte{0x82, 0x3, 0x1, 0x7, 0x8}, - wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x7, 0x8}, cl)...), - width: 10, - height: 20, - firstWrite: true, + wantRecorded: fakes.CastLine(t, []byte{0x7, 0x8}, cl), }, { name: "two_writes_stdout_data_message", @@ -148,15 +197,11 @@ func Test_conn_Write(t *testing.T) { wantRecorded: fakes.CastLine(t, []byte{0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5}, cl), }, { - name: "three_writes_stdout_data_message_with_split_fragment_cast_header_with_terminal", - inputs: [][]byte{{0x2, 0x3, 0x1, 0x7, 0x8}, {0x80, 0x6, 0x1, 0x1, 0x2, 0x3}, {0x4, 0x5}}, - wantForwarded: []byte{0x2, 0x3, 0x1, 0x7, 0x8, 0x80, 0x6, 0x1, 0x1, 0x2, 0x3, 0x4, 0x5}, - wantRecorded: append(fakes.AsciinemaResizeMsg(t, 10, 20), fakes.CastLine(t, []byte{0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5}, cl)...), - height: 20, - width: 10, - hasTerm: true, - firstWrite: true, - sendInitialResize: true, + name: "three_writes_stdout_data_message_with_split_fragment_cast_header_with_terminal", + inputs: [][]byte{{0x2, 0x3, 0x1, 0x7, 0x8}, {0x80, 0x6, 0x1, 0x1, 0x2, 0x3}, {0x4, 0x5}}, + wantForwarded: []byte{0x2, 0x3, 0x1, 0x7, 0x8, 0x80, 0x6, 0x1, 0x1, 0x2, 0x3, 0x4, 0x5}, + wantRecorded: fakes.CastLine(t, []byte{0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5}, cl), + hasTerm: true, }, } for _, tt := range tests { @@ -164,24 +209,22 @@ func Test_conn_Write(t *testing.T) { tc := &fakes.TestConn{} sr := &fakes.TestSessionRecorder{} rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar()) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() c := &conn{ - Conn: tc, - log: zl.Sugar(), - ch: sessionrecording.CastHeader{ - Width: tt.width, - Height: tt.height, - }, - rec: rec, - initialTermSizeSet: make(chan struct{}), - hasTerm: tt.hasTerm, - } - if !tt.firstWrite { - // This test case does not intend to test that cast header gets written once. - c.writeCastHeaderOnce.Do(func() {}) - } - if tt.sendInitialResize { - close(c.initialTermSizeSet) + Conn: tc, + ctx: ctx, + log: zl.Sugar(), + ch: sessionrecording.CastHeader{}, + rec: rec, + initialCastHeaderSent: make(chan struct{}), + hasTerm: tt.hasTerm, } + + c.writeCastHeaderOnce.Do(func() { + close(c.initialCastHeaderSent) + }) + for i, input := range tt.inputs { _, err := c.Write(input) if err != nil { @@ -242,19 +285,28 @@ func Test_conn_WriteRand(t *testing.T) { sr := &fakes.TestSessionRecorder{} rec := tsrecorder.New(sr, cl, cl.Now(), true, zl.Sugar()) for i := range 100 { - tc := &fakes.TestConn{} - c := &conn{ - Conn: tc, - log: zl.Sugar(), - rec: rec, - } - bb := fakes.RandomBytes(t) - for j, input := range bb { - f := func() { - c.Write(input) + t.Run(fmt.Sprintf("test_%d", i), func(t *testing.T) { + tc := &fakes.TestConn{} + c := &conn{ + Conn: tc, + log: zl.Sugar(), + rec: rec, + + ctx: context.Background(), // ctx must be non-nil. + initialCastHeaderSent: make(chan struct{}), } - testPanic(t, f, fmt.Sprintf("[%d %d] Write: panic parsing input of length %d first bytes %b current write message %+#v", i, j, len(input), firstBytes(input), c.currentWriteMsg)) - } + // Never block for random data. + c.writeCastHeaderOnce.Do(func() { + close(c.initialCastHeaderSent) + }) + bb := fakes.RandomBytes(t) + for j, input := range bb { + f := func() { + c.Write(input) + } + testPanic(t, f, fmt.Sprintf("[%d %d] Write: panic parsing input of length %d first bytes %b current write message %+#v", i, j, len(input), firstBytes(input), c.currentWriteMsg)) + } + }) } } @@ -262,7 +314,7 @@ func testPanic(t *testing.T, f func(), msg string) { t.Helper() defer func() { if r := recover(); r != nil { - t.Fatal(msg, r) + t.Fatal(msg, r, string(debug.Stack())) } }() f() diff --git a/k8s-operator/sessionrecording/ws/message.go b/k8s-operator/sessionrecording/ws/message.go index 713febec76ae8..35667ae21a5d0 100644 --- a/k8s-operator/sessionrecording/ws/message.go +++ b/k8s-operator/sessionrecording/ws/message.go @@ -7,10 +7,10 @@ package ws import ( "encoding/binary" + "errors" "fmt" "sync/atomic" - "github.com/pkg/errors" "go.uber.org/zap" "golang.org/x/net/websocket" @@ -139,6 +139,8 @@ func (msg *message) Parse(b []byte, log *zap.SugaredLogger) (bool, error) { return false, errors.New("[unexpected] received a message fragment with no stream ID") } + // Stream ID will be one of the constants from: + // https://github.com/kubernetes/kubernetes/blob/f9ed14bf9b1119a2e091f4b487a3b54930661034/staging/src/k8s.io/apimachinery/pkg/util/remotecommand/constants.go#L57-L64 streamID := uint32(msgPayload[0]) if !isInitialFragment && msg.streamID.Load() != streamID { return false, fmt.Errorf("[unexpected] received message fragments with mismatched streamIDs %d and %d", msg.streamID.Load(), streamID) diff --git a/cmd/containerboot/certs.go b/kube/certs/certs.go similarity index 60% rename from cmd/containerboot/certs.go rename to kube/certs/certs.go index 504ef7988072b..8e2e5fb43a8ac 100644 --- a/cmd/containerboot/certs.go +++ b/kube/certs/certs.go @@ -1,29 +1,32 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux - -package main +// Package certs implements logic to help multiple Kubernetes replicas share TLS +// certs for a common Tailscale Service. +package certs import ( "context" "fmt" - "log" "net" + "slices" "sync" "time" "tailscale.com/ipn" + "tailscale.com/kube/localclient" + "tailscale.com/types/logger" "tailscale.com/util/goroutines" "tailscale.com/util/mak" ) -// certManager is responsible for issuing certificates for known domains and for +// CertManager is responsible for issuing certificates for known domains and for // maintaining a loop that re-attempts issuance daily. // Currently cert manager logic is only run on ingress ProxyGroup replicas that are responsible for managing certs for // HA Ingress HTTPS endpoints ('write' replicas). -type certManager struct { - lc localClient +type CertManager struct { + lc localclient.LocalClient + logf logger.Logf tracker goroutines.Tracker // tracks running goroutines mu sync.Mutex // guards the following // certLoops contains a map of DNS names, for which we currently need to @@ -32,11 +35,18 @@ type certManager struct { certLoops map[string]context.CancelFunc } -// ensureCertLoops ensures that, for all currently managed Service HTTPS +func NewCertManager(lc localclient.LocalClient, logf logger.Logf) *CertManager { + return &CertManager{ + lc: lc, + logf: logf, + } +} + +// EnsureCertLoops ensures that, for all currently managed Service HTTPS // endpoints, there is a cert loop responsible for issuing and ensuring the // renewal of the TLS certs. // ServeConfig must not be nil. -func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error { +func (cm *CertManager) EnsureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error { if sc == nil { return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig") } @@ -87,12 +97,18 @@ func (cm *certManager) ensureCertLoops(ctx context.Context, sc *ipn.ServeConfig) // renewed at that point. Renewal here is needed to prevent the shared certs from expiry in edge cases where the 'write' // replica does not get any HTTPS requests. // https://letsencrypt.org/docs/integration-guide/#retrying-failures -func (cm *certManager) runCertLoop(ctx context.Context, domain string) { +func (cm *CertManager) runCertLoop(ctx context.Context, domain string) { const ( normalInterval = 24 * time.Hour // regular renewal check initialRetry = 1 * time.Minute // initial backoff after a failure maxRetryInterval = 24 * time.Hour // max backoff period ) + + if err := cm.waitForCertDomain(ctx, domain); err != nil { + // Best-effort, log and continue with the issuing loop. + cm.logf("error waiting for cert domain %s: %v", domain, err) + } + timer := time.NewTimer(0) // fire off timer immediately defer timer.Stop() retryCount := 0 @@ -101,38 +117,31 @@ func (cm *certManager) runCertLoop(ctx context.Context, domain string) { case <-ctx.Done(): return case <-timer.C: - // We call the certificate endpoint, but don't do anything - // with the returned certs here. - // The call to the certificate endpoint will ensure that - // certs are issued/renewed as needed and stored in the - // relevant state store. For example, for HA Ingress - // 'write' replica, the cert and key will be stored in a - // Kubernetes Secret named after the domain for which we - // are issuing. - // Note that renewals triggered by the call to the - // certificates endpoint here and by renewal check - // triggered during a call to node's HTTPS endpoint - // share the same state/renewal lock mechanism, so we - // should not run into redundant issuances during - // concurrent renewal checks. - // TODO(irbekrm): maybe it is worth adding a new - // issuance endpoint that explicitly only triggers - // issuance and stores certs in the relevant store, but - // does not return certs to the caller? + // We call the certificate endpoint, but don't do anything with the + // returned certs here. The call to the certificate endpoint will + // ensure that certs are issued/renewed as needed and stored in the + // relevant state store. For example, for HA Ingress 'write' replica, + // the cert and key will be stored in a Kubernetes Secret named after + // the domain for which we are issuing. + // + // Note that renewals triggered by the call to the certificates + // endpoint here and by renewal check triggered during a call to + // node's HTTPS endpoint share the same state/renewal lock mechanism, + // so we should not run into redundant issuances during concurrent + // renewal checks. - // An issuance holds a shared lock, so we need to avoid - // a situation where other services cannot issue certs - // because a single one is holding the lock. + // An issuance holds a shared lock, so we need to avoid a situation + // where other services cannot issue certs because a single one is + // holding the lock. ctxT, cancel := context.WithTimeout(ctx, time.Second*300) - defer cancel() _, _, err := cm.lc.CertPair(ctxT, domain) + cancel() if err != nil { - log.Printf("error refreshing certificate for %s: %v", domain, err) + cm.logf("error refreshing certificate for %s: %v", domain, err) } var nextInterval time.Duration - // TODO(irbekrm): distinguish between LE rate limit - // errors and other error types like transient network - // errors. + // TODO(irbekrm): distinguish between LE rate limit errors and other + // error types like transient network errors. if err == nil { retryCount = 0 nextInterval = normalInterval @@ -147,10 +156,34 @@ func (cm *certManager) runCertLoop(ctx context.Context, domain string) { backoff = maxRetryInterval } nextInterval = backoff - log.Printf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n", + cm.logf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n", domain, retryCount, err, nextInterval) } timer.Reset(nextInterval) } } } + +// waitForCertDomain ensures the requested domain is in the list of allowed +// domains before issuing the cert for the first time. +func (cm *CertManager) waitForCertDomain(ctx context.Context, domain string) error { + w, err := cm.lc.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) + if err != nil { + return fmt.Errorf("error watching IPN bus: %w", err) + } + defer w.Close() + + for { + n, err := w.Next() + if err != nil { + return err + } + if n.NetMap == nil { + continue + } + + if slices.Contains(n.NetMap.DNS.CertDomains, domain) { + return nil + } + } +} diff --git a/cmd/containerboot/certs_test.go b/kube/certs/certs_test.go similarity index 89% rename from cmd/containerboot/certs_test.go rename to kube/certs/certs_test.go index 577311ea36a64..8434f21ae6976 100644 --- a/cmd/containerboot/certs_test.go +++ b/kube/certs/certs_test.go @@ -1,17 +1,18 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux - -package main +package certs import ( "context" + "log" "testing" "time" "tailscale.com/ipn" + "tailscale.com/kube/localclient" "tailscale.com/tailcfg" + "tailscale.com/types/netmap" ) // TestEnsureCertLoops tests that the certManager correctly starts and stops @@ -161,8 +162,28 @@ func TestEnsureCertLoops(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cm := &certManager{ - lc: &fakeLocalClient{}, + notifyChan := make(chan ipn.Notify) + go func() { + for { + notifyChan <- ipn.Notify{ + NetMap: &netmap.NetworkMap{ + DNS: tailcfg.DNSConfig{ + CertDomains: []string{ + "my-app.tailnetxyz.ts.net", + "my-other-app.tailnetxyz.ts.net", + }, + }, + }, + } + } + }() + cm := &CertManager{ + lc: &localclient.FakeLocalClient{ + FakeIPNBusWatcher: localclient.FakeIPNBusWatcher{ + NotifyChan: notifyChan, + }, + }, + logf: log.Printf, certLoops: make(map[string]context.CancelFunc), } @@ -179,7 +200,7 @@ func TestEnsureCertLoops(t *testing.T) { } })() - err := cm.ensureCertLoops(ctx, tt.initialConfig) + err := cm.EnsureCertLoops(ctx, tt.initialConfig) if (err != nil) != tt.wantErr { t.Fatalf("ensureCertLoops() error = %v", err) } @@ -189,7 +210,7 @@ func TestEnsureCertLoops(t *testing.T) { } if tt.updatedConfig != nil { - if err := cm.ensureCertLoops(ctx, tt.updatedConfig); err != nil { + if err := cm.EnsureCertLoops(ctx, tt.updatedConfig); err != nil { t.Fatalf("ensureCertLoops() error on update = %v", err) } diff --git a/kube/health/healthz.go b/kube/health/healthz.go new file mode 100644 index 0000000000000..c8cfcc7ec01b4 --- /dev/null +++ b/kube/health/healthz.go @@ -0,0 +1,84 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package health contains shared types and underlying methods for serving +// a `/healthz` endpoint for containerboot and k8s-proxy. +package health + +import ( + "context" + "fmt" + "net/http" + "sync" + + "tailscale.com/client/local" + "tailscale.com/ipn" + "tailscale.com/kube/kubetypes" + "tailscale.com/types/logger" +) + +// Healthz is a simple health check server, if enabled it returns 200 OK if +// this tailscale node currently has at least one tailnet IP address else +// returns 503. +type Healthz struct { + sync.Mutex + hasAddrs bool + podIPv4 string + logger logger.Logf +} + +func (h *Healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.Lock() + defer h.Unlock() + + if h.hasAddrs { + w.Header().Add(kubetypes.PodIPv4Header, h.podIPv4) + if _, err := w.Write([]byte("ok")); err != nil { + http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError) + } + } else { + http.Error(w, "node currently has no tailscale IPs", http.StatusServiceUnavailable) + } +} + +func (h *Healthz) Update(healthy bool) { + h.Lock() + defer h.Unlock() + + if h.hasAddrs != healthy { + h.logger("Setting healthy %v", healthy) + } + h.hasAddrs = healthy +} + +func (h *Healthz) MonitorHealth(ctx context.Context, lc *local.Client) error { + w, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) + if err != nil { + return fmt.Errorf("failed to watch IPN bus: %w", err) + } + + for { + n, err := w.Next() + if err != nil { + return err + } + + if n.NetMap != nil { + h.Update(n.NetMap.SelfNode.Addresses().Len() != 0) + } + } +} + +// RegisterHealthHandlers registers a simple health handler at /healthz. +// A containerized tailscale instance is considered healthy if +// it has at least one tailnet IP address. +func RegisterHealthHandlers(mux *http.ServeMux, podIPv4 string, logger logger.Logf) *Healthz { + h := &Healthz{ + podIPv4: podIPv4, + logger: logger, + } + mux.Handle("GET /healthz", h) + return h +} diff --git a/kube/k8s-proxy/conf/conf.go b/kube/k8s-proxy/conf/conf.go new file mode 100644 index 0000000000000..5294952438896 --- /dev/null +++ b/kube/k8s-proxy/conf/conf.go @@ -0,0 +1,128 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package conf contains code to load, manipulate, and access config file +// settings for k8s-proxy. +package conf + +import ( + "encoding/json" + "errors" + "fmt" + "net/netip" + + "github.com/tailscale/hujson" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" + "tailscale.com/types/opt" +) + +const v1Alpha1 = "v1alpha1" + +// Config describes a config file. +type Config struct { + Raw []byte // raw bytes, in HuJSON form + Std []byte // standardized JSON form + Version string // "v1alpha1" + + // Parsed is the parsed config, converted from its raw bytes version to the + // latest known format. + Parsed ConfigV1Alpha1 +} + +// VersionedConfig allows specifying config at the root of the object, or in +// a versioned sub-object. +// e.g. {"version": "v1alpha1", "authKey": "abc123"} +// or {"version": "v1beta1", "a-beta-config": "a-beta-value", "v1alpha1": {"authKey": "abc123"}} +type VersionedConfig struct { + Version string `json:",omitempty"` // "v1alpha1" + + // Latest version of the config. + *ConfigV1Alpha1 + + // Backwards compatibility version(s) of the config. Fields and sub-fields + // from here should only be added to, never changed in place. + V1Alpha1 *ConfigV1Alpha1 `json:",omitempty"` + // V1Beta1 *ConfigV1Beta1 `json:",omitempty"` // Not yet used. +} + +type ConfigV1Alpha1 struct { + AuthKey *string `json:",omitempty"` // Tailscale auth key to use. + State *string `json:",omitempty"` // Path to the Tailscale state. + LogLevel *string `json:",omitempty"` // "debug", "info". Defaults to "info". + App *string `json:",omitempty"` // e.g. kubetypes.AppProxyGroupKubeAPIServer + ServerURL *string `json:",omitempty"` // URL of the Tailscale coordination server. + LocalAddr *string `json:",omitempty"` // The address to use for serving HTTP health checks and metrics (defaults to all interfaces). + LocalPort *uint16 `json:",omitempty"` // The port to use for serving HTTP health checks and metrics (defaults to 9002). + MetricsEnabled opt.Bool `json:",omitempty"` // Serve metrics on :/metrics. + HealthCheckEnabled opt.Bool `json:",omitempty"` // Serve health check on :/metrics. + + // TODO(tomhjp): The remaining fields should all be reloadable during + // runtime, but currently missing most of the APIServerProxy fields. + Hostname *string `json:",omitempty"` // Tailscale device hostname. + AcceptRoutes opt.Bool `json:",omitempty"` // Accepts routes advertised by other Tailscale nodes. + AdvertiseServices []string `json:",omitempty"` // Tailscale Services to advertise. + APIServerProxy *APIServerProxyConfig `json:",omitempty"` // Config specific to the API Server proxy. + StaticEndpoints []netip.AddrPort `json:",omitempty"` // StaticEndpoints are additional, user-defined endpoints that this node should advertise amongst its wireguard endpoints. +} + +type APIServerProxyConfig struct { + Enabled opt.Bool `json:",omitempty"` // Whether to enable the API Server proxy. + Mode *kubetypes.APIServerProxyMode `json:",omitempty"` // "auth" or "noauth" mode. + ServiceName *tailcfg.ServiceName `json:",omitempty"` // Name of the Tailscale Service to advertise. + IssueCerts opt.Bool `json:",omitempty"` // Whether this replica should issue TLS certs for the Tailscale Service. +} + +// Load reads and parses the config file at the provided path on disk. +func Load(raw []byte) (c Config, err error) { + c.Raw = raw + c.Std, err = hujson.Standardize(c.Raw) + if err != nil { + return c, fmt.Errorf("error parsing config as HuJSON/JSON: %w", err) + } + var ver VersionedConfig + if err := json.Unmarshal(c.Std, &ver); err != nil { + return c, fmt.Errorf("error parsing config: %w", err) + } + rootV1Alpha1 := (ver.Version == v1Alpha1) + backCompatV1Alpha1 := (ver.V1Alpha1 != nil) + switch { + case ver.Version == "": + return c, errors.New("error parsing config: no \"version\" field provided") + case rootV1Alpha1 && backCompatV1Alpha1: + // Exactly one of these should be set. + return c, errors.New("error parsing config: both root and v1alpha1 config provided") + case rootV1Alpha1 != backCompatV1Alpha1: + c.Version = v1Alpha1 + switch { + case rootV1Alpha1 && ver.ConfigV1Alpha1 != nil: + c.Parsed = *ver.ConfigV1Alpha1 + case backCompatV1Alpha1: + c.Parsed = *ver.V1Alpha1 + default: + c.Parsed = ConfigV1Alpha1{} + } + default: + return c, fmt.Errorf("error parsing config: unsupported \"version\" value %q; want \"%s\"", ver.Version, v1Alpha1) + } + + return c, nil +} + +func (c *Config) GetLocalAddr() string { + if c.Parsed.LocalAddr == nil { + return "[::]" + } + + return *c.Parsed.LocalAddr +} + +func (c *Config) GetLocalPort() uint16 { + if c.Parsed.LocalPort == nil { + return uint16(9002) + } + + return *c.Parsed.LocalPort +} diff --git a/kube/k8s-proxy/conf/conf_test.go b/kube/k8s-proxy/conf/conf_test.go new file mode 100644 index 0000000000000..3082be1ba9dcd --- /dev/null +++ b/kube/k8s-proxy/conf/conf_test.go @@ -0,0 +1,79 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package conf + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "tailscale.com/types/ptr" +) + +// Test that the config file can be at the root of the object, or in a versioned sub-object. +// or {"version": "v1beta1", "a-beta-config": "a-beta-value", "v1alpha1": {"authKey": "abc123"}} +func TestVersionedConfig(t *testing.T) { + testCases := map[string]struct { + inputConfig string + expectedConfig ConfigV1Alpha1 + expectedError string + }{ + "root_config_v1alpha1": { + inputConfig: `{"version": "v1alpha1", "authKey": "abc123"}`, + expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")}, + }, + "backwards_compat_v1alpha1_config": { + // Client doesn't know about v1beta1, so it should read in v1alpha1. + inputConfig: `{"version": "v1beta1", "beta-key": "beta-value", "authKey": "def456", "v1alpha1": {"authKey": "abc123"}}`, + expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")}, + }, + "unknown_key_allowed": { + // Adding new keys to the config doesn't require a version bump. + inputConfig: `{"version": "v1alpha1", "unknown-key": "unknown-value", "authKey": "abc123"}`, + expectedConfig: ConfigV1Alpha1{AuthKey: ptr.To("abc123")}, + }, + "version_only_no_authkey": { + inputConfig: `{"version": "v1alpha1"}`, + expectedConfig: ConfigV1Alpha1{}, + }, + "both_config_v1alpha1": { + inputConfig: `{"version": "v1alpha1", "authKey": "abc123", "v1alpha1": {"authKey": "def456"}}`, + expectedError: "both root and v1alpha1 config provided", + }, + "empty_config": { + inputConfig: `{}`, + expectedError: `no "version" field provided`, + }, + "v1beta1_without_backwards_compat": { + inputConfig: `{"version": "v1beta1", "beta-key": "beta-value", "authKey": "def456"}`, + expectedError: `unsupported "version" value "v1beta1"; want "v1alpha1"`, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + cfg, err := Load([]byte(tc.inputConfig)) + switch { + case tc.expectedError == "" && err != nil: + t.Fatalf("unexpected error: %v", err) + case tc.expectedError != "": + if err == nil { + t.Fatalf("expected error %q, got nil", tc.expectedError) + } else if !strings.Contains(err.Error(), tc.expectedError) { + t.Fatalf("expected error %q, got %q", tc.expectedError, err.Error()) + } + return + } + if cfg.Version != "v1alpha1" { + t.Fatalf("expected version %q, got %q", "v1alpha1", cfg.Version) + } + // Diff actual vs expected config. + if diff := cmp.Diff(cfg.Parsed, tc.expectedConfig); diff != "" { + t.Fatalf("Unexpected parsed config (-got +want):\n%s", diff) + } + }) + } +} diff --git a/kube/kubeclient/fake_client.go b/kube/kubeclient/fake_client.go index c21dc2bf89e61..15ebb5f443f2a 100644 --- a/kube/kubeclient/fake_client.go +++ b/kube/kubeclient/fake_client.go @@ -13,12 +13,13 @@ import ( var _ Client = &FakeClient{} type FakeClient struct { - GetSecretImpl func(context.Context, string) (*kubeapi.Secret, error) - CheckSecretPermissionsImpl func(ctx context.Context, name string) (bool, bool, error) - CreateSecretImpl func(context.Context, *kubeapi.Secret) error - UpdateSecretImpl func(context.Context, *kubeapi.Secret) error - JSONPatchResourceImpl func(context.Context, string, string, []JSONPatch) error - ListSecretsImpl func(context.Context, map[string]string) (*kubeapi.SecretList, error) + GetSecretImpl func(context.Context, string) (*kubeapi.Secret, error) + CheckSecretPermissionsImpl func(ctx context.Context, name string) (bool, bool, error) + CreateSecretImpl func(context.Context, *kubeapi.Secret) error + UpdateSecretImpl func(context.Context, *kubeapi.Secret) error + JSONPatchResourceImpl func(context.Context, string, string, []JSONPatch) error + ListSecretsImpl func(context.Context, map[string]string) (*kubeapi.SecretList, error) + StrategicMergePatchSecretImpl func(context.Context, string, *kubeapi.Secret, string) error } func (fc *FakeClient) CheckSecretPermissions(ctx context.Context, name string) (bool, bool, error) { @@ -30,8 +31,8 @@ func (fc *FakeClient) GetSecret(ctx context.Context, name string) (*kubeapi.Secr func (fc *FakeClient) SetURL(_ string) {} func (fc *FakeClient) SetDialer(dialer func(ctx context.Context, network, addr string) (net.Conn, error)) { } -func (fc *FakeClient) StrategicMergePatchSecret(context.Context, string, *kubeapi.Secret, string) error { - return nil +func (fc *FakeClient) StrategicMergePatchSecret(ctx context.Context, name string, s *kubeapi.Secret, fieldManager string) error { + return fc.StrategicMergePatchSecretImpl(ctx, name, s, fieldManager) } func (fc *FakeClient) Event(context.Context, string, string, string) error { return nil diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index 6f96875dddd0f..44b01fe1ad1f5 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -3,16 +3,19 @@ package kubetypes +import "fmt" + const ( // Hostinfo App values for the Tailscale Kubernetes Operator components. - AppOperator = "k8s-operator" - AppAPIServerProxy = "k8s-operator-proxy" - AppIngressProxy = "k8s-operator-ingress-proxy" - AppIngressResource = "k8s-operator-ingress-resource" - AppEgressProxy = "k8s-operator-egress-proxy" - AppConnector = "k8s-operator-connector-resource" - AppProxyGroupEgress = "k8s-operator-proxygroup-egress" - AppProxyGroupIngress = "k8s-operator-proxygroup-ingress" + AppOperator = "k8s-operator" + AppInProcessAPIServerProxy = "k8s-operator-proxy" + AppIngressProxy = "k8s-operator-ingress-proxy" + AppIngressResource = "k8s-operator-ingress-resource" + AppEgressProxy = "k8s-operator-egress-proxy" + AppConnector = "k8s-operator-connector-resource" + AppProxyGroupEgress = "k8s-operator-proxygroup-egress" + AppProxyGroupIngress = "k8s-operator-proxygroup-ingress" + AppProxyGroupKubeAPIServer = "k8s-operator-proxygroup-kube-apiserver" // Clientmetrics for Tailscale Kubernetes Operator components MetricIngressProxyCount = "k8s_ingress_proxies" // L3 @@ -29,6 +32,7 @@ const ( MetricEgressServiceCount = "k8s_egress_service_resources" MetricProxyGroupEgressCount = "k8s_proxygroup_egress_resources" MetricProxyGroupIngressCount = "k8s_proxygroup_ingress_resources" + MetricProxyGroupAPIServerCount = "k8s_proxygroup_kube_apiserver_resources" // Keys that containerboot writes to state file that can be used to determine its state. // fields set in Tailscale state Secret. These are mostly used by the Tailscale Kubernetes operator to determine @@ -52,4 +56,29 @@ const ( LabelManaged = "tailscale.com/managed" LabelSecretType = "tailscale.com/secret-type" // "config", "state" "certs" + + LabelSecretTypeConfig = "config" + LabelSecretTypeState = "state" + LabelSecretTypeCerts = "certs" + + KubeAPIServerConfigFile = "config.hujson" + APIServerProxyModeAuth APIServerProxyMode = "auth" + APIServerProxyModeNoAuth APIServerProxyMode = "noauth" ) + +// APIServerProxyMode specifies whether the API server proxy will add +// impersonation headers to requests based on the caller's Tailscale identity. +// May be "auth" or "noauth". +type APIServerProxyMode string + +func (a *APIServerProxyMode) UnmarshalJSON(data []byte) error { + switch string(data) { + case `"auth"`: + *a = APIServerProxyModeAuth + case `"noauth"`: + *a = APIServerProxyModeNoAuth + default: + return fmt.Errorf("unknown APIServerProxyMode %q", data) + } + return nil +} diff --git a/kube/kubetypes/types_test.go b/kube/kubetypes/types_test.go new file mode 100644 index 0000000000000..ea1846b3253e8 --- /dev/null +++ b/kube/kubetypes/types_test.go @@ -0,0 +1,42 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package kubetypes + +import ( + "encoding/json" + "testing" +) + +func TestUnmarshalAPIServerProxyMode(t *testing.T) { + tests := []struct { + data string + expected APIServerProxyMode + }{ + {data: `{"mode":"auth"}`, expected: APIServerProxyModeAuth}, + {data: `{"mode":"noauth"}`, expected: APIServerProxyModeNoAuth}, + {data: `{"mode":""}`, expected: ""}, + {data: `{"mode":"Auth"}`, expected: ""}, + {data: `{"mode":"unknown"}`, expected: ""}, + } + + for _, tc := range tests { + var s struct { + Mode *APIServerProxyMode `json:",omitempty"` + } + err := json.Unmarshal([]byte(tc.data), &s) + if tc.expected == "" { + if err == nil { + t.Errorf("expected error for %q, got none", tc.data) + } + continue + } + if err != nil { + t.Errorf("unexpected error for %q: %v", tc.data, err) + continue + } + if *s.Mode != tc.expected { + t.Errorf("for %q expected %q, got %q", tc.data, tc.expected, *s.Mode) + } + } +} diff --git a/kube/localclient/fake-client.go b/kube/localclient/fake-client.go new file mode 100644 index 0000000000000..7f0a08316634e --- /dev/null +++ b/kube/localclient/fake-client.go @@ -0,0 +1,35 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package localclient + +import ( + "context" + "fmt" + + "tailscale.com/ipn" +) + +type FakeLocalClient struct { + FakeIPNBusWatcher +} + +func (f *FakeLocalClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (IPNBusWatcher, error) { + return &f.FakeIPNBusWatcher, nil +} + +func (f *FakeLocalClient) CertPair(ctx context.Context, domain string) ([]byte, []byte, error) { + return nil, nil, fmt.Errorf("CertPair not implemented") +} + +type FakeIPNBusWatcher struct { + NotifyChan chan ipn.Notify +} + +func (f *FakeIPNBusWatcher) Close() error { + return nil +} + +func (f *FakeIPNBusWatcher) Next() (ipn.Notify, error) { + return <-f.NotifyChan, nil +} diff --git a/kube/localclient/local-client.go b/kube/localclient/local-client.go new file mode 100644 index 0000000000000..5d541e3655ddb --- /dev/null +++ b/kube/localclient/local-client.go @@ -0,0 +1,49 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package localclient provides an interface for all the local.Client methods +// kube needs to use, so that we can easily mock it in tests. +package localclient + +import ( + "context" + "io" + + "tailscale.com/client/local" + "tailscale.com/ipn" +) + +// LocalClient is roughly a subset of the local.Client struct's methods, used +// for easier testing. +type LocalClient interface { + WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (IPNBusWatcher, error) + CertIssuer +} + +// IPNBusWatcher is local.IPNBusWatcher's methods restated in an interface to +// allow for easier mocking in tests. +type IPNBusWatcher interface { + io.Closer + Next() (ipn.Notify, error) +} + +type CertIssuer interface { + CertPair(context.Context, string) ([]byte, []byte, error) +} + +// New returns a LocalClient that wraps the provided local.Client. +func New(lc *local.Client) LocalClient { + return &localClient{lc: lc} +} + +type localClient struct { + lc *local.Client +} + +func (l *localClient) WatchIPNBus(ctx context.Context, mask ipn.NotifyWatchOpt) (IPNBusWatcher, error) { + return l.lc.WatchIPNBus(ctx, mask) +} + +func (l *localClient) CertPair(ctx context.Context, domain string) ([]byte, []byte, error) { + return l.lc.CertPair(ctx, domain) +} diff --git a/cmd/containerboot/metrics.go b/kube/metrics/metrics.go similarity index 90% rename from cmd/containerboot/metrics.go rename to kube/metrics/metrics.go index bbd050de6df26..0db683008f91e 100644 --- a/cmd/containerboot/metrics.go +++ b/kube/metrics/metrics.go @@ -1,9 +1,11 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux +//go:build !plan9 -package main +// Package metrics contains shared types and underlying methods for serving +// localapi metrics. This is primarily consumed by containerboot and k8s-proxy. +package metrics import ( "fmt" @@ -68,7 +70,7 @@ func (m *metrics) handleDebug(w http.ResponseWriter, r *http.Request) { // In 1.78.x and 1.80.x, it also proxies debug paths to tailscaled's debug // endpoint if configured to ease migration for a breaking change serving user // metrics instead of debug metrics on the "metrics" port. -func registerMetricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) { +func RegisterMetricsHandlers(mux *http.ServeMux, lc *local.Client, debugAddrPort string) { m := &metrics{ lc: lc, debugEndpoint: debugAddrPort, diff --git a/cmd/containerboot/services.go b/kube/services/services.go similarity index 74% rename from cmd/containerboot/services.go rename to kube/services/services.go index 6079128c02b19..a9e50975ca9f1 100644 --- a/cmd/containerboot/services.go +++ b/kube/services/services.go @@ -1,25 +1,25 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux - -package main +// Package services manages graceful shutdown of Tailscale Services advertised +// by Kubernetes clients. +package services import ( "context" "fmt" - "log" "time" "tailscale.com/client/local" "tailscale.com/ipn" + "tailscale.com/types/logger" ) -// ensureServicesNotAdvertised is a function that gets called on containerboot -// termination and ensures that any currently advertised VIPServices get -// unadvertised to give clients time to switch to another node before this one -// is shut down. -func ensureServicesNotAdvertised(ctx context.Context, lc *local.Client) error { +// EnsureServicesNotAdvertised is a function that gets called on containerboot +// or k8s-proxy termination and ensures that any currently advertised Services +// get unadvertised to give clients time to switch to another node before this +// one is shut down. +func EnsureServicesNotAdvertised(ctx context.Context, lc *local.Client, logf logger.Logf) error { prefs, err := lc.GetPrefs(ctx) if err != nil { return fmt.Errorf("error getting prefs: %w", err) @@ -28,7 +28,7 @@ func ensureServicesNotAdvertised(ctx context.Context, lc *local.Client) error { return nil } - log.Printf("unadvertising services: %v", prefs.AdvertiseServices) + logf("unadvertising services: %v", prefs.AdvertiseServices) if _, err := lc.EditPrefs(ctx, &ipn.MaskedPrefs{ AdvertiseServicesSet: true, Prefs: ipn.Prefs{ diff --git a/kube/state/state.go b/kube/state/state.go new file mode 100644 index 0000000000000..2605f0952f708 --- /dev/null +++ b/kube/state/state.go @@ -0,0 +1,107 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package state updates state keys for tailnet client devices managed by the +// operator. These keys are used to signal readiness, metadata, and current +// configuration state to the operator. Client packages deployed by the operator +// include containerboot, tsrecorder, and k8s-proxy, but currently containerboot +// has its own implementation to manage the same keys. +package state + +import ( + "context" + "encoding/json" + "fmt" + + "tailscale.com/ipn" + "tailscale.com/kube/kubetypes" + klc "tailscale.com/kube/localclient" + "tailscale.com/tailcfg" + "tailscale.com/util/deephash" +) + +const ( + keyPodUID = ipn.StateKey(kubetypes.KeyPodUID) + keyCapVer = ipn.StateKey(kubetypes.KeyCapVer) + keyDeviceID = ipn.StateKey(kubetypes.KeyDeviceID) + keyDeviceIPs = ipn.StateKey(kubetypes.KeyDeviceIPs) + keyDeviceFQDN = ipn.StateKey(kubetypes.KeyDeviceFQDN) +) + +// SetInitialKeys sets Pod UID and cap ver and clears tailnet device state +// keys to help stop the operator using stale tailnet device state. +func SetInitialKeys(store ipn.StateStore, podUID string) error { + // Clear device state keys first so the operator knows if the pod UID + // matches, the other values are definitely not stale. + for _, key := range []ipn.StateKey{keyDeviceID, keyDeviceFQDN, keyDeviceIPs} { + if _, err := store.ReadState(key); err == nil { + if err := store.WriteState(key, nil); err != nil { + return fmt.Errorf("error writing %q to state store: %w", key, err) + } + } + } + + if err := store.WriteState(keyPodUID, []byte(podUID)); err != nil { + return fmt.Errorf("error writing pod UID to state store: %w", err) + } + if err := store.WriteState(keyCapVer, fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion)); err != nil { + return fmt.Errorf("error writing capability version to state store: %w", err) + } + + return nil +} + +// KeepKeysUpdated sets state store keys consistent with containerboot to +// signal proxy readiness to the operator. It runs until its context is +// cancelled or it hits an error. The passed in next function is expected to be +// from a local.IPNBusWatcher that is at least subscribed to +// ipn.NotifyInitialNetMap. +func KeepKeysUpdated(ctx context.Context, store ipn.StateStore, lc klc.LocalClient) error { + w, err := lc.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) + if err != nil { + return fmt.Errorf("error watching IPN bus: %w", err) + } + defer w.Close() + + var currentDeviceID, currentDeviceIPs, currentDeviceFQDN deephash.Sum + for { + n, err := w.Next() // Blocks on a streaming LocalAPI HTTP call. + if err != nil { + if err == ctx.Err() { + return nil + } + return err + } + if n.NetMap == nil { + continue + } + + if deviceID := n.NetMap.SelfNode.StableID(); deephash.Update(¤tDeviceID, &deviceID) { + if err := store.WriteState(keyDeviceID, []byte(deviceID)); err != nil { + return fmt.Errorf("failed to store device ID in state: %w", err) + } + } + + if fqdn := n.NetMap.SelfNode.Name(); deephash.Update(¤tDeviceFQDN, &fqdn) { + if err := store.WriteState(keyDeviceFQDN, []byte(fqdn)); err != nil { + return fmt.Errorf("failed to store device FQDN in state: %w", err) + } + } + + if addrs := n.NetMap.SelfNode.Addresses(); deephash.Update(¤tDeviceIPs, &addrs) { + var deviceIPs []string + for _, addr := range addrs.AsSlice() { + deviceIPs = append(deviceIPs, addr.Addr().String()) + } + deviceIPsValue, err := json.Marshal(deviceIPs) + if err != nil { + return err + } + if err := store.WriteState(keyDeviceIPs, deviceIPsValue); err != nil { + return fmt.Errorf("failed to store device IPs in state: %w", err) + } + } + } +} diff --git a/kube/state/state_test.go b/kube/state/state_test.go new file mode 100644 index 0000000000000..8701aa1b7fa65 --- /dev/null +++ b/kube/state/state_test.go @@ -0,0 +1,196 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package state + +import ( + "bytes" + "fmt" + "net/netip" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "tailscale.com/ipn" + "tailscale.com/ipn/store" + klc "tailscale.com/kube/localclient" + "tailscale.com/tailcfg" + "tailscale.com/types/logger" + "tailscale.com/types/netmap" +) + +func TestSetInitialStateKeys(t *testing.T) { + var ( + podUID = []byte("test-pod-uid") + expectedCapVer = fmt.Appendf(nil, "%d", tailcfg.CurrentCapabilityVersion) + ) + for name, tc := range map[string]struct { + initial map[ipn.StateKey][]byte + expected map[ipn.StateKey][]byte + }{ + "empty_initial": { + initial: map[ipn.StateKey][]byte{}, + expected: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + }, + }, + "existing_pod_uid_and_capver": { + initial: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + }, + expected: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + }, + }, + "all_keys_preexisting": { + initial: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + keyDeviceID: []byte("existing-device-id"), + keyDeviceFQDN: []byte("existing-device-fqdn"), + keyDeviceIPs: []byte(`["1.2.3.4"]`), + }, + expected: map[ipn.StateKey][]byte{ + keyPodUID: podUID, + keyCapVer: expectedCapVer, + keyDeviceID: nil, + keyDeviceFQDN: nil, + keyDeviceIPs: nil, + }, + }, + } { + t.Run(name, func(t *testing.T) { + store, err := store.New(logger.Discard, "mem:") + if err != nil { + t.Fatalf("error creating in-memory store: %v", err) + } + + for key, value := range tc.initial { + if err := store.WriteState(key, value); err != nil { + t.Fatalf("error writing initial state key %q: %v", key, err) + } + } + + if err := SetInitialKeys(store, string(podUID)); err != nil { + t.Fatalf("setInitialStateKeys failed: %v", err) + } + + actual := make(map[ipn.StateKey][]byte) + for expectedKey, expectedValue := range tc.expected { + actualValue, err := store.ReadState(expectedKey) + if err != nil { + t.Errorf("error reading state key %q: %v", expectedKey, err) + continue + } + + actual[expectedKey] = actualValue + if !bytes.Equal(actualValue, expectedValue) { + t.Errorf("state key %q mismatch: expected %q, got %q", expectedKey, expectedValue, actualValue) + } + } + if diff := cmp.Diff(actual, tc.expected); diff != "" { + t.Errorf("state keys mismatch (-got +want):\n%s", diff) + } + }) + } +} + +func TestKeepStateKeysUpdated(t *testing.T) { + store := fakeStore{ + writeChan: make(chan string), + } + + errs := make(chan error) + notifyChan := make(chan ipn.Notify) + lc := &klc.FakeLocalClient{ + FakeIPNBusWatcher: klc.FakeIPNBusWatcher{ + NotifyChan: notifyChan, + }, + } + + go func() { + err := KeepKeysUpdated(t.Context(), store, lc) + if err != nil { + errs <- fmt.Errorf("keepStateKeysUpdated returned with error: %w", err) + } + }() + + for _, tc := range []struct { + name string + notify ipn.Notify + expected []string + }{ + { + name: "initial_not_authed", + notify: ipn.Notify{}, + expected: nil, + }, + { + name: "authed", + notify: ipn.Notify{ + NetMap: &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{ + StableID: "TESTCTRL00000001", + Name: "test-node.test.ts.net", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.1/32"), netip.MustParsePrefix("fd7a:115c:a1e0:ab12:4843:cd96:0:1/128")}, + }).View(), + }, + }, + expected: []string{ + fmt.Sprintf("%s=%s", keyDeviceID, "TESTCTRL00000001"), + fmt.Sprintf("%s=%s", keyDeviceFQDN, "test-node.test.ts.net"), + fmt.Sprintf("%s=%s", keyDeviceIPs, `["100.64.0.1","fd7a:115c:a1e0:ab12:4843:cd96:0:1"]`), + }, + }, + { + name: "updated_fields", + notify: ipn.Notify{ + NetMap: &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{ + StableID: "TESTCTRL00000001", + Name: "updated.test.ts.net", + Addresses: []netip.Prefix{netip.MustParsePrefix("100.64.0.250/32")}, + }).View(), + }, + }, + expected: []string{ + fmt.Sprintf("%s=%s", keyDeviceFQDN, "updated.test.ts.net"), + fmt.Sprintf("%s=%s", keyDeviceIPs, `["100.64.0.250"]`), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + notifyChan <- tc.notify + for _, expected := range tc.expected { + select { + case got := <-store.writeChan: + if got != expected { + t.Errorf("expected %q, got %q", expected, got) + } + case err := <-errs: + t.Fatalf("unexpected error: %v", err) + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for expected write %q", expected) + } + } + }) + } +} + +type fakeStore struct { + writeChan chan string +} + +func (f fakeStore) ReadState(key ipn.StateKey) ([]byte, error) { + return nil, fmt.Errorf("ReadState not implemented") +} + +func (f fakeStore) WriteState(key ipn.StateKey, value []byte) error { + f.writeChan <- fmt.Sprintf("%s=%s", key, value) + return nil +} diff --git a/license_test.go b/license_test.go new file mode 100644 index 0000000000000..ec452a6e36be7 --- /dev/null +++ b/license_test.go @@ -0,0 +1,117 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package tailscaleroot + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "tailscale.com/util/set" +) + +func normalizeLineEndings(b []byte) []byte { + return bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n")) +} + +// TestLicenseHeaders checks that all Go files in the tree +// directory tree have a correct-looking Tailscale license header. +func TestLicenseHeaders(t *testing.T) { + want := normalizeLineEndings([]byte(strings.TrimLeft(` +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause +`, "\n"))) + + exceptions := set.Of( + // Subprocess test harness code + "util/winutil/testdata/testrestartableprocesses/main.go", + "util/winutil/subprocess_windows_test.go", + + // WireGuard copyright + "cmd/tailscale/cli/authenticode_windows.go", + "wgengine/router/ifconfig_windows.go", + + // noiseexplorer.com copyright + "control/controlbase/noiseexplorer_test.go", + + // Generated eBPF management code + "derp/xdp/bpf_bpfeb.go", + "derp/xdp/bpf_bpfel.go", + + // Generated kube deepcopy funcs file starts with a Go build tag + an empty line + "k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go", + ) + + err := filepath.Walk(".", func(path string, fi os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("path %s: %v", path, err) + } + if exceptions.Contains(filepath.ToSlash(path)) { + return nil + } + base := filepath.Base(path) + switch base { + case ".git", "node_modules", "tempfork": + return filepath.SkipDir + } + switch base { + case "zsyscall_windows.go": + // Generated code. + return nil + } + + if strings.HasSuffix(base, ".config.ts") { + return nil + } + if strings.HasSuffix(base, "_string.go") { + // Generated file from go:generate stringer + return nil + } + + ext := filepath.Ext(base) + switch ext { + default: + return nil + case ".go", ".ts", ".tsx": + } + + buf := make([]byte, 512) + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + if n, err := io.ReadAtLeast(f, buf, 512); err != nil && err != io.ErrUnexpectedEOF { + return err + } else { + buf = buf[:n] + } + + buf = normalizeLineEndings(buf) + + bufNoTrunc := buf + if i := bytes.Index(buf, []byte("\npackage ")); i != -1 { + buf = buf[:i] + } + + if bytes.Contains(buf, want) { + return nil + } + + if bytes.Contains(bufNoTrunc, []byte("BSD-3-Clause\npackage ")) { + t.Errorf("file %s has license header as a package doc; add a blank line before the package line", path) + return nil + } + + t.Errorf("file %s is missing Tailscale copyright header:\n\n%s", path, want) + return nil + }) + if err != nil { + t.Fatalf("Walk: %v", err) + } +} diff --git a/logpolicy/logpolicy.go b/logpolicy/logpolicy.go index fc259a417197b..f5c475712afe3 100644 --- a/logpolicy/logpolicy.go +++ b/logpolicy/logpolicy.go @@ -9,7 +9,6 @@ package logpolicy import ( "bufio" "bytes" - "cmp" "context" "crypto/tls" "encoding/json" @@ -225,6 +224,9 @@ func LogsDir(logf logger.Logf) string { logf("logpolicy: using LocalAppData dir %v", dir) return dir case "linux": + if distro.Get() == distro.JetKVM { + return "/userdata/tailscale/var" + } // STATE_DIRECTORY is set by systemd 240+ but we support older // systems-d. For example, Ubuntu 18.04 (Bionic Beaver) is 237. systemdStateDir := os.Getenv("STATE_DIRECTORY") @@ -911,8 +913,7 @@ func (opts TransportOptions) New() http.RoundTripper { tr.TLSNextProto = map[string]func(authority string, c *tls.Conn) http.RoundTripper{} } - host := cmp.Or(opts.Host, logtail.DefaultHost) - tr.TLSClientConfig = tlsdial.Config(host, opts.Health, tr.TLSClientConfig) + tr.TLSClientConfig = tlsdial.Config(opts.Health, tr.TLSClientConfig) // Force TLS 1.3 since we know log.tailscale.com supports it. tr.TLSClientConfig.MinVersion = tls.VersionTLS13 diff --git a/net/bakedroots/bakedroots.go b/net/bakedroots/bakedroots.go index 42e70c0dd2abb..b268b1546caac 100644 --- a/net/bakedroots/bakedroots.go +++ b/net/bakedroots/bakedroots.go @@ -7,6 +7,7 @@ package bakedroots import ( "crypto/x509" + "fmt" "sync" "tailscale.com/util/testenv" @@ -14,7 +15,7 @@ import ( // Get returns the baked-in roots. // -// As of 2025-01-21, this includes only the LetsEncrypt ISRG Root X1 root. +// As of 2025-01-21, this includes only the LetsEncrypt ISRG Root X1 & X2 roots. func Get() *x509.CertPool { roots.once.Do(func() { roots.parsePEM(append( @@ -25,16 +26,9 @@ func Get() *x509.CertPool { return roots.p } -// testingTB is a subset of testing.TB needed -// to verify the caller isn't in a parallel test. -type testingTB interface { - // Setenv panics if it's in a parallel test. - Setenv(k, v string) -} - // ResetForTest resets the cached roots for testing, // optionally setting them to caPEM if non-nil. -func ResetForTest(tb testingTB, caPEM []byte) { +func ResetForTest(tb testenv.TB, caPEM []byte) { if !testenv.InTest() { panic("not in test") } @@ -43,6 +37,10 @@ func ResetForTest(tb testingTB, caPEM []byte) { roots = rootsOnce{} if caPEM != nil { roots.once.Do(func() { roots.parsePEM(caPEM) }) + tb.Cleanup(func() { + // Reset the roots to real roots for any following test. + roots = rootsOnce{} + }) } } @@ -56,7 +54,7 @@ type rootsOnce struct { func (r *rootsOnce) parsePEM(caPEM []byte) { p := x509.NewCertPool() if !p.AppendCertsFromPEM(caPEM) { - panic("bogus PEM") + panic(fmt.Sprintf("bogus PEM: %q", caPEM)) } r.p = p } diff --git a/net/connectproxy/connectproxy.go b/net/connectproxy/connectproxy.go new file mode 100644 index 0000000000000..4bf6875029554 --- /dev/null +++ b/net/connectproxy/connectproxy.go @@ -0,0 +1,93 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package connectproxy contains some CONNECT proxy code. +package connectproxy + +import ( + "context" + "io" + "log" + "net" + "net/http" + "time" + + "tailscale.com/net/netx" + "tailscale.com/types/logger" +) + +// Handler is an HTTP CONNECT proxy handler. +type Handler struct { + // Dial, if non-nil, is an alternate dialer to use + // instead of the default dialer. + Dial netx.DialFunc + + // Logf, if non-nil, is an alterate logger to + // use instead of log.Printf. + Logf logger.Logf + + // Check, if non-nil, validates the CONNECT target. + Check func(hostPort string) error +} + +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + if r.Method != "CONNECT" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + dial := h.Dial + if dial == nil { + var d net.Dialer + dial = d.DialContext + } + logf := h.Logf + if logf == nil { + logf = log.Printf + } + + hostPort := r.RequestURI + if h.Check != nil { + if err := h.Check(hostPort); err != nil { + logf("CONNECT target %q not allowed: %v", hostPort, err) + http.Error(w, "Invalid CONNECT target", http.StatusForbidden) + return + } + } + + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + back, err := dial(ctx, "tcp", hostPort) + if err != nil { + logf("error CONNECT dialing %v: %v", hostPort, err) + http.Error(w, "Connect failure", http.StatusBadGateway) + return + } + defer back.Close() + + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "CONNECT hijack unavailable", http.StatusInternalServerError) + return + } + c, br, err := hj.Hijack() + if err != nil { + logf("CONNECT hijack: %v", err) + return + } + defer c.Close() + + io.WriteString(c, "HTTP/1.1 200 OK\r\n\r\n") + + errc := make(chan error, 2) + go func() { + _, err := io.Copy(c, back) + errc <- err + }() + go func() { + _, err := io.Copy(back, br) + errc <- err + }() + <-errc +} diff --git a/net/dns/manager_linux.go b/net/dns/manager_linux.go index 6bd368f50e330..643cc280af1e3 100644 --- a/net/dns/manager_linux.go +++ b/net/dns/manager_linux.go @@ -22,6 +22,7 @@ import ( "tailscale.com/types/logger" "tailscale.com/util/clientmetric" "tailscale.com/util/cmpver" + "tailscale.com/version/distro" ) type kv struct { @@ -38,6 +39,10 @@ var publishOnce sync.Once // // The health tracker may be nil; the knobs may be nil and are ignored on this platform. func NewOSConfigurator(logf logger.Logf, health *health.Tracker, _ *controlknobs.Knobs, interfaceName string) (ret OSConfigurator, err error) { + if distro.Get() == distro.JetKVM { + return NewNoopManager() + } + env := newOSConfigEnv{ fs: directFS{}, dbusPing: dbusPing, diff --git a/net/dns/recursive/recursive.go b/net/dns/recursive/recursive.go index eb23004d88190..fd865e37ab737 100644 --- a/net/dns/recursive/recursive.go +++ b/net/dns/recursive/recursive.go @@ -547,6 +547,7 @@ func (r *Resolver) queryNameserverProto( // Prepare a message asking for an appropriately-typed record // for the name we're querying. m := new(dns.Msg) + m.SetEdns0(1232, false /* no DNSSEC */) m.SetQuestion(name.WithTrailingDot(), uint16(qtype)) // Allow mocking out the network components with our exchange hook. diff --git a/net/dns/resolver/tsdns_test.go b/net/dns/resolver/tsdns_test.go index de08450d2e3eb..4bbfd4d6a417e 100644 --- a/net/dns/resolver/tsdns_test.go +++ b/net/dns/resolver/tsdns_test.go @@ -1106,10 +1106,6 @@ type linkSelFunc func(ip netip.Addr) string func (f linkSelFunc) PickLink(ip netip.Addr) string { return f(ip) } func TestHandleExitNodeDNSQueryWithNetPkg(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("skipping test on Windows; waiting for golang.org/issue/33097") - } - records := []any{ "no-records.test.", dnsHandler(), @@ -1405,9 +1401,6 @@ func TestHandleExitNodeDNSQueryWithNetPkg(t *testing.T) { // newWrapResolver returns a resolver that uses r (via handleExitNodeDNSQueryWithNetPkg) // to make DNS requests. func newWrapResolver(r *net.Resolver) *net.Resolver { - if runtime.GOOS == "windows" { - panic("doesn't work on Windows") // golang.org/issue/33097 - } return &net.Resolver{ PreferGo: true, Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { diff --git a/net/dnscache/dnscache.go b/net/dnscache/dnscache.go index 96550cbb17fca..d60e92f0b8bbc 100644 --- a/net/dnscache/dnscache.go +++ b/net/dnscache/dnscache.go @@ -24,6 +24,7 @@ import ( "tailscale.com/util/cloudenv" "tailscale.com/util/singleflight" "tailscale.com/util/slicesx" + "tailscale.com/util/testenv" ) var zaddr netip.Addr @@ -63,6 +64,10 @@ type Resolver struct { // If nil, net.DefaultResolver is used. Forward *net.Resolver + // LookupIPForTest, if non-nil and in tests, handles requests instead + // of the usual mechanisms. + LookupIPForTest func(ctx context.Context, host string) ([]netip.Addr, error) + // LookupIPFallback optionally provides a backup DNS mechanism // to use if Forward returns an error or no results. LookupIPFallback func(ctx context.Context, host string) ([]netip.Addr, error) @@ -284,7 +289,13 @@ func (r *Resolver) lookupIP(ctx context.Context, host string) (ip, ip6 netip.Add lookupCtx, lookupCancel := context.WithTimeout(ctx, r.lookupTimeoutForHost(host)) defer lookupCancel() - ips, err := r.fwd().LookupNetIP(lookupCtx, "ip", host) + + var ips []netip.Addr + if r.LookupIPForTest != nil && testenv.InTest() { + ips, err = r.LookupIPForTest(ctx, host) + } else { + ips, err = r.fwd().LookupNetIP(lookupCtx, "ip", host) + } if err != nil || len(ips) == 0 { if resolver, ok := r.cloudHostResolver(); ok { r.dlogf("resolving %q via cloud resolver", host) diff --git a/net/dnscache/messagecache_test.go b/net/dnscache/messagecache_test.go index 41fc334483f78..0bedfa5ad78e7 100644 --- a/net/dnscache/messagecache_test.go +++ b/net/dnscache/messagecache_test.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "net" - "runtime" "testing" "time" @@ -249,14 +248,6 @@ func TestGetDNSQueryCacheKey(t *testing.T) { } func getGoNetPacketDNSQuery(name string) []byte { - if runtime.GOOS == "windows" { - // On Windows, Go's net.Resolver doesn't use the DNS client. - // See https://github.com/golang/go/issues/33097 which - // was approved but not yet implemented. - // For now just pretend it's implemented to make this test - // pass on Windows with complicated the caller. - return makeQ(123, name) - } res := make(chan []byte, 1) r := &net.Resolver{ PreferGo: true, diff --git a/net/dnsfallback/dnsfallback.go b/net/dnsfallback/dnsfallback.go index 4c5d5fa2f2743..8e53c3b293cb4 100644 --- a/net/dnsfallback/dnsfallback.go +++ b/net/dnsfallback/dnsfallback.go @@ -286,7 +286,7 @@ func bootstrapDNSMap(ctx context.Context, serverName string, serverIP netip.Addr tr.DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) { return dialer.DialContext(ctx, "tcp", net.JoinHostPort(serverIP.String(), "443")) } - tr.TLSClientConfig = tlsdial.Config(serverName, ht, tr.TLSClientConfig) + tr.TLSClientConfig = tlsdial.Config(ht, tr.TLSClientConfig) c := &http.Client{Transport: tr} req, err := http.NewRequestWithContext(ctx, "GET", "https://"+serverName+"/bootstrap-dns?q="+url.QueryEscape(queryName), nil) if err != nil { diff --git a/net/netcheck/netcheck.go b/net/netcheck/netcheck.go index c9f03966beedd..cb622a339944d 100644 --- a/net/netcheck/netcheck.go +++ b/net/netcheck/netcheck.go @@ -23,6 +23,7 @@ import ( "syscall" "time" + "tailscale.com/derp" "tailscale.com/derp/derphttp" "tailscale.com/envknob" "tailscale.com/hostinfo" @@ -449,7 +450,7 @@ func makeProbePlan(dm *tailcfg.DERPMap, ifState *netmon.State, last *Report, pre // restoration back to the home DERP on the next full netcheck ~5 minutes later // - which is highly disruptive when it causes shifts in geo routed subnet // routers. By always including the home DERP in the incremental netcheck, we - // ensure that the home DERP is always probed, even if it observed a recenet + // ensure that the home DERP is always probed, even if it observed a recent // poor latency sample. This inclusion enables the latency history checks in // home DERP selection to still take effect. // planContainsHome indicates whether the home DERP has been added to the probePlan, @@ -753,6 +754,7 @@ func newReport() *Report { // GetReportOpts contains options that can be passed to GetReport. Unless // specified, all fields are optional and can be left as their zero value. +// At most one of OnlyTCP443 or OnlySTUN may be set. type GetReportOpts struct { // GetLastDERPActivity is a callback that, if provided, should return // the absolute time that the calling code last communicated with a @@ -765,6 +767,8 @@ type GetReportOpts struct { // OnlyTCP443 constrains netcheck reporting to measurements over TCP port // 443. OnlyTCP443 bool + // OnlySTUN constrains netcheck reporting to STUN measurements over UDP. + OnlySTUN bool } // getLastDERPActivity calls o.GetLastDERPActivity if both o and @@ -790,6 +794,13 @@ func (c *Client) SetForcePreferredDERP(region int) { // // It may not be called concurrently with itself. func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetReportOpts) (_ *Report, reterr error) { + onlySTUN := false + if opts != nil && opts.OnlySTUN { + if opts.OnlyTCP443 { + return nil, errors.New("netcheck: only one of OnlySTUN or OnlyTCP443 may be set in opts") + } + onlySTUN = true + } defer func() { if reterr != nil { metricNumGetReportError.Add(1) @@ -865,6 +876,9 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe }() if runtime.GOOS == "js" || runtime.GOOS == "tamago" || (runtime.GOOS == "plan9" && hostinfo.IsInVM86()) { + if onlySTUN { + return nil, errors.New("platform is restricted to HTTP, but OnlySTUN is set in opts") + } if err := c.runHTTPOnlyChecks(ctx, last, rs, dm); err != nil { return nil, err } @@ -896,7 +910,7 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe // it's unnecessary. captivePortalDone := syncs.ClosedChan() captivePortalStop := func() {} - if !rs.incremental { + if !rs.incremental && !onlySTUN { // NOTE(andrew): we can't simply add this goroutine to the // `NewWaitGroupChan` below, since we don't wait for that // waitgroup to finish when exiting this function and thus get @@ -970,13 +984,13 @@ func (c *Client) GetReport(ctx context.Context, dm *tailcfg.DERPMap, opts *GetRe rs.stopTimers() // Try HTTPS and ICMP latency check if all STUN probes failed due to - // UDP presumably being blocked. + // UDP presumably being blocked, and we are not constrained to only STUN. // TODO: this should be moved into the probePlan, using probeProto probeHTTPS. - if !rs.anyUDP() && ctx.Err() == nil { + if !rs.anyUDP() && ctx.Err() == nil && !onlySTUN { var wg sync.WaitGroup var need []*tailcfg.DERPRegion for rid, reg := range dm.Regions { - if !rs.haveRegionLatency(rid) && regionHasDERPNode(reg) { + if !rs.haveRegionLatency(rid) && regionHasDERPNode(reg) && !reg.Avoid && !reg.NoMeasureNoHome { need = append(need, reg) } } @@ -1358,6 +1372,15 @@ const ( // even without receiving a STUN response. // Note: must remain higher than the derp package frameReceiveRecordRate PreferredDERPFrameTime = 8 * time.Second + // PreferredDERPKeepAliveTimeout is 2x the DERP Keep Alive timeout. If there + // is no latency data to make judgements from, but we have heard from our + // current DERP region inside of 2x the KeepAlive window, don't switch DERP + // regions yet, keep the current region. This prevents region flapping / + // home DERP removal during short periods of packet loss where the DERP TCP + // connection may itself naturally recover. + // TODO(raggi): expose shared time bounds from the DERP package rather than + // duplicating them here. + PreferredDERPKeepAliveTimeout = 2 * derp.KeepAlive ) // addReportHistoryAndSetPreferredDERP adds r to the set of recent Reports @@ -1442,13 +1465,10 @@ func (c *Client) addReportHistoryAndSetPreferredDERP(rs *reportState, r *Report, // the STUN probe) since we started the netcheck, or in the past 2s, as // another signal for "this region is still working". heardFromOldRegionRecently := false + prevRegionLastHeard := rs.opts.getLastDERPActivity(prevDERP) if changingPreferred { - if lastHeard := rs.opts.getLastDERPActivity(prevDERP); !lastHeard.IsZero() { - now := c.timeNow() - - heardFromOldRegionRecently = lastHeard.After(rs.start) - heardFromOldRegionRecently = heardFromOldRegionRecently || lastHeard.After(now.Add(-PreferredDERPFrameTime)) - } + heardFromOldRegionRecently = prevRegionLastHeard.After(rs.start) + heardFromOldRegionRecently = heardFromOldRegionRecently || prevRegionLastHeard.After(now.Add(-PreferredDERPFrameTime)) } // The old region is accessible if we've heard from it via a non-STUN @@ -1475,17 +1495,20 @@ func (c *Client) addReportHistoryAndSetPreferredDERP(rs *reportState, r *Report, // If the forced DERP region probed successfully, or has recent traffic, // use it. _, haveLatencySample := r.RegionLatency[c.ForcePreferredDERP] - var recentActivity bool - if lastHeard := rs.opts.getLastDERPActivity(c.ForcePreferredDERP); !lastHeard.IsZero() { - now := c.timeNow() - recentActivity = lastHeard.After(rs.start) - recentActivity = recentActivity || lastHeard.After(now.Add(-PreferredDERPFrameTime)) - } + lastHeard := rs.opts.getLastDERPActivity(c.ForcePreferredDERP) + recentActivity := lastHeard.After(rs.start) + recentActivity = recentActivity || lastHeard.After(now.Add(-PreferredDERPFrameTime)) if haveLatencySample || recentActivity { r.PreferredDERP = c.ForcePreferredDERP } } + // If there was no latency data to make judgements on, but there is an + // active DERP connection that has at least been doing KeepAlive recently, + // keep it, rather than dropping it. + if r.PreferredDERP == 0 && prevRegionLastHeard.After(now.Add(-PreferredDERPKeepAliveTimeout)) { + r.PreferredDERP = prevDERP + } } func updateLatency(m map[int]time.Duration, regionID int, d time.Duration) { diff --git a/net/netcheck/netcheck_test.go b/net/netcheck/netcheck_test.go index 3affa614dae88..6830e7f27075c 100644 --- a/net/netcheck/netcheck_test.go +++ b/net/netcheck/netcheck_test.go @@ -18,6 +18,7 @@ import ( "testing" "time" + "tailscale.com/derp" "tailscale.com/net/netmon" "tailscale.com/net/stun/stuntest" "tailscale.com/tailcfg" @@ -419,6 +420,39 @@ func TestAddReportHistoryAndSetPreferredDERP(t *testing.T) { wantPrevLen: 2, wantDERP: 1, }, + { + name: "no_data_keep_home", + steps: []step{ + {0, report("d1", 2, "d2", 3)}, + {30 * time.Second, report()}, + {2 * time.Second, report()}, + {2 * time.Second, report()}, + {2 * time.Second, report()}, + {2 * time.Second, report()}, + }, + opts: &GetReportOpts{ + GetLastDERPActivity: mkLDAFunc(map[int]time.Time{ + 1: startTime, + }), + }, + wantPrevLen: 6, + wantDERP: 1, + }, + { + name: "no_data_home_expires", + steps: []step{ + {0, report("d1", 2, "d2", 3)}, + {30 * time.Second, report()}, + {2 * derp.KeepAlive, report()}, + }, + opts: &GetReportOpts{ + GetLastDERPActivity: mkLDAFunc(map[int]time.Time{ + 1: startTime, + }), + }, + wantPrevLen: 3, + wantDERP: 0, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/net/netmon/netmon_test.go b/net/netmon/netmon_test.go index a9af8fb004af3..b8ec1b75f97ec 100644 --- a/net/netmon/netmon_test.go +++ b/net/netmon/netmon_test.go @@ -12,6 +12,7 @@ import ( "time" "tailscale.com/util/eventbus" + "tailscale.com/util/eventbus/eventbustest" "tailscale.com/util/mak" ) @@ -68,6 +69,23 @@ func TestMonitorInjectEvent(t *testing.T) { } } +func TestMonitorInjectEventOnBus(t *testing.T) { + bus := eventbustest.NewBus(t) + + mon, err := New(bus, t.Logf) + if err != nil { + t.Fatal(err) + } + defer mon.Close() + tw := eventbustest.NewWatcher(t, bus) + + mon.Start() + mon.InjectEvent() + if err := eventbustest.Expect(tw, eventbustest.Type[*ChangeDelta]()); err != nil { + t.Error(err) + } +} + var ( monitor = flag.String("monitor", "", `go into monitor mode like 'route monitor'; test never terminates. Value can be either "raw" or "callback"`) monitorDuration = flag.Duration("monitor-duration", 0, "if non-zero, how long to run TestMonitorMode. Zero means forever.") @@ -77,13 +95,13 @@ func TestMonitorMode(t *testing.T) { switch *monitor { case "": t.Skip("skipping non-test without --monitor") - case "raw", "callback": + case "raw", "callback", "eventbus": default: - t.Skipf(`invalid --monitor value: must be "raw" or "callback"`) + t.Skipf(`invalid --monitor value: must be "raw", "callback" or "eventbus"`) } - bus := eventbus.New() - defer bus.Close() + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) mon, err := New(bus, t.Logf) if err != nil { @@ -124,6 +142,16 @@ func TestMonitorMode(t *testing.T) { mon.Start() <-done t.Logf("%v callbacks", n) + case "eventbus": + tw.TimeOut = *monitorDuration + n := 0 + mon.Start() + eventbustest.Expect(tw, func(event *ChangeDelta) (bool, error) { + n++ + t.Logf("cb: changed=%v, ifSt=%v", event.Major, event.New) + return false, nil // Return false, indicating we wanna look for more events + }) + t.Logf("%v events", n) } } diff --git a/net/packet/header.go b/net/packet/header.go index dbe84429adbd8..fa66a8641c6c4 100644 --- a/net/packet/header.go +++ b/net/packet/header.go @@ -8,6 +8,7 @@ import ( "math" ) +const igmpHeaderLength = 8 const tcpHeaderLength = 20 const sctpHeaderLength = 12 diff --git a/net/packet/packet.go b/net/packet/packet.go index b683b22126948..34b63aadd2c2e 100644 --- a/net/packet/packet.go +++ b/net/packet/packet.go @@ -51,10 +51,11 @@ type Parsed struct { IPVersion uint8 // IPProto is the IP subprotocol (UDP, TCP, etc.). Valid iff IPVersion != 0. IPProto ipproto.Proto - // SrcIP4 is the source address. Family matches IPVersion. Port is - // valid iff IPProto == TCP || IPProto == UDP. + // Src is the source address. Family matches IPVersion. Port is + // valid iff IPProto == TCP || IPProto == UDP || IPProto == SCTP. Src netip.AddrPort - // DstIP4 is the destination address. Family matches IPVersion. + // Dst is the destination address. Family matches IPVersion. Port is + // valid iff IPProto == TCP || IPProto == UDP || IPProto == SCTP. Dst netip.AddrPort // TCPFlags is the packet's TCP flag bits. Valid iff IPProto == TCP. TCPFlags TCPFlag @@ -160,14 +161,8 @@ func (q *Parsed) decode4(b []byte) { if fragOfs == 0 { // This is the first fragment - if moreFrags && len(sub) < minFragBlks { - // Suspiciously short first fragment, dump it. - q.IPProto = unknown - return - } - // otherwise, this is either non-fragmented (the usual case) - // or a big enough initial fragment that we can read the - // whole subprotocol header. + // Every protocol below MUST check that it has at least one entire + // transport header in order to protect against fragment confusion. switch q.IPProto { case ipproto.ICMPv4: if len(sub) < icmp4HeaderLength { @@ -179,6 +174,10 @@ func (q *Parsed) decode4(b []byte) { q.dataofs = q.subofs + icmp4HeaderLength return case ipproto.IGMP: + if len(sub) < igmpHeaderLength { + q.IPProto = unknown + return + } // Keep IPProto, but don't parse anything else // out. return @@ -211,6 +210,15 @@ func (q *Parsed) decode4(b []byte) { q.Dst = withPort(q.Dst, binary.BigEndian.Uint16(sub[2:4])) return case ipproto.TSMP: + // Strictly disallow fragmented TSMP + if moreFrags { + q.IPProto = unknown + return + } + if len(sub) < minTSMPSize { + q.IPProto = unknown + return + } // Inter-tailscale messages. q.dataofs = q.subofs return @@ -223,8 +231,11 @@ func (q *Parsed) decode4(b []byte) { } else { // This is a fragment other than the first one. if fragOfs < minFragBlks { - // First frag was suspiciously short, so we can't - // trust the followup either. + // disallow fragment offsets that are potentially inside of a + // transport header. This is notably asymmetric with the + // first-packet limit, that may allow a first-packet that requires a + // shorter offset than this limit, but without state to tie this + // to the first fragment we can not allow shorter packets. q.IPProto = unknown return } @@ -314,6 +325,10 @@ func (q *Parsed) decode6(b []byte) { q.Dst = withPort(q.Dst, binary.BigEndian.Uint16(sub[2:4])) return case ipproto.TSMP: + if len(sub) < minTSMPSize { + q.IPProto = unknown + return + } // Inter-tailscale messages. q.dataofs = q.subofs return diff --git a/net/packet/packet_test.go b/net/packet/packet_test.go index 4fc804a4fea21..09c2c101d66d9 100644 --- a/net/packet/packet_test.go +++ b/net/packet/packet_test.go @@ -385,6 +385,124 @@ var sctpDecode = Parsed{ Dst: mustIPPort("100.74.70.3:456"), } +var ipv4ShortFirstFragmentBuffer = []byte{ + // IP header (20 bytes) + 0x45, 0x00, 0x00, 0x4f, // Total length 79 bytes + 0x00, 0x01, 0x20, 0x00, // ID, Flags (MoreFragments set, offset 0) + 0x40, 0x06, 0x00, 0x00, // TTL, Protocol (TCP), Checksum + 0x01, 0x02, 0x03, 0x04, // Source IP + 0x05, 0x06, 0x07, 0x08, // Destination IP + // TCP header (20 bytes), but packet is truncated to 59 bytes of TCP data + // (total 79 bytes, 20 for IP) + 0x00, 0x7b, 0x02, 0x37, 0x00, 0x00, 0x12, 0x34, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + // Payload (39 bytes) + 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, +} + +var ipv4ShortFirstFragmentDecode = Parsed{ + b: ipv4ShortFirstFragmentBuffer, + subofs: 20, + dataofs: 40, + length: len(ipv4ShortFirstFragmentBuffer), + IPVersion: 4, + IPProto: ipproto.TCP, + Src: mustIPPort("1.2.3.4:123"), + Dst: mustIPPort("5.6.7.8:567"), + TCPFlags: 0x12, // SYN + ACK +} + +var ipv4SmallOffsetFragmentBuffer = []byte{ + // IP header (20 bytes) + 0x45, 0x00, 0x00, 0x28, // Total length 40 bytes + 0x00, 0x01, 0x20, 0x08, // ID, Flags (MoreFragments set, offset 8 bytes (0x08 / 8 = 1)) + 0x40, 0x06, 0x00, 0x00, // TTL, Protocol (TCP), Checksum + 0x01, 0x02, 0x03, 0x04, // Source IP + 0x05, 0x06, 0x07, 0x08, // Destination IP + // Payload (20 bytes) - this would be part of the TCP header in a real scenario + 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, 0x61, + 0x61, 0x61, 0x61, 0x61, +} + +var ipv4SmallOffsetFragmentDecode = Parsed{ + b: ipv4SmallOffsetFragmentBuffer, + subofs: 20, // subofs will still be set based on IHL + dataofs: 0, // It's unknown, so dataofs should be 0 + length: len(ipv4SmallOffsetFragmentBuffer), + IPVersion: 4, + IPProto: ipproto.Unknown, // Expected to be Unknown + Src: mustIPPort("1.2.3.4:0"), + Dst: mustIPPort("5.6.7.8:0"), +} + +// First fragment packet missing exactly one byte of the TCP header +var ipv4OneByteShortTCPHeaderBuffer = []byte{ + // IP header (20 bytes) + 0x45, 0x00, 0x00, 0x27, // Total length 51 bytes (20 IP + 19 TCP) + 0x00, 0x01, 0x20, 0x00, // ID, Flags (MoreFragments set, offset 0) + 0x40, 0x06, 0x00, 0x00, // TTL, Protocol (TCP), Checksum + 0x01, 0x02, 0x03, 0x04, // Source IP + 0x05, 0x06, 0x07, 0x08, // Destination IP + // TCP header - only 19 bytes (one byte short of the required 20) + 0x00, 0x7b, 0x02, 0x37, // Source port, Destination port + 0x00, 0x00, 0x12, 0x34, // Sequence number + 0x00, 0x00, 0x00, 0x00, // Acknowledgment number + 0x50, 0x12, 0x01, 0x00, // Data offset, flags, window size + 0x00, 0x00, 0x00, // Checksum (missing the last byte of urgent pointer) +} + +// IPv4 packet with maximum header length (60 bytes = 15 words) and a TCP header that's +// one byte short of being complete +var ipv4MaxHeaderShortTCPBuffer = []byte{ + // IP header with max options (60 bytes) + 0x4F, 0x00, 0x00, 0x4F, // Version (4) + IHL (15), ToS, Total length 79 bytes (60 IP + 19 TCP) + 0x00, 0x01, 0x20, 0x00, // ID, Flags (MoreFragments set, offset 0) + 0x40, 0x06, 0x00, 0x00, // TTL, Protocol (TCP), Checksum + 0x01, 0x02, 0x03, 0x04, // Source IP + 0x05, 0x06, 0x07, 0x08, // Destination IP + // IPv4 options (40 bytes) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + 0x01, 0x01, 0x01, 0x01, // 4 NOP options (padding) + // TCP header - only 19 bytes (one byte short of the required 20) + 0x00, 0x7b, 0x02, 0x37, // Source port, Destination port + 0x00, 0x00, 0x12, 0x34, // Sequence number + 0x00, 0x00, 0x00, 0x00, // Acknowledgment number + 0x50, 0x12, 0x01, 0x00, // Data offset, flags, window size + 0x00, 0x00, 0x00, // Checksum (missing the last byte of urgent pointer) +} + +var ipv4MaxHeaderShortTCPDecode = Parsed{ + b: ipv4MaxHeaderShortTCPBuffer, + subofs: 60, // 60 bytes for full IPv4 header with max options + dataofs: 0, // It's unknown, so dataofs should be 0 + length: len(ipv4MaxHeaderShortTCPBuffer), + IPVersion: 4, + IPProto: ipproto.Unknown, // Expected to be Unknown + Src: mustIPPort("1.2.3.4:0"), + Dst: mustIPPort("5.6.7.8:0"), +} + +var ipv4OneByteShortTCPHeaderDecode = Parsed{ + b: ipv4OneByteShortTCPHeaderBuffer, + subofs: 20, + dataofs: 0, // It's unknown, so dataofs should be 0 + length: len(ipv4OneByteShortTCPHeaderBuffer), + IPVersion: 4, + IPProto: ipproto.Unknown, // Expected to be Unknown + Src: mustIPPort("1.2.3.4:0"), + Dst: mustIPPort("5.6.7.8:0"), +} + func TestParsedString(t *testing.T) { tests := []struct { name string @@ -450,6 +568,10 @@ func TestDecode(t *testing.T) { {"ipv4_sctp", sctpBuffer, sctpDecode}, {"ipv4_frag", tcp4MediumFragmentBuffer, tcp4MediumFragmentDecode}, {"ipv4_fragtooshort", tcp4ShortFragmentBuffer, tcp4ShortFragmentDecode}, + {"ipv4_short_first_fragment", ipv4ShortFirstFragmentBuffer, ipv4ShortFirstFragmentDecode}, + {"ipv4_small_offset_fragment", ipv4SmallOffsetFragmentBuffer, ipv4SmallOffsetFragmentDecode}, + {"ipv4_one_byte_short_tcp_header", ipv4OneByteShortTCPHeaderBuffer, ipv4OneByteShortTCPHeaderDecode}, + {"ipv4_max_header_short_tcp", ipv4MaxHeaderShortTCPBuffer, ipv4MaxHeaderShortTCPDecode}, {"ip97", mustHexDecode("4500 0019 d186 4000 4061 751d 644a 4603 6449 e549 6865 6c6c 6f"), Parsed{ IPVersion: 4, diff --git a/net/packet/tsmp.go b/net/packet/tsmp.go index 4e004cca2cb7a..d78d10d36d3bb 100644 --- a/net/packet/tsmp.go +++ b/net/packet/tsmp.go @@ -19,6 +19,8 @@ import ( "tailscale.com/types/ipproto" ) +const minTSMPSize = 7 // the rejected body is 7 bytes + // TailscaleRejectedHeader is a TSMP message that says that one // Tailscale node has rejected the connection from another. Unlike a // TCP RST, this includes a reason. diff --git a/net/portmapper/portmapper.go b/net/portmapper/portmapper.go index 59f88e96604a5..c82fbf9da112a 100644 --- a/net/portmapper/portmapper.go +++ b/net/portmapper/portmapper.go @@ -507,6 +507,13 @@ func (c *Client) createMapping() { c.logf("createOrGetMapping: %v", err) } return + } else if mapping == nil { + return + + // TODO(creachadair): This was already logged in createOrGetMapping. + // It really should not happen at all, but we will need to untangle + // the control flow to eliminate that possibility. Meanwhile, this + // mitigates a panic downstream, cf. #16662. } if c.updates != nil { c.updates.Publish(Mapping{ @@ -515,7 +522,7 @@ func (c *Client) createMapping() { GoodUntil: mapping.GoodUntil(), }) } - if c.onChange != nil { + if c.onChange != nil && c.pubClient == nil { go c.onChange() } } diff --git a/net/portmapper/portmapper_test.go b/net/portmapper/portmapper_test.go index 515a0c28c993f..e66d3c159eccb 100644 --- a/net/portmapper/portmapper_test.go +++ b/net/portmapper/portmapper_test.go @@ -12,7 +12,7 @@ import ( "time" "tailscale.com/control/controlknobs" - "tailscale.com/util/eventbus" + "tailscale.com/util/eventbus/eventbustest" ) func TestCreateOrGetMapping(t *testing.T) { @@ -142,22 +142,15 @@ func TestUpdateEvent(t *testing.T) { t.Fatalf("Create test gateway: %v", err) } - bus := eventbus.New() - defer bus.Close() + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) - sub := eventbus.Subscribe[Mapping](bus.Client("TestUpdateEvent")) c := newTestClient(t, igd, bus) if _, err := c.Probe(t.Context()); err != nil { t.Fatalf("Probe failed: %v", err) } c.GetCachedMappingOrStartCreatingOne() - - select { - case evt := <-sub.Events(): - t.Logf("Received portmap update: %+v", evt) - case <-sub.Done(): - t.Error("Subscriber closed prematurely") - case <-time.After(5 * time.Second): - t.Error("Timed out waiting for an update event") + if err := eventbustest.Expect(tw, eventbustest.Type[Mapping]()); err != nil { + t.Error(err.Error()) } } diff --git a/net/tlsdial/tlsdial.go b/net/tlsdial/tlsdial.go index 1bd2450aa3c5d..80f3bfc06c4e8 100644 --- a/net/tlsdial/tlsdial.go +++ b/net/tlsdial/tlsdial.go @@ -59,18 +59,26 @@ var mitmBlockWarnable = health.Register(&health.Warnable{ ImpactsConnectivity: true, }) -// Config returns a tls.Config for connecting to a server. +// Config returns a tls.Config for connecting to a server that +// uses system roots for validation but, if those fail, also tries +// the baked-in LetsEncrypt roots as a fallback validation method. +// // If base is non-nil, it's cloned as the base config before // being configured and returned. // If ht is non-nil, it's used to report health errors. -func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { +func Config(ht *health.Tracker, base *tls.Config) *tls.Config { var conf *tls.Config if base == nil { conf = new(tls.Config) } else { conf = base.Clone() } - conf.ServerName = host + + // Note: we do NOT set conf.ServerName here (as we accidentally did + // previously), as this path is also used when dialing an HTTPS proxy server + // (through which we'll send a CONNECT request to get a TCP connection to do + // the real TCP connection) because host is the ultimate hostname, but this + // tls.Config is used for both the proxy and the ultimate target. if n := sslKeyLogFile; n != "" { f, err := os.OpenFile(n, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) @@ -93,7 +101,9 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { // (with the baked-in fallback root) in the VerifyConnection hook. conf.InsecureSkipVerify = true conf.VerifyConnection = func(cs tls.ConnectionState) (retErr error) { - if host == "log.tailscale.com" && hostinfo.IsNATLabGuestVM() { + dialedHost := cs.ServerName + + if dialedHost == "log.tailscale.com" && hostinfo.IsNATLabGuestVM() { // Allow log.tailscale.com TLS MITM for integration tests when // the client's running within a NATLab VM. return nil @@ -116,7 +126,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { // Show a dedicated warning. m, ok := blockblame.VerifyCertificate(cert) if ok { - log.Printf("tlsdial: server cert for %q looks like %q equipment (could be blocking Tailscale)", host, m.Name) + log.Printf("tlsdial: server cert seen while dialing %q looks like %q equipment (could be blocking Tailscale)", dialedHost, m.Name) ht.SetUnhealthy(mitmBlockWarnable, health.Args{"manufacturer": m.Name}) } else { ht.SetHealthy(mitmBlockWarnable) @@ -135,7 +145,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { ht.SetTLSConnectionError(cs.ServerName, nil) if selfSignedIssuer != "" { // Log the self-signed issuer, but don't treat it as an error. - log.Printf("tlsdial: warning: server cert for %q passed x509 validation but is self-signed by %q", host, selfSignedIssuer) + log.Printf("tlsdial: warning: server cert for %q passed x509 validation but is self-signed by %q", dialedHost, selfSignedIssuer) } } }() @@ -144,7 +154,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { // First try doing x509 verification with the system's // root CA pool. opts := x509.VerifyOptions{ - DNSName: cs.ServerName, + DNSName: dialedHost, Intermediates: x509.NewCertPool(), } for _, cert := range cs.PeerCertificates[1:] { @@ -152,7 +162,7 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { } _, errSys := cs.PeerCertificates[0].Verify(opts) if debug() { - log.Printf("tlsdial(sys %q): %v", host, errSys) + log.Printf("tlsdial(sys %q): %v", dialedHost, errSys) } // Always verify with our baked-in Let's Encrypt certificate, @@ -161,13 +171,11 @@ func Config(host string, ht *health.Tracker, base *tls.Config) *tls.Config { opts.Roots = bakedroots.Get() _, bakedErr := cs.PeerCertificates[0].Verify(opts) if debug() { - log.Printf("tlsdial(bake %q): %v", host, bakedErr) + log.Printf("tlsdial(bake %q): %v", dialedHost, bakedErr) } else if bakedErr != nil { - if _, loaded := tlsdialWarningPrinted.LoadOrStore(host, true); !loaded { - if errSys == nil { - log.Printf("tlsdial: warning: server cert for %q is not a Let's Encrypt cert", host) - } else { - log.Printf("tlsdial: error: server cert for %q failed to verify and is not a Let's Encrypt cert", host) + if _, loaded := tlsdialWarningPrinted.LoadOrStore(dialedHost, true); !loaded { + if errSys != nil { + log.Printf("tlsdial: error: server cert for %q failed both system roots & Let's Encrypt root validation", dialedHost) } } } @@ -202,9 +210,6 @@ func SetConfigExpectedCert(c *tls.Config, certDNSName string) { c.ServerName = certDNSName return } - if c.VerifyPeerCertificate != nil { - panic("refusing to override tls.Config.VerifyPeerCertificate") - } // Set InsecureSkipVerify to prevent crypto/tls from doing its // own cert verification, but do the same work that it'd do // (but using certDNSName) in the VerifyPeerCertificate hook. @@ -257,29 +262,30 @@ func SetConfigExpectedCertHash(c *tls.Config, wantFullCertSHA256Hex string) { if c.VerifyPeerCertificate != nil { panic("refusing to override tls.Config.VerifyPeerCertificate") } + // Set InsecureSkipVerify to prevent crypto/tls from doing its // own cert verification, but do the same work that it'd do - // (but using certDNSName) in the VerifyPeerCertificate hook. + // (but using certDNSName) in the VerifyConnection hook. c.InsecureSkipVerify = true - c.VerifyConnection = nil - c.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + + c.VerifyConnection = func(cs tls.ConnectionState) error { + dialedHost := cs.ServerName var sawGoodCert bool - for _, rawCert := range rawCerts { - cert, err := x509.ParseCertificate(rawCert) - if err != nil { - return fmt.Errorf("ParseCertificate: %w", err) - } + + for _, cert := range cs.PeerCertificates { if strings.HasPrefix(cert.Subject.CommonName, derpconst.MetaCertCommonNamePrefix) { continue } if sawGoodCert { return errors.New("unexpected multiple certs presented") } - if fmt.Sprintf("%02x", sha256.Sum256(rawCert)) != wantFullCertSHA256Hex { + if fmt.Sprintf("%02x", sha256.Sum256(cert.Raw)) != wantFullCertSHA256Hex { return fmt.Errorf("cert hash does not match expected cert hash") } - if err := cert.VerifyHostname(c.ServerName); err != nil { - return fmt.Errorf("cert does not match server name %q: %w", c.ServerName, err) + if dialedHost != "" { // it's empty when dialing a derper by IP with no hostname + if err := cert.VerifyHostname(dialedHost); err != nil { + return fmt.Errorf("cert does not match server name %q: %w", dialedHost, err) + } } now := time.Now() if now.After(cert.NotAfter) { @@ -302,12 +308,8 @@ func SetConfigExpectedCertHash(c *tls.Config, wantFullCertSHA256Hex string) { func NewTransport() *http.Transport { return &http.Transport{ DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - host, _, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } var d tls.Dialer - d.Config = Config(host, nil, nil) + d.Config = Config(nil, nil) return d.DialContext(ctx, network, addr) }, } diff --git a/net/tlsdial/tlsdial_test.go b/net/tlsdial/tlsdial_test.go index 6723b82e0d1c9..e2c4cdd4f51cb 100644 --- a/net/tlsdial/tlsdial_test.go +++ b/net/tlsdial/tlsdial_test.go @@ -86,7 +86,7 @@ func TestFallbackRootWorks(t *testing.T) { DisableKeepAlives: true, // for test cleanup ease } ht := new(health.Tracker) - tr.TLSClientConfig = Config("tlsdial.test", ht, tr.TLSClientConfig) + tr.TLSClientConfig = Config(ht, tr.TLSClientConfig) c := &http.Client{Transport: tr} ctr0 := atomic.LoadInt32(&counterFallbackOK) diff --git a/net/tsdial/tsdial.go b/net/tsdial/tsdial.go index 2492f666cf063..e4e4e9e8b0f92 100644 --- a/net/tsdial/tsdial.go +++ b/net/tsdial/tsdial.go @@ -322,7 +322,7 @@ func (d *Dialer) userDialResolve(ctx context.Context, network, addr string) (net } var r net.Resolver - if exitDNSDoH != "" && runtime.GOOS != "windows" { // Windows: https://github.com/golang/go/issues/33097 + if exitDNSDoH != "" { r.PreferGo = true r.Dial = func(ctx context.Context, network, address string) (net.Conn, error) { return &dohConn{ diff --git a/net/tshttpproxy/tshttpproxy_windows.go b/net/tshttpproxy/tshttpproxy_windows.go index 06a1f5ae445d0..7163c786307ac 100644 --- a/net/tshttpproxy/tshttpproxy_windows.go +++ b/net/tshttpproxy/tshttpproxy_windows.go @@ -18,6 +18,7 @@ import ( "unsafe" "github.com/alexbrainman/sspi/negotiate" + "github.com/dblohm7/wingoes" "golang.org/x/sys/windows" "tailscale.com/hostinfo" "tailscale.com/syncs" @@ -97,9 +98,7 @@ func proxyFromWinHTTPOrCache(req *http.Request) (*url.URL, error) { } if err == windows.ERROR_INVALID_PARAMETER { metricErrInvalidParameters.Add(1) - // Seen on Windows 8.1. (https://github.com/tailscale/tailscale/issues/879) - // TODO(bradfitz): figure this out. - setNoProxyUntil(time.Hour) + setNoProxyUntil(10 * time.Second) proxyErrorf("tshttpproxy: winhttp: GetProxyForURL(%q): ERROR_INVALID_PARAMETER [unexpected]", urlStr) return nil, nil } @@ -238,17 +237,30 @@ func (pi *winHTTPProxyInfo) free() { } } -var proxyForURLOpts = &winHTTPAutoProxyOptions{ - DwFlags: winHTTP_AUTOPROXY_ALLOW_AUTOCONFIG | winHTTP_AUTOPROXY_AUTO_DETECT, - DwAutoDetectFlags: winHTTP_AUTO_DETECT_TYPE_DHCP, // | winHTTP_AUTO_DETECT_TYPE_DNS_A, -} +var getProxyForURLOpts = sync.OnceValue(func() *winHTTPAutoProxyOptions { + opts := &winHTTPAutoProxyOptions{ + DwFlags: winHTTP_AUTOPROXY_AUTO_DETECT, + DwAutoDetectFlags: winHTTP_AUTO_DETECT_TYPE_DHCP | winHTTP_AUTO_DETECT_TYPE_DNS_A, + } + // Support for the WINHTTP_AUTOPROXY_ALLOW_AUTOCONFIG flag was added in Windows 10, version 1703. + // + // Using it on earlier versions causes GetProxyForURL to fail with ERROR_INVALID_PARAMETER, + // which prevents proxy detection and can lead to failures reaching the control server + // on environments where a proxy is required. + // + // https://web.archive.org/web/20250529044903/https://learn.microsoft.com/en-us/windows/win32/api/winhttp/ns-winhttp-winhttp_autoproxy_options + if wingoes.IsWin10BuildOrGreater(wingoes.Win10Build1703) { + opts.DwFlags |= winHTTP_AUTOPROXY_ALLOW_AUTOCONFIG + } + return opts +}) func (hi winHTTPInternet) GetProxyForURL(urlStr string) (string, error) { var out winHTTPProxyInfo err := winHTTPGetProxyForURL( hi, windows.StringToUTF16Ptr(urlStr), - proxyForURLOpts, + getProxyForURLOpts(), &out, ) if err != nil { diff --git a/net/tstun/mtu_test.go b/net/tstun/mtu_test.go index 8d165bfd341a9..ec31e45ce73f5 100644 --- a/net/tstun/mtu_test.go +++ b/net/tstun/mtu_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package tstun import ( diff --git a/net/tstun/tun.go b/net/tstun/tun.go index 88679daa24b6c..bfdaddf58b283 100644 --- a/net/tstun/tun.go +++ b/net/tstun/tun.go @@ -24,6 +24,9 @@ import ( // CrateTAP is the hook set by feature/tap. var CreateTAP feature.Hook[func(logf logger.Logf, tapName, bridgeName string) (tun.Device, error)] +// modprobeTunHook is a Linux-specific hook to run "/sbin/modprobe tun". +var modprobeTunHook feature.Hook[func() error] + // New returns a tun.Device for the requested device name, along with // the OS-dependent name that was allocated to the device. func New(logf logger.Logf, tunName string) (tun.Device, string, error) { @@ -51,7 +54,22 @@ func New(logf logger.Logf, tunName string) (tun.Device, string, error) { if runtime.GOOS == "plan9" { cleanUpPlan9Interfaces() } - dev, err = tun.CreateTUN(tunName, int(DefaultTUNMTU())) + // Try to create the TUN device up to two times. If it fails + // the first time and we're on Linux, try a desperate + // "modprobe tun" to load the tun module and try again. + for try := range 2 { + dev, err = tun.CreateTUN(tunName, int(DefaultTUNMTU())) + if err == nil || !modprobeTunHook.IsSet() { + if try > 0 { + logf("created TUN device %q after doing `modprobe tun`", tunName) + } + break + } + if modprobeTunHook.Get()() != nil { + // modprobe failed; no point trying again. + break + } + } } if err != nil { return nil, "", err diff --git a/net/tstun/tun_linux.go b/net/tstun/tun_linux.go index 9600ceb77328f..05cf58c17df8a 100644 --- a/net/tstun/tun_linux.go +++ b/net/tstun/tun_linux.go @@ -17,6 +17,14 @@ import ( func init() { tunDiagnoseFailure = diagnoseLinuxTUNFailure + modprobeTunHook.Set(func() error { + _, err := modprobeTun() + return err + }) +} + +func modprobeTun() ([]byte, error) { + return exec.Command("/sbin/modprobe", "tun").CombinedOutput() } func diagnoseLinuxTUNFailure(tunName string, logf logger.Logf, createErr error) { @@ -36,7 +44,7 @@ func diagnoseLinuxTUNFailure(tunName string, logf logger.Logf, createErr error) kernel := utsReleaseField(&un) logf("Linux kernel version: %s", kernel) - modprobeOut, err := exec.Command("/sbin/modprobe", "tun").CombinedOutput() + modprobeOut, err := modprobeTun() if err == nil { logf("'modprobe tun' successful") // Either tun is currently loaded, or it's statically diff --git a/net/udprelay/endpoint/endpoint.go b/net/udprelay/endpoint/endpoint.go index 2672a856b797b..0d2a14e965a4a 100644 --- a/net/udprelay/endpoint/endpoint.go +++ b/net/udprelay/endpoint/endpoint.go @@ -7,11 +7,16 @@ package endpoint import ( "net/netip" + "time" "tailscale.com/tstime" "tailscale.com/types/key" ) +// ServerRetryAfter is the default +// [tailscale.com/net/udprelay.ErrServerNotReady.RetryAfter] value. +const ServerRetryAfter = time.Second * 3 + // ServerEndpoint contains details for an endpoint served by a // [tailscale.com/net/udprelay.Server]. type ServerEndpoint struct { @@ -21,6 +26,10 @@ type ServerEndpoint struct { // unique ServerEndpoint allocation. ServerDisco key.DiscoPublic + // ClientDisco are the Disco public keys of the relay participants permitted + // to handshake with this endpoint. + ClientDisco [2]key.DiscoPublic + // LamportID is unique and monotonically non-decreasing across // ServerEndpoint allocations for the lifetime of Server. It enables clients // to dedup and resolve allocation event order. Clients may race to allocate diff --git a/net/udprelay/server.go b/net/udprelay/server.go index 7b63ec95e77c4..c34a4b5f6835c 100644 --- a/net/udprelay/server.go +++ b/net/udprelay/server.go @@ -8,6 +8,7 @@ package udprelay import ( "bytes" + "context" "crypto/rand" "errors" "fmt" @@ -19,11 +20,18 @@ import ( "time" "go4.org/mem" + "tailscale.com/client/local" "tailscale.com/disco" + "tailscale.com/net/netcheck" + "tailscale.com/net/netmon" "tailscale.com/net/packet" + "tailscale.com/net/stun" "tailscale.com/net/udprelay/endpoint" "tailscale.com/tstime" "tailscale.com/types/key" + "tailscale.com/types/logger" + "tailscale.com/util/eventbus" + "tailscale.com/util/set" ) const ( @@ -42,45 +50,30 @@ const ( // Server implements an experimental UDP relay server. type Server struct { - // disco keypair used as part of 3-way bind handshake - disco key.DiscoPrivate - discoPublic key.DiscoPublic - + // The following fields are initialized once and never mutated. + logf logger.Logf + disco key.DiscoPrivate + discoPublic key.DiscoPublic bindLifetime time.Duration steadyStateLifetime time.Duration - - // addrPorts contains the ip:port pairs returned as candidate server - // endpoints in response to an allocation request. - addrPorts []netip.AddrPort - - uc *net.UDPConn - - closeOnce sync.Once - wg sync.WaitGroup - closeCh chan struct{} - closed bool - - mu sync.Mutex // guards the following fields - lamportID uint64 - vniPool []uint32 // the pool of available VNIs - byVNI map[uint32]*serverEndpoint - byDisco map[pairOfDiscoPubKeys]*serverEndpoint -} - -// pairOfDiscoPubKeys is a pair of key.DiscoPublic. It must be constructed via -// newPairOfDiscoPubKeys to ensure lexicographical ordering. -type pairOfDiscoPubKeys [2]key.DiscoPublic - -func (p pairOfDiscoPubKeys) String() string { - return fmt.Sprintf("%s <=> %s", p[0].ShortString(), p[1].ShortString()) -} - -func newPairOfDiscoPubKeys(discoA, discoB key.DiscoPublic) pairOfDiscoPubKeys { - pair := pairOfDiscoPubKeys([2]key.DiscoPublic{discoA, discoB}) - slices.SortFunc(pair[:], func(a, b key.DiscoPublic) int { - return a.Compare(b) - }) - return pair + bus *eventbus.Bus + uc4 *net.UDPConn // always non-nil + uc4Port uint16 // always nonzero + uc6 *net.UDPConn // may be nil if IPv6 bind fails during initialization + uc6Port uint16 // may be zero if IPv6 bind fails during initialization + closeOnce sync.Once + wg sync.WaitGroup + closeCh chan struct{} + netChecker *netcheck.Client + + mu sync.Mutex // guards the following fields + addrDiscoveryOnce bool // addrDiscovery completed once (successfully or unsuccessfully) + addrPorts []netip.AddrPort // the ip:port pairs returned as candidate endpoints + closed bool + lamportID uint64 + vniPool []uint32 // the pool of available VNIs + byVNI map[uint32]*serverEndpoint + byDisco map[key.SortedPairOfDiscoPublic]*serverEndpoint } // serverEndpoint contains Server-internal [endpoint.ServerEndpoint] state. @@ -90,90 +83,99 @@ type serverEndpoint struct { // indexing of this array aligns with the following fields, e.g. // discoSharedSecrets[0] is the shared secret to use when sealing // Disco protocol messages for transmission towards discoPubKeys[0]. - discoPubKeys pairOfDiscoPubKeys - discoSharedSecrets [2]key.DiscoShared - handshakeState [2]disco.BindUDPRelayHandshakeState - addrPorts [2]netip.AddrPort - lastSeen [2]time.Time // TODO(jwhited): consider using mono.Time - challenge [2][disco.BindUDPRelayEndpointChallengeLen]byte + discoPubKeys key.SortedPairOfDiscoPublic + discoSharedSecrets [2]key.DiscoShared + handshakeGeneration [2]uint32 // or zero if a handshake has never started for that relay leg + handshakeAddrPorts [2]netip.AddrPort // or zero value if a handshake has never started for that relay leg + boundAddrPorts [2]netip.AddrPort // or zero value if a handshake has never completed for that relay leg + lastSeen [2]time.Time // TODO(jwhited): consider using mono.Time + challenge [2][disco.BindUDPRelayChallengeLen]byte lamportID uint64 vni uint32 allocatedAt time.Time } -func (e *serverEndpoint) handleDiscoControlMsg(from netip.AddrPort, senderIndex int, discoMsg disco.Message, uw udpWriter, serverDisco key.DiscoPublic) { +func (e *serverEndpoint) handleDiscoControlMsg(from netip.AddrPort, senderIndex int, discoMsg disco.Message, conn *net.UDPConn, serverDisco key.DiscoPublic) { if senderIndex != 0 && senderIndex != 1 { return } - handshakeState := e.handshakeState[senderIndex] - if handshakeState == disco.BindUDPRelayHandshakeStateAnswerReceived { - // this sender is already bound - return + + otherSender := 0 + if senderIndex == 0 { + otherSender = 1 + } + + validateVNIAndRemoteKey := func(common disco.BindUDPRelayEndpointCommon) error { + if common.VNI != e.vni { + return errors.New("mismatching VNI") + } + if common.RemoteKey.Compare(e.discoPubKeys.Get()[otherSender]) != 0 { + return errors.New("mismatching RemoteKey") + } + return nil } + switch discoMsg := discoMsg.(type) { case *disco.BindUDPRelayEndpoint: - switch handshakeState { - case disco.BindUDPRelayHandshakeStateInit: - // set sender addr - e.addrPorts[senderIndex] = from - fallthrough - case disco.BindUDPRelayHandshakeStateChallengeSent: - if from != e.addrPorts[senderIndex] { - // this is a later arriving bind from a different source, or - // a retransmit and the sender's source has changed, discard - return - } - m := new(disco.BindUDPRelayEndpointChallenge) - copy(m.Challenge[:], e.challenge[senderIndex][:]) - reply := make([]byte, packet.GeneveFixedHeaderLength, 512) - gh := packet.GeneveHeader{Control: true, VNI: e.vni, Protocol: packet.GeneveProtocolDisco} - err := gh.Encode(reply) - if err != nil { - return - } - reply = append(reply, disco.Magic...) - reply = serverDisco.AppendTo(reply) - box := e.discoSharedSecrets[senderIndex].Seal(m.AppendMarshal(nil)) - reply = append(reply, box...) - uw.WriteMsgUDPAddrPort(reply, nil, from) - // set new state - e.handshakeState[senderIndex] = disco.BindUDPRelayHandshakeStateChallengeSent + err := validateVNIAndRemoteKey(discoMsg.BindUDPRelayEndpointCommon) + if err != nil { + // silently drop return - default: - // disco.BindUDPRelayEndpoint is unexpected in all other handshake states + } + if discoMsg.Generation == 0 { + // Generation must be nonzero, silently drop + return + } + if e.handshakeGeneration[senderIndex] == discoMsg.Generation { + // we've seen this generation before, silently drop + return + } + e.handshakeGeneration[senderIndex] = discoMsg.Generation + e.handshakeAddrPorts[senderIndex] = from + m := new(disco.BindUDPRelayEndpointChallenge) + m.VNI = e.vni + m.Generation = discoMsg.Generation + m.RemoteKey = e.discoPubKeys.Get()[otherSender] + rand.Read(e.challenge[senderIndex][:]) + copy(m.Challenge[:], e.challenge[senderIndex][:]) + reply := make([]byte, packet.GeneveFixedHeaderLength, 512) + gh := packet.GeneveHeader{Control: true, VNI: e.vni, Protocol: packet.GeneveProtocolDisco} + err = gh.Encode(reply) + if err != nil { return } + reply = append(reply, disco.Magic...) + reply = serverDisco.AppendTo(reply) + box := e.discoSharedSecrets[senderIndex].Seal(m.AppendMarshal(nil)) + reply = append(reply, box...) + conn.WriteMsgUDPAddrPort(reply, nil, from) + return case *disco.BindUDPRelayEndpointAnswer: - switch handshakeState { - case disco.BindUDPRelayHandshakeStateChallengeSent: - if from != e.addrPorts[senderIndex] { - // sender source has changed - return - } - if !bytes.Equal(discoMsg.Answer[:], e.challenge[senderIndex][:]) { - // bad answer - return - } - // sender is now bound - // TODO: Consider installing a fast path via netfilter or similar to - // relay (NAT) data packets for this serverEndpoint. - e.handshakeState[senderIndex] = disco.BindUDPRelayHandshakeStateAnswerReceived - // record last seen as bound time - e.lastSeen[senderIndex] = time.Now() + err := validateVNIAndRemoteKey(discoMsg.BindUDPRelayEndpointCommon) + if err != nil { + // silently drop return - default: - // disco.BindUDPRelayEndpointAnswer is unexpected in all other handshake - // states, or we've already handled it + } + generation := e.handshakeGeneration[senderIndex] + if generation == 0 || // we have no active handshake + generation != discoMsg.Generation || // mismatching generation for the active handshake + e.handshakeAddrPorts[senderIndex] != from || // mismatching source for the active handshake + !bytes.Equal(e.challenge[senderIndex][:], discoMsg.Challenge[:]) { // mismatching answer for the active handshake + // silently drop return } + // Handshake complete. Update the binding for this sender. + e.boundAddrPorts[senderIndex] = from + e.lastSeen[senderIndex] = time.Now() // record last seen as bound time + return default: - // unexpected Disco message type + // unexpected message types, silently drop return } } -func (e *serverEndpoint) handleSealedDiscoControlMsg(from netip.AddrPort, b []byte, uw udpWriter, serverDisco key.DiscoPublic) { +func (e *serverEndpoint) handleSealedDiscoControlMsg(from netip.AddrPort, b []byte, conn *net.UDPConn, serverDisco key.DiscoPublic) { senderRaw, isDiscoMsg := disco.Source(b) if !isDiscoMsg { // Not a Disco message @@ -182,9 +184,9 @@ func (e *serverEndpoint) handleSealedDiscoControlMsg(from netip.AddrPort, b []by sender := key.DiscoPublicFromRaw32(mem.B(senderRaw)) senderIndex := -1 switch { - case sender.Compare(e.discoPubKeys[0]) == 0: + case sender.Compare(e.discoPubKeys.Get()[0]) == 0: senderIndex = 0 - case sender.Compare(e.discoPubKeys[1]) == 0: + case sender.Compare(e.discoPubKeys.Get()[1]) == 0: senderIndex = 1 default: // unknown Disco public key @@ -204,14 +206,10 @@ func (e *serverEndpoint) handleSealedDiscoControlMsg(from netip.AddrPort, b []by return } - e.handleDiscoControlMsg(from, senderIndex, discoMsg, uw, serverDisco) -} - -type udpWriter interface { - WriteMsgUDPAddrPort(b []byte, oob []byte, addr netip.AddrPort) (n, oobn int, err error) + e.handleDiscoControlMsg(from, senderIndex, discoMsg, conn, serverDisco) } -func (e *serverEndpoint) handlePacket(from netip.AddrPort, gh packet.GeneveHeader, b []byte, uw udpWriter, serverDisco key.DiscoPublic) { +func (e *serverEndpoint) handlePacket(from netip.AddrPort, gh packet.GeneveHeader, b []byte, rxSocket, otherAFSocket *net.UDPConn, serverDisco key.DiscoPublic) { if !gh.Control { if !e.isBound() { // not a control packet, but serverEndpoint isn't bound @@ -219,23 +217,26 @@ func (e *serverEndpoint) handlePacket(from netip.AddrPort, gh packet.GeneveHeade } var to netip.AddrPort switch { - case from == e.addrPorts[0]: + case from == e.boundAddrPorts[0]: e.lastSeen[0] = time.Now() - to = e.addrPorts[1] - case from == e.addrPorts[1]: + to = e.boundAddrPorts[1] + case from == e.boundAddrPorts[1]: e.lastSeen[1] = time.Now() - to = e.addrPorts[0] + to = e.boundAddrPorts[0] default: // unrecognized source return } - // relay packet - uw.WriteMsgUDPAddrPort(b, nil, to) - return - } - - if e.isBound() { - // control packet, but serverEndpoint is already bound + // Relay the packet towards the other party via the socket associated + // with the destination's address family. If source and destination + // address families are matching we tx on the same socket the packet + // was received (rxSocket), otherwise we use the "other" socket + // (otherAFSocket). [Server] makes no use of dual-stack sockets. + if from.Addr().Is4() == to.Addr().Is4() { + rxSocket.WriteMsgUDPAddrPort(b, nil, to) + } else if otherAFSocket != nil { + otherAFSocket.WriteMsgUDPAddrPort(b, nil, to) + } return } @@ -245,7 +246,7 @@ func (e *serverEndpoint) handlePacket(from netip.AddrPort, gh packet.GeneveHeade } msg := b[packet.GeneveFixedHeaderLength:] - e.handleSealedDiscoControlMsg(from, msg, uw, serverDisco) + e.handleSealedDiscoControlMsg(from, msg, rxSocket, serverDisco) } func (e *serverEndpoint) isExpired(now time.Time, bindLifetime, steadyStateLifetime time.Duration) bool { @@ -261,28 +262,25 @@ func (e *serverEndpoint) isExpired(now time.Time, bindLifetime, steadyStateLifet return false } -// isBound returns true if both clients have completed their 3-way handshake, +// isBound returns true if both clients have completed a 3-way handshake, // otherwise false. func (e *serverEndpoint) isBound() bool { - return e.handshakeState[0] == disco.BindUDPRelayHandshakeStateAnswerReceived && - e.handshakeState[1] == disco.BindUDPRelayHandshakeStateAnswerReceived + return e.boundAddrPorts[0].IsValid() && + e.boundAddrPorts[1].IsValid() } -// NewServer constructs a [Server] listening on 0.0.0.0:'port'. IPv6 is not yet -// supported. Port may be 0, and what ultimately gets bound is returned as -// 'boundPort'. Supplied 'addrs' are joined with 'boundPort' and returned as -// [endpoint.ServerEndpoint.AddrPorts] in response to Server.AllocateEndpoint() -// requests. -// -// TODO: IPv6 support -// TODO: dynamic addrs:port discovery -func NewServer(port int, addrs []netip.Addr) (s *Server, boundPort int, err error) { +// NewServer constructs a [Server] listening on port. If port is zero, then +// port selection is left up to the host networking stack. If +// len(overrideAddrs) > 0 these will be used in place of dynamic discovery, +// which is useful to override in tests. +func NewServer(logf logger.Logf, port int, overrideAddrs []netip.Addr) (s *Server, err error) { s = &Server{ + logf: logf, disco: key.NewDisco(), bindLifetime: defaultBindLifetime, steadyStateLifetime: defaultSteadyStateLifetime, closeCh: make(chan struct{}), - byDisco: make(map[pairOfDiscoPubKeys]*serverEndpoint), + byDisco: make(map[key.SortedPairOfDiscoPublic]*serverEndpoint), byVNI: make(map[uint32]*serverEndpoint), } s.discoPublic = s.disco.Public() @@ -292,57 +290,204 @@ func NewServer(port int, addrs []netip.Addr) (s *Server, boundPort int, err erro for i := 1; i < 1<<24; i++ { s.vniPool = append(s.vniPool, uint32(i)) } - boundPort, err = s.listenOn(port) + + bus := eventbus.New() + s.bus = bus + netMon, err := netmon.New(s.bus, logf) if err != nil { - return nil, 0, err + return nil, err } - addrPorts := make([]netip.AddrPort, 0, len(addrs)) - for _, addr := range addrs { - addrPort, err := netip.ParseAddrPort(net.JoinHostPort(addr.String(), strconv.Itoa(boundPort))) - if err != nil { - return nil, 0, err + s.netChecker = &netcheck.Client{ + NetMon: netMon, + Logf: logger.WithPrefix(logf, "netcheck: "), + SendPacket: func(b []byte, addrPort netip.AddrPort) (int, error) { + if addrPort.Addr().Is4() { + return s.uc4.WriteToUDPAddrPort(b, addrPort) + } else if s.uc6 != nil { + return s.uc6.WriteToUDPAddrPort(b, addrPort) + } else { + return 0, errors.New("IPv6 socket is not bound") + } + }, + } + + err = s.listenOn(port) + if err != nil { + return nil, err + } + + if len(overrideAddrs) > 0 { + addrPorts := make(set.Set[netip.AddrPort], len(overrideAddrs)) + for _, addr := range overrideAddrs { + if addr.IsValid() { + if addr.Is4() { + addrPorts.Add(netip.AddrPortFrom(addr, s.uc4Port)) + } else if s.uc6 != nil { + addrPorts.Add(netip.AddrPortFrom(addr, s.uc6Port)) + } + } } - addrPorts = append(addrPorts, addrPort) + s.addrPorts = addrPorts.Slice() + } else { + s.wg.Add(1) + go s.addrDiscoveryLoop() } - s.addrPorts = addrPorts - s.wg.Add(2) - go s.packetReadLoop() + + s.wg.Add(1) + go s.packetReadLoop(s.uc4, s.uc6) + if s.uc6 != nil { + s.wg.Add(1) + go s.packetReadLoop(s.uc6, s.uc4) + } + s.wg.Add(1) go s.endpointGCLoop() - return s, boundPort, nil + + return s, nil } -func (s *Server) listenOn(port int) (int, error) { - uc, err := net.ListenUDP("udp4", &net.UDPAddr{Port: port}) - if err != nil { - return 0, err +func (s *Server) addrDiscoveryLoop() { + defer s.wg.Done() + + timer := time.NewTimer(0) // fire immediately + defer timer.Stop() + + getAddrPorts := func() ([]netip.AddrPort, error) { + var addrPorts set.Set[netip.AddrPort] + addrPorts.Make() + + // get local addresses + ips, _, err := netmon.LocalAddresses() + if err != nil { + return nil, err + } + for _, ip := range ips { + if ip.IsValid() { + if ip.Is4() { + addrPorts.Add(netip.AddrPortFrom(ip, s.uc4Port)) + } else { + addrPorts.Add(netip.AddrPortFrom(ip, s.uc6Port)) + } + } + } + + // fetch DERPMap to feed to netcheck + derpMapCtx, derpMapCancel := context.WithTimeout(context.Background(), time.Second) + defer derpMapCancel() + localClient := &local.Client{} + // TODO(jwhited): We are in-process so use eventbus or similar. + // local.Client gets us going. + dm, err := localClient.CurrentDERPMap(derpMapCtx) + if err != nil { + return nil, err + } + + // get addrPorts as visible from DERP + netCheckerCtx, netCheckerCancel := context.WithTimeout(context.Background(), netcheck.ReportTimeout) + defer netCheckerCancel() + rep, err := s.netChecker.GetReport(netCheckerCtx, dm, &netcheck.GetReportOpts{ + OnlySTUN: true, + }) + if err != nil { + return nil, err + } + if rep.GlobalV4.IsValid() { + addrPorts.Add(rep.GlobalV4) + } + if rep.GlobalV6.IsValid() { + addrPorts.Add(rep.GlobalV6) + } + // TODO(jwhited): consider logging if rep.MappingVariesByDestIP as + // that's a hint we are not well-positioned to operate as a UDP relay. + return addrPorts.Slice(), nil } - // TODO: set IP_PKTINFO sockopt - _, boundPortStr, err := net.SplitHostPort(uc.LocalAddr().String()) - if err != nil { - s.uc.Close() - return 0, err + + for { + select { + case <-timer.C: + // Mirror magicsock behavior for duration between STUN. We consider + // 30s a min bound for NAT timeout. + timer.Reset(tstime.RandomDurationBetween(20*time.Second, 26*time.Second)) + addrPorts, err := getAddrPorts() + if err != nil { + s.logf("error discovering IP:port candidates: %v", err) + } + s.mu.Lock() + s.addrPorts = addrPorts + s.addrDiscoveryOnce = true + s.mu.Unlock() + case <-s.closeCh: + return + } } - boundPort, err := strconv.Atoi(boundPortStr) - if err != nil { - s.uc.Close() - return 0, err +} + +// listenOn binds an IPv4 and IPv6 socket to port. We consider it successful if +// we manage to bind the IPv4 socket. +// +// The requested port may be zero, in which case port selection is left up to +// the host networking stack. We make no attempt to bind a consistent port +// across IPv4 and IPv6 if the requested port is zero. +// +// TODO: make these "re-bindable" in similar fashion to magicsock as a means to +// deal with EDR software closing them. http://go/corp/30118 +func (s *Server) listenOn(port int) error { + for _, network := range []string{"udp4", "udp6"} { + uc, err := net.ListenUDP(network, &net.UDPAddr{Port: port}) + if err != nil { + if network == "udp4" { + return err + } else { + s.logf("ignoring IPv6 bind failure: %v", err) + break + } + } + // TODO: set IP_PKTINFO sockopt + _, boundPortStr, err := net.SplitHostPort(uc.LocalAddr().String()) + if err != nil { + uc.Close() + if s.uc4 != nil { + s.uc4.Close() + } + return err + } + portUint, err := strconv.ParseUint(boundPortStr, 10, 16) + if err != nil { + uc.Close() + if s.uc4 != nil { + s.uc4.Close() + } + return err + } + if network == "udp4" { + s.uc4 = uc + s.uc4Port = uint16(portUint) + } else { + s.uc6 = uc + s.uc6Port = uint16(portUint) + } } - s.uc = uc - return boundPort, nil + return nil } // Close closes the server. func (s *Server) Close() error { s.closeOnce.Do(func() { - s.mu.Lock() - defer s.mu.Unlock() - s.uc.Close() + s.uc4.Close() + if s.uc6 != nil { + s.uc6.Close() + } close(s.closeCh) s.wg.Wait() + // s.mu must not be held while s.wg.Wait'ing, otherwise we can + // deadlock. The goroutines we are waiting on to return can also + // acquire s.mu. + s.mu.Lock() + defer s.mu.Unlock() clear(s.byVNI) clear(s.byDisco) s.vniPool = nil s.closed = true + s.bus.Close() }) return nil } @@ -377,7 +522,14 @@ func (s *Server) endpointGCLoop() { } } -func (s *Server) handlePacket(from netip.AddrPort, b []byte, uw udpWriter) { +func (s *Server) handlePacket(from netip.AddrPort, b []byte, rxSocket, otherAFSocket *net.UDPConn) { + if stun.Is(b) && b[1] == 0x01 { + // A b[1] value of 0x01 (STUN method binding) is sufficiently + // non-overlapping with the Geneve header where the LSB is always 0 + // (part of 6 "reserved" bits). + s.netChecker.ReceiveSTUNPacket(b, from) + return + } gh := packet.GeneveHeader{} err := gh.Decode(b) if err != nil { @@ -394,31 +546,50 @@ func (s *Server) handlePacket(from netip.AddrPort, b []byte, uw udpWriter) { return } - e.handlePacket(from, gh, b, uw, s.discoPublic) + e.handlePacket(from, gh, b, rxSocket, otherAFSocket, s.discoPublic) } -func (s *Server) packetReadLoop() { +func (s *Server) packetReadLoop(readFromSocket, otherSocket *net.UDPConn) { defer func() { + // We intentionally close the [Server] if we encounter a socket read + // error below, at least until socket "re-binding" is implemented as + // part of http://go/corp/30118. + // + // Decrementing this [sync.WaitGroup] _before_ calling [Server.Close] is + // intentional as [Server.Close] waits on it. s.wg.Done() s.Close() }() b := make([]byte, 1<<16-1) for { // TODO: extract laddr from IP_PKTINFO for use in reply - n, from, err := s.uc.ReadFromUDPAddrPort(b) + n, from, err := readFromSocket.ReadFromUDPAddrPort(b) if err != nil { + s.logf("error reading from socket(%v): %v", readFromSocket.LocalAddr(), err) return } - s.handlePacket(from, b[:n], s.uc) + s.handlePacket(from, b[:n], readFromSocket, otherSocket) } } var ErrServerClosed = errors.New("server closed") +// ErrServerNotReady indicates the server is not ready. Allocation should be +// requested after waiting for at least RetryAfter duration. +type ErrServerNotReady struct { + RetryAfter time.Duration +} + +func (e ErrServerNotReady) Error() string { + return fmt.Sprintf("server not ready, retry after %v", e.RetryAfter) +} + // AllocateEndpoint allocates an [endpoint.ServerEndpoint] for the provided pair // of [key.DiscoPublic]'s. If an allocation already exists for discoA and discoB // it is returned without modification/reallocation. AllocateEndpoint returns -// [ErrServerClosed] if the server has been closed. +// the following notable errors: +// 1. [ErrServerClosed] if the server has been closed. +// 2. [ErrServerNotReady] if the server is not ready. func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.ServerEndpoint, error) { s.mu.Lock() defer s.mu.Unlock() @@ -426,11 +597,18 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv return endpoint.ServerEndpoint{}, ErrServerClosed } + if len(s.addrPorts) == 0 { + if !s.addrDiscoveryOnce { + return endpoint.ServerEndpoint{}, ErrServerNotReady{RetryAfter: endpoint.ServerRetryAfter} + } + return endpoint.ServerEndpoint{}, errors.New("server addrPorts are not yet known") + } + if discoA.Compare(s.discoPublic) == 0 || discoB.Compare(s.discoPublic) == 0 { return endpoint.ServerEndpoint{}, fmt.Errorf("client disco equals server disco: %s", s.discoPublic.ShortString()) } - pair := newPairOfDiscoPubKeys(discoA, discoB) + pair := key.NewSortedPairOfDiscoPublic(discoA, discoB) e, ok := s.byDisco[pair] if ok { // Return the existing allocation. Clients can resolve duplicate @@ -439,8 +617,14 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv // TODO: consider ServerEndpoint.BindLifetime -= time.Now()-e.allocatedAt // to give the client a more accurate picture of the bind window. return endpoint.ServerEndpoint{ - ServerDisco: s.discoPublic, - AddrPorts: s.addrPorts, + ServerDisco: s.discoPublic, + // Returning the "latest" addrPorts for an existing allocation is + // the simple choice. It may not be the best depending on client + // behaviors and endpoint state (bound or not). We might want to + // consider storing them (maybe interning) in the [*serverEndpoint] + // at allocation time. + ClientDisco: pair.Get(), + AddrPorts: slices.Clone(s.addrPorts), VNI: e.vni, LamportID: e.lamportID, BindLifetime: tstime.GoDuration{Duration: s.bindLifetime}, @@ -458,18 +642,18 @@ func (s *Server) AllocateEndpoint(discoA, discoB key.DiscoPublic) (endpoint.Serv lamportID: s.lamportID, allocatedAt: time.Now(), } - e.discoSharedSecrets[0] = s.disco.Shared(e.discoPubKeys[0]) - e.discoSharedSecrets[1] = s.disco.Shared(e.discoPubKeys[1]) + e.discoSharedSecrets[0] = s.disco.Shared(e.discoPubKeys.Get()[0]) + e.discoSharedSecrets[1] = s.disco.Shared(e.discoPubKeys.Get()[1]) e.vni, s.vniPool = s.vniPool[0], s.vniPool[1:] - rand.Read(e.challenge[0][:]) - rand.Read(e.challenge[1][:]) s.byDisco[pair] = e s.byVNI[e.vni] = e + s.logf("allocated endpoint vni=%d lamportID=%d disco[0]=%v disco[1]=%v", e.vni, e.lamportID, pair.Get()[0].ShortString(), pair.Get()[1].ShortString()) return endpoint.ServerEndpoint{ ServerDisco: s.discoPublic, - AddrPorts: s.addrPorts, + ClientDisco: pair.Get(), + AddrPorts: slices.Clone(s.addrPorts), VNI: e.vni, LamportID: e.lamportID, BindLifetime: tstime.GoDuration{Duration: s.bindLifetime}, diff --git a/net/udprelay/server_test.go b/net/udprelay/server_test.go index 38c7ae5d9a749..de1c293644992 100644 --- a/net/udprelay/server_test.go +++ b/net/udprelay/server_test.go @@ -19,23 +19,27 @@ import ( ) type testClient struct { - vni uint32 - local key.DiscoPrivate - server key.DiscoPublic - uc *net.UDPConn + vni uint32 + handshakeGeneration uint32 + local key.DiscoPrivate + remote key.DiscoPublic + server key.DiscoPublic + uc *net.UDPConn } -func newTestClient(t *testing.T, vni uint32, serverEndpoint netip.AddrPort, local key.DiscoPrivate, server key.DiscoPublic) *testClient { +func newTestClient(t *testing.T, vni uint32, serverEndpoint netip.AddrPort, local key.DiscoPrivate, remote, server key.DiscoPublic) *testClient { rAddr := &net.UDPAddr{IP: serverEndpoint.Addr().AsSlice(), Port: int(serverEndpoint.Port())} - uc, err := net.DialUDP("udp4", nil, rAddr) + uc, err := net.DialUDP("udp", nil, rAddr) if err != nil { t.Fatal(err) } return &testClient{ - vni: vni, - local: local, - server: server, - uc: uc, + vni: vni, + handshakeGeneration: 1, + local: local, + remote: remote, + server: server, + uc: uc, } } @@ -137,13 +141,35 @@ func (c *testClient) readControlDiscoMsg(t *testing.T) disco.Message { } func (c *testClient) handshake(t *testing.T) { - c.writeControlDiscoMsg(t, &disco.BindUDPRelayEndpoint{}) + generation := c.handshakeGeneration + c.handshakeGeneration++ + common := disco.BindUDPRelayEndpointCommon{ + VNI: c.vni, + Generation: generation, + RemoteKey: c.remote, + } + c.writeControlDiscoMsg(t, &disco.BindUDPRelayEndpoint{ + BindUDPRelayEndpointCommon: common, + }) msg := c.readControlDiscoMsg(t) challenge, ok := msg.(*disco.BindUDPRelayEndpointChallenge) if !ok { - t.Fatal("unexepcted disco message type") + t.Fatal("unexpected disco message type") + } + if challenge.Generation != common.Generation { + t.Fatalf("rx'd challenge.Generation (%d) != %d", challenge.Generation, common.Generation) + } + if challenge.VNI != common.VNI { + t.Fatalf("rx'd challenge.VNI (%d) != %d", challenge.VNI, common.VNI) + } + if challenge.RemoteKey != common.RemoteKey { + t.Fatalf("rx'd challenge.RemoteKey (%v) != %v", challenge.RemoteKey, common.RemoteKey) } - c.writeControlDiscoMsg(t, &disco.BindUDPRelayEndpointAnswer{Answer: challenge.Challenge}) + answer := &disco.BindUDPRelayEndpointAnswer{ + BindUDPRelayEndpointCommon: common, + } + answer.Challenge = challenge.Challenge + c.writeControlDiscoMsg(t, answer) } func (c *testClient) close() { @@ -154,59 +180,140 @@ func TestServer(t *testing.T) { discoA := key.NewDisco() discoB := key.NewDisco() - ipv4LoopbackAddr := netip.MustParseAddr("127.0.0.1") - - server, _, err := NewServer(0, []netip.Addr{ipv4LoopbackAddr}) - if err != nil { - t.Fatal(err) + cases := []struct { + name string + overrideAddrs []netip.Addr + forceClientsMixedAF bool + }{ + { + name: "over ipv4", + overrideAddrs: []netip.Addr{netip.MustParseAddr("127.0.0.1")}, + }, + { + name: "over ipv6", + overrideAddrs: []netip.Addr{netip.MustParseAddr("::1")}, + }, + { + name: "mixed address families", + overrideAddrs: []netip.Addr{netip.MustParseAddr("127.0.0.1"), netip.MustParseAddr("::1")}, + forceClientsMixedAF: true, + }, } - defer server.Close() - endpoint, err := server.AllocateEndpoint(discoA.Public(), discoB.Public()) - if err != nil { - t.Fatal(err) - } - dupEndpoint, err := server.AllocateEndpoint(discoA.Public(), discoB.Public()) - if err != nil { - t.Fatal(err) - } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + server, err := NewServer(t.Logf, 0, tt.overrideAddrs) + if err != nil { + t.Fatal(err) + } + defer server.Close() - // We expect the same endpoint details pre-handshake. - if diff := cmp.Diff(dupEndpoint, endpoint, cmpopts.EquateComparable(netip.AddrPort{}, key.DiscoPublic{})); diff != "" { - t.Fatalf("wrong dupEndpoint (-got +want)\n%s", diff) - } + endpoint, err := server.AllocateEndpoint(discoA.Public(), discoB.Public()) + if err != nil { + t.Fatal(err) + } + dupEndpoint, err := server.AllocateEndpoint(discoA.Public(), discoB.Public()) + if err != nil { + t.Fatal(err) + } - if len(endpoint.AddrPorts) != 1 { - t.Fatalf("unexpected endpoint.AddrPorts: %v", endpoint.AddrPorts) - } - tcA := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoA, endpoint.ServerDisco) - defer tcA.close() - tcB := newTestClient(t, endpoint.VNI, endpoint.AddrPorts[0], discoB, endpoint.ServerDisco) - defer tcB.close() + // We expect the same endpoint details pre-handshake. + if diff := cmp.Diff(dupEndpoint, endpoint, cmpopts.EquateComparable(netip.AddrPort{}, key.DiscoPublic{})); diff != "" { + t.Fatalf("wrong dupEndpoint (-got +want)\n%s", diff) + } - tcA.handshake(t) - tcB.handshake(t) + if len(endpoint.AddrPorts) < 1 { + t.Fatalf("unexpected endpoint.AddrPorts: %v", endpoint.AddrPorts) + } + tcAServerEndpointAddr := endpoint.AddrPorts[0] + tcA := newTestClient(t, endpoint.VNI, tcAServerEndpointAddr, discoA, discoB.Public(), endpoint.ServerDisco) + defer tcA.close() + tcBServerEndpointAddr := tcAServerEndpointAddr + if tt.forceClientsMixedAF { + foundMixedAF := false + for _, addr := range endpoint.AddrPorts { + if addr.Addr().Is4() != tcBServerEndpointAddr.Addr().Is4() { + tcBServerEndpointAddr = addr + foundMixedAF = true + } + } + if !foundMixedAF { + t.Fatal("force clients to mixed address families is set, but relay server lacks address family diversity") + } + } + tcB := newTestClient(t, endpoint.VNI, tcBServerEndpointAddr, discoB, discoA.Public(), endpoint.ServerDisco) + defer tcB.close() - dupEndpoint, err = server.AllocateEndpoint(discoA.Public(), discoB.Public()) - if err != nil { - t.Fatal(err) - } - // We expect the same endpoint details post-handshake. - if diff := cmp.Diff(dupEndpoint, endpoint, cmpopts.EquateComparable(netip.AddrPort{}, key.DiscoPublic{})); diff != "" { - t.Fatalf("wrong dupEndpoint (-got +want)\n%s", diff) - } + for i := 0; i < 2; i++ { + // We handshake both clients twice to guarantee server-side + // packet reading goroutines, which are independent across + // address families, have seen an answer from both clients + // before proceeding. This is needed because the test assumes + // that B's handshake is complete (the first send is A->B below), + // but the server may not have handled B's handshake answer + // before it handles A's data pkt towards B. + // + // Data transmissions following "re-handshakes" orient so that + // the sender is the same as the party that performed the + // handshake, for the same reasons. + // + // [magicsock.relayManager] is not prone to this issue as both + // parties transmit data packets immediately following their + // handshake answer. + tcA.handshake(t) + tcB.handshake(t) + } - txToB := []byte{1, 2, 3} - tcA.writeDataPkt(t, txToB) - rxFromA := tcB.readDataPkt(t) - if !bytes.Equal(txToB, rxFromA) { - t.Fatal("unexpected msg A->B") - } + dupEndpoint, err = server.AllocateEndpoint(discoA.Public(), discoB.Public()) + if err != nil { + t.Fatal(err) + } + // We expect the same endpoint details post-handshake. + if diff := cmp.Diff(dupEndpoint, endpoint, cmpopts.EquateComparable(netip.AddrPort{}, key.DiscoPublic{})); diff != "" { + t.Fatalf("wrong dupEndpoint (-got +want)\n%s", diff) + } + + txToB := []byte{1, 2, 3} + tcA.writeDataPkt(t, txToB) + rxFromA := tcB.readDataPkt(t) + if !bytes.Equal(txToB, rxFromA) { + t.Fatal("unexpected msg A->B") + } + + txToA := []byte{4, 5, 6} + tcB.writeDataPkt(t, txToA) + rxFromB := tcA.readDataPkt(t) + if !bytes.Equal(txToA, rxFromB) { + t.Fatal("unexpected msg B->A") + } + + tcAOnNewPort := newTestClient(t, endpoint.VNI, tcAServerEndpointAddr, discoA, discoB.Public(), endpoint.ServerDisco) + tcAOnNewPort.handshakeGeneration = tcA.handshakeGeneration + 1 + defer tcAOnNewPort.close() + + // Handshake client A on a new source IP:port, verify we can send packets on the new binding + tcAOnNewPort.handshake(t) + + fromAOnNewPort := []byte{7, 8, 9} + tcAOnNewPort.writeDataPkt(t, fromAOnNewPort) + rxFromA = tcB.readDataPkt(t) + if !bytes.Equal(fromAOnNewPort, rxFromA) { + t.Fatal("unexpected msg A->B") + } + + tcBOnNewPort := newTestClient(t, endpoint.VNI, tcBServerEndpointAddr, discoB, discoA.Public(), endpoint.ServerDisco) + tcBOnNewPort.handshakeGeneration = tcB.handshakeGeneration + 1 + defer tcBOnNewPort.close() + + // Handshake client B on a new source IP:port, verify we can send packets on the new binding + tcBOnNewPort.handshake(t) - txToA := []byte{4, 5, 6} - tcB.writeDataPkt(t, txToA) - rxFromB := tcA.readDataPkt(t) - if !bytes.Equal(txToA, rxFromB) { - t.Fatal("unexpected msg B->A") + fromBOnNewPort := []byte{7, 8, 9} + tcBOnNewPort.writeDataPkt(t, fromBOnNewPort) + rxFromB = tcAOnNewPort.readDataPkt(t) + if !bytes.Equal(fromBOnNewPort, rxFromB) { + t.Fatal("unexpected msg B->A") + } + }) } } diff --git a/paths/paths.go b/paths/paths.go index 28c3be02a9c86..6c9c3fa6c9dea 100644 --- a/paths/paths.go +++ b/paths/paths.go @@ -6,6 +6,7 @@ package paths import ( + "log" "os" "path/filepath" "runtime" @@ -70,6 +71,37 @@ func DefaultTailscaledStateFile() string { return "" } +// DefaultTailscaledStateDir returns the default state directory +// to use for tailscaled, for use when the user provided neither +// a state directory or state file path to use. +// +// It returns the empty string if there's no reasonable default. +func DefaultTailscaledStateDir() string { + if runtime.GOOS == "plan9" { + home, err := os.UserHomeDir() + if err != nil { + log.Fatalf("failed to get home directory: %v", err) + } + return filepath.Join(home, "tailscale-state") + } + return filepath.Dir(DefaultTailscaledStateFile()) +} + +// MakeAutomaticStateDir reports whether the platform +// automatically creates the state directory for tailscaled +// when it's absent. +func MakeAutomaticStateDir() bool { + switch runtime.GOOS { + case "plan9": + return true + case "linux": + if distro.Get() == distro.JetKVM { + return true + } + } + return false +} + // MkStateDir ensures that dirPath, the daemon's configuration directory // containing machine keys etc, both exists and has the correct permissions. // We want it to only be accessible to the user the daemon is running under. diff --git a/paths/paths_unix.go b/paths/paths_unix.go index 50a8b7ca502f7..d317921d59cd9 100644 --- a/paths/paths_unix.go +++ b/paths/paths_unix.go @@ -21,6 +21,9 @@ func init() { } func statePath() string { + if runtime.GOOS == "linux" && distro.Get() == distro.JetKVM { + return "/userdata/tailscale/var/tailscaled.state" + } switch runtime.GOOS { case "linux", "illumos", "solaris": return "/var/lib/tailscale/tailscaled.state" diff --git a/posture/serialnumber_stub.go b/posture/serialnumber_stub.go index cdabf03e5a417..4cc84fa133489 100644 --- a/posture/serialnumber_stub.go +++ b/posture/serialnumber_stub.go @@ -1,13 +1,12 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -// android: not implemented // js: not implemented // plan9: not implemented // solaris: currently unsupported by go-smbios: // https://github.com/digitalocean/go-smbios/pull/21 -//go:build android || solaris || plan9 || js || wasm || tamago || aix || (darwin && !cgo && !ios) +//go:build solaris || plan9 || js || wasm || tamago || aix || (darwin && !cgo && !ios) package posture diff --git a/posture/serialnumber_ios.go b/posture/serialnumber_syspolicy.go similarity index 75% rename from posture/serialnumber_ios.go rename to posture/serialnumber_syspolicy.go index 55d0e438b54d5..d6491ff214b95 100644 --- a/posture/serialnumber_ios.go +++ b/posture/serialnumber_syspolicy.go @@ -1,6 +1,8 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause +//go:build android || ios + package posture import ( @@ -10,9 +12,9 @@ import ( "tailscale.com/util/syspolicy" ) -// GetSerialNumbers returns the serial number of the iOS/tvOS device as reported by an +// GetSerialNumbers returns the serial number of the device as reported by an // MDM solution. It requires configuration via the DeviceSerialNumber system policy. -// This is the only way to gather serial numbers on iOS and tvOS. +// This is the only way to gather serial numbers on iOS, tvOS and Android. func GetSerialNumbers(_ logger.Logf) ([]string, error) { s, err := syspolicy.GetString(syspolicy.DeviceSerialNumber, "") if err != nil { diff --git a/prober/derp.go b/prober/derp.go index 98e61ff54b89a..c7a82317dcabc 100644 --- a/prober/derp.go +++ b/prober/derp.go @@ -47,6 +47,7 @@ import ( type derpProber struct { p *Prober derpMapURL string // or "local" + meshKey key.DERPMesh udpInterval time.Duration meshInterval time.Duration tlsInterval time.Duration @@ -71,7 +72,7 @@ type derpProber struct { udpProbeFn func(string, int) ProbeClass meshProbeFn func(string, string) ProbeClass bwProbeFn func(string, string, int64) ProbeClass - qdProbeFn func(string, string, int, time.Duration) ProbeClass + qdProbeFn func(string, string, int, time.Duration, key.DERPMesh) ProbeClass sync.Mutex lastDERPMap *tailcfg.DERPMap @@ -143,6 +144,12 @@ func WithRegionCodeOrID(regionCode string) DERPOpt { } } +func WithMeshKey(meshKey key.DERPMesh) DERPOpt { + return func(d *derpProber) { + d.meshKey = meshKey + } +} + // DERP creates a new derpProber. // // If derpMapURL is "local", the DERPMap is fetched via @@ -250,7 +257,7 @@ func (d *derpProber) probeMapFn(ctx context.Context) error { wantProbes[n] = true if d.probes[n] == nil { log.Printf("adding DERP queuing delay probe for %s->%s (%s)", server.Name, to.Name, region.RegionName) - d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout)) + d.probes[n] = d.p.Run(n, -10*time.Second, labels, d.qdProbeFn(server.Name, to.Name, d.qdPacketsPerSecond, d.qdPacketTimeout, d.meshKey)) } } } @@ -284,7 +291,7 @@ func (d *derpProber) probeMesh(from, to string) ProbeClass { } dm := d.lastDERPMap - return derpProbeNodePair(ctx, dm, fromN, toN) + return derpProbeNodePair(ctx, dm, fromN, toN, d.meshKey) }, Class: "derp_mesh", Labels: Labels{"derp_path": derpPath}, @@ -308,7 +315,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { if err != nil { return err } - return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix) + return derpProbeBandwidth(ctx, d.lastDERPMap, fromN, toN, size, &transferTimeSeconds, &totalBytesTransferred, d.bwTUNIPv4Prefix, d.meshKey) }, Class: "derp_bw", Labels: Labels{ @@ -336,7 +343,7 @@ func (d *derpProber) probeBandwidth(from, to string, size int64) ProbeClass { // to the queuing delay measurement and are recorded as dropped. 'from' and 'to' are // expected to be names (DERPNode.Name) of two DERP servers in the same region, // and may refer to the same server. -func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration) ProbeClass { +func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, packetTimeout time.Duration, meshKey key.DERPMesh) ProbeClass { derpPath := "mesh" if from == to { derpPath = "single" @@ -349,7 +356,7 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa if err != nil { return err } - return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh) + return derpProbeQueuingDelay(ctx, d.lastDERPMap, fromN, toN, packetsPerSecond, packetTimeout, &packetsDropped, qdh, meshKey) }, Class: "derp_qd", Labels: Labels{"derp_path": derpPath}, @@ -368,15 +375,15 @@ func (d *derpProber) probeQueuingDelay(from, to string, packetsPerSecond int, pa // derpProbeQueuingDelay continuously sends data between two local DERP clients // connected to two DERP servers in order to measure queuing delays. From and to // can be the same server. -func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram) (err error) { +func derpProbeQueuingDelay(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, packetsPerSecond int, packetTimeout time.Duration, packetsDropped *expvar.Float, qdh *histogram, meshKey key.DERPMesh) (err error) { // This probe uses clients with isProber=false to avoid spamming the derper // logs with every packet sent by the queuing delay probe. - fromc, err := newConn(ctx, dm, from, false) + fromc, err := newConn(ctx, dm, from, false, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, false) + toc, err := newConn(ctx, dm, to, false, meshKey) if err != nil { return err } @@ -418,6 +425,24 @@ func runDerpProbeQueuingDelayContinously(ctx context.Context, from, to *tailcfg. txRecords := make([]txRecord, 0, packetsPerSecond*int(packetTimeout.Seconds())) var txRecordsMu sync.Mutex + // applyTimeouts walks over txRecords and expires any records that are older + // than packetTimeout, recording in metrics that they were removed. + applyTimeouts := func() { + txRecordsMu.Lock() + defer txRecordsMu.Unlock() + + now := time.Now() + recs := txRecords[:0] + for _, r := range txRecords { + if now.Sub(r.at) > packetTimeout { + packetsDropped.Add(1) + } else { + recs = append(recs, r) + } + } + txRecords = recs + } + // Send the packets. sendErrC := make(chan error, 1) // TODO: construct a disco CallMeMaybe in the same fashion as magicsock, e.g. magic bytes, src pub, seal payload. @@ -438,10 +463,12 @@ func runDerpProbeQueuingDelayContinously(ctx context.Context, from, to *tailcfg. case <-ctx.Done(): return case <-t.C: + applyTimeouts() txRecordsMu.Lock() if len(txRecords) == cap(txRecords) { txRecords = slices.Delete(txRecords, 0, 1) packetsDropped.Add(1) + log.Printf("unexpected: overflow in txRecords") } txRecords = append(txRecords, txRecord{time.Now(), seq}) txRecordsMu.Unlock() @@ -674,15 +701,15 @@ func derpProbeUDP(ctx context.Context, ipStr string, port int) error { // DERP clients connected to two DERP servers.If tunIPv4Address is specified, // probes will use a TCP connection over a TUN device at this address in order // to exercise TCP-in-TCP in similar fashion to TCP over Tailscale via DERP. -func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix) (err error) { +func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, size int64, transferTimeSeconds, totalBytesTransferred *expvar.Float, tunIPv4Prefix *netip.Prefix, meshKey key.DERPMesh) (err error) { // This probe uses clients with isProber=false to avoid spamming the derper logs with every packet // sent by the bandwidth probe. - fromc, err := newConn(ctx, dm, from, false) + fromc, err := newConn(ctx, dm, from, false, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, false) + toc, err := newConn(ctx, dm, to, false, meshKey) if err != nil { return err } @@ -712,13 +739,13 @@ func derpProbeBandwidth(ctx context.Context, dm *tailcfg.DERPMap, from, to *tail // derpProbeNodePair sends a small packet between two local DERP clients // connected to two DERP servers. -func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode) (err error) { - fromc, err := newConn(ctx, dm, from, true) +func derpProbeNodePair(ctx context.Context, dm *tailcfg.DERPMap, from, to *tailcfg.DERPNode, meshKey key.DERPMesh) (err error) { + fromc, err := newConn(ctx, dm, from, true, meshKey) if err != nil { return err } defer fromc.Close() - toc, err := newConn(ctx, dm, to, true) + toc, err := newConn(ctx, dm, to, true, meshKey) if err != nil { return err } @@ -1116,7 +1143,7 @@ func derpProbeBandwidthTUN(ctx context.Context, transferTimeSeconds, totalBytesT return nil } -func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool) (*derphttp.Client, error) { +func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isProber bool, meshKey key.DERPMesh) (*derphttp.Client, error) { // To avoid spamming the log with regular connection messages. l := logger.Filtered(log.Printf, func(s string) bool { return !strings.Contains(s, "derphttp.Client.Connect: connecting to") @@ -1132,6 +1159,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr } }) dc.IsProber = isProber + dc.MeshKey = meshKey err := dc.Connect(ctx) if err != nil { return nil, err @@ -1165,7 +1193,7 @@ func newConn(ctx context.Context, dm *tailcfg.DERPMap, n *tailcfg.DERPNode, isPr case derp.ServerInfoMessage: errc <- nil default: - errc <- fmt.Errorf("unexpected first message type %T", errc) + errc <- fmt.Errorf("unexpected first message type %T", m) } }() select { diff --git a/prober/tls_test.go b/prober/tls_test.go index 9ba17f79da911..f6ca4aeb19be6 100644 --- a/prober/tls_test.go +++ b/prober/tls_test.go @@ -6,6 +6,7 @@ package prober import ( "bytes" "context" + "crypto/ecdsa" "crypto/rand" "crypto/rsa" "crypto/tls" @@ -140,16 +141,60 @@ func (s *CRLServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Write(s.crlBytes) } -func TestCRL(t *testing.T) { - // Generate CA key and self-signed CA cert - caKey, err := rsa.GenerateKey(rand.Reader, 4096) +// someECDSAKey{1,2,3} are different EC private keys in PEM format +// as generated by: +// +// openssl ecparam -name prime256v1 -genkey -noout -out - +// +// They're used in tests to avoid burning CPU at test time to just +// to make some arbitrary test keys. +const ( + someECDSAKey1 = ` +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIDKggO47Si0/JgqF0q9m0HfQ92lbERWsBaKS5YihtuheoAoGCCqGSM49 +AwEHoUQDQgAE/JtNZkfFmAGQJHW5Xgz0Eoyi9MKVxl77sXjIFDMX233QDIWPEM/B +vmNMvdFkuYBjwbq6H+SNf1NXRNladEGU/Q== +-----END EC PRIVATE KEY----- +` + someECDSAKey2 = ` +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIPIJhRf4MpzLil1ZKcRqMx+jPeJXw96KtYYzV2AcgBzgoAoGCCqGSM49 +AwEHoUQDQgAEhA9CSWFmUvdvXMzyt+as+6f+0luydHU1x/gEksVByYIgYxahaGts +xbSKj6F2WgAN/ok1gFLqhH3UWMNVthM1wA== +-----END EC PRIVATE KEY----- +` + someECDSAKey3 = ` +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIKgZ1OJjK2St9O0i52N1K+IgSiu2/NSMk9Yt2+kDMHd7oAoGCCqGSM49 +AwEHoUQDQgAExFp80etkjy/AEUtSgJjXRA39jTU7eiEmCGRREewFQhwcEscBEfrg +6NN31r9YlEs+hZ8gXE1L3Deu6jn5jW3pig== +-----END EC PRIVATE KEY----- +` +) + +// parseECKey parses an EC private key from a PEM-encoded string. +func parseECKey(t *testing.T, pemPriv string) *ecdsa.PrivateKey { + t.Helper() + block, _ := pem.Decode([]byte(pemPriv)) + if block == nil { + t.Fatal("failed to decode PEM") + } + key, err := x509.ParseECPrivateKey(block.Bytes) if err != nil { - t.Fatal(err) + t.Fatalf("failed to parse EC key: %v", err) } + return key +} + +func TestCRL(t *testing.T) { + // Generate CA key and self-signed CA cert + caKey := parseECKey(t, someECDSAKey1) + caTpl := issuerCertTpl caTpl.BasicConstraintsValid = true caTpl.IsCA = true caTpl.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature + caTpl.SignatureAlgorithm = x509.ECDSAWithSHA256 caBytes, err := x509.CreateCertificate(rand.Reader, &caTpl, &caTpl, &caKey.PublicKey, caKey) if err != nil { t.Fatal(err) @@ -162,11 +207,9 @@ func TestCRL(t *testing.T) { // Issue a leaf cert signed by the CA leaf := leafCert leaf.SerialNumber = big.NewInt(20001) + leaf.SignatureAlgorithm = x509.ECDSAWithSHA256 leaf.Issuer = caCert.Subject - leafKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - t.Fatal(err) - } + leafKey := parseECKey(t, someECDSAKey2) leafBytes, err := x509.CreateCertificate(rand.Reader, &leaf, caCert, &leafKey.PublicKey, caKey) if err != nil { t.Fatal(err) @@ -182,10 +225,8 @@ func TestCRL(t *testing.T) { noCRLCert.CRLDistributionPoints = []string{} noCRLCert.NotBefore = time.Unix(letsEncryptStartedStaplingCRL, 0).Add(-48 * time.Hour) noCRLCert.Issuer = caCert.Subject - noCRLCertKey, err := rsa.GenerateKey(rand.Reader, 4096) - if err != nil { - t.Fatal(err) - } + noCRLCert.SignatureAlgorithm = x509.ECDSAWithSHA256 + noCRLCertKey := parseECKey(t, someECDSAKey3) noCRLStapledBytes, err := x509.CreateCertificate(rand.Reader, &noCRLCert, caCert, &noCRLCertKey.PublicKey, caKey) if err != nil { t.Fatal(err) diff --git a/release/dist/qnap/files/scripts/Dockerfile.qpkg b/release/dist/qnap/files/scripts/Dockerfile.qpkg index 1f4c2406d7642..dbcaac11668f0 100644 --- a/release/dist/qnap/files/scripts/Dockerfile.qpkg +++ b/release/dist/qnap/files/scripts/Dockerfile.qpkg @@ -10,7 +10,7 @@ RUN apt-get update -y && \ patch # Install QNAP QDK (force a specific version to pick up updates) -RUN git clone https://github.com/tailscale/QDK.git && cd /QDK && git reset --hard 9a31a67387c583d19a81a378dcf7c25e2abe231d +RUN git clone https://github.com/tailscale/QDK.git && cd /QDK && git reset --hard 6aba74f6b4c8ea0c30b8aec9f3476f428f6a58a1 RUN cd /QDK && ./InstallToUbuntu.sh install ENV PATH="/usr/share/QDK/bin:${PATH}" diff --git a/scripts/check_license_headers.sh b/scripts/check_license_headers.sh deleted file mode 100755 index 8345afab76508..0000000000000 --- a/scripts/check_license_headers.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh -# -# Copyright (c) Tailscale Inc & AUTHORS -# SPDX-License-Identifier: BSD-3-Clause -# -# check_license_headers.sh checks that all Go files in the given -# directory tree have a correct-looking Tailscale license header. - -check_file() { - got=$1 - - want=$(cat <&2 - exit 1 -fi - -fail=0 -for file in $(find $1 \( -name '*.go' -or -name '*.tsx' -or -name '*.ts' -not -name '*.config.ts' \) -not -path '*/.git/*' -not -path '*/node_modules/*'); do - case $file in - $1/tempfork/*) - # Skip, tempfork of third-party code - ;; - $1/wgengine/router/ifconfig_windows.go) - # WireGuard copyright. - ;; - $1/cmd/tailscale/cli/authenticode_windows.go) - # WireGuard copyright. - ;; - *_string.go) - # Generated file from go:generate stringer - ;; - $1/control/controlbase/noiseexplorer_test.go) - # Noiseexplorer.com copyright. - ;; - */zsyscall_windows.go) - # Generated syscall wrappers - ;; - $1/util/winutil/subprocess_windows_test.go) - # Subprocess test harness code - ;; - $1/util/winutil/testdata/testrestartableprocesses/main.go) - # Subprocess test harness code - ;; - *$1/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go) - # Generated kube deepcopy funcs file starts with a Go build tag + an empty line - header="$(head -5 $file | tail -n+3 )" - ;; - $1/derp/xdp/bpf_bpfe*.go) - # Generated eBPF management code - ;; - *) - header="$(head -2 $file)" - ;; - esac - if [ ! -z "$header" ]; then - if ! check_file "$header"; then - fail=1 - echo "${file#$1/} doesn't have the right copyright header:" - echo "$header" | sed -e 's/^/ /g' - fi - fi -done - -if [ $fail -ne 0 ]; then - exit 1 -fi diff --git a/sessionrecording/header.go b/sessionrecording/header.go index 4806f6585f976..545bf06bd5984 100644 --- a/sessionrecording/header.go +++ b/sessionrecording/header.go @@ -66,13 +66,15 @@ type CastHeader struct { Kubernetes *Kubernetes `json:"kubernetes,omitempty"` } -// Kubernetes contains 'kubectl exec' session specific information for +// Kubernetes contains 'kubectl exec/attach' session specific information for // tsrecorder. type Kubernetes struct { - // PodName is the name of the Pod being exec-ed. + // PodName is the name of the Pod the session was recorded for. PodName string - // Namespace is the namespace in which is the Pod that is being exec-ed. + // Namespace is the namespace in which the Pod the session was recorded for exists in. Namespace string - // Container is the container being exec-ed. + // Container is the container the session was recorded for. Container string + // SessionType is the type of session that was executed (e.g., exec, attach) + SessionType string } diff --git a/shell.nix b/shell.nix index bb8eacb67ee18..2eb5b441a2d87 100644 --- a/shell.nix +++ b/shell.nix @@ -16,4 +16,4 @@ ) { src = ./.; }).shellNix -# nix-direnv cache busting line: sha256-av4kr09rjNRmag94ziNjJuI/cg8b8lAD3Tk24t/ezH4= +# nix-direnv cache busting line: sha256-4QTSspHLYJfzlontQ7msXyOB5gzq7ZwSvWmKuYY5klA= diff --git a/ssh/tailssh/incubator.go b/ssh/tailssh/incubator.go index 442fedcf22242..dd280143e36e3 100644 --- a/ssh/tailssh/incubator.go +++ b/ssh/tailssh/incubator.go @@ -7,7 +7,7 @@ // and groups to the specified `--uid`, `--gid` and `--groups`, and // then launches the requested `--cmd`. -//go:build linux || (darwin && !ios) || freebsd || openbsd +//go:build (linux && !android) || (darwin && !ios) || freebsd || openbsd package tailssh @@ -51,6 +51,7 @@ const ( darwin = "darwin" freebsd = "freebsd" openbsd = "openbsd" + windows = "windows" ) func init() { @@ -80,20 +81,22 @@ func tryExecInDir(ctx context.Context, dir string) error { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() + run := func(path string) error { + cmd := exec.CommandContext(ctx, path) + cmd.Dir = dir + return cmd.Run() + } + // Assume that the following executables exist, are executable, and // immediately return. - var name string - switch runtime.GOOS { - case "windows": + if runtime.GOOS == windows { windir := os.Getenv("windir") - name = filepath.Join(windir, "system32", "doskey.exe") - default: - name = "/bin/true" + return run(filepath.Join(windir, "system32", "doskey.exe")) } - - cmd := exec.CommandContext(ctx, name) - cmd.Dir = dir - return cmd.Run() + if err := run("/bin/true"); !errors.Is(err, exec.ErrNotFound) { // including nil + return err + } + return run("/usr/bin/true") } // newIncubatorCommand returns a new exec.Cmd configured with @@ -107,7 +110,7 @@ func tryExecInDir(ctx context.Context, dir string) error { // The returned Cmd.Env is guaranteed to be nil; the caller populates it. func (ss *sshSession) newIncubatorCommand(logf logger.Logf) (cmd *exec.Cmd, err error) { defer func() { - if cmd.Env != nil { + if cmd != nil && cmd.Env != nil { panic("internal error") } }() diff --git a/ssh/tailssh/incubator_linux.go b/ssh/tailssh/incubator_linux.go index bcbe0e240a24d..4dfb9f27cc097 100644 --- a/ssh/tailssh/incubator_linux.go +++ b/ssh/tailssh/incubator_linux.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux +//go:build linux && !android package tailssh diff --git a/ssh/tailssh/tailssh.go b/ssh/tailssh/tailssh.go index e42f09bdfb4e4..b249a10639c30 100644 --- a/ssh/tailssh/tailssh.go +++ b/ssh/tailssh/tailssh.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux || (darwin && !ios) || freebsd || openbsd || plan9 +//go:build (linux && !android) || (darwin && !ios) || freebsd || openbsd || plan9 // Package tailssh is an SSH server integrated into Tailscale. package tailssh @@ -281,7 +281,7 @@ func (c *conn) errBanner(message string, err error) error { if err != nil { c.logf("%s: %s", message, err) } - if err := c.spac.SendAuthBanner("tailscale: " + message); err != nil { + if err := c.spac.SendAuthBanner("tailscale: " + message + "\n"); err != nil { c.logf("failed to send auth banner: %s", err) } return errTerminal @@ -324,9 +324,16 @@ func (c *conn) clientAuth(cm gossh.ConnMetadata) (perms *gossh.Permissions, retE return nil, c.errBanner("failed to get connection info", err) } - action, localUser, acceptEnv, err := c.evaluatePolicy() - if err != nil { - return nil, c.errBanner("failed to evaluate SSH policy", err) + action, localUser, acceptEnv, result := c.evaluatePolicy() + switch result { + case accepted: + // do nothing + case rejectedUser: + return nil, c.errBanner(fmt.Sprintf("tailnet policy does not permit you to SSH as user %q", c.info.sshUser), nil) + case rejected, noPolicy: + return nil, c.errBanner("tailnet policy does not permit you to SSH to this node", fmt.Errorf("failed to evaluate policy, result: %s", result)) + default: + return nil, c.errBanner("failed to evaluate tailnet policy", fmt.Errorf("failed to evaluate policy, result: %s", result)) } c.action0 = action @@ -597,18 +604,23 @@ func (c *conn) setInfo(cm gossh.ConnMetadata) error { return nil } +type evalResult string + +const ( + noPolicy evalResult = "no policy" + rejected evalResult = "rejected" + rejectedUser evalResult = "rejected user" + accepted evalResult = "accept" +) + // evaluatePolicy returns the SSHAction and localUser after evaluating // the SSHPolicy for this conn. -func (c *conn) evaluatePolicy() (_ *tailcfg.SSHAction, localUser string, acceptEnv []string, _ error) { +func (c *conn) evaluatePolicy() (_ *tailcfg.SSHAction, localUser string, acceptEnv []string, result evalResult) { pol, ok := c.sshPolicy() if !ok { - return nil, "", nil, fmt.Errorf("tailssh: rejecting connection; no SSH policy") - } - a, localUser, acceptEnv, ok := c.evalSSHPolicy(pol) - if !ok { - return nil, "", nil, fmt.Errorf("tailssh: rejecting connection; no matching policy") + return nil, "", nil, noPolicy } - return a, localUser, acceptEnv, nil + return c.evalSSHPolicy(pol) } // handleSessionPostSSHAuth runs an SSH session after the SSH-level authentication, @@ -706,9 +718,9 @@ func (c *conn) newSSHSession(s ssh.Session) *sshSession { // isStillValid reports whether the conn is still valid. func (c *conn) isStillValid() bool { - a, localUser, _, err := c.evaluatePolicy() - c.vlogf("stillValid: %+v %v %v", a, localUser, err) - if err != nil { + a, localUser, _, result := c.evaluatePolicy() + c.vlogf("stillValid: %+v %v %v", a, localUser, result) + if result != accepted { return false } if !a.Accept && a.HoldAndDelegate == "" { @@ -1089,13 +1101,20 @@ func (c *conn) ruleExpired(r *tailcfg.SSHRule) bool { return r.RuleExpires.Before(c.srv.now()) } -func (c *conn) evalSSHPolicy(pol *tailcfg.SSHPolicy) (a *tailcfg.SSHAction, localUser string, acceptEnv []string, ok bool) { +func (c *conn) evalSSHPolicy(pol *tailcfg.SSHPolicy) (a *tailcfg.SSHAction, localUser string, acceptEnv []string, result evalResult) { + failedOnUser := false for _, r := range pol.Rules { if a, localUser, acceptEnv, err := c.matchRule(r); err == nil { - return a, localUser, acceptEnv, true + return a, localUser, acceptEnv, accepted + } else if errors.Is(err, errUserMatch) { + failedOnUser = true } } - return nil, "", nil, false + result = rejected + if failedOnUser { + result = rejectedUser + } + return nil, "", nil, result } // internal errors for testing; they don't escape to callers or logs. @@ -1129,6 +1148,9 @@ func (c *conn) matchRule(r *tailcfg.SSHRule) (a *tailcfg.SSHAction, localUser st if c.ruleExpired(r) { return nil, "", nil, errRuleExpired } + if !c.anyPrincipalMatches(r.Principals) { + return nil, "", nil, errPrincipalMatch + } if !r.Action.Reject { // For all but Reject rules, SSHUsers is required. // If SSHUsers is nil or empty, mapLocalUser will return an @@ -1138,9 +1160,6 @@ func (c *conn) matchRule(r *tailcfg.SSHRule) (a *tailcfg.SSHAction, localUser st return nil, "", nil, errUserMatch } } - if !c.anyPrincipalMatches(r.Principals) { - return nil, "", nil, errPrincipalMatch - } return r.Action, localUser, r.AcceptEnv, nil } diff --git a/ssh/tailssh/tailssh_test.go b/ssh/tailssh/tailssh_test.go index 79479d7fbf5c7..96fb87f4903c0 100644 --- a/ssh/tailssh/tailssh_test.go +++ b/ssh/tailssh/tailssh_test.go @@ -253,7 +253,7 @@ func TestEvalSSHPolicy(t *testing.T) { name string policy *tailcfg.SSHPolicy ci *sshConnInfo - wantMatch bool + wantResult evalResult wantUser string wantAcceptEnv []string }{ @@ -299,10 +299,20 @@ func TestEvalSSHPolicy(t *testing.T) { ci: &sshConnInfo{sshUser: "alice"}, wantUser: "thealice", wantAcceptEnv: []string{"EXAMPLE", "?_?", "TEST_*"}, - wantMatch: true, + wantResult: accepted, }, { - name: "no-matches-returns-failure", + name: "no-matches-returns-rejected", + policy: &tailcfg.SSHPolicy{ + Rules: []*tailcfg.SSHRule{}, + }, + ci: &sshConnInfo{sshUser: "alice"}, + wantUser: "", + wantAcceptEnv: nil, + wantResult: rejected, + }, + { + name: "no-user-matches-returns-rejected-user", policy: &tailcfg.SSHPolicy{ Rules: []*tailcfg.SSHRule{ { @@ -340,7 +350,7 @@ func TestEvalSSHPolicy(t *testing.T) { ci: &sshConnInfo{sshUser: "alice"}, wantUser: "", wantAcceptEnv: nil, - wantMatch: false, + wantResult: rejectedUser, }, } for _, tt := range tests { @@ -349,14 +359,14 @@ func TestEvalSSHPolicy(t *testing.T) { info: tt.ci, srv: &server{logf: tstest.WhileTestRunningLogger(t)}, } - got, gotUser, gotAcceptEnv, match := c.evalSSHPolicy(tt.policy) - if match != tt.wantMatch { - t.Errorf("match = %v; want %v", match, tt.wantMatch) + got, gotUser, gotAcceptEnv, result := c.evalSSHPolicy(tt.policy) + if result != tt.wantResult { + t.Errorf("result = %v; want %v", result, tt.wantResult) } if gotUser != tt.wantUser { t.Errorf("user = %q; want %q", gotUser, tt.wantUser) } - if tt.wantMatch == true && got == nil { + if tt.wantResult == accepted && got == nil { t.Errorf("expected non-nil action on success") } if !slices.Equal(gotAcceptEnv, tt.wantAcceptEnv) { @@ -467,7 +477,7 @@ func (ts *localState) NodeKey() key.NodePublic { func newSSHRule(action *tailcfg.SSHAction) *tailcfg.SSHRule { return &tailcfg.SSHRule{ SSHUsers: map[string]string{ - "*": currentUser, + "alice": currentUser, }, Action: action, Principals: []*tailcfg.SSHPrincipal{ @@ -789,6 +799,11 @@ func TestSSHAuthFlow(t *testing.T) { Accept: true, Message: "Welcome to Tailscale SSH!", }) + bobRule := newSSHRule(&tailcfg.SSHAction{ + Accept: true, + Message: "Welcome to Tailscale SSH!", + }) + bobRule.SSHUsers = map[string]string{"bob": "bob"} rejectRule := newSSHRule(&tailcfg.SSHAction{ Reject: true, Message: "Go Away!", @@ -808,7 +823,16 @@ func TestSSHAuthFlow(t *testing.T) { sshEnabled: true, }, authErr: true, - wantBanners: []string{"tailscale: failed to evaluate SSH policy"}, + wantBanners: []string{"tailscale: tailnet policy does not permit you to SSH to this node\n"}, + }, + { + name: "user-mismatch", + state: &localState{ + sshEnabled: true, + matchingRule: bobRule, + }, + authErr: true, + wantBanners: []string{`tailscale: tailnet policy does not permit you to SSH as user "alice"` + "\n"}, }, { name: "accept", diff --git a/ssh/tailssh/user.go b/ssh/tailssh/user.go index 097f0d296e92c..ac92c762a875e 100644 --- a/ssh/tailssh/user.go +++ b/ssh/tailssh/user.go @@ -1,7 +1,7 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause -//go:build linux || (darwin && !ios) || freebsd || openbsd || plan9 +//go:build (linux && !android) || (darwin && !ios) || freebsd || openbsd || plan9 package tailssh diff --git a/syncs/syncs.go b/syncs/syncs.go index 337fca7557f34..cf0be919b5b6b 100644 --- a/syncs/syncs.go +++ b/syncs/syncs.go @@ -67,12 +67,18 @@ func (v *AtomicValue[T]) Swap(x T) (old T) { if oldV != nil { return oldV.(wrappedValue[T]).v } - return old + return old // zero value of T } // CompareAndSwap executes the compare-and-swap operation for the Value. +// It panics if T is not comparable. func (v *AtomicValue[T]) CompareAndSwap(oldV, newV T) (swapped bool) { - return v.v.CompareAndSwap(wrappedValue[T]{oldV}, wrappedValue[T]{newV}) + var zero T + return v.v.CompareAndSwap(wrappedValue[T]{oldV}, wrappedValue[T]{newV}) || + // In the edge-case where [atomic.Value.Store] is uninitialized + // and trying to compare with the zero value of T, + // then compare-and-swap with the nil any value. + (any(oldV) == any(zero) && v.v.CompareAndSwap(any(nil), wrappedValue[T]{newV})) } // MutexValue is a value protected by a mutex. diff --git a/syncs/syncs_test.go b/syncs/syncs_test.go index 901d429486d13..2439b6068391b 100644 --- a/syncs/syncs_test.go +++ b/syncs/syncs_test.go @@ -64,6 +64,23 @@ func TestAtomicValue(t *testing.T) { t.Fatalf("LoadOk = (%v, %v), want (nil, true)", got, gotOk) } } + + { + c1, c2, c3 := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var v AtomicValue[chan struct{}] + if v.CompareAndSwap(c1, c2) != false { + t.Fatalf("CompareAndSwap = true, want false") + } + if v.CompareAndSwap(nil, c1) != true { + t.Fatalf("CompareAndSwap = false, want true") + } + if v.CompareAndSwap(c2, c3) != false { + t.Fatalf("CompareAndSwap = true, want false") + } + if v.CompareAndSwap(c1, c2) != true { + t.Fatalf("CompareAndSwap = false, want true") + } + } } func TestMutexValue(t *testing.T) { diff --git a/tailcfg/tailcfg.go b/tailcfg/tailcfg.go index 0a58d8f0cc229..5e3c4e5720a92 100644 --- a/tailcfg/tailcfg.go +++ b/tailcfg/tailcfg.go @@ -161,7 +161,14 @@ type CapabilityVersion int // - 114: 2025-01-30: NodeAttrMaxKeyDuration CapMap defined, clients might use it (no tailscaled code change) (#14829) // - 115: 2025-03-07: Client understands DERPRegion.NoMeasureNoHome. // - 116: 2025-05-05: Client serves MagicDNS "AAAA" if NodeAttrMagicDNSPeerAAAA set on self node -const CurrentCapabilityVersion CapabilityVersion = 116 +// - 117: 2025-05-28: Client understands DisplayMessages (structured health messages), but not necessarily PrimaryAction. +// - 118: 2025-07-01: Client sends Hostinfo.StateEncrypted to report whether the state file is encrypted at rest (#15830) +// - 119: 2025-07-10: Client uses Hostinfo.Location.Priority to prioritize one route over another. +// - 120: 2025-07-15: Client understands peer relay disco messages, and implements peer client and relay server functions +// - 121: 2025-07-19: Client understands peer relay endpoint alloc with [disco.AllocateUDPRelayEndpointRequest] & [disco.AllocateUDPRelayEndpointResponse] +// - 122: 2025-07-21: Client sends Hostinfo.ExitNodeID to report which exit node it has selected, if any. +// - 123: 2025-07-28: fix deadlock regression from cryptokey routing change (issue #16651) +const CurrentCapabilityVersion CapabilityVersion = 123 // ID is an integer ID for a user, node, or login allocated by the // control plane. @@ -870,6 +877,7 @@ type Hostinfo struct { UserspaceRouter opt.Bool `json:",omitempty"` // if the client's subnet router is running in userspace (netstack) mode AppConnector opt.Bool `json:",omitempty"` // if the client is running the app-connector service ServicesHash string `json:",omitempty"` // opaque hash of the most recent list of tailnet services, change in hash indicates config should be fetched via c2n + ExitNodeID StableNodeID `json:",omitzero"` // the client’s selected exit node, empty when unselected. // Location represents geographical location data about a // Tailscale host. Location is optional and only set if @@ -877,6 +885,12 @@ type Hostinfo struct { Location *Location `json:",omitempty"` TPM *TPMInfo `json:",omitempty"` // TPM device metadata, if available + // StateEncrypted reports whether the node state is stored encrypted on + // disk. The actual mechanism is platform-specific: + // * Apple nodes use the Keychain + // * Linux and Windows nodes use the TPM + // * Android apps use EncryptedSharedPreferences + StateEncrypted opt.Bool `json:",omitempty"` // NOTE: any new fields containing pointers in this type // require changes to Hostinfo.Equal. @@ -907,6 +921,9 @@ type TPMInfo struct { SpecRevision int `json:",omitempty"` } +// Present reports whether a TPM device is present on this machine. +func (t *TPMInfo) Present() bool { return t != nil } + // ServiceName is the name of a service, of the form `svc:dns-label`. Services // represent some kind of application provided for users of the tailnet with a // MagicDNS name and possibly dedicated IP addresses. Currently (2024-01-21), @@ -914,6 +931,16 @@ type TPMInfo struct { // This is not related to the older [Service] used in [Hostinfo.Services]. type ServiceName string +// AsServiceName reports whether the given string is a valid service name. +// If so returns the name as a [tailcfg.ServiceName], otherwise returns "". +func AsServiceName(s string) ServiceName { + svcName := ServiceName(s) + if err := svcName.Validate(); err != nil { + return "" + } + return svcName +} + // Validate validates if the service name is formatted correctly. // We only allow valid DNS labels, since the expectation is that these will be // used as parts of domain names. All errors are [vizerror.Error]. @@ -1843,10 +1870,14 @@ type PingResponse struct { // omitted, Err should contain information as to the cause. LatencySeconds float64 `json:",omitempty"` - // Endpoint is the ip:port if direct UDP was used. - // It is not currently set for TSMP pings. + // Endpoint is a string of the form "{ip}:{port}" if direct UDP was used. It + // is not currently set for TSMP. Endpoint string `json:",omitempty"` + // PeerRelay is a string of the form "{ip}:{port}:vni:{vni}" if a peer + // relay was used. It is not currently set for TSMP. + PeerRelay string `json:",omitempty"` + // DERPRegionID is non-zero DERP region ID if DERP was used. // It is not currently set for TSMP pings. DERPRegionID int `json:",omitempty"` @@ -2028,13 +2059,31 @@ type MapResponse struct { // plane's perspective. A nil value means no change from the previous // MapResponse. A non-nil 0-length slice restores the health to good (no // known problems). A non-zero length slice are the list of problems that - // the control place sees. + // the control plane sees. + // + // Either this will be set, or DisplayMessages will be set, but not both. // // Note that this package's type, due its use of a slice and omitempty, is // unable to marshal a zero-length non-nil slice. The control server needs // to marshal this type using a separate type. See MapResponse docs. Health []string `json:",omitempty"` + // DisplayMessages sets the health state of the node from the control + // plane's perspective. + // + // Either this will be set, or Health will be set, but not both. + // + // The map keys are IDs that uniquely identify the type of health issue. The + // map values are the messages. If the server sends down a map with entries, + // the client treats it as a patch: new entries are added, keys with a value + // of nil are deleted, existing entries with new values are updated. A nil + // map and an empty map both mean no change has occurred since the last + // update. + // + // As a special case, the map key "*" with a value of nil means to clear all + // prior display messages before processing the other map entries. + DisplayMessages map[DisplayMessageID]*DisplayMessage `json:",omitempty"` + // SSHPolicy, if non-nil, updates the SSH policy for how incoming // SSH connections should be handled. SSHPolicy *SSHPolicy `json:",omitempty"` @@ -2078,6 +2127,88 @@ type MapResponse struct { DefaultAutoUpdate opt.Bool `json:",omitempty"` } +// DisplayMessage represents a health state of the node from the control plane's +// perspective. It is deliberately similar to [health.Warnable] as both get +// converted into [health.UnhealthyState] to be sent to the GUI. +type DisplayMessage struct { + // Title is a string that the GUI uses as title for this message. The title + // should be short and fit in a single line. It should not end in a period. + // + // Example: "Network may be blocking Tailscale". + // + // See the various instantiations of [health.Warnable] for more examples. + Title string + + // Text is an extended string that the GUI will display to the user. This + // could be multiple sentences explaining the issue in more detail. + // + // Example: "macOS Screen Time seems to be blocking Tailscale. Try disabling + // Screen Time in System Settings > Screen Time > Content & Privacy > Access + // to Web Content." + // + // See the various instantiations of [health.Warnable] for more examples. + Text string + + // Severity is the severity of the DisplayMessage, which the GUI can use to + // determine how to display it. Maps to [health.Severity]. + Severity DisplayMessageSeverity + + // ImpactsConnectivity is whether the health problem will impact the user's + // ability to connect to the Internet or other nodes on the tailnet, which + // the GUI can use to determine how to display it. + ImpactsConnectivity bool `json:",omitempty"` + + // Primary action, if present, represents the action to allow the user to + // take when interacting with this message. For example, if the + // DisplayMessage is shown via a notification, the action label might be a + // button on that notification and clicking the button would open the URL. + PrimaryAction *DisplayMessageAction `json:",omitempty"` +} + +// DisplayMessageAction represents an action (URL and link) to be presented to +// the user associated with a [DisplayMessage]. +type DisplayMessageAction struct { + // URL is the URL to navigate to when the user interacts with this action + URL string + + // Label is the call to action for the UI to display on the UI element that + // will open the URL (such as a button or link). For example, "Sign in" or + // "Learn more". + Label string +} + +// DisplayMessageID is a string that uniquely identifies the kind of health +// issue (e.g. "session-expired"). +type DisplayMessageID string + +// Equal returns true iff all fields are equal. +func (m DisplayMessage) Equal(o DisplayMessage) bool { + return m.Title == o.Title && + m.Text == o.Text && + m.Severity == o.Severity && + m.ImpactsConnectivity == o.ImpactsConnectivity && + (m.PrimaryAction == nil) == (o.PrimaryAction == nil) && + (m.PrimaryAction == nil || (m.PrimaryAction.URL == o.PrimaryAction.URL && + m.PrimaryAction.Label == o.PrimaryAction.Label)) +} + +// DisplayMessageSeverity represents how serious a [DisplayMessage] is. Analogous +// to health.Severity. +type DisplayMessageSeverity string + +const ( + // SeverityHigh is the highest severity level, used for critical errors that need immediate attention. + // On platforms where the client GUI can deliver notifications, a SeverityHigh message will trigger + // a modal notification. + SeverityHigh DisplayMessageSeverity = "high" + // SeverityMedium is used for errors that are important but not critical. This won't trigger a modal + // notification, however it will be displayed in a more visible way than a SeverityLow message. + SeverityMedium DisplayMessageSeverity = "medium" + // SeverityLow is used for less important notices that don't need immediate attention. The user will + // have to go to a Settings window, or another "hidden" GUI location to see these messages. + SeverityLow DisplayMessageSeverity = "low" +) + // ClientVersion is information about the latest client version that's available // for the client (and whether they're already running it). // @@ -2164,10 +2295,10 @@ type Debug struct { Exit *int `json:",omitempty"` } -func (id ID) String() string { return fmt.Sprintf("id:%x", int64(id)) } -func (id UserID) String() string { return fmt.Sprintf("userid:%x", int64(id)) } -func (id LoginID) String() string { return fmt.Sprintf("loginid:%x", int64(id)) } -func (id NodeID) String() string { return fmt.Sprintf("nodeid:%x", int64(id)) } +func (id ID) String() string { return fmt.Sprintf("id:%d", int64(id)) } +func (id UserID) String() string { return fmt.Sprintf("userid:%d", int64(id)) } +func (id LoginID) String() string { return fmt.Sprintf("loginid:%d", int64(id)) } +func (id NodeID) String() string { return fmt.Sprintf("nodeid:%d", int64(id)) } // Equal reports whether n and n2 are equal. func (n *Node) Equal(n2 *Node) bool { @@ -2266,6 +2397,7 @@ type NodeCapability string const ( CapabilityFileSharing NodeCapability = "https://tailscale.com/cap/file-sharing" CapabilityAdmin NodeCapability = "https://tailscale.com/cap/is-admin" + CapabilityOwner NodeCapability = "https://tailscale.com/cap/is-owner" CapabilitySSH NodeCapability = "https://tailscale.com/cap/ssh" // feature enabled/available CapabilitySSHRuleIn NodeCapability = "https://tailscale.com/cap/ssh-rule-in" // some SSH rule reach this node CapabilityDataPlaneAuditLogs NodeCapability = "https://tailscale.com/cap/data-plane-audit-logs" // feature enabled @@ -2493,17 +2625,30 @@ const ( // peer node list. NodeAttrNativeIPV4 NodeCapability = "native-ipv4" - // NodeAttrRelayServer permits the node to act as an underlay UDP relay - // server. There are no expected values for this key in NodeCapMap. - NodeAttrRelayServer NodeCapability = "relay:server" - - // NodeAttrRelayClient permits the node to act as an underlay UDP relay - // client. There are no expected values for this key in NodeCapMap. - NodeAttrRelayClient NodeCapability = "relay:client" + // NodeAttrDisableRelayServer prevents the node from acting as an underlay + // UDP relay server. There are no expected values for this key; the key + // only needs to be present in [NodeCapMap] to take effect. + NodeAttrDisableRelayServer NodeCapability = "disable-relay-server" + + // NodeAttrDisableRelayClient prevents the node from both allocating UDP + // relay server endpoints itself, and from using endpoints allocated by + // its peers. This attribute can be added to the node dynamically; if added + // while the node is already running, the node will be unable to allocate + // endpoints after it next updates its network map, and will be immediately + // unable to use new paths via a UDP relay server. Setting this attribute + // dynamically does not remove any existing paths, including paths that + // traverse a UDP relay server. There are no expected values for this key + // in [NodeCapMap]; the key only needs to be present in [NodeCapMap] to + // take effect. + NodeAttrDisableRelayClient NodeCapability = "disable-relay-client" // NodeAttrMagicDNSPeerAAAA is a capability that tells the node's MagicDNS // server to answer AAAA queries about its peers. See tailscale/tailscale#1152. NodeAttrMagicDNSPeerAAAA NodeCapability = "magicdns-aaaa" + + // NodeAttrTrafficSteering configures the node to use the traffic + // steering subsystem for via routes. See tailscale/corp#29966. + NodeAttrTrafficSteering NodeCapability = "traffic-steering" ) // SetDNSRequest is a request to add a DNS record. diff --git a/tailcfg/tailcfg_clone.go b/tailcfg/tailcfg_clone.go index 2c7941d51d7e3..95f8905b84e69 100644 --- a/tailcfg/tailcfg_clone.go +++ b/tailcfg/tailcfg_clone.go @@ -186,8 +186,10 @@ var _HostinfoCloneNeedsRegeneration = Hostinfo(struct { UserspaceRouter opt.Bool AppConnector opt.Bool ServicesHash string + ExitNodeID StableNodeID Location *Location TPM *TPMInfo + StateEncrypted opt.Bool }{}) // Clone makes a deep copy of NetInfo. diff --git a/tailcfg/tailcfg_test.go b/tailcfg/tailcfg_test.go index 079162a150191..addd2330ba239 100644 --- a/tailcfg/tailcfg_test.go +++ b/tailcfg/tailcfg_test.go @@ -67,8 +67,10 @@ func TestHostinfoEqual(t *testing.T) { "UserspaceRouter", "AppConnector", "ServicesHash", + "ExitNodeID", "Location", "TPM", + "StateEncrypted", } if have := fieldsOf(reflect.TypeFor[Hostinfo]()); !reflect.DeepEqual(have, hiHandles) { t.Errorf("Hostinfo.Equal check might be out of sync\nfields: %q\nhandled: %q\n", @@ -272,6 +274,21 @@ func TestHostinfoEqual(t *testing.T) { &Hostinfo{IngressEnabled: true}, false, }, + { + &Hostinfo{ExitNodeID: "stable-exit"}, + &Hostinfo{ExitNodeID: "stable-exit"}, + true, + }, + { + &Hostinfo{ExitNodeID: ""}, + &Hostinfo{}, + true, + }, + { + &Hostinfo{ExitNodeID: ""}, + &Hostinfo{ExitNodeID: "stable-exit"}, + false, + }, } for i, tt := range tests { got := tt.a.Equal(tt.b) @@ -878,3 +895,132 @@ func TestCheckTag(t *testing.T) { }) } } + +func TestDisplayMessageEqual(t *testing.T) { + type test struct { + name string + value1 DisplayMessage + value2 DisplayMessage + wantEqual bool + } + + for _, test := range []test{ + { + name: "same", + value1: DisplayMessage{ + Title: "title", + Text: "text", + Severity: SeverityHigh, + ImpactsConnectivity: false, + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Open", + }, + }, + value2: DisplayMessage{ + Title: "title", + Text: "text", + Severity: SeverityHigh, + ImpactsConnectivity: false, + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Open", + }, + }, + wantEqual: true, + }, + { + name: "different-title", + value1: DisplayMessage{ + Title: "title", + }, + value2: DisplayMessage{ + Title: "different title", + }, + wantEqual: false, + }, + { + name: "different-text", + value1: DisplayMessage{ + Text: "some text", + }, + value2: DisplayMessage{ + Text: "different text", + }, + wantEqual: false, + }, + { + name: "different-severity", + value1: DisplayMessage{ + Severity: SeverityHigh, + }, + value2: DisplayMessage{ + Severity: SeverityMedium, + }, + wantEqual: false, + }, + { + name: "different-impactsConnectivity", + value1: DisplayMessage{ + ImpactsConnectivity: true, + }, + value2: DisplayMessage{ + ImpactsConnectivity: false, + }, + wantEqual: false, + }, + { + name: "different-primaryAction-nil-non-nil", + value1: DisplayMessage{}, + value2: DisplayMessage{ + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Open", + }, + }, + wantEqual: false, + }, + { + name: "different-primaryAction-url", + value1: DisplayMessage{ + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Open", + }, + }, + value2: DisplayMessage{ + PrimaryAction: &DisplayMessageAction{ + URL: "https://zombo.com", + Label: "Open", + }, + }, + wantEqual: false, + }, + { + name: "different-primaryAction-label", + value1: DisplayMessage{ + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Open", + }, + }, + value2: DisplayMessage{ + PrimaryAction: &DisplayMessageAction{ + URL: "https://example.com", + Label: "Learn more", + }, + }, + wantEqual: false, + }, + } { + t.Run(test.name, func(t *testing.T) { + got := test.value1.Equal(test.value2) + + if got != test.wantEqual { + value1 := must.Get(json.MarshalIndent(test.value1, "", " ")) + value2 := must.Get(json.MarshalIndent(test.value2, "", " ")) + t.Errorf("value1.Equal(value2): got %t, want %t\nvalue1:\n%s\nvalue2:\n%s", got, test.wantEqual, value1, value2) + } + }) + } +} diff --git a/tailcfg/tailcfg_view.go b/tailcfg/tailcfg_view.go index c76654887f8ab..c407800210a5e 100644 --- a/tailcfg/tailcfg_view.go +++ b/tailcfg/tailcfg_view.go @@ -300,9 +300,11 @@ func (v HostinfoView) Userspace() opt.Bool { return v.ж.User func (v HostinfoView) UserspaceRouter() opt.Bool { return v.ж.UserspaceRouter } func (v HostinfoView) AppConnector() opt.Bool { return v.ж.AppConnector } func (v HostinfoView) ServicesHash() string { return v.ж.ServicesHash } +func (v HostinfoView) ExitNodeID() StableNodeID { return v.ж.ExitNodeID } func (v HostinfoView) Location() LocationView { return v.ж.Location.View() } func (v HostinfoView) TPM() views.ValuePointer[TPMInfo] { return views.ValuePointerOf(v.ж.TPM) } +func (v HostinfoView) StateEncrypted() opt.Bool { return v.ж.StateEncrypted } func (v HostinfoView) Equal(v2 HostinfoView) bool { return v.ж.Equal(v2.ж) } // A compilation failure here means this code must be regenerated, with the command at the top of this file. @@ -344,8 +346,10 @@ var _HostinfoViewNeedsRegeneration = Hostinfo(struct { UserspaceRouter opt.Bool AppConnector opt.Bool ServicesHash string + ExitNodeID StableNodeID Location *Location TPM *TPMInfo + StateEncrypted opt.Bool }{}) // View returns a read-only view of NetInfo. diff --git a/tka/builder_test.go b/tka/builder_test.go index 666af9ad07daf..3dbd4347abf06 100644 --- a/tka/builder_test.go +++ b/tka/builder_test.go @@ -5,6 +5,7 @@ package tka import ( "crypto/ed25519" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -90,6 +91,20 @@ func TestAuthorityBuilderRemoveKey(t *testing.T) { if _, err := a.state.GetKey(key2.MustID()); err != ErrNoSuchKey { t.Errorf("GetKey(key2).err = %v, want %v", err, ErrNoSuchKey) } + + // Check that removing the remaining key errors out. + b = a.NewUpdater(signer25519(priv)) + if err := b.RemoveKey(key.MustID()); err != nil { + t.Fatalf("RemoveKey(%v) failed: %v", key, err) + } + updates, err = b.Finalize(storage) + if err != nil { + t.Fatalf("Finalize() failed: %v", err) + } + wantErr := "cannot remove the last key" + if err := a.Inform(storage, updates); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("expected Inform() to return error %q, got: %v", wantErr, err) + } } func TestAuthorityBuilderSetKeyVote(t *testing.T) { diff --git a/tka/tka.go b/tka/tka.go index 04b712660d270..ade621bc689e3 100644 --- a/tka/tka.go +++ b/tka/tka.go @@ -440,6 +440,13 @@ func aumVerify(aum AUM, state State, isGenesisAUM bool) error { return fmt.Errorf("signature %d: %v", i, err) } } + + if aum.MessageKind == AUMRemoveKey && len(state.Keys) == 1 { + if kid, err := state.Keys[0].ID(); err == nil && bytes.Equal(aum.KeyID, kid) { + return errors.New("cannot remove the last key in the state") + } + } + return nil } diff --git a/tool/go-win.ps1 b/tool/go-win.ps1 new file mode 100644 index 0000000000000..49313ffbabee9 --- /dev/null +++ b/tool/go-win.ps1 @@ -0,0 +1,64 @@ +<# + go.ps1 – Tailscale Go toolchain fetching wrapper for Windows/PowerShell + • Reads go.toolchain.rev one dir above this script + • If the requested commit hash isn't cached, downloads and unpacks + https://github.com/tailscale/go/releases/download/build-${REV}/${OS}-${ARCH}.tar.gz + • Finally execs the toolchain's "go" binary, forwarding all args & exit-code +#> + +param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]] $Args +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if ($env:CI -eq 'true' -and $env:NODEBUG -ne 'true') { + $VerbosePreference = 'Continue' +} + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..') +$REV = (Get-Content (Join-Path $repoRoot 'go.toolchain.rev') -Raw).Trim() + +if ([IO.Path]::IsPathRooted($REV)) { + $toolchain = $REV +} else { + if (-not [string]::IsNullOrWhiteSpace($env:TSGO_CACHE_ROOT)) { + $cacheRoot = $env:TSGO_CACHE_ROOT + } else { + $cacheRoot = Join-Path $env:USERPROFILE '.cache\tsgo' + } + + $toolchain = Join-Path $cacheRoot $REV + $marker = "$toolchain.extracted" + + if (-not (Test-Path $marker)) { + Write-Host "# Downloading Go toolchain $REV" -ForegroundColor Cyan + if (Test-Path $toolchain) { Remove-Item -Recurse -Force $toolchain } + + # Removing the marker file again (even though it shouldn't still exist) + # because the equivalent Bash script also does so (to guard against + # concurrent cache fills?). + # TODO(bradfitz): remove this and add some proper locking instead? + if (Test-Path $marker ) { Remove-Item -Force $marker } + + New-Item -ItemType Directory -Path $cacheRoot -Force | Out-Null + + $url = "https://github.com/tailscale/go/releases/download/build-$REV/windows-amd64.tar.gz" + $tgz = "$toolchain.tar.gz" + Invoke-WebRequest -Uri $url -OutFile $tgz -UseBasicParsing -ErrorAction Stop + + New-Item -ItemType Directory -Path $toolchain -Force | Out-Null + tar --strip-components=1 -xzf $tgz -C $toolchain + Remove-Item $tgz + Set-Content -Path $marker -Value $REV + } +} + +$goExe = Join-Path $toolchain 'bin\go.exe' +if (-not (Test-Path $goExe)) { throw "go executable not found at $goExe" } + +& $goExe @Args +exit $LASTEXITCODE + diff --git a/tool/go.cmd b/tool/go.cmd new file mode 100644 index 0000000000000..04172a28d5b25 --- /dev/null +++ b/tool/go.cmd @@ -0,0 +1,2 @@ +@echo off +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0go-win.ps1" %* diff --git a/tool/gocross/gocross-wrapper.sh b/tool/gocross/gocross-wrapper.sh index 366011fefdd6b..90485d31b95af 100755 --- a/tool/gocross/gocross-wrapper.sh +++ b/tool/gocross/gocross-wrapper.sh @@ -3,8 +3,11 @@ # SPDX-License-Identifier: BSD-3-Clause # # gocross-wrapper.sh is a wrapper that can be aliased to 'go', which -# transparently builds gocross using a "bootstrap" Go toolchain, and -# then invokes gocross. +# transparently runs the version of github.com/tailscale/go as specified repo's +# go.toolchain.rev file. +# +# It also conditionally (if TS_USE_GOCROSS=1) builds gocross and uses it as a go +# wrapper to inject certain go flags. set -euo pipefail @@ -71,11 +74,20 @@ case "$REV" in echo "# Cleaning up old Go toolchain $hash" >&2 rm -rf "$HOME/.cache/tsgo/$hash" rm -rf "$HOME/.cache/tsgo/$hash.extracted" + rm -rf "$HOME/.cache/tsgoroot/$hash" done fi ;; esac +# gocross is opt-in as of 2025-06-16. See tailscale/corp#26717. +# It's primarily used for xcode builds, and a bit still for Windows. +# In the past we needed it for git version stamping on Linux etc, but +# Go does that itself nowadays. +if [ "${TS_USE_GOCROSS:-}" != "1" ]; then + exit 0 # out of subshell +fi + if [[ -d "$toolchain" ]]; then # A toolchain exists, but is it recent enough to compile gocross? If not, # wipe it out so that the next if block fetches a usable one. @@ -119,4 +131,29 @@ if [[ "$gocross_ok" == "0" ]]; then fi ) # End of the subshell execution. -exec "${BASH_SOURCE%/*}/../../gocross" "$@" +repo_root="${BASH_SOURCE%/*}/../.." + +# Some scripts/package systems set GOROOT even though they should only be +# setting $PATH. Stop them from breaking builds - go(1) respects GOROOT and +# so if it is left on here, compilation units depending on our Go fork will +# fail (such as those which depend on our net/ patches). +unset GOROOT + +# gocross is opt-in as of 2025-06-16. See tailscale/corp#26717 +# and comment above in this file. +if [ "${TS_USE_GOCROSS:-}" != "1" ]; then + read -r REV <"${repo_root}/go.toolchain.rev" + case "$REV" in + /*) + toolchain="$REV" + ;; + *) + # If the prior subshell completed successfully, this toolchain location + # should be valid at this point. + toolchain="$HOME/.cache/tsgo/$REV" + ;; + esac + exec "$toolchain/bin/go" "$@" +fi + +exec "${repo_root}/gocross" "$@" diff --git a/tool/gocross/gocross_wrapper_test.go b/tool/gocross/gocross_wrapper_test.go index 2b0f016a29d57..f4dcec4292695 100644 --- a/tool/gocross/gocross_wrapper_test.go +++ b/tool/gocross/gocross_wrapper_test.go @@ -15,7 +15,7 @@ import ( func TestGocrossWrapper(t *testing.T) { for i := range 2 { // once to build gocross; second to test it's cached cmd := exec.Command("./gocross-wrapper.sh", "version") - cmd.Env = append(os.Environ(), "CI=true", "NOBASHDEBUG=false") // for "set -x" verbosity + cmd.Env = append(os.Environ(), "CI=true", "NOBASHDEBUG=false", "TS_USE_GOCROSS=1") // for "set -x" verbosity out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("gocross-wrapper.sh failed: %v\n%s", err, out) diff --git a/tool/gocross/toolchain.go b/tool/gocross/toolchain.go index e701662f5b1e8..f422e289e3571 100644 --- a/tool/gocross/toolchain.go +++ b/tool/gocross/toolchain.go @@ -62,7 +62,7 @@ func getToolchain() (toolchainDir, gorootDir string, err error) { cache := filepath.Join(os.Getenv("HOME"), ".cache") toolchainDir = filepath.Join(cache, "tsgo", rev) - gorootDir = filepath.Join(toolchainDir, "gocross-goroot") + gorootDir = filepath.Join(cache, "tsgoroot", rev) // You might wonder why getting the toolchain also provisions and returns a // path suitable for use as GOROOT. Wonder no longer! diff --git a/tool/listpkgs/listpkgs.go b/tool/listpkgs/listpkgs.go new file mode 100644 index 0000000000000..400bf90c18315 --- /dev/null +++ b/tool/listpkgs/listpkgs.go @@ -0,0 +1,206 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// listpkgs prints the import paths that match the Go package patterns +// given on the command line and conditionally filters them in various ways. +package main + +import ( + "bufio" + "flag" + "fmt" + "go/build/constraint" + "log" + "os" + "slices" + "strings" + "sync" + + "golang.org/x/tools/go/packages" +) + +var ( + ignore3p = flag.Bool("ignore-3p", false, "ignore third-party packages forked/vendored into Tailscale") + goos = flag.String("goos", "", "GOOS to use for loading packages (default: current OS)") + goarch = flag.String("goarch", "", "GOARCH to use for loading packages (default: current architecture)") + withTagsAllStr = flag.String("with-tags-all", "", "if non-empty, a comma-separated list of builds tags to require (a package will only be listed if it contains all of these build tags)") + withoutTagsAnyStr = flag.String("without-tags-any", "", "if non-empty, a comma-separated list of build constraints to exclude (a package will be omitted if it contains any of these build tags)") + shard = flag.String("shard", "", "if non-empty, a string of the form 'N/M' to only print packages in shard N of M (e.g. '1/3', '2/3', '3/3/' for different thirds of the list)") +) + +func main() { + flag.Parse() + + patterns := flag.Args() + if len(patterns) == 0 { + flag.Usage() + os.Exit(1) + } + + cfg := &packages.Config{ + Mode: packages.LoadFiles, + Env: os.Environ(), + } + if *goos != "" { + cfg.Env = append(cfg.Env, "GOOS="+*goos) + } + if *goarch != "" { + cfg.Env = append(cfg.Env, "GOARCH="+*goarch) + } + + pkgs, err := packages.Load(cfg, patterns...) + if err != nil { + log.Fatalf("loading packages: %v", err) + } + + var withoutAny []string + if *withoutTagsAnyStr != "" { + withoutAny = strings.Split(*withoutTagsAnyStr, ",") + } + var withAll []string + if *withTagsAllStr != "" { + withAll = strings.Split(*withTagsAllStr, ",") + } + + seen := map[string]bool{} + matches := 0 +Pkg: + for _, pkg := range pkgs { + if pkg.PkgPath == "" { // malformed (shouldn’t happen) + continue + } + if seen[pkg.PkgPath] { + continue // suppress duplicates when patterns overlap + } + seen[pkg.PkgPath] = true + + pkgPath := pkg.PkgPath + + if *ignore3p && isThirdParty(pkgPath) { + continue + } + if withAll != nil { + for _, t := range withAll { + if !hasBuildTag(pkg, t) { + continue Pkg + } + } + } + for _, t := range withoutAny { + if hasBuildTag(pkg, t) { + continue Pkg + } + } + matches++ + + if *shard != "" { + var n, m int + if _, err := fmt.Sscanf(*shard, "%d/%d", &n, &m); err != nil || n < 1 || m < 1 { + log.Fatalf("invalid shard format %q; expected 'N/M'", *shard) + } + if m > 0 && (matches-1)%m != n-1 { + continue // not in this shard + } + } + fmt.Println(pkgPath) + } + + // If any package had errors (e.g. missing deps) report them via packages.PrintErrors. + // This mirrors `go list` behaviour when -e is *not* supplied. + if packages.PrintErrors(pkgs) > 0 { + os.Exit(1) + } +} + +func isThirdParty(pkg string) bool { + return strings.HasPrefix(pkg, "tailscale.com/tempfork/") +} + +// hasBuildTag reports whether any source file in pkg mentions `tag` +// in a //go:build constraint. +func hasBuildTag(pkg *packages.Package, tag string) bool { + all := slices.Concat(pkg.CompiledGoFiles, pkg.OtherFiles, pkg.IgnoredFiles) + suffix := "_" + tag + ".go" + for _, name := range all { + if strings.HasSuffix(name, suffix) { + return true + } + ok, err := fileMentionsTag(name, tag) + if err != nil { + log.Printf("reading %s: %v", name, err) + continue + } + if ok { + return true + } + } + return false +} + +// tagSet is a set of build tags. +// The values are always true. We avoid non-std set types +// to make this faster to "go run" on empty caches. +type tagSet map[string]bool + +var ( + mu sync.Mutex + fileTags = map[string]tagSet{} // abs path -> set of build tags mentioned in file +) + +func getFileTags(filename string) (tagSet, error) { + mu.Lock() + tags, ok := fileTags[filename] + mu.Unlock() + if ok { + return tags, nil + } + + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + ts := make(tagSet) + s := bufio.NewScanner(f) + for s.Scan() { + line := s.Text() + if strings.TrimSpace(line) == "" { + continue // still in leading blank lines + } + if !strings.HasPrefix(line, "//") { + // hit real code – done with header comments + // TODO(bradfitz): care about /* */ comments? + break + } + if !strings.HasPrefix(line, "//go:build") { + continue // some other comment + } + expr, err := constraint.Parse(line) + if err != nil { + return nil, fmt.Errorf("parsing %q: %w", line, err) + } + // Call Eval to populate ts with the tags mentioned in the expression. + // We don't care about the result, just the side effect of populating ts. + expr.Eval(func(tag string) bool { + ts[tag] = true + return true // arbitrary + }) + } + if err := s.Err(); err != nil { + return nil, fmt.Errorf("reading %s: %w", filename, err) + } + + mu.Lock() + defer mu.Unlock() + fileTags[filename] = ts + return tags, nil +} + +func fileMentionsTag(filename, tag string) (bool, error) { + tags, err := getFileTags(filename) + if err != nil { + return false, err + } + return tags[tag], nil +} diff --git a/tsconsensus/authorization.go b/tsconsensus/authorization.go index 1e0b70c0759d3..bd8e2f39a014b 100644 --- a/tsconsensus/authorization.go +++ b/tsconsensus/authorization.go @@ -87,29 +87,29 @@ func (a *authorization) Refresh(ctx context.Context) error { } func (a *authorization) AllowsHost(addr netip.Addr) bool { + a.mu.Lock() + defer a.mu.Unlock() if a.peers == nil { return false } - a.mu.Lock() - defer a.mu.Unlock() return a.peers.addrs.Contains(addr) } func (a *authorization) SelfAllowed() bool { + a.mu.Lock() + defer a.mu.Unlock() if a.peers == nil { return false } - a.mu.Lock() - defer a.mu.Unlock() return a.peers.status.Self.Tags != nil && views.SliceContains(*a.peers.status.Self.Tags, a.tag) } func (a *authorization) AllowedPeers() views.Slice[*ipnstate.PeerStatus] { + a.mu.Lock() + defer a.mu.Unlock() if a.peers == nil { return views.Slice[*ipnstate.PeerStatus]{} } - a.mu.Lock() - defer a.mu.Unlock() return views.SliceOf(a.peers.statuses) } diff --git a/tsconsensus/bolt_store.go b/tsconsensus/bolt_store.go new file mode 100644 index 0000000000000..ca347cfc049b2 --- /dev/null +++ b/tsconsensus/bolt_store.go @@ -0,0 +1,19 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !loong64 + +package tsconsensus + +import ( + "github.com/hashicorp/raft" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" +) + +func boltStore(path string) (raft.StableStore, raft.LogStore, error) { + store, err := raftboltdb.NewBoltStore(path) + if err != nil { + return nil, nil, err + } + return store, store, nil +} diff --git a/tsconsensus/bolt_store_no_bolt.go b/tsconsensus/bolt_store_no_bolt.go new file mode 100644 index 0000000000000..33b3bd6c7a29f --- /dev/null +++ b/tsconsensus/bolt_store_no_bolt.go @@ -0,0 +1,18 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +//go:build loong64 + +package tsconsensus + +import ( + "errors" + + "github.com/hashicorp/raft" +) + +func boltStore(path string) (raft.StableStore, raft.LogStore, error) { + // "github.com/hashicorp/raft-boltdb/v2" doesn't build on loong64 + // see https://github.com/hashicorp/raft-boltdb/issues/27 + return nil, nil, errors.New("not implemented") +} diff --git a/tsconsensus/tsconsensus.go b/tsconsensus/tsconsensus.go index 74094782f4383..b6bf373102aa6 100644 --- a/tsconsensus/tsconsensus.go +++ b/tsconsensus/tsconsensus.go @@ -32,6 +32,7 @@ import ( "net" "net/http" "net/netip" + "path/filepath" "time" "github.com/hashicorp/go-hclog" @@ -71,6 +72,7 @@ type Config struct { MaxConnPool int ConnTimeout time.Duration ServeDebugMonitor bool + StateDirPath string } // DefaultConfig returns a Config populated with default values ready for use. @@ -223,10 +225,31 @@ func Start(ctx context.Context, ts *tsnet.Server, fsm raft.FSM, clusterTag strin func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, self selfRaftNode, auth *authorization, cfg Config) (*raft.Raft, error) { cfg.Raft.LocalID = raft.ServerID(self.id) - // no persistence (for now?) - logStore := raft.NewInmemStore() - stableStore := raft.NewInmemStore() - snapshots := raft.NewInmemSnapshotStore() + var logStore raft.LogStore + var stableStore raft.StableStore + var snapStore raft.SnapshotStore + + if cfg.StateDirPath == "" { + // comments in raft code say to only use for tests + logStore = raft.NewInmemStore() + stableStore = raft.NewInmemStore() + snapStore = raft.NewInmemSnapshotStore() + } else { + var err error + stableStore, logStore, err = boltStore(filepath.Join(cfg.StateDirPath, "store")) + if err != nil { + return nil, err + } + snaplogger := hclog.New(&hclog.LoggerOptions{ + Name: "raft-snap", + Output: cfg.Raft.LogOutput, + Level: hclog.LevelFromString(cfg.Raft.LogLevel), + }) + snapStore, err = raft.NewFileSnapshotStoreWithLogger(filepath.Join(cfg.StateDirPath, "snapstore"), 2, snaplogger) + if err != nil { + return nil, err + } + } // opens the listener on the raft port, raft will close it when it thinks it's appropriate ln, err := ts.Listen("tcp", raftAddr(self.hostAddr, cfg)) @@ -234,7 +257,7 @@ func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, sel return nil, err } - logger := hclog.New(&hclog.LoggerOptions{ + transportLogger := hclog.New(&hclog.LoggerOptions{ Name: "raft-net", Output: cfg.Raft.LogOutput, Level: hclog.LevelFromString(cfg.Raft.LogLevel), @@ -248,9 +271,9 @@ func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, sel }, cfg.MaxConnPool, cfg.ConnTimeout, - logger) + transportLogger) - return raft.NewRaft(cfg.Raft, *fsm, logStore, stableStore, snapshots, transport) + return raft.NewRaft(cfg.Raft, *fsm, logStore, stableStore, snapStore, transport) } // A Consensus is the consensus algorithm for a tsnet.Server diff --git a/tsconsensus/tsconsensus_test.go b/tsconsensus/tsconsensus_test.go index d1b92f8a489f7..bfb6b3e0688cc 100644 --- a/tsconsensus/tsconsensus_test.go +++ b/tsconsensus/tsconsensus_test.go @@ -17,6 +17,7 @@ import ( "net/netip" "os" "path/filepath" + "runtime" "strings" "sync" "testing" @@ -37,6 +38,7 @@ import ( "tailscale.com/types/key" "tailscale.com/types/logger" "tailscale.com/types/views" + "tailscale.com/util/cibuild" "tailscale.com/util/racebuild" ) @@ -113,6 +115,9 @@ func (f *fsm) Restore(rc io.ReadCloser) error { } func testConfig(t *testing.T) { + if runtime.GOOS == "windows" && cibuild.On() { + t.Skip("cmd/natc isn't supported on Windows, so skipping tsconsensus tests on CI for now; see https://github.com/tailscale/tailscale/issues/16340") + } // -race AND Parallel makes things start to take too long. if !racebuild.On { t.Parallel() diff --git a/tsnet/depaware.txt b/tsnet/depaware.txt index 662752554d02e..da3175b8c42d2 100644 --- a/tsnet/depaware.txt +++ b/tsnet/depaware.txt @@ -113,8 +113,6 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) L github.com/google/nftables/internal/parseexprfunc from github.com/google/nftables+ L github.com/google/nftables/xt from github.com/google/nftables/expr+ DWI github.com/google/uuid from github.com/prometheus-community/pro-bing+ - LDW github.com/gorilla/csrf from tailscale.com/client/web - LDW github.com/gorilla/securecookie from github.com/gorilla/csrf github.com/hdevalence/ed25519consensus from tailscale.com/clientupdate/distsign+ L 💣 github.com/illarion/gonotify/v3 from tailscale.com/net/dns L github.com/illarion/gonotify/v3/syscallf from github.com/illarion/gonotify/v3 @@ -239,7 +237,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) tailscale.com/envknob/featureknob from tailscale.com/client/web+ tailscale.com/feature from tailscale.com/ipn/ipnext+ tailscale.com/health from tailscale.com/control/controlclient+ - tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal + tailscale.com/health/healthmsg from tailscale.com/ipn/ipnlocal+ tailscale.com/hostinfo from tailscale.com/client/web+ tailscale.com/internal/noiseconn from tailscale.com/control/controlclient tailscale.com/ipn from tailscale.com/client/local+ @@ -497,7 +495,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) crypto/internal/fips140/edwards25519/field from crypto/ecdh+ crypto/internal/fips140/hkdf from crypto/internal/fips140/tls13+ crypto/internal/fips140/hmac from crypto/hmac+ - crypto/internal/fips140/mlkem from crypto/tls + crypto/internal/fips140/mlkem from crypto/tls+ crypto/internal/fips140/nistec from crypto/elliptic+ crypto/internal/fips140/nistec/fiat from crypto/internal/fips140/nistec crypto/internal/fips140/rsa from crypto/rsa @@ -517,6 +515,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) crypto/internal/randutil from crypto/dsa+ crypto/internal/sysrand from crypto/internal/entropy+ crypto/md5 from crypto/tls+ + LD crypto/mlkem from golang.org/x/crypto/ssh crypto/rand from crypto/ed25519+ crypto/rc4 from crypto/tls+ crypto/rsa from crypto/tls+ @@ -534,12 +533,11 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) W debug/dwarf from debug/pe W debug/pe from github.com/dblohm7/wingoes/pe embed from github.com/tailscale/web-client-prebuilt+ - encoding from encoding/gob+ + encoding from encoding/json+ encoding/asn1 from crypto/x509+ encoding/base32 from github.com/fxamacker/cbor/v2+ encoding/base64 from encoding/json+ encoding/binary from compress/gzip+ - LDW encoding/gob from github.com/gorilla/securecookie encoding/hex from crypto/x509+ encoding/json from expvar+ encoding/pem from crypto/tls+ @@ -553,7 +551,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) hash/crc32 from compress/gzip+ hash/maphash from go4.org/mem html from html/template+ - LDW html/template from github.com/gorilla/csrf+ + LDW html/template from tailscale.com/util/eventbus internal/abi from crypto/x509/internal/macos+ internal/asan from internal/runtime/maps+ internal/bisect from internal/godebug @@ -584,7 +582,7 @@ tailscale.com/tsnet dependencies: (generated by github.com/tailscale/depaware) internal/runtime/math from internal/runtime/maps+ internal/runtime/sys from crypto/subtle+ LA internal/runtime/syscall from runtime+ - LDW internal/saferio from debug/pe+ + W internal/saferio from debug/pe internal/singleflight from net internal/stringslite from embed+ internal/sync from sync+ diff --git a/tsnet/tsnet.go b/tsnet/tsnet.go index 4664a66a796d4..65367f235482f 100644 --- a/tsnet/tsnet.go +++ b/tsnet/tsnet.go @@ -536,10 +536,7 @@ func (s *Server) start() (reterr error) { if err != nil { return err } - s.rootPath, err = getTSNetDir(s.logf, confDir, prog) - if err != nil { - return err - } + s.rootPath = filepath.Join(confDir, "tsnet-"+prog) } if err := os.MkdirAll(s.rootPath, 0700); err != nil { return err @@ -897,49 +894,6 @@ func (s *Server) getUDPHandlerForFlow(src, dst netip.AddrPort) (handler func(net return func(c nettype.ConnPacketConn) { ln.handle(c) }, true } -// getTSNetDir usually just returns filepath.Join(confDir, "tsnet-"+prog) -// with no error. -// -// One special case is that it renames old "tslib-" directories to -// "tsnet-", and that rename might return an error. -// -// TODO(bradfitz): remove this maybe 6 months after 2022-03-17, -// once people (notably Tailscale corp services) have updated. -func getTSNetDir(logf logger.Logf, confDir, prog string) (string, error) { - oldPath := filepath.Join(confDir, "tslib-"+prog) - newPath := filepath.Join(confDir, "tsnet-"+prog) - - fi, err := os.Lstat(oldPath) - if os.IsNotExist(err) { - // Common path. - return newPath, nil - } - if err != nil { - return "", err - } - if !fi.IsDir() { - return "", fmt.Errorf("expected old tslib path %q to be a directory; got %v", oldPath, fi.Mode()) - } - - // At this point, oldPath exists and is a directory. But does - // the new path exist? - - fi, err = os.Lstat(newPath) - if err == nil && fi.IsDir() { - // New path already exists somehow. Ignore the old one and - // don't try to migrate it. - return newPath, nil - } - if err != nil && !os.IsNotExist(err) { - return "", err - } - if err := os.Rename(oldPath, newPath); err != nil { - return "", err - } - logf("renamed old tsnet state storage directory %q to %q", oldPath, newPath) - return newPath, nil -} - // APIClient returns a tailscale.Client that can be used to make authenticated // requests to the Tailscale control server. // It requires the user to set tailscale.I_Acknowledge_This_API_Is_Unstable. diff --git a/tstest/integration/capmap_test.go b/tstest/integration/capmap_test.go new file mode 100644 index 0000000000000..0ee05be2f57d7 --- /dev/null +++ b/tstest/integration/capmap_test.go @@ -0,0 +1,147 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package integration + +import ( + "errors" + "testing" + "time" + + "tailscale.com/tailcfg" + "tailscale.com/tstest" +) + +// TestPeerCapMap tests that the node capability map (CapMap) is included in peer information. +func TestPeerCapMap(t *testing.T) { + tstest.Shard(t) + tstest.Parallel(t) + env := NewTestEnv(t) + + // Spin up two nodes. + n1 := NewTestNode(t, env) + d1 := n1.StartDaemon() + n1.AwaitListening() + n1.MustUp() + n1.AwaitRunning() + + n2 := NewTestNode(t, env) + d2 := n2.StartDaemon() + n2.AwaitListening() + n2.MustUp() + n2.AwaitRunning() + + n1.AwaitIP4() + n2.AwaitIP4() + + // Get the nodes from the control server. + nodes := env.Control.AllNodes() + if len(nodes) != 2 { + t.Fatalf("expected 2 nodes, got %d nodes", len(nodes)) + } + + // Figure out which node is which by comparing keys. + st1 := n1.MustStatus() + var tn1, tn2 *tailcfg.Node + for _, n := range nodes { + if n.Key == st1.Self.PublicKey { + tn1 = n + } else { + tn2 = n + } + } + + // Set CapMap on both nodes. + caps := make(tailcfg.NodeCapMap) + caps["example:custom"] = []tailcfg.RawMessage{`"value"`} + caps["example:enabled"] = []tailcfg.RawMessage{`true`} + + env.Control.SetNodeCapMap(tn1.Key, caps) + env.Control.SetNodeCapMap(tn2.Key, caps) + + // Check that nodes see each other's CapMap. + if err := tstest.WaitFor(10*time.Second, func() error { + st1 := n1.MustStatus() + st2 := n2.MustStatus() + + if len(st1.Peer) == 0 || len(st2.Peer) == 0 { + return errors.New("no peers") + } + + // Check n1 sees n2's CapMap. + p1 := st1.Peer[st1.Peers()[0]] + if p1.CapMap == nil { + return errors.New("peer CapMap is nil") + } + if p1.CapMap["example:custom"] == nil || p1.CapMap["example:enabled"] == nil { + return errors.New("peer CapMap missing entries") + } + + // Check n2 sees n1's CapMap. + p2 := st2.Peer[st2.Peers()[0]] + if p2.CapMap == nil { + return errors.New("peer CapMap is nil") + } + if p2.CapMap["example:custom"] == nil || p2.CapMap["example:enabled"] == nil { + return errors.New("peer CapMap missing entries") + } + + return nil + }); err != nil { + t.Fatal(err) + } + + d1.MustCleanShutdown(t) + d2.MustCleanShutdown(t) +} + +// TestSetNodeCapMap tests that SetNodeCapMap updates are propagated to peers. +func TestSetNodeCapMap(t *testing.T) { + tstest.Shard(t) + tstest.Parallel(t) + env := NewTestEnv(t) + + n1 := NewTestNode(t, env) + d1 := n1.StartDaemon() + n1.AwaitListening() + n1.MustUp() + n1.AwaitRunning() + + nodes := env.Control.AllNodes() + if len(nodes) != 1 { + t.Fatalf("expected 1 node, got %d nodes", len(nodes)) + } + node1 := nodes[0] + + // Set initial CapMap. + caps := make(tailcfg.NodeCapMap) + caps["test:state"] = []tailcfg.RawMessage{`"initial"`} + env.Control.SetNodeCapMap(node1.Key, caps) + + // Start second node and verify it sees the first node's CapMap. + n2 := NewTestNode(t, env) + d2 := n2.StartDaemon() + n2.AwaitListening() + n2.MustUp() + n2.AwaitRunning() + + if err := tstest.WaitFor(5*time.Second, func() error { + st := n2.MustStatus() + if len(st.Peer) == 0 { + return errors.New("no peers") + } + p := st.Peer[st.Peers()[0]] + if p.CapMap == nil || p.CapMap["test:state"] == nil { + return errors.New("peer CapMap not set") + } + if string(p.CapMap["test:state"][0]) != `"initial"` { + return errors.New("wrong CapMap value") + } + return nil + }); err != nil { + t.Fatal(err) + } + + d1.MustCleanShutdown(t) + d2.MustCleanShutdown(t) +} diff --git a/tstest/integration/integration.go b/tstest/integration/integration.go index d64bfbbd9d755..987bb569a4f66 100644 --- a/tstest/integration/integration.go +++ b/tstest/integration/integration.go @@ -569,11 +569,12 @@ type TestNode struct { env *TestEnv tailscaledParser *nodeOutputParser - dir string // temp dir for sock & state - configFile string // or empty for none - sockFile string - stateFile string - upFlagGOOS string // if non-empty, sets TS_DEBUG_UP_FLAG_GOOS for cmd/tailscale CLI + dir string // temp dir for sock & state + configFile string // or empty for none + sockFile string + stateFile string + upFlagGOOS string // if non-empty, sets TS_DEBUG_UP_FLAG_GOOS for cmd/tailscale CLI + encryptState bool mu sync.Mutex onLogLine []func([]byte) @@ -640,7 +641,7 @@ func (n *TestNode) diskPrefs() *ipn.Prefs { if _, err := os.ReadFile(n.stateFile); err != nil { t.Fatalf("reading prefs: %v", err) } - fs, err := store.NewFileStore(nil, n.stateFile) + fs, err := store.New(nil, n.stateFile) if err != nil { t.Fatalf("reading prefs, NewFileStore: %v", err) } @@ -822,6 +823,9 @@ func (n *TestNode) StartDaemonAsIPNGOOS(ipnGOOS string) *Daemon { if n.configFile != "" { cmd.Args = append(cmd.Args, "--config="+n.configFile) } + if n.encryptState { + cmd.Args = append(cmd.Args, "--encrypt-state") + } cmd.Env = append(os.Environ(), "TS_DEBUG_PERMIT_HTTP_C2N=1", "TS_LOG_TARGET="+n.env.LogCatcherServer.URL, diff --git a/tstest/integration/integration_test.go b/tstest/integration/integration_test.go index 90cc7e443b5d3..7cb251f31c344 100644 --- a/tstest/integration/integration_test.go +++ b/tstest/integration/integration_test.go @@ -21,6 +21,7 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "strconv" "sync/atomic" "testing" @@ -32,6 +33,7 @@ import ( "tailscale.com/client/tailscale" "tailscale.com/clientupdate" "tailscale.com/cmd/testwrapper/flakytest" + "tailscale.com/hostinfo" "tailscale.com/ipn" "tailscale.com/net/tsaddr" "tailscale.com/net/tstun" @@ -1470,3 +1472,60 @@ func TestNetstackUDPLoopback(t *testing.T) { d1.MustCleanShutdown(t) } + +func TestEncryptStateMigration(t *testing.T) { + if !hostinfo.New().TPM.Present() { + t.Skip("TPM not available") + } + if runtime.GOOS != "linux" && runtime.GOOS != "windows" { + t.Skip("--encrypt-state for tailscaled state not supported on this platform") + } + tstest.Shard(t) + tstest.Parallel(t) + env := NewTestEnv(t) + n := NewTestNode(t, env) + + runNode := func(t *testing.T, wantStateKeys []string) { + t.Helper() + + // Run the node. + d := n.StartDaemon() + n.AwaitResponding() + n.MustUp() + n.AwaitRunning() + + // Check the contents of the state file. + buf, err := os.ReadFile(n.stateFile) + if err != nil { + t.Fatalf("reading %q: %v", n.stateFile, err) + } + t.Logf("state file content:\n%s", buf) + var content map[string]any + if err := json.Unmarshal(buf, &content); err != nil { + t.Fatalf("parsing %q: %v", n.stateFile, err) + } + for _, k := range wantStateKeys { + if _, ok := content[k]; !ok { + t.Errorf("state file is missing key %q", k) + } + } + + // Stop the node. + d.MustCleanShutdown(t) + } + + wantPlaintextStateKeys := []string{"_machinekey", "_current-profile", "_profiles"} + wantEncryptedStateKeys := []string{"key", "nonce", "data"} + t.Run("regular-state", func(t *testing.T) { + n.encryptState = false + runNode(t, wantPlaintextStateKeys) + }) + t.Run("migrate-to-encrypted", func(t *testing.T) { + n.encryptState = true + runNode(t, wantEncryptedStateKeys) + }) + t.Run("migrate-to-plaintext", func(t *testing.T) { + n.encryptState = false + runNode(t, wantPlaintextStateKeys) + }) +} diff --git a/tstest/integration/tailscaled_deps_test_darwin.go b/tstest/integration/tailscaled_deps_test_darwin.go index 321ba25668c1f..a73c6ebf649f2 100644 --- a/tstest/integration/tailscaled_deps_test_darwin.go +++ b/tstest/integration/tailscaled_deps_test_darwin.go @@ -51,6 +51,7 @@ import ( _ "tailscale.com/util/eventbus" _ "tailscale.com/util/multierr" _ "tailscale.com/util/osshare" + _ "tailscale.com/util/syspolicy" _ "tailscale.com/version" _ "tailscale.com/version/distro" _ "tailscale.com/wgengine" diff --git a/tstest/integration/tailscaled_deps_test_freebsd.go b/tstest/integration/tailscaled_deps_test_freebsd.go index 321ba25668c1f..a73c6ebf649f2 100644 --- a/tstest/integration/tailscaled_deps_test_freebsd.go +++ b/tstest/integration/tailscaled_deps_test_freebsd.go @@ -51,6 +51,7 @@ import ( _ "tailscale.com/util/eventbus" _ "tailscale.com/util/multierr" _ "tailscale.com/util/osshare" + _ "tailscale.com/util/syspolicy" _ "tailscale.com/version" _ "tailscale.com/version/distro" _ "tailscale.com/wgengine" diff --git a/tstest/integration/tailscaled_deps_test_linux.go b/tstest/integration/tailscaled_deps_test_linux.go index 321ba25668c1f..a73c6ebf649f2 100644 --- a/tstest/integration/tailscaled_deps_test_linux.go +++ b/tstest/integration/tailscaled_deps_test_linux.go @@ -51,6 +51,7 @@ import ( _ "tailscale.com/util/eventbus" _ "tailscale.com/util/multierr" _ "tailscale.com/util/osshare" + _ "tailscale.com/util/syspolicy" _ "tailscale.com/version" _ "tailscale.com/version/distro" _ "tailscale.com/wgengine" diff --git a/tstest/integration/tailscaled_deps_test_openbsd.go b/tstest/integration/tailscaled_deps_test_openbsd.go index 321ba25668c1f..a73c6ebf649f2 100644 --- a/tstest/integration/tailscaled_deps_test_openbsd.go +++ b/tstest/integration/tailscaled_deps_test_openbsd.go @@ -51,6 +51,7 @@ import ( _ "tailscale.com/util/eventbus" _ "tailscale.com/util/multierr" _ "tailscale.com/util/osshare" + _ "tailscale.com/util/syspolicy" _ "tailscale.com/version" _ "tailscale.com/version/distro" _ "tailscale.com/wgengine" diff --git a/tstest/integration/testcontrol/testcontrol.go b/tstest/integration/testcontrol/testcontrol.go index 71205f897aad8..739795bb3d245 100644 --- a/tstest/integration/testcontrol/testcontrol.go +++ b/tstest/integration/testcontrol/testcontrol.go @@ -1000,7 +1000,11 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse, s.mu.Lock() peerAddress := s.masquerades[p.Key][node.Key] routes := s.nodeSubnetRoutes[p.Key] + peerCapMap := maps.Clone(s.nodeCapMaps[p.Key]) s.mu.Unlock() + if peerCapMap != nil { + p.CapMap = peerCapMap + } if peerAddress.IsValid() { if peerAddress.Is6() { p.Addresses[1] = netip.PrefixFrom(peerAddress, peerAddress.BitLen()) diff --git a/tstest/tlstest/tlstest.go b/tstest/tlstest/tlstest.go new file mode 100644 index 0000000000000..76ec0e7e2dfad --- /dev/null +++ b/tstest/tlstest/tlstest.go @@ -0,0 +1,187 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package tlstest contains code to help test Tailscale's TLS support without +// depending on real WebPKI roots or certificates during tests. +package tlstest + +import ( + "bytes" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + _ "embed" + "encoding/pem" + "fmt" + "math/big" + "sync" + "time" +) + +// TestRootCA returns a self-signed ECDSA root CA certificate (as PEM) for +// testing purposes. +// +// Typical use in a test is like: +// +// bakedroots.ResetForTest(t, tlstest.TestRootCA()) +func TestRootCA() []byte { + return bytes.Clone(testRootCAOncer()) +} + +// cache for [privateKey], so it always returns the same key for a given domain. +var ( + mu sync.Mutex + privateKeys = make(map[string][]byte) // domain -> private key PEM +) + +// caDomain is a fake domain name to repreesnt the private key for the root CA. +const caDomain = "_root" + +// privateKey returns a PEM-encoded test ECDSA private key for the given domain. +func privateKey(domain string) (pemBytes []byte) { + mu.Lock() + defer mu.Unlock() + if pemBytes, ok := privateKeys[domain]; ok { + return bytes.Clone(pemBytes) + } + defer func() { privateKeys[domain] = bytes.Clone(pemBytes) }() + + k, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + panic(fmt.Sprintf("failed to generate ECDSA key for %q: %v", domain, err)) + } + der, err := x509.MarshalECPrivateKey(k) + if err != nil { + panic(fmt.Sprintf("failed to marshal ECDSA key for %q: %v", domain, err)) + } + var buf bytes.Buffer + if err := pem.Encode(&buf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: der}); err != nil { + panic(fmt.Sprintf("failed to encode PEM: %v", err)) + } + return buf.Bytes() +} + +var testRootCAOncer = sync.OnceValue(func() []byte { + key := rootCAKey() + now := time.Now().Add(-time.Hour) + tpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "Tailscale Unit Test ECDSA Root", + Organization: []string{"Tailscale Test Org"}, + }, + NotBefore: now, + NotAfter: now.AddDate(5, 0, 0), + + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + SubjectKeyId: mustSKID(&key.PublicKey), + } + + der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key) + if err != nil { + panic(err) + } + return pemCert(der) +}) + +func pemCert(der []byte) []byte { + var buf bytes.Buffer + if err := pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil { + panic(fmt.Sprintf("failed to encode PEM: %v", err)) + } + return buf.Bytes() +} + +var rootCAKey = sync.OnceValue(func() *ecdsa.PrivateKey { + return mustParsePEM(privateKey(caDomain), x509.ParseECPrivateKey) +}) + +func mustParsePEM[T any](pemBytes []byte, parse func([]byte) (T, error)) T { + block, rest := pem.Decode(pemBytes) + if block == nil || len(rest) > 0 { + panic("invalid PEM") + } + v, err := parse(block.Bytes) + if err != nil { + panic(fmt.Sprintf("invalid PEM: %v", err)) + } + return v +} + +// Domain is a fake domain name used in TLS tests. +// +// They don't have real DNS records. Tests are expected to fake DNS +// lookups and dials for these domains. +type Domain string + +// ProxyServer is a domain name for a hypothetical proxy server. +const ( + ProxyServer = Domain("proxy.tstest") + + // ControlPlane is a domain name for a test control plane server. + ControlPlane = Domain("controlplane.tstest") + + // Derper is a domain name for a test DERP server. + Derper = Domain("derp.tstest") +) + +// ServerTLSConfig returns a TLS configuration suitable for a server +// using the KeyPair's certificate and private key. +func (d Domain) ServerTLSConfig() *tls.Config { + cert, err := tls.X509KeyPair(d.CertPEM(), privateKey(string(d))) + if err != nil { + panic("invalid TLS key pair: " + err.Error()) + } + return &tls.Config{ + Certificates: []tls.Certificate{cert}, + } +} + +// KeyPEM returns a PEM-encoded private key for the domain. +func (d Domain) KeyPEM() []byte { + return privateKey(string(d)) +} + +// CertPEM returns a PEM-encoded certificate for the domain. +func (d Domain) CertPEM() []byte { + caCert := mustParsePEM(TestRootCA(), x509.ParseCertificate) + caPriv := mustParsePEM(privateKey(caDomain), x509.ParseECPrivateKey) + leafKey := mustParsePEM(d.KeyPEM(), x509.ParseECPrivateKey) + + serial, err := rand.Int(rand.Reader, big.NewInt(0).Lsh(big.NewInt(1), 128)) + if err != nil { + panic(err) + } + + now := time.Now().Add(-time.Hour) + tpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: string(d)}, + NotBefore: now, + NotAfter: now.AddDate(2, 0, 0), + + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: []string{string(d)}, + } + + der, err := x509.CreateCertificate(rand.Reader, tpl, caCert, &leafKey.PublicKey, caPriv) + if err != nil { + panic(err) + } + return pemCert(der) +} + +func mustSKID(pub *ecdsa.PublicKey) []byte { + skid, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + panic(err) + } + return skid[:20] // same as x509 library +} diff --git a/tstest/tlstest/tlstest_test.go b/tstest/tlstest/tlstest_test.go new file mode 100644 index 0000000000000..8497b872ec7c5 --- /dev/null +++ b/tstest/tlstest/tlstest_test.go @@ -0,0 +1,21 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package tlstest + +import ( + "testing" +) + +func TestPrivateKey(t *testing.T) { + a := privateKey("a.tstest") + a2 := privateKey("a.tstest") + b := privateKey("b.tstest") + + if string(a) != string(a2) { + t.Errorf("a and a2 should be equal") + } + if string(a) == string(b) { + t.Errorf("a and b should not be equal") + } +} diff --git a/tsweb/promvarz/promvarz_test.go b/tsweb/promvarz/promvarz_test.go index 9f91b5d12380e..cffbbec2273c8 100644 --- a/tsweb/promvarz/promvarz_test.go +++ b/tsweb/promvarz/promvarz_test.go @@ -1,5 +1,6 @@ // Copyright (c) Tailscale Inc & AUTHORS // SPDX-License-Identifier: BSD-3-Clause + package promvarz import ( diff --git a/tsweb/varz/varz.go b/tsweb/varz/varz.go index c6d66fbe2beda..aca2878b74f29 100644 --- a/tsweb/varz/varz.go +++ b/tsweb/varz/varz.go @@ -11,6 +11,8 @@ import ( "fmt" "io" "net/http" + "os" + "path/filepath" "reflect" "runtime" "sort" @@ -189,7 +191,11 @@ func writePromExpVar(w io.Writer, prefix string, kv expvar.KeyValue) { return } if vs, ok := v.(string); ok && strings.HasSuffix(name, "version") { - fmt.Fprintf(w, "%s{version=%q} 1\n", name, vs) + if name == "version" { + fmt.Fprintf(w, "%s{version=%q,binary=%q} 1\n", name, vs, binaryName()) + } else { + fmt.Fprintf(w, "%s{version=%q} 1\n", name, vs) + } return } switch v := v.(type) { @@ -308,6 +314,18 @@ func ExpvarDoHandler(expvarDoFunc func(f func(expvar.KeyValue))) func(http.Respo } } +var binaryName = sync.OnceValue(func() string { + exe, err := os.Executable() + if err != nil { + return "" + } + exe2, err := filepath.EvalSymlinks(exe) + if err != nil { + return filepath.Base(exe) + } + return filepath.Base(exe2) +}) + // PrometheusMetricsReflectRooter is an optional interface that expvar.Var implementations // can implement to indicate that they should be walked recursively with reflect to find // sets of fields to export. diff --git a/types/geo/doc.go b/types/geo/doc.go new file mode 100644 index 0000000000000..749c6308093f6 --- /dev/null +++ b/types/geo/doc.go @@ -0,0 +1,6 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package geo provides functionality to represent and process geographical +// locations on a spherical Earth. +package geo diff --git a/types/geo/point.go b/types/geo/point.go new file mode 100644 index 0000000000000..d7160ac593338 --- /dev/null +++ b/types/geo/point.go @@ -0,0 +1,279 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo + +import ( + "encoding/binary" + "errors" + "fmt" + "math" + "strconv" +) + +// ErrBadPoint indicates that the point is malformed. +var ErrBadPoint = errors.New("not a valid point") + +// Point represents a pair of latitude and longitude coordinates. +type Point struct { + lat Degrees + // lng180 is the longitude offset by +180° so the zero value is invalid + // and +0+0/ is Point{lat: +0.0, lng180: +180.0}. + lng180 Degrees +} + +// MakePoint returns a Point representing a given latitude and longitude on +// a WGS 84 ellipsoid. The Coordinate Reference System is EPSG:4326. +// Latitude is wrapped to [-90°, +90°] and longitude to (-180°, +180°]. +func MakePoint(latitude, longitude Degrees) Point { + lat, lng := float64(latitude), float64(longitude) + + switch { + case math.IsNaN(lat) || math.IsInf(lat, 0): + // don’t wrap + case lat < -90 || lat > 90: + // Latitude wraps by flipping the longitude + lat = math.Mod(lat, 360.0) + switch { + case lat == 0.0: + lat = 0.0 // -0.0 == 0.0, but -0° is not valid + case lat < -270.0: + lat = +360.0 + lat + case lat < -90.0: + lat = -180.0 - lat + lng += 180.0 + case lat > +270.0: + lat = -360.0 + lat + case lat > +90.0: + lat = +180.0 - lat + lng += 180.0 + } + } + + switch { + case lat == -90.0 || lat == +90.0: + // By convention, the north and south poles have longitude 0°. + lng = 0 + case math.IsNaN(lng) || math.IsInf(lng, 0): + // don’t wrap + case lng <= -180.0 || lng > 180.0: + // Longitude wraps around normally + lng = math.Mod(lng, 360.0) + switch { + case lng == 0.0: + lng = 0.0 // -0.0 == 0.0, but -0° is not valid + case lng <= -180.0: + lng = +360.0 + lng + case lng > +180.0: + lng = -360.0 + lng + } + } + + return Point{ + lat: Degrees(lat), + lng180: Degrees(lng + 180.0), + } +} + +// Valid reports if p is a valid point. +func (p Point) Valid() bool { + return !p.IsZero() +} + +// LatLng reports the latitude and longitude. +func (p Point) LatLng() (lat, lng Degrees, err error) { + if p.IsZero() { + return 0 * Degree, 0 * Degree, ErrBadPoint + } + return p.lat, p.lng180 - 180.0*Degree, nil +} + +// LatLng reports the latitude and longitude in float64. If err is nil, then lat +// and lng will never both be 0.0 to disambiguate between an empty struct and +// Null Island (0° 0°). +func (p Point) LatLngFloat64() (lat, lng float64, err error) { + dlat, dlng, err := p.LatLng() + if err != nil { + return 0.0, 0.0, err + } + if dlat == 0.0 && dlng == 0.0 { + // dlng must survive conversion to float32. + dlng = math.SmallestNonzeroFloat32 + } + return float64(dlat), float64(dlng), err +} + +// SphericalAngleTo returns the angular distance from p to q, calculated on a +// spherical Earth. +func (p Point) SphericalAngleTo(q Point) (Radians, error) { + pLat, pLng, pErr := p.LatLng() + qLat, qLng, qErr := q.LatLng() + switch { + case pErr != nil && qErr != nil: + return 0.0, fmt.Errorf("spherical distance from %v to %v: %w", p, q, errors.Join(pErr, qErr)) + case pErr != nil: + return 0.0, fmt.Errorf("spherical distance from %v: %w", p, pErr) + case qErr != nil: + return 0.0, fmt.Errorf("spherical distance to %v: %w", q, qErr) + } + // The spherical law of cosines is accurate enough for close points when + // using float64. + // + // The haversine formula is an alternative, but it is poorly behaved + // when points are on opposite sides of the sphere. + rLat, rLng := float64(pLat.Radians()), float64(pLng.Radians()) + sLat, sLng := float64(qLat.Radians()), float64(qLng.Radians()) + cosA := math.Sin(rLat)*math.Sin(sLat) + + math.Cos(rLat)*math.Cos(sLat)*math.Cos(rLng-sLng) + return Radians(math.Acos(cosA)), nil +} + +// DistanceTo reports the great-circle distance between p and q, in meters. +func (p Point) DistanceTo(q Point) (Distance, error) { + r, err := p.SphericalAngleTo(q) + if err != nil { + return 0, err + } + return DistanceOnEarth(r.Turns()), nil +} + +// String returns a space-separated pair of latitude and longitude, in decimal +// degrees. Positive latitudes are in the northern hemisphere, and positive +// longitudes are east of the prime meridian. If p was not initialized, this +// will return "nowhere". +func (p Point) String() string { + lat, lng, err := p.LatLng() + if err != nil { + if err == ErrBadPoint { + return "nowhere" + } + panic(err) + } + + return lat.String() + " " + lng.String() +} + +// AppendBinary implements [encoding.BinaryAppender]. The output consists of two +// float32s in big-endian byte order: latitude and longitude offset by 180°. +// If p is not a valid, the output will be an 8-byte zero value. +func (p Point) AppendBinary(b []byte) ([]byte, error) { + end := binary.BigEndian + b = end.AppendUint32(b, math.Float32bits(float32(p.lat))) + b = end.AppendUint32(b, math.Float32bits(float32(p.lng180))) + return b, nil +} + +// MarshalBinary implements [encoding.BinaryMarshaller]. The output matches that +// of calling [Point.AppendBinary]. +func (p Point) MarshalBinary() ([]byte, error) { + var b [8]byte + return p.AppendBinary(b[:0]) +} + +// UnmarshalBinary implements [encoding.BinaryUnmarshaler]. It expects input +// that was formatted by [Point.AppendBinary]: in big-endian byte order, a +// float32 representing latitude followed by a float32 representing longitude +// offset by 180°. If latitude and longitude fall outside valid ranges, then +// an error is returned. +func (p *Point) UnmarshalBinary(data []byte) error { + if len(data) < 8 { // Two uint32s are 8 bytes long + return fmt.Errorf("%w: not enough data: %q", ErrBadPoint, data) + } + + end := binary.BigEndian + lat := Degrees(math.Float32frombits(end.Uint32(data[0:]))) + if lat < -90*Degree || lat > 90*Degree { + return fmt.Errorf("%w: latitude outside [-90°, +90°]: %s", ErrBadPoint, lat) + } + lng180 := Degrees(math.Float32frombits(end.Uint32(data[4:]))) + if lng180 != 0 && (lng180 < 0*Degree || lng180 > 360*Degree) { + // lng180 == 0 is OK: the zero value represents invalid points. + lng := lng180 - 180*Degree + return fmt.Errorf("%w: longitude outside (-180°, +180°]: %s", ErrBadPoint, lng) + } + + p.lat = lat + p.lng180 = lng180 + return nil +} + +// AppendText implements [encoding.TextAppender]. The output is a point +// formatted as OGC Well-Known Text, as "POINT (longitude latitude)" where +// longitude and latitude are in decimal degrees. If p is not valid, the output +// will be "POINT EMPTY". +func (p Point) AppendText(b []byte) ([]byte, error) { + if p.IsZero() { + b = append(b, []byte("POINT EMPTY")...) + return b, nil + } + + lat, lng, err := p.LatLng() + if err != nil { + return b, err + } + + b = append(b, []byte("POINT (")...) + b = strconv.AppendFloat(b, float64(lng), 'f', -1, 64) + b = append(b, ' ') + b = strconv.AppendFloat(b, float64(lat), 'f', -1, 64) + b = append(b, ')') + return b, nil +} + +// MarshalText implements [encoding.TextMarshaller]. The output matches that +// of calling [Point.AppendText]. +func (p Point) MarshalText() ([]byte, error) { + var b [8]byte + return p.AppendText(b[:0]) +} + +// MarshalUint64 produces the same output as MashalBinary, encoded in a uint64. +func (p Point) MarshalUint64() (uint64, error) { + b, err := p.MarshalBinary() + return binary.NativeEndian.Uint64(b), err +} + +// UnmarshalUint64 expects input formatted by MarshalUint64. +func (p *Point) UnmarshalUint64(v uint64) error { + b := binary.NativeEndian.AppendUint64(nil, v) + return p.UnmarshalBinary(b) +} + +// IsZero reports if p is the zero value. +func (p Point) IsZero() bool { + return p == Point{} +} + +// EqualApprox reports if p and q are approximately equal: that is the absolute +// difference of both latitude and longitude are less than tol. If tol is +// negative, then tol defaults to a reasonably small number (10⁻⁵). If tol is +// zero, then p and q must be exactly equal. +func (p Point) EqualApprox(q Point, tol float64) bool { + if tol == 0 { + return p == q + } + + if p.IsZero() && q.IsZero() { + return true + } else if p.IsZero() || q.IsZero() { + return false + } + + plat, plng, err := p.LatLng() + if err != nil { + panic(err) + } + qlat, qlng, err := q.LatLng() + if err != nil { + panic(err) + } + + if tol < 0 { + tol = 1e-5 + } + + dlat := float64(plat) - float64(qlat) + dlng := float64(plng) - float64(qlng) + return ((dlat < 0 && -dlat < tol) || (dlat >= 0 && dlat < tol)) && + ((dlng < 0 && -dlng < tol) || (dlng >= 0 && dlng < tol)) +} diff --git a/types/geo/point_test.go b/types/geo/point_test.go new file mode 100644 index 0000000000000..308c1a1834377 --- /dev/null +++ b/types/geo/point_test.go @@ -0,0 +1,541 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo_test + +import ( + "fmt" + "math" + "testing" + "testing/quick" + + "tailscale.com/types/geo" +) + +func TestPointZero(t *testing.T) { + var zero geo.Point + + if got := zero.IsZero(); !got { + t.Errorf("IsZero() got %t", got) + } + + if got := zero.Valid(); got { + t.Errorf("Valid() got %t", got) + } + + wantErr := geo.ErrBadPoint.Error() + if _, _, err := zero.LatLng(); err.Error() != wantErr { + t.Errorf("LatLng() err %q, want %q", err, wantErr) + } + + wantStr := "nowhere" + if got := zero.String(); got != wantStr { + t.Errorf("String() got %q, want %q", got, wantStr) + } + + wantB := []byte{0, 0, 0, 0, 0, 0, 0, 0} + if b, err := zero.MarshalBinary(); err != nil { + t.Errorf("MarshalBinary() err %q, want nil", err) + } else if string(b) != string(wantB) { + t.Errorf("MarshalBinary got %q, want %q", b, wantB) + } + + wantI := uint64(0x00000000) + if i, err := zero.MarshalUint64(); err != nil { + t.Errorf("MarshalUint64() err %q, want nil", err) + } else if i != wantI { + t.Errorf("MarshalUint64 got %v, want %v", i, wantI) + } +} + +func TestPoint(t *testing.T) { + for _, tt := range []struct { + name string + lat geo.Degrees + lng geo.Degrees + wantLat geo.Degrees + wantLng geo.Degrees + wantString string + wantText string + }{ + { + name: "null-island", + lat: +0.0, + lng: +0.0, + wantLat: +0.0, + wantLng: +0.0, + wantString: "+0° +0°", + wantText: "POINT (0 0)", + }, + { + name: "north-pole", + lat: +90.0, + lng: +0.0, + wantLat: +90.0, + wantLng: +0.0, + wantString: "+90° +0°", + wantText: "POINT (0 90)", + }, + { + name: "south-pole", + lat: -90.0, + lng: +0.0, + wantLat: -90.0, + wantLng: +0.0, + wantString: "-90° +0°", + wantText: "POINT (0 -90)", + }, + { + name: "north-pole-weird-longitude", + lat: +90.0, + lng: +1.0, + wantLat: +90.0, + wantLng: +0.0, + wantString: "+90° +0°", + wantText: "POINT (0 90)", + }, + { + name: "south-pole-weird-longitude", + lat: -90.0, + lng: +1.0, + wantLat: -90.0, + wantLng: +0.0, + wantString: "-90° +0°", + wantText: "POINT (0 -90)", + }, + { + name: "almost-north", + lat: +89.0, + lng: +0.0, + wantLat: +89.0, + wantLng: +0.0, + wantString: "+89° +0°", + wantText: "POINT (0 89)", + }, + { + name: "past-north", + lat: +91.0, + lng: +0.0, + wantLat: +89.0, + wantLng: +180.0, + wantString: "+89° +180°", + wantText: "POINT (180 89)", + }, + { + name: "almost-south", + lat: -89.0, + lng: +0.0, + wantLat: -89.0, + wantLng: +0.0, + wantString: "-89° +0°", + wantText: "POINT (0 -89)", + }, + { + name: "past-south", + lat: -91.0, + lng: +0.0, + wantLat: -89.0, + wantLng: +180.0, + wantString: "-89° +180°", + wantText: "POINT (180 -89)", + }, + { + name: "antimeridian-north", + lat: +180.0, + lng: +0.0, + wantLat: +0.0, + wantLng: +180.0, + wantString: "+0° +180°", + wantText: "POINT (180 0)", + }, + { + name: "antimeridian-south", + lat: -180.0, + lng: +0.0, + wantLat: +0.0, + wantLng: +180.0, + wantString: "+0° +180°", + wantText: "POINT (180 0)", + }, + { + name: "almost-antimeridian-north", + lat: +179.0, + lng: +0.0, + wantLat: +1.0, + wantLng: +180.0, + wantString: "+1° +180°", + wantText: "POINT (180 1)", + }, + { + name: "past-antimeridian-north", + lat: +181.0, + lng: +0.0, + wantLat: -1.0, + wantLng: +180.0, + wantString: "-1° +180°", + wantText: "POINT (180 -1)", + }, + { + name: "almost-antimeridian-south", + lat: -179.0, + lng: +0.0, + wantLat: -1.0, + wantLng: +180.0, + wantString: "-1° +180°", + wantText: "POINT (180 -1)", + }, + { + name: "past-antimeridian-south", + lat: -181.0, + lng: +0.0, + wantLat: +1.0, + wantLng: +180.0, + wantString: "+1° +180°", + wantText: "POINT (180 1)", + }, + { + name: "circumnavigate-north", + lat: +360.0, + lng: +1.0, + wantLat: +0.0, + wantLng: +1.0, + wantString: "+0° +1°", + wantText: "POINT (1 0)", + }, + { + name: "circumnavigate-south", + lat: -360.0, + lng: +1.0, + wantLat: +0.0, + wantLng: +1.0, + wantString: "+0° +1°", + wantText: "POINT (1 0)", + }, + { + name: "almost-circumnavigate-north", + lat: +359.0, + lng: +1.0, + wantLat: -1.0, + wantLng: +1.0, + wantString: "-1° +1°", + wantText: "POINT (1 -1)", + }, + { + name: "past-circumnavigate-north", + lat: +361.0, + lng: +1.0, + wantLat: +1.0, + wantLng: +1.0, + wantString: "+1° +1°", + wantText: "POINT (1 1)", + }, + { + name: "almost-circumnavigate-south", + lat: -359.0, + lng: +1.0, + wantLat: +1.0, + wantLng: +1.0, + wantString: "+1° +1°", + wantText: "POINT (1 1)", + }, + { + name: "past-circumnavigate-south", + lat: -361.0, + lng: +1.0, + wantLat: -1.0, + wantLng: +1.0, + wantString: "-1° +1°", + wantText: "POINT (1 -1)", + }, + { + name: "antimeridian-east", + lat: +0.0, + lng: +180.0, + wantLat: +0.0, + wantLng: +180.0, + wantString: "+0° +180°", + wantText: "POINT (180 0)", + }, + { + name: "antimeridian-west", + lat: +0.0, + lng: -180.0, + wantLat: +0.0, + wantLng: +180.0, + wantString: "+0° +180°", + wantText: "POINT (180 0)", + }, + { + name: "almost-antimeridian-east", + lat: +0.0, + lng: +179.0, + wantLat: +0.0, + wantLng: +179.0, + wantString: "+0° +179°", + wantText: "POINT (179 0)", + }, + { + name: "past-antimeridian-east", + lat: +0.0, + lng: +181.0, + wantLat: +0.0, + wantLng: -179.0, + wantString: "+0° -179°", + wantText: "POINT (-179 0)", + }, + { + name: "almost-antimeridian-west", + lat: +0.0, + lng: -179.0, + wantLat: +0.0, + wantLng: -179.0, + wantString: "+0° -179°", + wantText: "POINT (-179 0)", + }, + { + name: "past-antimeridian-west", + lat: +0.0, + lng: -181.0, + wantLat: +0.0, + wantLng: +179.0, + wantString: "+0° +179°", + wantText: "POINT (179 0)", + }, + { + name: "montreal", + lat: +45.508888, + lng: -73.561668, + wantLat: +45.508888, + wantLng: -73.561668, + wantString: "+45.508888° -73.561668°", + wantText: "POINT (-73.561668 45.508888)", + }, + { + name: "canada", + lat: 57.550480044655636, + lng: -98.41680517868062, + wantLat: 57.550480044655636, + wantLng: -98.41680517868062, + wantString: "+57.550480044655636° -98.41680517868062°", + wantText: "POINT (-98.41680517868062 57.550480044655636)", + }, + } { + t.Run(tt.name, func(t *testing.T) { + p := geo.MakePoint(tt.lat, tt.lng) + + lat, lng, err := p.LatLng() + if !approx(lat, tt.wantLat) { + t.Errorf("MakePoint: lat %v, want %v", lat, tt.wantLat) + } + if !approx(lng, tt.wantLng) { + t.Errorf("MakePoint: lng %v, want %v", lng, tt.wantLng) + } + if err != nil { + t.Fatalf("LatLng: err %q, expected nil", err) + } + + if got := p.String(); got != tt.wantString { + t.Errorf("String: got %q, wantString %q", got, tt.wantString) + } + + txt, err := p.MarshalText() + if err != nil { + t.Errorf("Text: err %q, expected nil", err) + } else if string(txt) != tt.wantText { + t.Errorf("Text: got %q, wantText %q", txt, tt.wantText) + } + + b, err := p.MarshalBinary() + if err != nil { + t.Fatalf("MarshalBinary: err %q, expected nil", err) + } + + var q geo.Point + if err := q.UnmarshalBinary(b); err != nil { + t.Fatalf("UnmarshalBinary: err %q, expected nil", err) + } + if !q.EqualApprox(p, -1) { + t.Errorf("UnmarshalBinary: roundtrip failed: %#v != %#v", q, p) + } + + i, err := p.MarshalUint64() + if err != nil { + t.Fatalf("MarshalUint64: err %q, expected nil", err) + } + + var r geo.Point + if err := r.UnmarshalUint64(i); err != nil { + t.Fatalf("UnmarshalUint64: err %r, expected nil", err) + } + if !q.EqualApprox(r, -1) { + t.Errorf("UnmarshalUint64: roundtrip failed: %#v != %#v", r, p) + } + }) + } +} + +func TestPointMarshalBinary(t *testing.T) { + roundtrip := func(p geo.Point) error { + b, err := p.MarshalBinary() + if err != nil { + return fmt.Errorf("marshal: %v", err) + } + var q geo.Point + if err := q.UnmarshalBinary(b); err != nil { + return fmt.Errorf("unmarshal: %v", err) + } + if q != p { + return fmt.Errorf("%#v != %#v", q, p) + } + return nil + } + + t.Run("nowhere", func(t *testing.T) { + var nowhere geo.Point + if err := roundtrip(nowhere); err != nil { + t.Errorf("roundtrip: %v", err) + } + }) + + t.Run("quick-check", func(t *testing.T) { + f := func(lat geo.Degrees, lng geo.Degrees) (ok bool) { + pt := geo.MakePoint(lat, lng) + if err := roundtrip(pt); err != nil { + t.Errorf("roundtrip: %v", err) + } + return !t.Failed() + } + if err := quick.Check(f, nil); err != nil { + t.Error(err) + } + }) +} + +func TestPointMarshalUint64(t *testing.T) { + t.Skip("skip") + roundtrip := func(p geo.Point) error { + i, err := p.MarshalUint64() + if err != nil { + return fmt.Errorf("marshal: %v", err) + } + var q geo.Point + if err := q.UnmarshalUint64(i); err != nil { + return fmt.Errorf("unmarshal: %v", err) + } + if q != p { + return fmt.Errorf("%#v != %#v", q, p) + } + return nil + } + + t.Run("nowhere", func(t *testing.T) { + var nowhere geo.Point + if err := roundtrip(nowhere); err != nil { + t.Errorf("roundtrip: %v", err) + } + }) + + t.Run("quick-check", func(t *testing.T) { + f := func(lat geo.Degrees, lng geo.Degrees) (ok bool) { + if err := roundtrip(geo.MakePoint(lat, lng)); err != nil { + t.Errorf("roundtrip: %v", err) + } + return !t.Failed() + } + if err := quick.Check(f, nil); err != nil { + t.Error(err) + } + }) +} + +func TestPointSphericalAngleTo(t *testing.T) { + const earthRadius = 6371.000 // volumetric mean radius (km) + const kmToRad = 1 / earthRadius + for _, tt := range []struct { + name string + x geo.Point + y geo.Point + want geo.Radians + wantErr string + }{ + { + name: "same-point-null-island", + x: geo.MakePoint(0, 0), + y: geo.MakePoint(0, 0), + want: 0.0 * geo.Radian, + }, + { + name: "same-point-north-pole", + x: geo.MakePoint(+90, 0), + y: geo.MakePoint(+90, +90), + want: 0.0 * geo.Radian, + }, + { + name: "same-point-south-pole", + x: geo.MakePoint(-90, 0), + y: geo.MakePoint(-90, -90), + want: 0.0 * geo.Radian, + }, + { + name: "north-pole-to-south-pole", + x: geo.MakePoint(+90, 0), + y: geo.MakePoint(-90, -90), + want: math.Pi * geo.Radian, + }, + { + name: "toronto-to-montreal", + x: geo.MakePoint(+43.6532, -79.3832), + y: geo.MakePoint(+45.5019, -73.5674), + want: 504.26 * kmToRad * geo.Radian, + }, + { + name: "sydney-to-san-francisco", + x: geo.MakePoint(-33.8727, +151.2057), + y: geo.MakePoint(+37.7749, -122.4194), + want: 11948.18 * kmToRad * geo.Radian, + }, + { + name: "new-york-to-paris", + x: geo.MakePoint(+40.7128, -74.0060), + y: geo.MakePoint(+48.8575, +2.3514), + want: 5837.15 * kmToRad * geo.Radian, + }, + { + name: "seattle-to-tokyo", + x: geo.MakePoint(+47.6061, -122.3328), + y: geo.MakePoint(+35.6764, +139.6500), + want: 7700.00 * kmToRad * geo.Radian, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.x.SphericalAngleTo(tt.y) + if tt.wantErr == "" && err != nil { + t.Fatalf("err %q, expected nil", err) + } + if tt.wantErr != "" && (err == nil || err.Error() != tt.wantErr) { + t.Fatalf("err %q, expected %q", err, tt.wantErr) + } + if tt.wantErr != "" { + return + } + + if !approx(got, tt.want) { + t.Errorf("x to y: got %v, want %v", got, tt.want) + } + + // Distance should be commutative + got, err = tt.y.SphericalAngleTo(tt.x) + if err != nil { + t.Fatalf("err %q, expected nil", err) + } + if !approx(got, tt.want) { + t.Errorf("y to x: got %v, want %v", got, tt.want) + } + t.Logf("x to y: %v km", got/kmToRad) + }) + } +} + +func approx[T ~float64](x, y T) bool { + return math.Abs(float64(x)-float64(y)) <= 1e-5 +} diff --git a/types/geo/quantize.go b/types/geo/quantize.go new file mode 100644 index 0000000000000..18ec11f9f119c --- /dev/null +++ b/types/geo/quantize.go @@ -0,0 +1,106 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo + +import ( + "math" + "sync" +) + +// MinSeparation is the minimum separation between two points after quantizing. +// [Point.Quantize] guarantees that two points will either be snapped to exactly +// the same point, which conflates multiple positions together, or that the two +// points will be far enough apart that successfully performing most reverse +// lookups would be highly improbable. +const MinSeparation = 50_000 * Meter + +// Latitude +var ( + // numSepsEquatorToPole is the number of separations between a point on + // the equator to a point on a pole, that satisfies [minPointSep]. In + // other words, the number of separations between 0° and +90° degrees + // latitude. + numSepsEquatorToPole = int(math.Floor(float64( + earthPolarCircumference / MinSeparation / 4))) + + // latSep is the number of degrees between two adjacent latitudinal + // points. In other words, the next point going straight north of + // 0° would be latSep°. + latSep = Degrees(90.0 / float64(numSepsEquatorToPole)) +) + +// snapToLat returns the number of the nearest latitudinal separation to +// lat. A positive result is north of the equator, a negative result is south, +// and zero is the equator itself. For example, a result of -1 would mean a +// point that is [latSep] south of the equator. +func snapToLat(lat Degrees) int { + return int(math.Round(float64(lat / latSep))) +} + +// lngSep is a lookup table for the number of degrees between two adjacent +// longitudinal separations. where the index corresponds to the absolute value +// of the latitude separation. The first value corresponds to the equator and +// the last value corresponds to the separation before the pole. There is no +// value for the pole itself, because longitude has no meaning there. +// +// [lngSep] is calculated on init, which is so quick and will be used so often +// that the startup cost is negligible. +var lngSep = sync.OnceValue(func() []Degrees { + lut := make([]Degrees, numSepsEquatorToPole) + + // i ranges from the equator to a pole + for i := range len(lut) { + // lat ranges from [0°, 90°], because the southern hemisphere is + // a reflection of the northern one. + lat := Degrees(i) * latSep + ratio := math.Cos(float64(lat.Radians())) + circ := Distance(ratio) * earthEquatorialCircumference + num := int(math.Floor(float64(circ / MinSeparation))) + // We define lut[0] as 0°, lut[len(lut)] to be the north pole, + // which means -lut[len(lut)] is the south pole. + lut[i] = Degrees(360.0 / float64(num)) + } + return lut +}) + +// snapToLatLng returns the number of the nearest latitudinal separation to lat, +// and the nearest longitudinal separation to lng. +func snapToLatLng(lat, lng Degrees) (Degrees, Degrees) { + latN := snapToLat(lat) + + // absolute index into lngSep + n := latN + if n < 0 { + n = -latN + } + + lngSep := lngSep() + if n < len(lngSep) { + sep := lngSep[n] + lngN := int(math.Round(float64(lng / sep))) + return Degrees(latN) * latSep, Degrees(lngN) * sep + } + if latN < 0 { // south pole + return -90 * Degree, 0 * Degree + } else { // north pole + return +90 * Degree, 0 * Degree + } +} + +// Quantize returns a new [Point] after throwing away enough location data in p +// so that it would be difficult to distinguish a node among all the other nodes +// in its general vicinity. One caveat is that if there’s only one point in an +// obscure location, someone could triangulate the node using additional data. +// +// This method is stable: given the same p, it will always return the same +// result. It is equivalent to snapping to points on Earth that are at least +// [MinSeparation] apart. +func (p Point) Quantize() Point { + if p.IsZero() { + return p + } + + lat, lng := snapToLatLng(p.lat, p.lng180-180) + return MakePoint(lat, lng) +} diff --git a/types/geo/quantize_test.go b/types/geo/quantize_test.go new file mode 100644 index 0000000000000..3c707e303c250 --- /dev/null +++ b/types/geo/quantize_test.go @@ -0,0 +1,130 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo_test + +import ( + "testing" + "testing/quick" + + "tailscale.com/types/geo" +) + +func TestPointAnonymize(t *testing.T) { + t.Run("nowhere", func(t *testing.T) { + var zero geo.Point + p := zero.Quantize() + want := zero.Valid() + if got := p.Valid(); got != want { + t.Fatalf("zero.Valid %t, want %t", got, want) + } + }) + + t.Run("separation", func(t *testing.T) { + // Walk from the south pole to the north pole and check that each + // point on the latitude is approximately MinSeparation apart. + const southPole = -90 * geo.Degree + const northPole = 90 * geo.Degree + const dateLine = 180 * geo.Degree + + llat := southPole + for lat := llat; lat <= northPole; lat += 0x1p-4 { + last := geo.MakePoint(llat, 0) + cur := geo.MakePoint(lat, 0) + anon := cur.Quantize() + switch l, g, err := anon.LatLng(); { + case err != nil: + t.Fatal(err) + case lat == southPole: + // initialize llng, to the first snapped longitude + llat = l + goto Lng + case g != 0: + t.Fatalf("%v is west or east of %v", anon, last) + case l < llat: + t.Fatalf("%v is south of %v", anon, last) + case l == llat: + continue + case l > llat: + switch dist, err := last.DistanceTo(anon); { + case err != nil: + t.Fatal(err) + case dist == 0.0: + continue + case dist < geo.MinSeparation: + t.Logf("lat=%v last=%v cur=%v anon=%v", lat, last, cur, anon) + t.Fatalf("%v is too close to %v", anon, last) + default: + llat = l + } + } + + Lng: + llng := dateLine + for lng := llng; lng <= dateLine && lng >= -dateLine; lng -= 0x1p-3 { + last := geo.MakePoint(llat, llng) + cur := geo.MakePoint(lat, lng) + anon := cur.Quantize() + switch l, g, err := anon.LatLng(); { + case err != nil: + t.Fatal(err) + case lng == dateLine: + // initialize llng, to the first snapped longitude + llng = g + continue + case l != llat: + t.Fatalf("%v is north or south of %v", anon, last) + case g != llng: + const tolerance = geo.MinSeparation * 0x1p-9 + switch dist, err := last.DistanceTo(anon); { + case err != nil: + t.Fatal(err) + case dist < tolerance: + continue + case dist < (geo.MinSeparation - tolerance): + t.Logf("lat=%v lng=%v last=%v cur=%v anon=%v", lat, lng, last, cur, anon) + t.Fatalf("%v is too close to %v: %v", anon, last, dist) + default: + llng = g + } + + } + } + } + if llat == southPole { + t.Fatal("llat never incremented") + } + }) + + t.Run("quick-check", func(t *testing.T) { + f := func(lat, lng geo.Degrees) bool { + p := geo.MakePoint(lat, lng) + q := p.Quantize() + t.Logf("quantize %v = %v", p, q) + + lat, lng, err := q.LatLng() + if err != nil { + t.Errorf("err %v, want nil", err) + return !t.Failed() + } + + if lat < -90*geo.Degree || lat > 90*geo.Degree { + t.Errorf("lat outside [-90°, +90°]: %v", lat) + } + if lng < -180*geo.Degree || lng > 180*geo.Degree { + t.Errorf("lng outside [-180°, +180°], %v", lng) + } + + if dist, err := p.DistanceTo(q); err != nil { + t.Error(err) + } else if dist > (geo.MinSeparation * 2) { + t.Errorf("moved too far: %v", dist) + } + + return !t.Failed() + } + if err := quick.Check(f, nil); err != nil { + t.Fatal(err) + } + }) +} diff --git a/types/geo/units.go b/types/geo/units.go new file mode 100644 index 0000000000000..76a4c02f79f34 --- /dev/null +++ b/types/geo/units.go @@ -0,0 +1,191 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo + +import ( + "math" + "strconv" + "strings" + "unicode" +) + +const ( + Degree Degrees = 1 + Radian Radians = 1 + Turn Turns = 1 + Meter Distance = 1 +) + +// Degrees represents a latitude or longitude, in decimal degrees. +type Degrees float64 + +// ParseDegrees parses s as decimal degrees. +func ParseDegrees(s string) (Degrees, error) { + s = strings.TrimSuffix(s, "°") + f, err := strconv.ParseFloat(s, 64) + return Degrees(f), err +} + +// MustParseDegrees parses s as decimal degrees, but panics on error. +func MustParseDegrees(s string) Degrees { + d, err := ParseDegrees(s) + if err != nil { + panic(err) + } + return d +} + +// String implements the [Stringer] interface. The output is formatted in +// decimal degrees, prefixed by either the appropriate + or - sign, and suffixed +// by a ° degree symbol. +func (d Degrees) String() string { + b, _ := d.AppendText(nil) + b = append(b, []byte("°")...) + return string(b) +} + +// AppendText implements [encoding.TextAppender]. The output is formatted in +// decimal degrees, prefixed by either the appropriate + or - sign. +func (d Degrees) AppendText(b []byte) ([]byte, error) { + b = d.AppendZeroPaddedText(b, 0) + return b, nil +} + +// AppendZeroPaddedText appends d formatted as decimal degrees to b. The number of +// integer digits will be zero-padded to nint. +func (d Degrees) AppendZeroPaddedText(b []byte, nint int) []byte { + n := float64(d) + + if math.IsInf(n, 0) || math.IsNaN(n) { + return strconv.AppendFloat(b, n, 'f', -1, 64) + } + + sign := byte('+') + if math.Signbit(n) { + sign = '-' + n = -n + } + b = append(b, sign) + + pad := nint - 1 + for nn := n / 10; nn >= 1 && pad > 0; nn /= 10 { + pad-- + } + for range pad { + b = append(b, '0') + } + return strconv.AppendFloat(b, n, 'f', -1, 64) +} + +// Radians converts d into radians. +func (d Degrees) Radians() Radians { + return Radians(d * math.Pi / 180.0) +} + +// Turns converts d into a number of turns. +func (d Degrees) Turns() Turns { + return Turns(d / 360.0) +} + +// Radians represents a latitude or longitude, in radians. +type Radians float64 + +// ParseRadians parses s as radians. +func ParseRadians(s string) (Radians, error) { + s = strings.TrimSuffix(s, "rad") + s = strings.TrimRightFunc(s, unicode.IsSpace) + f, err := strconv.ParseFloat(s, 64) + return Radians(f), err +} + +// MustParseRadians parses s as radians, but panics on error. +func MustParseRadians(s string) Radians { + r, err := ParseRadians(s) + if err != nil { + panic(err) + } + return r +} + +// String implements the [Stringer] interface. +func (r Radians) String() string { + return strconv.FormatFloat(float64(r), 'f', -1, 64) + " rad" +} + +// Degrees converts r into decimal degrees. +func (r Radians) Degrees() Degrees { + return Degrees(r * 180.0 / math.Pi) +} + +// Turns converts r into a number of turns. +func (r Radians) Turns() Turns { + return Turns(r / 2 / math.Pi) +} + +// Turns represents a number of complete revolutions around a sphere. +type Turns float64 + +// String implements the [Stringer] interface. +func (o Turns) String() string { + return strconv.FormatFloat(float64(o), 'f', -1, 64) +} + +// Degrees converts t into decimal degrees. +func (o Turns) Degrees() Degrees { + return Degrees(o * 360.0) +} + +// Radians converts t into radians. +func (o Turns) Radians() Radians { + return Radians(o * 2 * math.Pi) +} + +// Distance represents a great-circle distance in meters. +type Distance float64 + +// ParseDistance parses s as distance in meters. +func ParseDistance(s string) (Distance, error) { + s = strings.TrimSuffix(s, "m") + s = strings.TrimRightFunc(s, unicode.IsSpace) + f, err := strconv.ParseFloat(s, 64) + return Distance(f), err +} + +// MustParseDistance parses s as distance in meters, but panics on error. +func MustParseDistance(s string) Distance { + d, err := ParseDistance(s) + if err != nil { + panic(err) + } + return d +} + +// String implements the [Stringer] interface. +func (d Distance) String() string { + return strconv.FormatFloat(float64(d), 'f', -1, 64) + "m" +} + +// DistanceOnEarth converts t turns into the great-circle distance, in meters. +func DistanceOnEarth(t Turns) Distance { + return Distance(t) * EarthMeanCircumference +} + +// Earth Fact Sheet +// https://nssdc.gsfc.nasa.gov/planetary/factsheet/earthfact.html +const ( + // EarthMeanRadius is the volumetric mean radius of the Earth. + EarthMeanRadius = 6_371_000 * Meter + // EarthMeanCircumference is the volumetric mean circumference of the Earth. + EarthMeanCircumference = 2 * math.Pi * EarthMeanRadius + + // earthEquatorialRadius is the equatorial radius of the Earth. + earthEquatorialRadius = 6_378_137 * Meter + // earthEquatorialCircumference is the equatorial circumference of the Earth. + earthEquatorialCircumference = 2 * math.Pi * earthEquatorialRadius + + // earthPolarRadius is the polar radius of the Earth. + earthPolarRadius = 6_356_752 * Meter + // earthPolarCircumference is the polar circumference of the Earth. + earthPolarCircumference = 2 * math.Pi * earthPolarRadius +) diff --git a/types/geo/units_test.go b/types/geo/units_test.go new file mode 100644 index 0000000000000..b6f724ce0d9b3 --- /dev/null +++ b/types/geo/units_test.go @@ -0,0 +1,395 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package geo_test + +import ( + "math" + "strings" + "testing" + + "tailscale.com/types/geo" +) + +func TestDegrees(t *testing.T) { + for _, tt := range []struct { + name string + degs geo.Degrees + wantStr string + wantText string + wantPad string + wantRads geo.Radians + wantTurns geo.Turns + }{ + { + name: "zero", + degs: 0.0 * geo.Degree, + wantStr: "+0°", + wantText: "+0", + wantPad: "+000", + wantRads: 0.0 * geo.Radian, + wantTurns: 0 * geo.Turn, + }, + { + name: "quarter-turn", + degs: 90.0 * geo.Degree, + wantStr: "+90°", + wantText: "+90", + wantPad: "+090", + wantRads: 0.5 * math.Pi * geo.Radian, + wantTurns: 0.25 * geo.Turn, + }, + { + name: "half-turn", + degs: 180.0 * geo.Degree, + wantStr: "+180°", + wantText: "+180", + wantPad: "+180", + wantRads: 1.0 * math.Pi * geo.Radian, + wantTurns: 0.5 * geo.Turn, + }, + { + name: "full-turn", + degs: 360.0 * geo.Degree, + wantStr: "+360°", + wantText: "+360", + wantPad: "+360", + wantRads: 2.0 * math.Pi * geo.Radian, + wantTurns: 1.0 * geo.Turn, + }, + { + name: "negative-zero", + degs: geo.MustParseDegrees("-0.0"), + wantStr: "-0°", + wantText: "-0", + wantPad: "-000", + wantRads: 0 * geo.Radian * -1, + wantTurns: 0 * geo.Turn * -1, + }, + { + name: "small-degree", + degs: -1.2003 * geo.Degree, + wantStr: "-1.2003°", + wantText: "-1.2003", + wantPad: "-001.2003", + wantRads: -0.020949187011687936 * geo.Radian, + wantTurns: -0.0033341666666666663 * geo.Turn, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if got := tt.degs.String(); got != tt.wantStr { + t.Errorf("String got %q, want %q", got, tt.wantStr) + } + + d, err := geo.ParseDegrees(tt.wantStr) + if err != nil { + t.Fatalf("ParseDegrees err %q, want nil", err.Error()) + } + if d != tt.degs { + t.Errorf("ParseDegrees got %q, want %q", d, tt.degs) + } + + b, err := tt.degs.AppendText(nil) + if err != nil { + t.Fatalf("AppendText err %q, want nil", err.Error()) + } + if string(b) != tt.wantText { + t.Errorf("AppendText got %q, want %q", b, tt.wantText) + } + + b = tt.degs.AppendZeroPaddedText(nil, 3) + if string(b) != tt.wantPad { + t.Errorf("AppendZeroPaddedText got %q, want %q", b, tt.wantPad) + } + + r := tt.degs.Radians() + if r != tt.wantRads { + t.Errorf("Radian got %v, want %v", r, tt.wantRads) + } + if d := r.Degrees(); d != tt.degs { // Roundtrip + t.Errorf("Degrees got %v, want %v", d, tt.degs) + } + + o := tt.degs.Turns() + if o != tt.wantTurns { + t.Errorf("Turns got %v, want %v", o, tt.wantTurns) + } + }) + } +} + +func TestRadians(t *testing.T) { + for _, tt := range []struct { + name string + rads geo.Radians + wantStr string + wantText string + wantDegs geo.Degrees + wantTurns geo.Turns + }{ + { + name: "zero", + rads: 0.0 * geo.Radian, + wantStr: "0 rad", + wantDegs: 0.0 * geo.Degree, + wantTurns: 0 * geo.Turn, + }, + { + name: "quarter-turn", + rads: 0.5 * math.Pi * geo.Radian, + wantStr: "1.5707963267948966 rad", + wantDegs: 90.0 * geo.Degree, + wantTurns: 0.25 * geo.Turn, + }, + { + name: "half-turn", + rads: 1.0 * math.Pi * geo.Radian, + wantStr: "3.141592653589793 rad", + wantDegs: 180.0 * geo.Degree, + wantTurns: 0.5 * geo.Turn, + }, + { + name: "full-turn", + rads: 2.0 * math.Pi * geo.Radian, + wantStr: "6.283185307179586 rad", + wantDegs: 360.0 * geo.Degree, + wantTurns: 1.0 * geo.Turn, + }, + { + name: "negative-zero", + rads: geo.MustParseRadians("-0"), + wantStr: "-0 rad", + wantDegs: 0 * geo.Degree * -1, + wantTurns: 0 * geo.Turn * -1, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if got := tt.rads.String(); got != tt.wantStr { + t.Errorf("String got %q, want %q", got, tt.wantStr) + } + + r, err := geo.ParseRadians(tt.wantStr) + if err != nil { + t.Fatalf("ParseDegrees err %q, want nil", err.Error()) + } + if r != tt.rads { + t.Errorf("ParseDegrees got %q, want %q", r, tt.rads) + } + + d := tt.rads.Degrees() + if d != tt.wantDegs { + t.Errorf("Degrees got %v, want %v", d, tt.wantDegs) + } + if r := d.Radians(); r != tt.rads { // Roundtrip + t.Errorf("Radians got %v, want %v", r, tt.rads) + } + + o := tt.rads.Turns() + if o != tt.wantTurns { + t.Errorf("Turns got %v, want %v", o, tt.wantTurns) + } + }) + } +} + +func TestTurns(t *testing.T) { + for _, tt := range []struct { + name string + turns geo.Turns + wantStr string + wantText string + wantDegs geo.Degrees + wantRads geo.Radians + }{ + { + name: "zero", + turns: 0.0, + wantStr: "0", + wantDegs: 0.0 * geo.Degree, + wantRads: 0 * geo.Radian, + }, + { + name: "quarter-turn", + turns: 0.25, + wantStr: "0.25", + wantDegs: 90.0 * geo.Degree, + wantRads: 0.5 * math.Pi * geo.Radian, + }, + { + name: "half-turn", + turns: 0.5, + wantStr: "0.5", + wantDegs: 180.0 * geo.Degree, + wantRads: 1.0 * math.Pi * geo.Radian, + }, + { + name: "full-turn", + turns: 1.0, + wantStr: "1", + wantDegs: 360.0 * geo.Degree, + wantRads: 2.0 * math.Pi * geo.Radian, + }, + { + name: "negative-zero", + turns: geo.Turns(math.Copysign(0, -1)), + wantStr: "-0", + wantDegs: 0 * geo.Degree * -1, + wantRads: 0 * geo.Radian * -1, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if got := tt.turns.String(); got != tt.wantStr { + t.Errorf("String got %q, want %q", got, tt.wantStr) + } + + d := tt.turns.Degrees() + if d != tt.wantDegs { + t.Errorf("Degrees got %v, want %v", d, tt.wantDegs) + } + if o := d.Turns(); o != tt.turns { // Roundtrip + t.Errorf("Turns got %v, want %v", o, tt.turns) + } + + r := tt.turns.Radians() + if r != tt.wantRads { + t.Errorf("Turns got %v, want %v", r, tt.wantRads) + } + }) + } +} + +func TestDistance(t *testing.T) { + for _, tt := range []struct { + name string + dist geo.Distance + wantStr string + }{ + { + name: "zero", + dist: 0.0 * geo.Meter, + wantStr: "0m", + }, + { + name: "random", + dist: 4 * geo.Meter, + wantStr: "4m", + }, + { + name: "light-second", + dist: 299_792_458 * geo.Meter, + wantStr: "299792458m", + }, + { + name: "planck-length", + dist: 1.61625518e-35 * geo.Meter, + wantStr: "0.0000000000000000000000000000000000161625518m", + }, + { + name: "negative-zero", + dist: geo.Distance(math.Copysign(0, -1)), + wantStr: "-0m", + }, + } { + t.Run(tt.name, func(t *testing.T) { + if got := tt.dist.String(); got != tt.wantStr { + t.Errorf("String got %q, want %q", got, tt.wantStr) + } + + r, err := geo.ParseDistance(tt.wantStr) + if err != nil { + t.Fatalf("ParseDegrees err %q, want nil", err.Error()) + } + if r != tt.dist { + t.Errorf("ParseDegrees got %q, want %q", r, tt.dist) + } + }) + } +} + +func TestDistanceOnEarth(t *testing.T) { + for _, tt := range []struct { + name string + here geo.Point + there geo.Point + want geo.Distance + wantErr string + }{ + { + name: "no-points", + here: geo.Point{}, + there: geo.Point{}, + wantErr: "not a valid point", + }, + { + name: "not-here", + here: geo.Point{}, + there: geo.MakePoint(0, 0), + wantErr: "not a valid point", + }, + { + name: "not-there", + here: geo.MakePoint(0, 0), + there: geo.Point{}, + wantErr: "not a valid point", + }, + { + name: "null-island", + here: geo.MakePoint(0, 0), + there: geo.MakePoint(0, 0), + want: 0 * geo.Meter, + }, + { + name: "equator-to-south-pole", + here: geo.MakePoint(0, 0), + there: geo.MakePoint(-90, 0), + want: geo.EarthMeanCircumference / 4, + }, + { + name: "north-pole-to-south-pole", + here: geo.MakePoint(+90, 0), + there: geo.MakePoint(-90, 0), + want: geo.EarthMeanCircumference / 2, + }, + { + name: "meridian-to-antimeridian", + here: geo.MakePoint(0, 0), + there: geo.MakePoint(0, -180), + want: geo.EarthMeanCircumference / 2, + }, + { + name: "positive-to-negative-antimeridian", + here: geo.MakePoint(0, 180), + there: geo.MakePoint(0, -180), + want: 0 * geo.Meter, + }, + { + name: "toronto-to-montreal", + here: geo.MakePoint(+43.70011, -79.41630), + there: geo.MakePoint(+45.50884, -73.58781), + want: 503_200 * geo.Meter, + }, + { + name: "montreal-to-san-francisco", + here: geo.MakePoint(+45.50884, -73.58781), + there: geo.MakePoint(+37.77493, -122.41942), + want: 4_082_600 * geo.Meter, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.here.DistanceTo(tt.there) + if tt.wantErr == "" && err != nil { + t.Fatalf("err %q, want nil", err) + } + if tt.wantErr != "" && !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err %q, want %q", err, tt.wantErr) + } + + approx := func(x, y geo.Distance) bool { + return math.Abs(float64(x)-float64(y)) <= 10 + } + if !approx(got, tt.want) { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} diff --git a/types/key/derp.go b/types/key/derp.go new file mode 100644 index 0000000000000..1466b85bc5288 --- /dev/null +++ b/types/key/derp.go @@ -0,0 +1,90 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package key + +import ( + "crypto/subtle" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + + "go4.org/mem" + "tailscale.com/types/structs" +) + +var ErrInvalidMeshKey = errors.New("invalid mesh key") + +// DERPMesh is a mesh key, used for inter-DERP-node communication and for +// privileged DERP clients. +type DERPMesh struct { + _ structs.Incomparable // == isn't constant-time + k [32]byte // 64-digit hexadecimal numbers fit in 32 bytes +} + +// MarshalJSON implements the [encoding/json.Marshaler] interface. +func (k DERPMesh) MarshalJSON() ([]byte, error) { + return json.Marshal(k.String()) +} + +// UnmarshalJSON implements the [encoding/json.Unmarshaler] interface. +func (k *DERPMesh) UnmarshalJSON(data []byte) error { + var s string + json.Unmarshal(data, &s) + + if hex.DecodedLen(len(s)) != len(k.k) { + return fmt.Errorf("types/key/derp: cannot unmarshal, incorrect size mesh key len: %d, must be %d, %w", hex.DecodedLen(len(s)), len(k.k), ErrInvalidMeshKey) + } + _, err := hex.Decode(k.k[:], []byte(s)) + if err != nil { + return fmt.Errorf("types/key/derp: cannot unmarshal, invalid mesh key: %w", err) + } + + return nil +} + +// DERPMeshFromRaw32 parses a 32-byte raw value as a DERP mesh key. +func DERPMeshFromRaw32(raw mem.RO) DERPMesh { + if raw.Len() != 32 { + panic("input has wrong size") + } + var ret DERPMesh + raw.Copy(ret.k[:]) + return ret +} + +// ParseDERPMesh parses a DERP mesh key from a string. +// This function trims whitespace around the string. +// If the key is not a 64-digit hexadecimal number, ErrInvalidMeshKey is returned. +func ParseDERPMesh(key string) (DERPMesh, error) { + key = strings.TrimSpace(key) + if len(key) != 64 { + return DERPMesh{}, fmt.Errorf("%w: must be 64-digit hexadecimal number", ErrInvalidMeshKey) + } + decoded, err := hex.DecodeString(key) + if err != nil { + return DERPMesh{}, fmt.Errorf("%w: %v", ErrInvalidMeshKey, err) + } + return DERPMeshFromRaw32(mem.B(decoded)), nil +} + +// IsZero reports whether k is the zero value. +func (k DERPMesh) IsZero() bool { + return k.Equal(DERPMesh{}) +} + +// Equal reports whether k and other are the same key. +func (k DERPMesh) Equal(other DERPMesh) bool { + // Compare mesh keys in constant time to prevent timing attacks. + // Since mesh keys are a fixed length, we don’t need to be concerned + // about timing attacks on client mesh keys that are the wrong length. + // See https://github.com/tailscale/corp/issues/28720 + return subtle.ConstantTimeCompare(k.k[:], other.k[:]) == 1 +} + +// String returns k as a hex-encoded 64-digit number. +func (k DERPMesh) String() string { + return hex.EncodeToString(k.k[:]) +} diff --git a/types/key/derp_test.go b/types/key/derp_test.go new file mode 100644 index 0000000000000..b91cbbf8c4e01 --- /dev/null +++ b/types/key/derp_test.go @@ -0,0 +1,133 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package key + +import ( + "errors" + "testing" + + "go4.org/mem" +) + +func TestDERPMeshIsValid(t *testing.T) { + for name, tt := range map[string]struct { + input string + want string + wantErr error + }{ + "good": { + input: "0123456789012345678901234567890123456789012345678901234567890123", + want: "0123456789012345678901234567890123456789012345678901234567890123", + wantErr: nil, + }, + "hex": { + input: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + want: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + wantErr: nil, + }, + "uppercase": { + input: "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF", + want: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + wantErr: nil, + }, + "whitespace": { + input: " 0123456789012345678901234567890123456789012345678901234567890123 ", + want: "0123456789012345678901234567890123456789012345678901234567890123", + wantErr: nil, + }, + "short": { + input: "0123456789abcdef", + wantErr: ErrInvalidMeshKey, + }, + "long": { + input: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0", + wantErr: ErrInvalidMeshKey, + }, + } { + t.Run(name, func(t *testing.T) { + k, err := ParseDERPMesh(tt.input) + if !errors.Is(err, tt.wantErr) { + t.Errorf("err %v, want %v", err, tt.wantErr) + } + + got := k.String() + if got != tt.want && tt.wantErr == nil { + t.Errorf("got %q, want %q", got, tt.want) + } + + }) + } + +} + +func TestDERPMesh(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + str string + hex []byte + equal bool // are str and hex equal? + }{ + "zero": { + str: "0000000000000000000000000000000000000000000000000000000000000000", + hex: []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + equal: true, + }, + "equal": { + str: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + hex: []byte{ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, + }, + equal: true, + }, + "unequal": { + str: "0badc0de00000000000000000000000000000000000000000000000000000000", + hex: []byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + equal: false, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + k, err := ParseDERPMesh(tt.str) + if err != nil { + t.Fatal(err) + } + + // string representation should round-trip + s := k.String() + if s != tt.str { + t.Fatalf("string %s, want %s", s, tt.str) + } + + // if tt.equal, then tt.hex is intended to be equal + if k.k != [32]byte(tt.hex) && tt.equal { + t.Fatalf("decoded %x, want %x", k.k, tt.hex) + } + + h := DERPMeshFromRaw32(mem.B(tt.hex)) + if k.Equal(h) != tt.equal { + if tt.equal { + t.Fatalf("%v != %v", k, h) + } else { + t.Fatalf("%v == %v", k, h) + } + } + + }) + } +} diff --git a/types/key/disco.go b/types/key/disco.go index 1013ce5bf89af..ce5f9b36fd9a1 100644 --- a/types/key/disco.go +++ b/types/key/disco.go @@ -73,6 +73,44 @@ func (k DiscoPrivate) Shared(p DiscoPublic) DiscoShared { return ret } +// SortedPairOfDiscoPublic is a lexicographically sorted container of two +// [DiscoPublic] keys. +type SortedPairOfDiscoPublic struct { + k [2]DiscoPublic +} + +// Get returns the underlying keys. +func (s SortedPairOfDiscoPublic) Get() [2]DiscoPublic { + return s.k +} + +// NewSortedPairOfDiscoPublic returns a SortedPairOfDiscoPublic from a and b. +func NewSortedPairOfDiscoPublic(a, b DiscoPublic) SortedPairOfDiscoPublic { + s := SortedPairOfDiscoPublic{} + if a.Compare(b) < 0 { + s.k[0] = a + s.k[1] = b + } else { + s.k[0] = b + s.k[1] = a + } + return s +} + +func (s SortedPairOfDiscoPublic) String() string { + return fmt.Sprintf("%s <=> %s", s.k[0].ShortString(), s.k[1].ShortString()) +} + +// Equal returns true if s and b are equal, otherwise it returns false. +func (s SortedPairOfDiscoPublic) Equal(b SortedPairOfDiscoPublic) bool { + for i := range s.k { + if s.k[i].Compare(b.k[i]) != 0 { + return false + } + } + return true +} + // DiscoPublic is the public portion of a DiscoPrivate. type DiscoPublic struct { k [32]byte diff --git a/types/key/disco_test.go b/types/key/disco_test.go index c62c13cbf8970..131fe350f508a 100644 --- a/types/key/disco_test.go +++ b/types/key/disco_test.go @@ -81,3 +81,21 @@ func TestDiscoShared(t *testing.T) { t.Error("k1.Shared(k2) != k2.Shared(k1)") } } + +func TestSortedPairOfDiscoPublic(t *testing.T) { + pubA := DiscoPublic{} + pubA.k[0] = 0x01 + pubB := DiscoPublic{} + pubB.k[0] = 0x02 + sortedInput := NewSortedPairOfDiscoPublic(pubA, pubB) + unsortedInput := NewSortedPairOfDiscoPublic(pubB, pubA) + if sortedInput.Get() != unsortedInput.Get() { + t.Fatal("sortedInput.Get() != unsortedInput.Get()") + } + if unsortedInput.Get()[0] != pubA { + t.Fatal("unsortedInput.Get()[0] != pubA") + } + if unsortedInput.Get()[1] != pubB { + t.Fatal("unsortedInput.Get()[1] != pubB") + } +} diff --git a/types/lazy/map.go b/types/lazy/map.go new file mode 100644 index 0000000000000..75a1dd739d3bc --- /dev/null +++ b/types/lazy/map.go @@ -0,0 +1,62 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package lazy + +import "tailscale.com/util/mak" + +// GMap is a map of lazily computed [GValue] pointers, keyed by a comparable +// type. +// +// Use either Get or GetErr, depending on whether your fill function returns an +// error. +// +// GMap is not safe for concurrent use. +type GMap[K comparable, V any] struct { + store map[K]*GValue[V] +} + +// Len returns the number of entries in the map. +func (s *GMap[K, V]) Len() int { + return len(s.store) +} + +// Set attempts to set the value of k to v, and reports whether it succeeded. +// Set only succeeds if k has never been called with Get/GetErr/Set before. +func (s *GMap[K, V]) Set(k K, v V) bool { + z, ok := s.store[k] + if !ok { + z = new(GValue[V]) + mak.Set(&s.store, k, z) + } + return z.Set(v) +} + +// MustSet sets the value of k to v, or panics if k already has a value. +func (s *GMap[K, V]) MustSet(k K, v V) { + if !s.Set(k, v) { + panic("Set after already filled") + } +} + +// Get returns the value for k, computing it with fill if it's not already +// present. +func (s *GMap[K, V]) Get(k K, fill func() V) V { + z, ok := s.store[k] + if !ok { + z = new(GValue[V]) + mak.Set(&s.store, k, z) + } + return z.Get(fill) +} + +// GetErr returns the value for k, computing it with fill if it's not already +// present. +func (s *GMap[K, V]) GetErr(k K, fill func() (V, error)) (V, error) { + z, ok := s.store[k] + if !ok { + z = new(GValue[V]) + mak.Set(&s.store, k, z) + } + return z.GetErr(fill) +} diff --git a/types/lazy/map_test.go b/types/lazy/map_test.go new file mode 100644 index 0000000000000..ec1152b0b802c --- /dev/null +++ b/types/lazy/map_test.go @@ -0,0 +1,95 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package lazy + +import ( + "errors" + "testing" +) + +func TestGMap(t *testing.T) { + var gm GMap[string, int] + n := int(testing.AllocsPerRun(1000, func() { + got := gm.Get("42", fortyTwo) + if got != 42 { + t.Fatalf("got %v; want 42", got) + } + })) + if n != 0 { + t.Errorf("allocs = %v; want 0", n) + } +} + +func TestGMapErr(t *testing.T) { + var gm GMap[string, int] + n := int(testing.AllocsPerRun(1000, func() { + got, err := gm.GetErr("42", func() (int, error) { + return 42, nil + }) + if got != 42 || err != nil { + t.Fatalf("got %v, %v; want 42, nil", got, err) + } + })) + if n != 0 { + t.Errorf("allocs = %v; want 0", n) + } + + var gmErr GMap[string, int] + wantErr := errors.New("test error") + n = int(testing.AllocsPerRun(1000, func() { + got, err := gmErr.GetErr("42", func() (int, error) { + return 0, wantErr + }) + if got != 0 || err != wantErr { + t.Fatalf("got %v, %v; want 0, %v", got, err, wantErr) + } + })) + if n != 0 { + t.Errorf("allocs = %v; want 0", n) + } +} + +func TestGMapSet(t *testing.T) { + var gm GMap[string, int] + if !gm.Set("42", 42) { + t.Fatalf("Set failed") + } + if gm.Set("42", 43) { + t.Fatalf("Set succeeded after first Set") + } + n := int(testing.AllocsPerRun(1000, func() { + got := gm.Get("42", fortyTwo) + if got != 42 { + t.Fatalf("got %v; want 42", got) + } + })) + if n != 0 { + t.Errorf("allocs = %v; want 0", n) + } +} + +func TestGMapMustSet(t *testing.T) { + var gm GMap[string, int] + gm.MustSet("42", 42) + defer func() { + if e := recover(); e == nil { + t.Errorf("unexpected success; want panic") + } + }() + gm.MustSet("42", 43) +} + +func TestGMapRecursivePanic(t *testing.T) { + defer func() { + if e := recover(); e != nil { + t.Logf("got panic, as expected") + } else { + t.Errorf("unexpected success; want panic") + } + }() + gm := GMap[string, int]{} + gm.Get("42", func() int { + return gm.Get("42", func() int { return 42 }) + }) +} diff --git a/types/netmap/netmap.go b/types/netmap/netmap.go index c6250c49ce9c9..963f80a441ee4 100644 --- a/types/netmap/netmap.go +++ b/types/netmap/netmap.go @@ -54,12 +54,12 @@ type NetworkMap struct { // between updates and should not be modified. DERPMap *tailcfg.DERPMap - // ControlHealth are the list of health check problems for this + // DisplayMessages are the list of health check problems for this // node from the perspective of the control plane. // If empty, there are no known problems from the control plane's // point of view, but the node might know about its own health // check problems. - ControlHealth []string + DisplayMessages map[tailcfg.DisplayMessageID]tailcfg.DisplayMessage // TKAEnabled indicates whether the tailnet key authority should be // enabled, from the perspective of the control plane. diff --git a/types/netmap/nodemut.go b/types/netmap/nodemut.go index e31c731becbf1..f4de1bf0b8f02 100644 --- a/types/netmap/nodemut.go +++ b/types/netmap/nodemut.go @@ -37,7 +37,7 @@ func (m NodeMutationDERPHome) Apply(n *tailcfg.Node) { n.HomeDERP = m.DERPRegion } -// NodeMutation is a NodeMutation that says a node's endpoints have changed. +// NodeMutationEndpoints is a NodeMutation that says a node's endpoints have changed. type NodeMutationEndpoints struct { mutatingNodeID Endpoints []netip.AddrPort @@ -163,6 +163,7 @@ func mapResponseContainsNonPatchFields(res *tailcfg.MapResponse) bool { res.PacketFilters != nil || res.UserProfiles != nil || res.Health != nil || + res.DisplayMessages != nil || res.SSHPolicy != nil || res.TKAInfo != nil || res.DomainDataPlaneAuditLogID != "" || diff --git a/util/eventbus/bus.go b/util/eventbus/bus.go index 45d12da2f3736..e5bf7329a67ee 100644 --- a/util/eventbus/bus.go +++ b/util/eventbus/bus.go @@ -40,8 +40,8 @@ type Bus struct { clients set.Set[*Client] } -// New returns a new bus. Use [PublisherOf] to make event publishers, -// and [Bus.Queue] and [Subscribe] to make event subscribers. +// New returns a new bus. Use [Publish] to make event publishers, +// and [Subscribe] and [SubscribeFunc] to make event subscribers. func New() *Bus { ret := &Bus{ write: make(chan PublishedEvent), diff --git a/util/eventbus/client.go b/util/eventbus/client.go index a7a88c0a158c7..f4261b13c9f45 100644 --- a/util/eventbus/client.go +++ b/util/eventbus/client.go @@ -113,15 +113,16 @@ func (c *Client) shouldPublish(t reflect.Type) bool { // Subscribe requests delivery of events of type T through the given // Queue. Panics if the queue already has a subscriber for T. func Subscribe[T any](c *Client) *Subscriber[T] { - return newSubscriber[T](c.subscribeState()) + r := c.subscribeState() + s := newSubscriber[T](r) + r.addSubscriber(s) + return s } // Publisher returns a publisher for event type T using the given // client. func Publish[T any](c *Client) *Publisher[T] { - ret := newPublisher[T](c) - c.mu.Lock() - defer c.mu.Unlock() - c.pub.Add(ret) - return ret + p := newPublisher[T](c) + c.addPublisher(p) + return p } diff --git a/util/eventbus/debug.go b/util/eventbus/debug.go index 832d72ac07dda..a055f078fc4f2 100644 --- a/util/eventbus/debug.go +++ b/util/eventbus/debug.go @@ -186,3 +186,25 @@ type hookFn[T any] struct { ID uint64 Fn func(T) } + +// DebugEvent is a representation of an event used for debug clients. +type DebugEvent struct { + Count int + Type string + From string + To []string + Event any +} + +// DebugTopics provides the JSON encoding as a wrapper for a collection of [DebugTopic]. +type DebugTopics struct { + Topics []DebugTopic +} + +// DebugTopic provides the JSON encoding of publishers and subscribers for a +// given topic. +type DebugTopic struct { + Name string + Publisher string + Subscribers []string +} diff --git a/util/eventbus/doc.go b/util/eventbus/doc.go index 964a686eae109..f95f9398c8de9 100644 --- a/util/eventbus/doc.go +++ b/util/eventbus/doc.go @@ -89,4 +89,14 @@ // The [Debugger], obtained through [Bus.Debugger], provides // introspection facilities to monitor events flowing through the bus, // and inspect publisher and subscriber state. +// +// Additionally, a debug command exists for monitoring the eventbus: +// +// tailscale debug daemon-bus-events +// +// # Testing facilities +// +// Helpers for testing code with the eventbus can be found in: +// +// eventbus/eventbustest package eventbus diff --git a/util/eventbus/eventbustest/doc.go b/util/eventbus/eventbustest/doc.go new file mode 100644 index 0000000000000..9e39504a83521 --- /dev/null +++ b/util/eventbus/eventbustest/doc.go @@ -0,0 +1,45 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package eventbustest provides helper methods for testing an [eventbus.Bus]. +// +// # Usage +// +// A [Watcher] presents a set of generic helpers for testing events. +// +// To test code that generates events, create a [Watcher] from the [eventbus.Bus] +// used by the code under test, run the code to generate events, then use the watcher +// to verify that the expected events were produced. In outline: +// +// bus := eventbustest.NewBus(t) +// tw := eventbustest.NewWatcher(t, bus) +// somethingThatEmitsSomeEvent() +// if err := eventbustest.Expect(tw, eventbustest.Type[EventFoo]()); err != nil { +// t.Error(err.Error()) +// } +// +// As shown, [Expect] checks that at least one event of the given type occurs +// in the stream generated by the code under test. +// +// The following functions all take an any parameter representing a function. +// This function will take an argument of the expected type and is used to test +// for the events on the eventbus being of the given type. The function can +// take the shape described in [Expect]. +// +// [Type] is a helper for only testing event type. +// +// To check for specific properties of an event, use [Expect], and pass a function +// as the second argument that tests for those properties. +// +// To test for multiple events, use [Expect], which checks that the stream +// contains the given events in the given order, possibly with other events +// interspersed. +// +// To test the complete contents of the stream, use [ExpectExactly], which +// checks that the stream contains exactly the given events in the given order, +// and no others. +// +// See the [usage examples]. +// +// [usage examples]: https://github.com/tailscale/tailscale/blob/main/util/eventbus/eventbustest/examples_test.go +package eventbustest diff --git a/util/eventbus/eventbustest/eventbustest.go b/util/eventbus/eventbustest/eventbustest.go new file mode 100644 index 0000000000000..75d430d53683e --- /dev/null +++ b/util/eventbus/eventbustest/eventbustest.go @@ -0,0 +1,203 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package eventbustest + +import ( + "errors" + "fmt" + "reflect" + "testing" + "time" + + "tailscale.com/util/eventbus" +) + +// NewBus constructs an [eventbus.Bus] that will be shut automatically when +// its controlling test ends. +func NewBus(t *testing.T) *eventbus.Bus { + bus := eventbus.New() + t.Cleanup(bus.Close) + return bus +} + +// NewTestWatcher constructs a [Watcher] that can be used to check the stream of +// events generated by code under test. After construction the caller may use +// [Expect] and [ExpectExactly], to verify that the desired events were captured. +func NewWatcher(t *testing.T, bus *eventbus.Bus) *Watcher { + tw := &Watcher{ + mon: bus.Debugger().WatchBus(), + TimeOut: 5 * time.Second, + chDone: make(chan bool, 1), + events: make(chan any, 100), + } + if deadline, ok := t.Deadline(); ok { + tw.TimeOut = deadline.Sub(time.Now()) + } + t.Cleanup(tw.done) + go tw.watch() + return tw +} + +// Watcher monitors and holds events for test expectations. +type Watcher struct { + mon *eventbus.Subscriber[eventbus.RoutedEvent] + events chan any + chDone chan bool + // TimeOut defines when the Expect* functions should stop looking for events + // coming from the Watcher. The value is set by [NewWatcher] and defaults to + // the deadline passed in by [testing.T]. If looking to verify the absence + // of an event, the TimeOut can be set to a lower value after creating the + // Watcher. + TimeOut time.Duration +} + +// Type is a helper representing the expectation to see an event of type T, without +// caring about the content of the event. +// It makes it possible to use helpers like: +// +// eventbustest.ExpectFilter(tw, eventbustest.Type[EventFoo]()) +func Type[T any]() func(T) { return func(T) {} } + +// Expect verifies that the given events are a subsequence of the events +// observed by tw. That is, tw must contain at least one event matching the type +// of each argument in the given order, other event types are allowed to occur in +// between without error. The given events are represented by a function +// that must have one of the following forms: +// +// // Tests for the event type only +// func(e ExpectedType) +// +// // Tests for event type and whatever is defined in the body. +// // If return is false, the test will look for other events of that type +// // If return is true, the test will look for the next given event +// // if a list is given +// func(e ExpectedType) bool +// +// // Tests for event type and whatever is defined in the body. +// // The boolean return works as above. +// // The if error != nil, the test helper will return that error immediately. +// func(e ExpectedType) (bool, error) +// +// If the list of events must match exactly with no extra events, +// use [ExpectExactly]. +func Expect(tw *Watcher, filters ...any) error { + if len(filters) == 0 { + return errors.New("no event filters were provided") + } + eventCount := 0 + head := 0 + for head < len(filters) { + eventFunc := eventFilter(filters[head]) + select { + case event := <-tw.events: + eventCount++ + if ok, err := eventFunc(event); err != nil { + return err + } else if ok { + head++ + } + case <-time.After(tw.TimeOut): + return fmt.Errorf( + "timed out waiting for event, saw %d events, %d was expected", + eventCount, head) + case <-tw.chDone: + return errors.New("watcher closed while waiting for events") + } + } + return nil +} + +// ExpectExactly checks for some number of events showing up on the event bus +// in a given order, returning an error if the events does not match the given list +// exactly. The given events are represented by a function as described in +// [Expect]. Use [Expect] if other events are allowed. +func ExpectExactly(tw *Watcher, filters ...any) error { + if len(filters) == 0 { + return errors.New("no event filters were provided") + } + eventCount := 0 + for pos, next := range filters { + eventFunc := eventFilter(next) + fnType := reflect.TypeOf(next) + argType := fnType.In(0) + select { + case event := <-tw.events: + eventCount++ + typeEvent := reflect.TypeOf(event) + if typeEvent != argType { + return fmt.Errorf( + "expected event type %s, saw %s, at index %d", + argType, typeEvent, pos) + } else if ok, err := eventFunc(event); err != nil { + return err + } else if !ok { + return fmt.Errorf( + "expected test ok for type %s, at index %d", argType, pos) + } + case <-time.After(tw.TimeOut): + return fmt.Errorf( + "timed out waiting for event, saw %d events, %d was expected", + eventCount, pos) + case <-tw.chDone: + return errors.New("watcher closed while waiting for events") + } + } + return nil +} + +func (tw *Watcher) watch() { + for { + select { + case event := <-tw.mon.Events(): + tw.events <- event.Event + case <-tw.chDone: + tw.mon.Close() + return + } + } +} + +// done tells the watcher to stop monitoring for new events. +func (tw *Watcher) done() { + close(tw.chDone) +} + +type filter = func(any) (bool, error) + +func eventFilter(f any) filter { + ft := reflect.TypeOf(f) + if ft.Kind() != reflect.Func { + panic("filter is not a function") + } else if ft.NumIn() != 1 { + panic(fmt.Sprintf("function takes %d arguments, want 1", ft.NumIn())) + } + var fixup func([]reflect.Value) []reflect.Value + switch ft.NumOut() { + case 0: + fixup = func([]reflect.Value) []reflect.Value { + return []reflect.Value{reflect.ValueOf(true), reflect.Zero(reflect.TypeFor[error]())} + } + case 1: + if ft.Out(0) != reflect.TypeFor[bool]() { + panic(fmt.Sprintf("result is %T, want bool", ft.Out(0))) + } + fixup = func(vals []reflect.Value) []reflect.Value { + return append(vals, reflect.Zero(reflect.TypeFor[error]())) + } + case 2: + if ft.Out(0) != reflect.TypeFor[bool]() || ft.Out(1) != reflect.TypeFor[error]() { + panic(fmt.Sprintf("results are %T, %T; want bool, error", ft.Out(0), ft.Out(1))) + } + fixup = func(vals []reflect.Value) []reflect.Value { return vals } + default: + panic(fmt.Sprintf("function returns %d values", ft.NumOut())) + } + fv := reflect.ValueOf(f) + return reflect.MakeFunc(reflect.TypeFor[filter](), func(args []reflect.Value) []reflect.Value { + if !args[0].IsValid() || args[0].Elem().Type() != ft.In(0) { + return []reflect.Value{reflect.ValueOf(false), reflect.Zero(reflect.TypeFor[error]())} + } + return fixup(fv.Call([]reflect.Value{args[0].Elem()})) + }).Interface().(filter) +} diff --git a/util/eventbus/eventbustest/eventbustest_test.go b/util/eventbus/eventbustest/eventbustest_test.go new file mode 100644 index 0000000000000..fd95973e5538d --- /dev/null +++ b/util/eventbus/eventbustest/eventbustest_test.go @@ -0,0 +1,366 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package eventbustest_test + +import ( + "fmt" + "testing" + "time" + + "tailscale.com/util/eventbus" + "tailscale.com/util/eventbus/eventbustest" +) + +type EventFoo struct { + Value int +} + +type EventBar struct { + Value string +} + +type EventBaz struct { + Value []float64 +} + +func TestExpectFilter(t *testing.T) { + tests := []struct { + name string + events []int + expectFunc any + wantErr bool + }{ + { + name: "single event", + events: []int{42}, + expectFunc: eventbustest.Type[EventFoo](), + wantErr: false, + }, + { + name: "multiple events, single expectation", + events: []int{42, 1, 2, 3, 4, 5}, + expectFunc: eventbustest.Type[EventFoo](), + wantErr: false, + }, + { + name: "filter on event with function", + events: []int{24, 42}, + expectFunc: func(event EventFoo) (bool, error) { + if event.Value == 42 { + return true, nil + } + return false, nil + }, + wantErr: false, + }, + { + name: "first event has to be func", + events: []int{24, 42}, + expectFunc: func(event EventFoo) (bool, error) { + if event.Value != 42 { + return false, fmt.Errorf("expected 42, got %d", event.Value) + } + return false, nil + }, + wantErr: true, + }, + { + name: "no events", + events: []int{}, + expectFunc: func(event EventFoo) (bool, error) { + return true, nil + }, + wantErr: true, + }, + } + + bus := eventbustest.NewBus(t) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tw := eventbustest.NewWatcher(t, bus) + // TODO(cmol): When synctest is out of experimental, use that instead: + // https://go.dev/blog/synctest + tw.TimeOut = 10 * time.Millisecond + + client := bus.Client("testClient") + defer client.Close() + updater := eventbus.Publish[EventFoo](client) + + for _, i := range tt.events { + updater.Publish(EventFoo{i}) + } + + if err := eventbustest.Expect(tw, tt.expectFunc); (err != nil) != tt.wantErr { + t.Errorf("ExpectFilter[EventFoo]: error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestExpectEvents(t *testing.T) { + tests := []struct { + name string + events []any + expectEvents []any + wantErr bool + }{ + { + name: "No expectations", + events: []any{EventFoo{}}, + expectEvents: []any{}, + wantErr: true, + }, + { + name: "One event", + events: []any{EventFoo{}}, + expectEvents: []any{eventbustest.Type[EventFoo]()}, + wantErr: false, + }, + { + name: "Two events", + events: []any{EventFoo{}, EventBar{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: false, + }, + { + name: "Two expected events with another in the middle", + events: []any{EventFoo{}, EventBaz{}, EventBar{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: false, + }, + { + name: "Missing event", + events: []any{EventFoo{}, EventBaz{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: true, + }, + { + name: "One event with specific value", + events: []any{EventFoo{42}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: false, + }, + { + name: "Two event with one specific value", + events: []any{EventFoo{43}, EventFoo{42}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: false, + }, + { + name: "One event with wrong value", + events: []any{EventFoo{43}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: true, + }, + { + name: "Two events with specific values", + events: []any{EventFoo{42}, EventFoo{42}, EventBar{"42"}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + func(ev EventBar) (bool, error) { + if ev.Value == "42" { + return true, nil + } + return false, nil + }, + }, + wantErr: false, + }, + } + + bus := eventbustest.NewBus(t) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tw := eventbustest.NewWatcher(t, bus) + // TODO(cmol): When synctest is out of experimental, use that instead: + // https://go.dev/blog/synctest + tw.TimeOut = 10 * time.Millisecond + + client := bus.Client("testClient") + defer client.Close() + updaterFoo := eventbus.Publish[EventFoo](client) + updaterBar := eventbus.Publish[EventBar](client) + updaterBaz := eventbus.Publish[EventBaz](client) + + for _, ev := range tt.events { + switch ev.(type) { + case EventFoo: + evCast := ev.(EventFoo) + updaterFoo.Publish(evCast) + case EventBar: + evCast := ev.(EventBar) + updaterBar.Publish(evCast) + case EventBaz: + evCast := ev.(EventBaz) + updaterBaz.Publish(evCast) + } + } + + if err := eventbustest.Expect(tw, tt.expectEvents...); (err != nil) != tt.wantErr { + t.Errorf("ExpectEvents: error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestExpectExactlyEventsFilter(t *testing.T) { + tests := []struct { + name string + events []any + expectEvents []any + wantErr bool + }{ + { + name: "No expectations", + events: []any{EventFoo{}}, + expectEvents: []any{}, + wantErr: true, + }, + { + name: "One event", + events: []any{EventFoo{}}, + expectEvents: []any{eventbustest.Type[EventFoo]()}, + wantErr: false, + }, + { + name: "Two events", + events: []any{EventFoo{}, EventBar{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: false, + }, + { + name: "Two expected events with another in the middle", + events: []any{EventFoo{}, EventBaz{}, EventBar{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: true, + }, + { + name: "Missing event", + events: []any{EventFoo{}, EventBaz{}}, + expectEvents: []any{eventbustest.Type[EventFoo](), eventbustest.Type[EventBar]()}, + wantErr: true, + }, + { + name: "One event with value", + events: []any{EventFoo{42}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: false, + }, + { + name: "Two event with one specific value", + events: []any{EventFoo{43}, EventFoo{42}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: true, + }, + { + name: "One event with wrong value", + events: []any{EventFoo{43}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + }, + wantErr: true, + }, + { + name: "Two events with specific values", + events: []any{EventFoo{42}, EventFoo{42}, EventBar{"42"}}, + expectEvents: []any{ + func(ev EventFoo) (bool, error) { + if ev.Value == 42 { + return true, nil + } + return false, nil + }, + func(ev EventBar) (bool, error) { + if ev.Value == "42" { + return true, nil + } + return false, nil + }, + }, + wantErr: true, + }, + } + + bus := eventbustest.NewBus(t) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tw := eventbustest.NewWatcher(t, bus) + // TODO(cmol): When synctest is out of experimental, use that instead: + // https://go.dev/blog/synctest + tw.TimeOut = 10 * time.Millisecond + + client := bus.Client("testClient") + defer client.Close() + updaterFoo := eventbus.Publish[EventFoo](client) + updaterBar := eventbus.Publish[EventBar](client) + updaterBaz := eventbus.Publish[EventBaz](client) + + for _, ev := range tt.events { + switch ev.(type) { + case EventFoo: + evCast := ev.(EventFoo) + updaterFoo.Publish(evCast) + case EventBar: + evCast := ev.(EventBar) + updaterBar.Publish(evCast) + case EventBaz: + evCast := ev.(EventBaz) + updaterBaz.Publish(evCast) + } + } + + if err := eventbustest.ExpectExactly(tw, tt.expectEvents...); (err != nil) != tt.wantErr { + t.Errorf("ExpectEvents: error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/util/eventbus/eventbustest/examples_test.go b/util/eventbus/eventbustest/examples_test.go new file mode 100644 index 0000000000000..914e29933b2a2 --- /dev/null +++ b/util/eventbus/eventbustest/examples_test.go @@ -0,0 +1,201 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package eventbustest_test + +import ( + "testing" + + "tailscale.com/util/eventbus" + "tailscale.com/util/eventbus/eventbustest" +) + +func TestExample_Expect(t *testing.T) { + type eventOfInterest struct{} + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updater := eventbus.Publish[eventOfInterest](client) + updater.Publish(eventOfInterest{}) + + if err := eventbustest.Expect(tw, eventbustest.Type[eventOfInterest]()); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } + // Output: + // OK +} + +func TestExample_Expect_WithFunction(t *testing.T) { + type eventOfInterest struct { + value int + } + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updater := eventbus.Publish[eventOfInterest](client) + updater.Publish(eventOfInterest{43}) + updater.Publish(eventOfInterest{42}) + + // Look for an event of eventOfInterest with a specific value + if err := eventbustest.Expect(tw, func(event eventOfInterest) (bool, error) { + if event.value != 42 { + return false, nil // Look for another event with the expected value. + // You could alternatively return an error here to ensure that the + // first seen eventOfInterest matches the value: + // return false, fmt.Errorf("expected 42, got %d", event.value) + } + return true, nil + }); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } + // Output: + // OK +} + +func TestExample_Expect_MultipleEvents(t *testing.T) { + type eventOfInterest struct{} + type eventOfNoConcern struct{} + type eventOfCuriosity struct{} + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updaterInterest := eventbus.Publish[eventOfInterest](client) + updaterConcern := eventbus.Publish[eventOfNoConcern](client) + updaterCuriosity := eventbus.Publish[eventOfCuriosity](client) + updaterInterest.Publish(eventOfInterest{}) + updaterConcern.Publish(eventOfNoConcern{}) + updaterCuriosity.Publish(eventOfCuriosity{}) + + // Even though three events was published, we just care about the two + if err := eventbustest.Expect(tw, + eventbustest.Type[eventOfInterest](), + eventbustest.Type[eventOfCuriosity]()); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } + // Output: + // OK +} + +func TestExample_ExpectExactly_MultipleEvents(t *testing.T) { + type eventOfInterest struct{} + type eventOfNoConcern struct{} + type eventOfCuriosity struct{} + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updaterInterest := eventbus.Publish[eventOfInterest](client) + updaterConcern := eventbus.Publish[eventOfNoConcern](client) + updaterCuriosity := eventbus.Publish[eventOfCuriosity](client) + updaterInterest.Publish(eventOfInterest{}) + updaterConcern.Publish(eventOfNoConcern{}) + updaterCuriosity.Publish(eventOfCuriosity{}) + + // Will fail as more events than the two expected comes in + if err := eventbustest.ExpectExactly(tw, + eventbustest.Type[eventOfInterest](), + eventbustest.Type[eventOfCuriosity]()); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } +} + +func TestExample_Expect_WithMultipleFunctions(t *testing.T) { + type eventOfInterest struct { + value int + } + type eventOfNoConcern struct{} + type eventOfCuriosity struct { + value string + } + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updaterInterest := eventbus.Publish[eventOfInterest](client) + updaterConcern := eventbus.Publish[eventOfNoConcern](client) + updaterCuriosity := eventbus.Publish[eventOfCuriosity](client) + updaterInterest.Publish(eventOfInterest{42}) + updaterConcern.Publish(eventOfNoConcern{}) + updaterCuriosity.Publish(eventOfCuriosity{"42"}) + + interest := func(event eventOfInterest) (bool, error) { + if event.value == 42 { + return true, nil + } + return false, nil + } + curiosity := func(event eventOfCuriosity) (bool, error) { + if event.value == "42" { + return true, nil + } + return false, nil + } + + // Will fail as more events than the two expected comes in + if err := eventbustest.Expect(tw, interest, curiosity); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } + // Output: + // OK +} + +func TestExample_ExpectExactly_WithMultipleFuncions(t *testing.T) { + type eventOfInterest struct { + value int + } + type eventOfNoConcern struct{} + type eventOfCuriosity struct { + value string + } + + bus := eventbustest.NewBus(t) + tw := eventbustest.NewWatcher(t, bus) + + client := bus.Client("testClient") + updaterInterest := eventbus.Publish[eventOfInterest](client) + updaterConcern := eventbus.Publish[eventOfNoConcern](client) + updaterCuriosity := eventbus.Publish[eventOfCuriosity](client) + updaterInterest.Publish(eventOfInterest{42}) + updaterConcern.Publish(eventOfNoConcern{}) + updaterCuriosity.Publish(eventOfCuriosity{"42"}) + + interest := func(event eventOfInterest) (bool, error) { + if event.value == 42 { + return true, nil + } + return false, nil + } + curiosity := func(event eventOfCuriosity) (bool, error) { + if event.value == "42" { + return true, nil + } + return false, nil + } + + // Will fail as more events than the two expected comes in + if err := eventbustest.ExpectExactly(tw, interest, curiosity); err != nil { + t.Log(err.Error()) + } else { + t.Log("OK") + } + // Output: + // expected event type eventbustest.eventOfCuriosity, saw eventbustest.eventOfNoConcern, at index 1 +} diff --git a/util/eventbus/publish.go b/util/eventbus/publish.go index 9897114b64973..4a4bdfb7eda11 100644 --- a/util/eventbus/publish.go +++ b/util/eventbus/publish.go @@ -21,11 +21,7 @@ type Publisher[T any] struct { } func newPublisher[T any](c *Client) *Publisher[T] { - ret := &Publisher[T]{ - client: c, - } - c.addPublisher(ret) - return ret + return &Publisher[T]{client: c} } // Close closes the publisher. diff --git a/util/eventbus/subscribe.go b/util/eventbus/subscribe.go index ba17e85484655..ee534781a2cce 100644 --- a/util/eventbus/subscribe.go +++ b/util/eventbus/subscribe.go @@ -91,7 +91,7 @@ func (q *subscribeState) pump(ctx context.Context) { } } else { // Keep the cases in this select in sync with - // Subscriber.dispatch below. The only different should be + // Subscriber.dispatch below. The only difference should be // that this select doesn't deliver queued values to // anyone, and unconditionally accepts new values. select { @@ -134,9 +134,10 @@ func (s *subscribeState) subscribeTypes() []reflect.Type { return ret } -func (s *subscribeState) addSubscriber(t reflect.Type, sub subscriber) { +func (s *subscribeState) addSubscriber(sub subscriber) { s.outputsMu.Lock() defer s.outputsMu.Unlock() + t := sub.subscribeType() if s.outputs[t] != nil { panic(fmt.Errorf("double subscription for event %s", t)) } @@ -183,15 +184,10 @@ type Subscriber[T any] struct { } func newSubscriber[T any](r *subscribeState) *Subscriber[T] { - t := reflect.TypeFor[T]() - - ret := &Subscriber[T]{ + return &Subscriber[T]{ read: make(chan T), - unregister: func() { r.deleteSubscriber(t) }, + unregister: func() { r.deleteSubscriber(reflect.TypeFor[T]()) }, } - r.addSubscriber(t, ret) - - return ret } func newMonitor[T any](attach func(fn func(T)) (cancel func())) *Subscriber[T] { diff --git a/util/jsonutil/types.go b/util/jsonutil/types.go deleted file mode 100644 index 057473249f258..0000000000000 --- a/util/jsonutil/types.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package jsonutil - -// Bytes is a byte slice in a json-encoded struct. -// encoding/json assumes that []byte fields are hex-encoded. -// Bytes are not hex-encoded; they are treated the same as strings. -// This can avoid unnecessary allocations due to a round trip through strings. -type Bytes []byte - -func (b *Bytes) UnmarshalText(text []byte) error { - // Copy the contexts of text. - *b = append(*b, text...) - return nil -} diff --git a/util/jsonutil/unmarshal.go b/util/jsonutil/unmarshal.go deleted file mode 100644 index b1eb4ea873e67..0000000000000 --- a/util/jsonutil/unmarshal.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -// Package jsonutil provides utilities to improve JSON performance. -// It includes an Unmarshal wrapper that amortizes allocated garbage over subsequent runs -// and a Bytes type to reduce allocations when unmarshalling a non-hex-encoded string into a []byte. -package jsonutil - -import ( - "bytes" - "encoding/json" - "sync" -) - -// decoder is a re-usable json decoder. -type decoder struct { - dec *json.Decoder - r *bytes.Reader -} - -var readerPool = sync.Pool{ - New: func() any { - return bytes.NewReader(nil) - }, -} - -var decoderPool = sync.Pool{ - New: func() any { - var d decoder - d.r = readerPool.Get().(*bytes.Reader) - d.dec = json.NewDecoder(d.r) - return &d - }, -} - -// Unmarshal is similar to encoding/json.Unmarshal. -// There are three major differences: -// -// On error, encoding/json.Unmarshal zeros v. -// This Unmarshal may leave partial data in v. -// Always check the error before using v! -// (Future improvements may remove this bug.) -// -// The errors they return don't always match perfectly. -// If you do error matching more precise than err != nil, -// don't use this Unmarshal. -// -// This Unmarshal allocates considerably less memory. -func Unmarshal(b []byte, v any) error { - d := decoderPool.Get().(*decoder) - d.r.Reset(b) - off := d.dec.InputOffset() - err := d.dec.Decode(v) - d.r.Reset(nil) // don't keep a reference to b - // In case of error, report the offset in this byte slice, - // instead of in the totality of all bytes this decoder has processed. - // It is not possible to make all errors match json.Unmarshal exactly, - // but we can at least try. - switch jsonerr := err.(type) { - case *json.SyntaxError: - jsonerr.Offset -= off - case *json.UnmarshalTypeError: - jsonerr.Offset -= off - case nil: - // json.Unmarshal fails if there's any extra junk in the input. - // json.Decoder does not; see https://github.com/golang/go/issues/36225. - // We need to check for anything left over in the buffer. - if d.dec.More() { - // TODO: Provide a better error message. - // Unfortunately, we can't set the msg field. - // The offset doesn't perfectly match json: - // Ours is at the end of the valid data, - // and theirs is at the beginning of the extra data after whitespace. - // Close enough, though. - err = &json.SyntaxError{Offset: d.dec.InputOffset() - off} - - // TODO: zero v. This is hard; see encoding/json.indirect. - } - } - if err == nil { - decoderPool.Put(d) - } else { - // There might be junk left in the decoder's buffer. - // There's no way to flush it, no Reset method. - // Abandoned the decoder but reuse the reader. - readerPool.Put(d.r) - } - return err -} diff --git a/util/jsonutil/unmarshal_test.go b/util/jsonutil/unmarshal_test.go deleted file mode 100644 index 32f8402f02e58..0000000000000 --- a/util/jsonutil/unmarshal_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause - -package jsonutil - -import ( - "encoding/json" - "reflect" - "testing" -) - -func TestCompareToStd(t *testing.T) { - tests := []string{ - `{}`, - `{"a": 1}`, - `{]`, - `"abc"`, - `5`, - `{"a": 1} `, - `{"a": 1} {}`, - `{} bad data`, - `{"a": 1} "hello"`, - `[]`, - ` {"x": {"t": [3,4,5]}}`, - } - - for _, test := range tests { - b := []byte(test) - var ourV, stdV any - ourErr := Unmarshal(b, &ourV) - stdErr := json.Unmarshal(b, &stdV) - if (ourErr == nil) != (stdErr == nil) { - t.Errorf("Unmarshal(%q): our err = %#[2]v (%[2]T), std err = %#[3]v (%[3]T)", test, ourErr, stdErr) - } - // if !reflect.DeepEqual(ourErr, stdErr) { - // t.Logf("Unmarshal(%q): our err = %#[2]v (%[2]T), std err = %#[3]v (%[3]T)", test, ourErr, stdErr) - // } - if ourErr != nil { - // TODO: if we zero ourV on error, remove this continue. - continue - } - if !reflect.DeepEqual(ourV, stdV) { - t.Errorf("Unmarshal(%q): our val = %v, std val = %v", test, ourV, stdV) - } - } -} - -func BenchmarkUnmarshal(b *testing.B) { - var m any - j := []byte("5") - b.ReportAllocs() - for range b.N { - Unmarshal(j, &m) - } -} - -func BenchmarkStdUnmarshal(b *testing.B) { - var m any - j := []byte("5") - b.ReportAllocs() - for range b.N { - json.Unmarshal(j, &m) - } -} diff --git a/util/linuxfw/detector.go b/util/linuxfw/detector.go index f3ee4aa0b84f0..fffa523afdcf4 100644 --- a/util/linuxfw/detector.go +++ b/util/linuxfw/detector.go @@ -23,6 +23,11 @@ func detectFirewallMode(logf logger.Logf, prefHint string) FirewallMode { hostinfo.SetFirewallMode("nft-gokrazy") return FirewallModeNfTables } + if distro.Get() == distro.JetKVM { + // JetKVM doesn't have iptables. + hostinfo.SetFirewallMode("nft-jetkvm") + return FirewallModeNfTables + } mode := envknob.String("TS_DEBUG_FIREWALL_MODE") // If the envknob isn't set, fall back to the pref suggested by c2n or diff --git a/util/linuxfw/iptables_runner.go b/util/linuxfw/iptables_runner.go index a4fea85c8a99a..ba1d7c1574cd1 100644 --- a/util/linuxfw/iptables_runner.go +++ b/util/linuxfw/iptables_runner.go @@ -700,8 +700,9 @@ func (i *iptablesRunner) DelMagicsockPortRule(port uint16, network string) error // IPTablesCleanUp removes all Tailscale added iptables rules. // Any errors that occur are logged to the provided logf. func IPTablesCleanUp(logf logger.Logf) { - if distro.Get() == distro.Gokrazy { - // Gokrazy uses nftables and doesn't have the "iptables" command. + switch distro.Get() { + case distro.Gokrazy, distro.JetKVM: + // These use nftables and don't have the "iptables" command. // Avoid log spam on cleanup. (#12277) return } diff --git a/util/must/must.go b/util/must/must.go index 21965daa9b038..a292da2268c27 100644 --- a/util/must/must.go +++ b/util/must/must.go @@ -23,3 +23,11 @@ func Get[T any](v T, err error) T { } return v } + +// Get2 returns v1 and v2 as is. It panics if err is non-nil. +func Get2[T any, U any](v1 T, v2 U, err error) (T, U) { + if err != nil { + panic(err) + } + return v1, v2 +} diff --git a/util/prompt/prompt.go b/util/prompt/prompt.go new file mode 100644 index 0000000000000..4e589ceb32b52 --- /dev/null +++ b/util/prompt/prompt.go @@ -0,0 +1,24 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +// Package prompt provides a simple way to prompt the user for input. +package prompt + +import ( + "fmt" + "strings" +) + +// YesNo takes a question and prompts the user to answer the +// question with a yes or no. It appends a [y/n] to the message. +func YesNo(msg string) bool { + fmt.Print(msg + " [y/n] ") + var resp string + fmt.Scanln(&resp) + resp = strings.ToLower(resp) + switch resp { + case "y", "yes", "sure": + return true + } + return false +} diff --git a/util/set/intset.go b/util/set/intset.go new file mode 100644 index 0000000000000..b747d3bffa9fd --- /dev/null +++ b/util/set/intset.go @@ -0,0 +1,172 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package set + +import ( + "iter" + "maps" + "math/bits" + "math/rand/v2" + + "golang.org/x/exp/constraints" + "tailscale.com/util/mak" +) + +// IntSet is a set optimized for integer values close to zero +// or set of integers that are close in value. +type IntSet[T constraints.Integer] struct { + // bits is a [bitSet] for numbers less than [bits.UintSize]. + bits bitSet + + // extra is a mapping of [bitSet] for numbers not in bits, + // where the key is a number modulo [bits.UintSize]. + extra map[uint64]bitSet + + // extraLen is the count of numbers in extra since len(extra) + // does not reflect that each bitSet may have multiple numbers. + extraLen int +} + +// Values returns an iterator over the elements of the set. +// The iterator will yield the elements in no particular order. +func (s IntSet[T]) Values() iter.Seq[T] { + return func(yield func(T) bool) { + if s.bits != 0 { + for i := range s.bits.values() { + if !yield(decodeZigZag[T](i)) { + return + } + } + } + if s.extra != nil { + for hi, bs := range s.extra { + for lo := range bs.values() { + if !yield(decodeZigZag[T](hi*bits.UintSize + lo)) { + return + } + } + } + } + } +} + +// Contains reports whether e is in the set. +func (s IntSet[T]) Contains(e T) bool { + if v := encodeZigZag(e); v < bits.UintSize { + return s.bits.contains(v) + } else { + hi, lo := v/uint64(bits.UintSize), v%uint64(bits.UintSize) + return s.extra[hi].contains(lo) + } +} + +// Add adds e to the set. +// +// When storing a IntSet in a map as a value type, +// it is important to re-assign the map entry after calling Add or Delete, +// as the IntSet's representation may change. +func (s *IntSet[T]) Add(e T) { + if v := encodeZigZag(e); v < bits.UintSize { + s.bits.add(v) + } else { + hi, lo := v/uint64(bits.UintSize), v%uint64(bits.UintSize) + if bs := s.extra[hi]; !bs.contains(lo) { + bs.add(lo) + mak.Set(&s.extra, hi, bs) + s.extra[hi] = bs + s.extraLen++ + } + } +} + +// AddSeq adds the values from seq to the set. +func (s *IntSet[T]) AddSeq(seq iter.Seq[T]) { + for e := range seq { + s.Add(e) + } +} + +// Len reports the number of elements in the set. +func (s IntSet[T]) Len() int { + return s.bits.len() + s.extraLen +} + +// Delete removes e from the set. +// +// When storing a IntSet in a map as a value type, +// it is important to re-assign the map entry after calling Add or Delete, +// as the IntSet's representation may change. +func (s *IntSet[T]) Delete(e T) { + if v := encodeZigZag(e); v < bits.UintSize { + s.bits.delete(v) + } else { + hi, lo := v/uint64(bits.UintSize), v%uint64(bits.UintSize) + if bs := s.extra[hi]; bs.contains(lo) { + bs.delete(lo) + mak.Set(&s.extra, hi, bs) + s.extra[hi] = bs + s.extraLen-- + } + } +} + +// Clone returns a copy of s that doesn't alias the original. +func (s IntSet[T]) Clone() IntSet[T] { + return IntSet[T]{ + bits: s.bits, + extra: maps.Clone(s.extra), + extraLen: s.extraLen, + } +} + +type bitSet uint + +func (s bitSet) values() iter.Seq[uint64] { + return func(yield func(uint64) bool) { + // Hyrum-proofing: randomly iterate in forwards or reverse. + if rand.Uint64()%2 == 0 { + for i := 0; i < bits.UintSize; i++ { + if s.contains(uint64(i)) && !yield(uint64(i)) { + return + } + } + } else { + for i := bits.UintSize; i >= 0; i-- { + if s.contains(uint64(i)) && !yield(uint64(i)) { + return + } + } + } + } +} +func (s bitSet) len() int { return bits.OnesCount(uint(s)) } +func (s bitSet) contains(i uint64) bool { return s&(1< 0 } +func (s *bitSet) add(i uint64) { *s |= 1 << i } +func (s *bitSet) delete(i uint64) { *s &= ^(1 << i) } + +// encodeZigZag encodes an integer as an unsigned integer ensuring that +// negative integers near zero still have a near zero positive value. +// For unsigned integers, it returns the value verbatim. +func encodeZigZag[T constraints.Integer](v T) uint64 { + var zero T + if ^zero >= 0 { // must be constraints.Unsigned + return uint64(v) + } else { // must be constraints.Signed + // See [google.golang.org/protobuf/encoding/protowire.EncodeZigZag] + return uint64(int64(v)<<1) ^ uint64(int64(v)>>63) + } +} + +// decodeZigZag decodes an unsigned integer as an integer ensuring that +// negative integers near zero still have a near zero positive value. +// For unsigned integers, it returns the value verbatim. +func decodeZigZag[T constraints.Integer](v uint64) T { + var zero T + if ^zero >= 0 { // must be constraints.Unsigned + return T(v) + } else { // must be constraints.Signed + // See [google.golang.org/protobuf/encoding/protowire.DecodeZigZag] + return T(int64(v>>1) ^ int64(v)<<63>>63) + } +} diff --git a/util/set/intset_test.go b/util/set/intset_test.go new file mode 100644 index 0000000000000..9523fe88db127 --- /dev/null +++ b/util/set/intset_test.go @@ -0,0 +1,174 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package set + +import ( + "maps" + "math" + "slices" + "testing" + + "golang.org/x/exp/constraints" +) + +func TestIntSet(t *testing.T) { + t.Run("Int64", func(t *testing.T) { + ss := make(Set[int64]) + var si IntSet[int64] + intValues(t, ss, si) + deleteInt(t, ss, &si, -5) + deleteInt(t, ss, &si, 2) + deleteInt(t, ss, &si, 75) + intValues(t, ss, si) + addInt(t, ss, &si, 2) + addInt(t, ss, &si, 75) + addInt(t, ss, &si, 75) + addInt(t, ss, &si, -3) + addInt(t, ss, &si, -3) + addInt(t, ss, &si, -3) + addInt(t, ss, &si, math.MinInt64) + addInt(t, ss, &si, 8) + intValues(t, ss, si) + addInt(t, ss, &si, 77) + addInt(t, ss, &si, 76) + addInt(t, ss, &si, 76) + addInt(t, ss, &si, 76) + intValues(t, ss, si) + addInt(t, ss, &si, -5) + addInt(t, ss, &si, 7) + addInt(t, ss, &si, -83) + addInt(t, ss, &si, math.MaxInt64) + intValues(t, ss, si) + deleteInt(t, ss, &si, -5) + deleteInt(t, ss, &si, 2) + deleteInt(t, ss, &si, 75) + intValues(t, ss, si) + deleteInt(t, ss, &si, math.MinInt64) + deleteInt(t, ss, &si, math.MaxInt64) + intValues(t, ss, si) + }) + + t.Run("Uint64", func(t *testing.T) { + ss := make(Set[uint64]) + var si IntSet[uint64] + intValues(t, ss, si) + deleteInt(t, ss, &si, 5) + deleteInt(t, ss, &si, 2) + deleteInt(t, ss, &si, 75) + intValues(t, ss, si) + addInt(t, ss, &si, 2) + addInt(t, ss, &si, 75) + addInt(t, ss, &si, 75) + addInt(t, ss, &si, 3) + addInt(t, ss, &si, 3) + addInt(t, ss, &si, 8) + intValues(t, ss, si) + addInt(t, ss, &si, 77) + addInt(t, ss, &si, 76) + addInt(t, ss, &si, 76) + addInt(t, ss, &si, 76) + intValues(t, ss, si) + addInt(t, ss, &si, 5) + addInt(t, ss, &si, 7) + addInt(t, ss, &si, 83) + addInt(t, ss, &si, math.MaxInt64) + intValues(t, ss, si) + deleteInt(t, ss, &si, 5) + deleteInt(t, ss, &si, 2) + deleteInt(t, ss, &si, 75) + intValues(t, ss, si) + deleteInt(t, ss, &si, math.MaxInt64) + intValues(t, ss, si) + }) +} + +func intValues[T constraints.Integer](t testing.TB, ss Set[T], si IntSet[T]) { + got := slices.Collect(maps.Keys(ss)) + slices.Sort(got) + want := slices.Collect(si.Values()) + slices.Sort(want) + if !slices.Equal(got, want) { + t.Fatalf("Values mismatch:\n\tgot %v\n\twant %v", got, want) + } + if got, want := si.Len(), ss.Len(); got != want { + t.Fatalf("Len() = %v, want %v", got, want) + } +} + +func addInt[T constraints.Integer](t testing.TB, ss Set[T], si *IntSet[T], v T) { + t.Helper() + if got, want := si.Contains(v), ss.Contains(v); got != want { + t.Fatalf("Contains(%v) = %v, want %v", v, got, want) + } + ss.Add(v) + si.Add(v) + if !si.Contains(v) { + t.Fatalf("Contains(%v) = false, want true", v) + } + if got, want := si.Len(), ss.Len(); got != want { + t.Fatalf("Len() = %v, want %v", got, want) + } +} + +func deleteInt[T constraints.Integer](t testing.TB, ss Set[T], si *IntSet[T], v T) { + t.Helper() + if got, want := si.Contains(v), ss.Contains(v); got != want { + t.Fatalf("Contains(%v) = %v, want %v", v, got, want) + } + ss.Delete(v) + si.Delete(v) + if si.Contains(v) { + t.Fatalf("Contains(%v) = true, want false", v) + } + if got, want := si.Len(), ss.Len(); got != want { + t.Fatalf("Len() = %v, want %v", got, want) + } +} + +func TestZigZag(t *testing.T) { + t.Run("Int64", func(t *testing.T) { + for _, tt := range []struct { + decoded int64 + encoded uint64 + }{ + {math.MinInt64, math.MaxUint64}, + {-2, 3}, + {-1, 1}, + {0, 0}, + {1, 2}, + {2, 4}, + {math.MaxInt64, math.MaxUint64 - 1}, + } { + encoded := encodeZigZag(tt.decoded) + if encoded != tt.encoded { + t.Errorf("encodeZigZag(%v) = %v, want %v", tt.decoded, encoded, tt.encoded) + } + decoded := decodeZigZag[int64](tt.encoded) + if decoded != tt.decoded { + t.Errorf("decodeZigZag(%v) = %v, want %v", tt.encoded, decoded, tt.decoded) + } + } + }) + t.Run("Uint64", func(t *testing.T) { + for _, tt := range []struct { + decoded uint64 + encoded uint64 + }{ + {0, 0}, + {1, 1}, + {2, 2}, + {math.MaxInt64, math.MaxInt64}, + {math.MaxUint64, math.MaxUint64}, + } { + encoded := encodeZigZag(tt.decoded) + if encoded != tt.encoded { + t.Errorf("encodeZigZag(%v) = %v, want %v", tt.decoded, encoded, tt.encoded) + } + decoded := decodeZigZag[uint64](tt.encoded) + if decoded != tt.decoded { + t.Errorf("decodeZigZag(%v) = %v, want %v", tt.encoded, decoded, tt.decoded) + } + } + }) +} diff --git a/util/set/smallset.go b/util/set/smallset.go new file mode 100644 index 0000000000000..1b77419d27dc9 --- /dev/null +++ b/util/set/smallset.go @@ -0,0 +1,148 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package set + +import ( + "iter" + "maps" + + "tailscale.com/types/structs" +) + +// SmallSet is a set that is optimized for reducing memory overhead when the +// expected size of the set is 0 or 1 elements. +// +// The zero value of SmallSet is a usable empty set. +// +// When storing a SmallSet in a map as a value type, it is important to re-assign +// the map entry after calling Add or Delete, as the SmallSet's representation +// may change. +// +// Copying a SmallSet by value may alias the previous value. Use the Clone method +// to create a new SmallSet with the same contents. +type SmallSet[T comparable] struct { + _ structs.Incomparable // to prevent == mistakes + one T // if non-zero, then single item in set + m Set[T] // if non-nil, the set of items, which might be size 1 if it's the zero value of T +} + +// Values returns an iterator over the elements of the set. +// The iterator will yield the elements in no particular order. +func (s SmallSet[T]) Values() iter.Seq[T] { + if s.m != nil { + return maps.Keys(s.m) + } + var zero T + return func(yield func(T) bool) { + if s.one != zero { + yield(s.one) + } + } +} + +// Contains reports whether e is in the set. +func (s SmallSet[T]) Contains(e T) bool { + if s.m != nil { + return s.m.Contains(e) + } + var zero T + return e != zero && s.one == e +} + +// SoleElement returns the single value in the set, if the set has exactly one +// element. +// +// If the set is empty or has more than one element, ok will be false and e will +// be the zero value of T. +func (s SmallSet[T]) SoleElement() (e T, ok bool) { + return s.one, s.Len() == 1 +} + +// Add adds e to the set. +// +// When storing a SmallSet in a map as a value type, it is important to +// re-assign the map entry after calling Add or Delete, as the SmallSet's +// representation may change. +func (s *SmallSet[T]) Add(e T) { + var zero T + if s.m != nil { + s.m.Add(e) + return + } + // Non-zero elements can go into s.one. + if e != zero { + if s.one == zero { + s.one = e // Len 0 to Len 1 + return + } + if s.one == e { + return // dup + } + } + // Need to make a multi map, either + // because we now have two items, or + // because e is the zero value. + s.m = Set[T]{} + if s.one != zero { + s.m.Add(s.one) // move single item to multi + } + s.m.Add(e) // add new item, possibly zero + s.one = zero +} + +// Len reports the number of elements in the set. +func (s SmallSet[T]) Len() int { + var zero T + if s.m != nil { + return s.m.Len() + } + if s.one != zero { + return 1 + } + return 0 +} + +// Delete removes e from the set. +// +// When storing a SmallSet in a map as a value type, it is important to +// re-assign the map entry after calling Add or Delete, as the SmallSet's +// representation may change. +func (s *SmallSet[T]) Delete(e T) { + var zero T + if s.m == nil { + if s.one == e { + s.one = zero + } + return + } + s.m.Delete(e) + + // If the map size drops to zero, that means + // it only contained the zero value of T. + if s.m.Len() == 0 { + s.m = nil + return + } + + // If the map size drops to one element and doesn't + // contain the zero value, we can switch back to the + // single-item representation. + if s.m.Len() == 1 { + for v := range s.m { + if v != zero { + s.one = v + s.m = nil + } + } + } + return +} + +// Clone returns a copy of s that doesn't alias the original. +func (s SmallSet[T]) Clone() SmallSet[T] { + return SmallSet[T]{ + one: s.one, + m: maps.Clone(s.m), // preserves nilness + } +} diff --git a/util/set/smallset_test.go b/util/set/smallset_test.go new file mode 100644 index 0000000000000..d6f446df08e81 --- /dev/null +++ b/util/set/smallset_test.go @@ -0,0 +1,126 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package set + +import ( + "fmt" + "iter" + "maps" + "reflect" + "slices" + "testing" +) + +func TestSmallSet(t *testing.T) { + t.Parallel() + + wantSize := reflect.TypeFor[int64]().Size() + reflect.TypeFor[map[int]struct{}]().Size() + if wantSize > 16 { + t.Errorf("wantSize should be no more than 16") // it might be smaller on 32-bit systems + } + if size := reflect.TypeFor[SmallSet[int64]]().Size(); size != wantSize { + t.Errorf("SmallSet[int64] size is %d, want %v", size, wantSize) + } + + type op struct { + add bool + v int + } + ops := iter.Seq[op](func(yield func(op) bool) { + for _, add := range []bool{false, true} { + for v := range 4 { + if !yield(op{add: add, v: v}) { + return + } + } + } + }) + type setLike interface { + Add(int) + Delete(int) + } + apply := func(s setLike, o op) { + if o.add { + s.Add(o.v) + } else { + s.Delete(o.v) + } + } + + // For all combinations of 4 operations, + // apply them to both a regular map and SmallSet + // and make sure all the invariants hold. + + for op1 := range ops { + for op2 := range ops { + for op3 := range ops { + for op4 := range ops { + + normal := Set[int]{} + small := &SmallSet[int]{} + for _, op := range []op{op1, op2, op3, op4} { + apply(normal, op) + apply(small, op) + } + + name := func() string { + return fmt.Sprintf("op1=%v, op2=%v, op3=%v, op4=%v", op1, op2, op3, op4) + } + if normal.Len() != small.Len() { + t.Errorf("len mismatch after ops %s: normal=%d, small=%d", name(), normal.Len(), small.Len()) + } + if got := small.Clone().Len(); normal.Len() != got { + t.Errorf("len mismatch after ops %s: normal=%d, clone=%d", name(), normal.Len(), got) + } + + normalEle := slices.Sorted(maps.Keys(normal)) + smallEle := slices.Sorted(small.Values()) + if !slices.Equal(normalEle, smallEle) { + t.Errorf("elements mismatch after ops %s: normal=%v, small=%v", name(), normalEle, smallEle) + } + for e := range 5 { + if normal.Contains(e) != small.Contains(e) { + t.Errorf("contains(%v) mismatch after ops %s: normal=%v, small=%v", e, name(), normal.Contains(e), small.Contains(e)) + } + } + + if err := small.checkInvariants(); err != nil { + t.Errorf("checkInvariants failed after ops %s: %v", name(), err) + } + + if !t.Failed() { + sole, ok := small.SoleElement() + if ok != (small.Len() == 1) { + t.Errorf("SoleElement ok mismatch after ops %s: SoleElement ok=%v, want=%v", name(), ok, !ok) + } + if ok && sole != smallEle[0] { + t.Errorf("SoleElement value mismatch after ops %s: SoleElement=%v, want=%v", name(), sole, smallEle[0]) + t.Errorf("Internals: %+v", small) + } + } + } + } + } + } +} + +func (s *SmallSet[T]) checkInvariants() error { + var zero T + if s.m != nil && s.one != zero { + return fmt.Errorf("both m and one are non-zero") + } + if s.m != nil { + switch len(s.m) { + case 0: + return fmt.Errorf("m is non-nil but empty") + case 1: + for k := range s.m { + if k != zero { + return fmt.Errorf("m contains exactly 1 non-zero element, %v", k) + } + } + } + } + return nil +} diff --git a/util/syspolicy/policy_keys.go b/util/syspolicy/policy_keys.go index 29b2dfd281c4a..cd5f8172c159a 100644 --- a/util/syspolicy/policy_keys.go +++ b/util/syspolicy/policy_keys.go @@ -54,6 +54,15 @@ const ( ExitNodeID Key = "ExitNodeID" ExitNodeIP Key = "ExitNodeIP" // default ""; if blank, no exit node is forced. Value is exit node IP. + // AllowExitNodeOverride is a boolean key that allows the user to override exit node policy settings + // and manually select an exit node. It does not allow disabling exit node usage entirely. + // It is typically used in conjunction with [ExitNodeID] set to "auto:any". + // + // Warning: This policy setting is experimental and may change, be renamed or removed in the future. + // It may also not be fully supported by all Tailscale clients until it is out of experimental status. + // See tailscale/corp#29969. + AllowExitNodeOverride Key = "ExitNode.AllowOverride" + // Keys with a string value that specifies an option: "always", "never", "user-decides". // The default is "user-decides" unless otherwise stated. Enforcement of // these policies is typically performed in ipnlocal.applySysPolicy(). GUIs @@ -120,14 +129,18 @@ const ( LogSCMInteractions Key = "LogSCMInteractions" FlushDNSOnSessionUnlock Key = "FlushDNSOnSessionUnlock" + // EncryptState is a boolean setting that specifies whether to encrypt the + // tailscaled state file with a TPM device. + EncryptState Key = "EncryptState" + // PostureChecking indicates if posture checking is enabled and the client shall gather // posture data. // Key is a string value that specifies an option: "always", "never", "user-decides". // The default is "user-decides" unless otherwise stated. PostureChecking Key = "PostureChecking" // DeviceSerialNumber is the serial number of the device that is running Tailscale. - // This is used on iOS/tvOS to allow IT administrators to manually give us a serial number via MDM. - // We are unable to programmatically get the serial number from IOKit due to sandboxing restrictions. + // This is used on Android, iOS and tvOS to allow IT administrators to manually give us a serial number via MDM. + // We are unable to programmatically get the serial number on mobile due to sandboxing restrictions. DeviceSerialNumber Key = "DeviceSerialNumber" // ManagedByOrganizationName indicates the name of the organization managing the Tailscale @@ -169,6 +182,7 @@ const ( var implicitDefinitions = []*setting.Definition{ // Device policy settings (can only be configured on a per-device basis): setting.NewDefinition(AllowedSuggestedExitNodes, setting.DeviceSetting, setting.StringListValue), + setting.NewDefinition(AllowExitNodeOverride, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(AlwaysOn, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(AlwaysOnOverrideWithReason, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(ApplyUpdates, setting.DeviceSetting, setting.PreferenceOptionValue), @@ -186,6 +200,7 @@ var implicitDefinitions = []*setting.Definition{ setting.NewDefinition(ExitNodeID, setting.DeviceSetting, setting.StringValue), setting.NewDefinition(ExitNodeIP, setting.DeviceSetting, setting.StringValue), setting.NewDefinition(FlushDNSOnSessionUnlock, setting.DeviceSetting, setting.BooleanValue), + setting.NewDefinition(EncryptState, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(Hostname, setting.DeviceSetting, setting.StringValue), setting.NewDefinition(LogSCMInteractions, setting.DeviceSetting, setting.BooleanValue), setting.NewDefinition(LogTarget, setting.DeviceSetting, setting.StringValue), diff --git a/util/syspolicy/rsop/change_callbacks.go b/util/syspolicy/rsop/change_callbacks.go index b962f30c008c1..87b45b654709d 100644 --- a/util/syspolicy/rsop/change_callbacks.go +++ b/util/syspolicy/rsop/change_callbacks.go @@ -59,6 +59,11 @@ func (c PolicyChange) HasChanged(key setting.Key) bool { } } +// HasChangedAnyOf reports whether any of the specified policy settings has changed. +func (c PolicyChange) HasChangedAnyOf(keys ...setting.Key) bool { + return slices.ContainsFunc(keys, c.HasChanged) +} + // policyChangeCallbacks are the callbacks to invoke when the effective policy changes. // It is safe for concurrent use. type policyChangeCallbacks struct { diff --git a/util/syspolicy/setting/snapshot.go b/util/syspolicy/setting/snapshot.go index 087325a04c6f1..3a40785dce9de 100644 --- a/util/syspolicy/setting/snapshot.go +++ b/util/syspolicy/setting/snapshot.go @@ -9,6 +9,7 @@ import ( "maps" "slices" "strings" + "time" jsonv2 "github.com/go-json-experiment/json" "github.com/go-json-experiment/json/jsontext" @@ -152,6 +153,24 @@ var ( _ jsonv2.UnmarshalerFrom = (*Snapshot)(nil) ) +// As of 2025-07-28, jsonv2 no longer has a default representation for [time.Duration], +// so we need to provide a custom marshaler. +// +// This is temporary until the decision on the default representation is made +// (see https://github.com/golang/go/issues/71631#issuecomment-2981670799). +// +// In the future, we might either use the default representation (if compatible with +// [time.Duration.String]) or specify something like json.WithFormat[time.Duration]("units") +// when golang/go#71664 is implemented. +// +// TODO(nickkhyl): revisit this when the decision on the default [time.Duration] +// representation is made in golang/go#71631 and/or golang/go#71664 is implemented. +var formatDurationAsUnits = jsonv2.JoinOptions( + jsonv2.WithMarshalers(jsonv2.MarshalToFunc(func(e *jsontext.Encoder, t time.Duration) error { + return e.WriteToken(jsontext.String(t.String())) + })), +) + // MarshalJSONTo implements [jsonv2.MarshalerTo]. func (s *Snapshot) MarshalJSONTo(out *jsontext.Encoder) error { data := &snapshotJSON{} @@ -159,7 +178,7 @@ func (s *Snapshot) MarshalJSONTo(out *jsontext.Encoder) error { data.Summary = s.summary data.Settings = s.m } - return jsonv2.MarshalEncode(out, data) + return jsonv2.MarshalEncode(out, data, formatDurationAsUnits) } // UnmarshalJSONFrom implements [jsonv2.UnmarshalerFrom]. diff --git a/util/syspolicy/setting/snapshot_test.go b/util/syspolicy/setting/snapshot_test.go index d41b362f06976..19f014acaa831 100644 --- a/util/syspolicy/setting/snapshot_test.go +++ b/util/syspolicy/setting/snapshot_test.go @@ -491,6 +491,18 @@ func TestMarshalUnmarshalSnapshot(t *testing.T) { snapshot: NewSnapshot(map[Key]RawItem{"ListPolicy": RawItemOf([]string{"Value1", "Value2"})}), wantJSON: `{"Settings": {"ListPolicy": {"Value": ["Value1", "Value2"]}}}`, }, + { + name: "Duration/Zero", + snapshot: NewSnapshot(map[Key]RawItem{"DurationPolicy": RawItemOf(time.Duration(0))}), + wantJSON: `{"Settings": {"DurationPolicy": {"Value": "0s"}}}`, + wantBack: NewSnapshot(map[Key]RawItem{"DurationPolicy": RawItemOf("0s")}), + }, + { + name: "Duration/NonZero", + snapshot: NewSnapshot(map[Key]RawItem{"DurationPolicy": RawItemOf(2 * time.Hour)}), + wantJSON: `{"Settings": {"DurationPolicy": {"Value": "2h0m0s"}}}`, + wantBack: NewSnapshot(map[Key]RawItem{"DurationPolicy": RawItemOf("2h0m0s")}), + }, { name: "Empty/With-Summary", snapshot: NewSnapshot( diff --git a/util/syspolicy/source/test_store.go b/util/syspolicy/source/test_store.go index 4b175611fef0d..efaf4cd5a7c0f 100644 --- a/util/syspolicy/source/test_store.go +++ b/util/syspolicy/source/test_store.go @@ -154,6 +154,13 @@ func (s *TestStore) RegisterChangeCallback(callback func()) (unregister func(), }, nil } +// IsEmpty reports whether the store does not contain any settings. +func (s *TestStore) IsEmpty() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.mr) == 0 +} + // ReadString implements [Store]. func (s *TestStore) ReadString(key setting.Key) (string, error) { defer s.recordRead(key, setting.StringValue) diff --git a/util/syspolicy/syspolicy.go b/util/syspolicy/syspolicy.go index afcc28ff1fd86..6555a58ac4564 100644 --- a/util/syspolicy/syspolicy.go +++ b/util/syspolicy/syspolicy.go @@ -56,6 +56,30 @@ func MustRegisterStoreForTest(tb testenv.TB, name string, scope setting.PolicySc return reg } +// HasAnyOf returns whether at least one of the specified policy settings is configured, +// or an error if no keys are provided or the check fails. +func HasAnyOf(keys ...Key) (bool, error) { + if len(keys) == 0 { + return false, errors.New("at least one key must be specified") + } + policy, err := rsop.PolicyFor(setting.DefaultScope()) + if err != nil { + return false, err + } + effective := policy.Get() + for _, k := range keys { + _, err := effective.GetErr(k) + if errors.Is(err, setting.ErrNotConfigured) || errors.Is(err, setting.ErrNoSuchKey) { + continue + } + if err != nil { + return false, err + } + return true, nil + } + return false, nil +} + // GetString returns a string policy setting with the specified key, // or defaultValue if it does not exist. func GetString(key Key, defaultValue string) (string, error) { diff --git a/version/distro/distro.go b/version/distro/distro.go index f7997e1d9f81b..0e88bdd2fa297 100644 --- a/version/distro/distro.go +++ b/version/distro/distro.go @@ -9,6 +9,7 @@ import ( "os" "runtime" "strconv" + "strings" "tailscale.com/types/lazy" "tailscale.com/util/lineiter" @@ -31,6 +32,7 @@ const ( Unraid = Distro("unraid") Alpine = Distro("alpine") UBNT = Distro("ubnt") // Ubiquiti Networks + JetKVM = Distro("jetkvm") ) var distro lazy.SyncValue[Distro] @@ -102,10 +104,20 @@ func linuxDistro() Distro { return Unraid case have("/etc/alpine-release"): return Alpine + case runtime.GOARCH == "arm" && isDeviceModel("JetKVM"): + return JetKVM } return "" } +func isDeviceModel(want string) bool { + if runtime.GOOS != "linux" { + return false + } + v, _ := os.ReadFile("/sys/firmware/devicetree/base/model") + return want == strings.Trim(string(v), "\x00\r\n\t ") +} + func freebsdDistro() Distro { switch { case have("/etc/pfSense-rc"): diff --git a/version/print.go b/version/print.go index be90432cc85df..43ee2b5591410 100644 --- a/version/print.go +++ b/version/print.go @@ -20,6 +20,7 @@ var stringLazy = sync.OnceValue(func() string { if gitCommit() != "" { fmt.Fprintf(&ret, " tailscale commit: %s%s\n", gitCommit(), dirtyString()) } + fmt.Fprintf(&ret, " long version: %s\n", Long()) if extraGitCommitStamp != "" { fmt.Fprintf(&ret, " other commit: %s\n", extraGitCommitStamp) } diff --git a/version_test.go b/version_test.go index 1f434e682f1d6..3d983a19d51db 100644 --- a/version_test.go +++ b/version_test.go @@ -6,21 +6,16 @@ package tailscaleroot import ( "fmt" "os" - "regexp" + "os/exec" + "runtime" "strings" "testing" + + "golang.org/x/mod/modfile" ) func TestDockerfileVersion(t *testing.T) { - goMod, err := os.ReadFile("go.mod") - if err != nil { - t.Fatal(err) - } - m := regexp.MustCompile(`(?m)^go (\d\.\d+)\r?($|\.)`).FindStringSubmatch(string(goMod)) - if m == nil { - t.Fatalf("didn't find go version in go.mod") - } - goVersion := m[1] + goVersion := mustGetGoModVersion(t, false) dockerFile, err := os.ReadFile("Dockerfile") if err != nil { @@ -31,3 +26,60 @@ func TestDockerfileVersion(t *testing.T) { t.Errorf("didn't find %q in Dockerfile", wantSub) } } + +// TestGoVersion tests that the Go version specified in go.mod matches ./tool/go version. +func TestGoVersion(t *testing.T) { + // We could special-case ./tool/go path for Windows, but really there is no + // need to run it there. + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows") + } + goModVersion := mustGetGoModVersion(t, true) + + goToolCmd := exec.Command("./tool/go", "version") + goToolOutput, err := goToolCmd.Output() + if err != nil { + t.Fatalf("Failed to get ./tool/go version: %v", err) + } + + // Version info will approximately look like 'go version go1.24.4 linux/amd64'. + parts := strings.Fields(string(goToolOutput)) + if len(parts) < 4 { + t.Fatalf("Unexpected ./tool/go version output format: %s", goToolOutput) + } + + goToolVersion := strings.TrimPrefix(parts[2], "go") + + if goModVersion != goToolVersion { + t.Errorf("Go version in go.mod (%q) does not match the version of ./tool/go (%q).\nEnsure that the go.mod refers to the same Go version as ./go.toolchain.rev.", + goModVersion, goToolVersion) + } +} + +func mustGetGoModVersion(t *testing.T, includePatchVersion bool) string { + t.Helper() + + goModBytes, err := os.ReadFile("go.mod") + if err != nil { + t.Fatal(err) + } + + modFile, err := modfile.Parse("go.mod", goModBytes, nil) + if err != nil { + t.Fatal(err) + } + + if modFile.Go == nil { + t.Fatal("no Go version found in go.mod") + } + + version := modFile.Go.Version + + parts := strings.Split(version, ".") + if !includePatchVersion { + if len(parts) >= 2 { + version = parts[0] + "." + parts[1] + } + } + return version +} diff --git a/wgengine/magicsock/batching_conn.go b/wgengine/magicsock/batching_conn.go index 5320d1cafa59a..b769907dbe88f 100644 --- a/wgengine/magicsock/batching_conn.go +++ b/wgengine/magicsock/batching_conn.go @@ -4,8 +4,6 @@ package magicsock import ( - "net/netip" - "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" "tailscale.com/types/nettype" @@ -21,5 +19,5 @@ var ( type batchingConn interface { nettype.PacketConn ReadBatch(msgs []ipv6.Message, flags int) (n int, err error) - WriteBatchTo(buffs [][]byte, addr netip.AddrPort) error + WriteBatchTo(buffs [][]byte, addr epAddr, offset int) error } diff --git a/wgengine/magicsock/batching_conn_linux.go b/wgengine/magicsock/batching_conn_linux.go index 25bf974b022ba..a0607c624445c 100644 --- a/wgengine/magicsock/batching_conn_linux.go +++ b/wgengine/magicsock/batching_conn_linux.go @@ -22,6 +22,7 @@ import ( "golang.org/x/sys/unix" "tailscale.com/hostinfo" "tailscale.com/net/neterror" + "tailscale.com/net/packet" "tailscale.com/types/nettype" ) @@ -92,9 +93,14 @@ const ( maxIPv6PayloadLen = 1<<16 - 1 - 8 ) -// coalesceMessages iterates msgs, coalescing them where possible while -// maintaining datagram order. All msgs have their Addr field set to addr. -func (c *linuxBatchingConn) coalesceMessages(addr *net.UDPAddr, buffs [][]byte, msgs []ipv6.Message) int { +// coalesceMessages iterates 'buffs', setting and coalescing them in 'msgs' +// where possible while maintaining datagram order. +// +// All msgs have their Addr field set to addr. +// +// All msgs[i].Buffers[0] are preceded by a Geneve header with vni.get() if +// vni.isSet(). +func (c *linuxBatchingConn) coalesceMessages(addr *net.UDPAddr, vni virtualNetworkID, buffs [][]byte, msgs []ipv6.Message, offset int) int { var ( base = -1 // index of msg we are currently coalescing into gsoSize int // segmentation size of msgs[base] @@ -105,7 +111,18 @@ func (c *linuxBatchingConn) coalesceMessages(addr *net.UDPAddr, buffs [][]byte, if addr.IP.To4() == nil { maxPayloadLen = maxIPv6PayloadLen } + vniIsSet := vni.isSet() + var gh packet.GeneveHeader + if vniIsSet { + gh.Protocol = packet.GeneveProtocolWireGuard + gh.VNI = vni.get() + } for i, buff := range buffs { + if vniIsSet { + gh.Encode(buffs[i]) + } else { + buff = buff[offset:] + } if i > 0 { msgLen := len(buff) baseLenBefore := len(msgs[base].Buffers[0]) @@ -162,29 +179,39 @@ func (c *linuxBatchingConn) putSendBatch(batch *sendBatch) { c.sendBatchPool.Put(batch) } -func (c *linuxBatchingConn) WriteBatchTo(buffs [][]byte, addr netip.AddrPort) error { +func (c *linuxBatchingConn) WriteBatchTo(buffs [][]byte, addr epAddr, offset int) error { batch := c.getSendBatch() defer c.putSendBatch(batch) - if addr.Addr().Is6() { - as16 := addr.Addr().As16() + if addr.ap.Addr().Is6() { + as16 := addr.ap.Addr().As16() copy(batch.ua.IP, as16[:]) batch.ua.IP = batch.ua.IP[:16] } else { - as4 := addr.Addr().As4() + as4 := addr.ap.Addr().As4() copy(batch.ua.IP, as4[:]) batch.ua.IP = batch.ua.IP[:4] } - batch.ua.Port = int(addr.Port()) + batch.ua.Port = int(addr.ap.Port()) var ( n int retried bool ) retry: if c.txOffload.Load() { - n = c.coalesceMessages(batch.ua, buffs, batch.msgs) + n = c.coalesceMessages(batch.ua, addr.vni, buffs, batch.msgs, offset) } else { + vniIsSet := addr.vni.isSet() + var gh packet.GeneveHeader + if vniIsSet { + gh.Protocol = packet.GeneveProtocolWireGuard + gh.VNI = addr.vni.get() + offset -= packet.GeneveFixedHeaderLength + } for i := range buffs { - batch.msgs[i].Buffers[0] = buffs[i] + if vniIsSet { + gh.Encode(buffs[i]) + } + batch.msgs[i].Buffers[0] = buffs[i][offset:] batch.msgs[i].Addr = batch.ua batch.msgs[i].OOB = batch.msgs[i].OOB[:0] } diff --git a/wgengine/magicsock/batching_conn_linux_test.go b/wgengine/magicsock/batching_conn_linux_test.go index 5c22bf1c73cf4..7e0ab8fc485eb 100644 --- a/wgengine/magicsock/batching_conn_linux_test.go +++ b/wgengine/magicsock/batching_conn_linux_test.go @@ -9,6 +9,7 @@ import ( "testing" "golang.org/x/net/ipv6" + "tailscale.com/net/packet" ) func setGSOSize(control *[]byte, gsoSize uint16) { @@ -154,58 +155,117 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { getGSOSizeFromControl: getGSOSize, } + withGeneveSpace := func(len, cap int) []byte { + return make([]byte, len+packet.GeneveFixedHeaderLength, cap+packet.GeneveFixedHeaderLength) + } + + vni1 := virtualNetworkID{} + vni1.set(1) + cases := []struct { name string buffs [][]byte + vni virtualNetworkID wantLens []int wantGSO []int }{ { name: "one message no coalesce", buffs: [][]byte{ - make([]byte, 1, 1), + withGeneveSpace(1, 1), }, wantLens: []int{1}, wantGSO: []int{0}, }, + { + name: "one message no coalesce vni.isSet", + buffs: [][]byte{ + withGeneveSpace(1, 1), + }, + vni: vni1, + wantLens: []int{1 + packet.GeneveFixedHeaderLength}, + wantGSO: []int{0}, + }, { name: "two messages equal len coalesce", buffs: [][]byte{ - make([]byte, 1, 2), - make([]byte, 1, 1), + withGeneveSpace(1, 2), + withGeneveSpace(1, 1), }, wantLens: []int{2}, wantGSO: []int{1}, }, + { + name: "two messages equal len coalesce vni.isSet", + buffs: [][]byte{ + withGeneveSpace(1, 2+packet.GeneveFixedHeaderLength), + withGeneveSpace(1, 1), + }, + vni: vni1, + wantLens: []int{2 + (2 * packet.GeneveFixedHeaderLength)}, + wantGSO: []int{1 + packet.GeneveFixedHeaderLength}, + }, { name: "two messages unequal len coalesce", buffs: [][]byte{ - make([]byte, 2, 3), - make([]byte, 1, 1), + withGeneveSpace(2, 3), + withGeneveSpace(1, 1), }, wantLens: []int{3}, wantGSO: []int{2}, }, + { + name: "two messages unequal len coalesce vni.isSet", + buffs: [][]byte{ + withGeneveSpace(2, 3+packet.GeneveFixedHeaderLength), + withGeneveSpace(1, 1), + }, + vni: vni1, + wantLens: []int{3 + (2 * packet.GeneveFixedHeaderLength)}, + wantGSO: []int{2 + packet.GeneveFixedHeaderLength}, + }, { name: "three messages second unequal len coalesce", buffs: [][]byte{ - make([]byte, 2, 3), - make([]byte, 1, 1), - make([]byte, 2, 2), + withGeneveSpace(2, 3), + withGeneveSpace(1, 1), + withGeneveSpace(2, 2), }, wantLens: []int{3, 2}, wantGSO: []int{2, 0}, }, + { + name: "three messages second unequal len coalesce vni.isSet", + buffs: [][]byte{ + withGeneveSpace(2, 3+(2*packet.GeneveFixedHeaderLength)), + withGeneveSpace(1, 1), + withGeneveSpace(2, 2), + }, + vni: vni1, + wantLens: []int{3 + (2 * packet.GeneveFixedHeaderLength), 2 + packet.GeneveFixedHeaderLength}, + wantGSO: []int{2 + packet.GeneveFixedHeaderLength, 0}, + }, { name: "three messages limited cap coalesce", buffs: [][]byte{ - make([]byte, 2, 4), - make([]byte, 2, 2), - make([]byte, 2, 2), + withGeneveSpace(2, 4), + withGeneveSpace(2, 2), + withGeneveSpace(2, 2), }, wantLens: []int{4, 2}, wantGSO: []int{2, 0}, }, + { + name: "three messages limited cap coalesce vni.isSet", + buffs: [][]byte{ + withGeneveSpace(2, 4+packet.GeneveFixedHeaderLength), + withGeneveSpace(2, 2), + withGeneveSpace(2, 2), + }, + vni: vni1, + wantLens: []int{4 + (2 * packet.GeneveFixedHeaderLength), 2 + packet.GeneveFixedHeaderLength}, + wantGSO: []int{2 + packet.GeneveFixedHeaderLength, 0}, + }, } for _, tt := range cases { @@ -219,7 +279,7 @@ func Test_linuxBatchingConn_coalesceMessages(t *testing.T) { msgs[i].Buffers = make([][]byte, 1) msgs[i].OOB = make([]byte, 0, 2) } - got := c.coalesceMessages(addr, tt.buffs, msgs) + got := c.coalesceMessages(addr, tt.vni, tt.buffs, msgs, packet.GeneveFixedHeaderLength) if got != len(tt.wantLens) { t.Fatalf("got len %d want: %d", got, len(tt.wantLens)) } diff --git a/wgengine/magicsock/debughttp.go b/wgengine/magicsock/debughttp.go index aa109c242e27c..cfdf8c1e12d78 100644 --- a/wgengine/magicsock/debughttp.go +++ b/wgengine/magicsock/debughttp.go @@ -72,18 +72,18 @@ func (c *Conn) ServeHTTPDebug(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "

# ip:port to endpoint

    ") { type kv struct { - ipp netip.AddrPort - pi *peerInfo + addr epAddr + pi *peerInfo } - ent := make([]kv, 0, len(c.peerMap.byIPPort)) - for k, v := range c.peerMap.byIPPort { + ent := make([]kv, 0, len(c.peerMap.byEpAddr)) + for k, v := range c.peerMap.byEpAddr { ent = append(ent, kv{k, v}) } - sort.Slice(ent, func(i, j int) bool { return ipPortLess(ent[i].ipp, ent[j].ipp) }) + sort.Slice(ent, func(i, j int) bool { return epAddrLess(ent[i].addr, ent[j].addr) }) for _, e := range ent { ep := e.pi.ep shortStr := ep.publicKey.ShortString() - fmt.Fprintf(w, "
  • %v: %v
    • \n", e.ipp, strings.Trim(shortStr, "[]"), shortStr) + fmt.Fprintf(w, "
  • %v: %v
    • \n", e.addr, strings.Trim(shortStr, "[]"), shortStr) } } @@ -148,11 +148,11 @@ func printEndpointHTML(w io.Writer, ep *endpoint) { for ipp := range ep.endpointState { eps = append(eps, ipp) } - sort.Slice(eps, func(i, j int) bool { return ipPortLess(eps[i], eps[j]) }) + sort.Slice(eps, func(i, j int) bool { return addrPortLess(eps[i], eps[j]) }) io.WriteString(w, "

    Endpoints:

      ") for _, ipp := range eps { s := ep.endpointState[ipp] - if ipp == ep.bestAddr.AddrPort { + if ipp == ep.bestAddr.ap && !ep.bestAddr.vni.isSet() { fmt.Fprintf(w, "
    • %s: (best)
        ", ipp) } else { fmt.Fprintf(w, "
      • %s: ...
          ", ipp) @@ -196,9 +196,19 @@ func peerDebugName(p tailcfg.NodeView) string { return p.Hostinfo().Hostname() } -func ipPortLess(a, b netip.AddrPort) bool { +func addrPortLess(a, b netip.AddrPort) bool { if v := a.Addr().Compare(b.Addr()); v != 0 { return v < 0 } return a.Port() < b.Port() } + +func epAddrLess(a, b epAddr) bool { + if v := a.ap.Addr().Compare(b.ap.Addr()); v != 0 { + return v < 0 + } + if a.ap.Port() == b.ap.Port() { + return a.vni.get() < b.vni.get() + } + return a.ap.Port() < b.ap.Port() +} diff --git a/wgengine/magicsock/derp.go b/wgengine/magicsock/derp.go index ffdff14a11d47..5afdbc6d8718b 100644 --- a/wgengine/magicsock/derp.go +++ b/wgengine/magicsock/derp.go @@ -740,8 +740,11 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep *en return 0, nil } - ipp := netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, uint16(regionID)) - if c.handleDiscoMessage(b[:n], ipp, dm.src, discoRXPathDERP) { + srcAddr := epAddr{ap: netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, uint16(regionID))} + pt, isGeneveEncap := packetLooksLike(b[:n]) + if pt == packetLooksLikeDisco && + !isGeneveEncap { // We should never receive Geneve-encapsulated disco over DERP. + c.handleDiscoMessage(b[:n], srcAddr, false, dm.src, discoRXPathDERP) return 0, nil } @@ -755,9 +758,9 @@ func (c *Conn) processDERPReadResult(dm derpReadResult, b []byte) (n int, ep *en return 0, nil } - ep.noteRecvActivity(ipp, mono.Now()) + ep.noteRecvActivity(srcAddr, mono.Now()) if stats := c.stats.Load(); stats != nil { - stats.UpdateRxPhysical(ep.nodeAddr, ipp, 1, dm.n) + stats.UpdateRxPhysical(ep.nodeAddr, srcAddr.ap, 1, dm.n) } c.metrics.inboundPacketsDERPTotal.Add(1) diff --git a/wgengine/magicsock/endpoint.go b/wgengine/magicsock/endpoint.go index 3788708a8260f..6381b021088b6 100644 --- a/wgengine/magicsock/endpoint.go +++ b/wgengine/magicsock/endpoint.go @@ -25,6 +25,7 @@ import ( "golang.org/x/net/ipv6" "tailscale.com/disco" "tailscale.com/ipn/ipnstate" + "tailscale.com/net/packet" "tailscale.com/net/stun" "tailscale.com/net/tstun" "tailscale.com/tailcfg" @@ -74,17 +75,18 @@ type endpoint struct { // mu protects all following fields. mu sync.Mutex // Lock ordering: Conn.mu, then endpoint.mu - heartBeatTimer *time.Timer // nil when idle - lastSendExt mono.Time // last time there were outgoing packets sent to this peer from an external trigger (e.g. wireguard-go or disco pingCLI) - lastSendAny mono.Time // last time there were outgoing packets sent this peer from any trigger, internal or external to magicsock - lastFullPing mono.Time // last time we pinged all disco or wireguard only endpoints - derpAddr netip.AddrPort // fallback/bootstrap path, if non-zero (non-zero for well-behaved clients) + heartBeatTimer *time.Timer // nil when idle + lastSendExt mono.Time // last time there were outgoing packets sent to this peer from an external trigger (e.g. wireguard-go or disco pingCLI) + lastSendAny mono.Time // last time there were outgoing packets sent this peer from any trigger, internal or external to magicsock + lastFullPing mono.Time // last time we pinged all disco or wireguard only endpoints + lastUDPRelayPathDiscovery mono.Time // last time we ran UDP relay path discovery + derpAddr netip.AddrPort // fallback/bootstrap path, if non-zero (non-zero for well-behaved clients) bestAddr addrQuality // best non-DERP path; zero if none; mutate via setBestAddrLocked() bestAddrAt mono.Time // time best address re-confirmed trustBestAddrUntil mono.Time // time when bestAddr expires sentPing map[stun.TxID]sentPing - endpointState map[netip.AddrPort]*endpointState + endpointState map[netip.AddrPort]*endpointState // netip.AddrPort type for key (instead of [epAddr]) as [endpointState] is irrelevant for Geneve-encapsulated paths isCallMeMaybeEP map[netip.AddrPort]bool // The following fields are related to the new "silent disco" @@ -98,8 +100,37 @@ type endpoint struct { relayCapable bool // whether the node is capable of speaking via a [tailscale.com/net/udprelay.Server] } +// udpRelayEndpointReady determines whether the given relay [addrQuality] should +// be installed as de.bestAddr. It is only called by [relayManager] once it has +// determined maybeBest is functional via [disco.Pong] reception. +func (de *endpoint) udpRelayEndpointReady(maybeBest addrQuality) { + de.mu.Lock() + defer de.mu.Unlock() + now := mono.Now() + curBestAddrTrusted := now.Before(de.trustBestAddrUntil) + sameRelayServer := de.bestAddr.vni.isSet() && maybeBest.relayServerDisco.Compare(de.bestAddr.relayServerDisco) == 0 + + if !curBestAddrTrusted || + sameRelayServer || + betterAddr(maybeBest, de.bestAddr) { + // We must set maybeBest as de.bestAddr if: + // 1. de.bestAddr is untrusted. betterAddr does not consider + // time-based trust. + // 2. maybeBest & de.bestAddr are on the same relay. If the maybeBest + // handshake happened to use a different source address/transport, + // the relay will drop packets from the 'old' de.bestAddr's. + // 3. maybeBest is a 'betterAddr'. + // + // TODO(jwhited): add observability around !curBestAddrTrusted and sameRelayServer + // TODO(jwhited): collapse path change logging with endpoint.handlePongConnLocked() + de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v", de.publicKey.ShortString(), de.discoShort(), maybeBest.epAddr, maybeBest.wireMTU) + de.setBestAddrLocked(maybeBest) + de.trustBestAddrUntil = now.Add(trustUDPAddrDuration) + } +} + func (de *endpoint) setBestAddrLocked(v addrQuality) { - if v.AddrPort != de.bestAddr.AddrPort { + if v.epAddr != de.bestAddr.epAddr { de.probeUDPLifetime.resetCycleEndpointLocked() } de.bestAddr = v @@ -135,11 +166,11 @@ type probeUDPLifetime struct { // timeout cliff in the future. timer *time.Timer - // bestAddr contains the endpoint.bestAddr.AddrPort at the time a cycle was + // bestAddr contains the endpoint.bestAddr.epAddr at the time a cycle was // scheduled to start. A probing cycle is 1:1 with the current - // endpoint.bestAddr.AddrPort in the interest of simplicity. When - // endpoint.bestAddr.AddrPort changes, any active probing cycle will reset. - bestAddr netip.AddrPort + // endpoint.bestAddr.epAddr in the interest of simplicity. When + // endpoint.bestAddr.epAddr changes, any active probing cycle will reset. + bestAddr epAddr // cycleStartedAt contains the time at which the first cliff // (ProbeUDPLifetimeConfig.Cliffs[0]) was pinged for the current/last cycle. cycleStartedAt time.Time @@ -191,7 +222,7 @@ func (p *probeUDPLifetime) resetCycleEndpointLocked() { } p.cycleActive = false p.currentCliff = 0 - p.bestAddr = netip.AddrPort{} + p.bestAddr = epAddr{} } // ProbeUDPLifetimeConfig represents the configuration for probing UDP path @@ -334,7 +365,7 @@ type endpointDisco struct { } type sentPing struct { - to netip.AddrPort + to epAddr at mono.Time timer *time.Timer // timeout timer purpose discoPingPurpose @@ -446,7 +477,8 @@ func (de *endpoint) deleteEndpointLocked(why string, ep netip.AddrPort) { From: ep, }) delete(de.endpointState, ep) - if de.bestAddr.AddrPort == ep { + asEpAddr := epAddr{ap: ep} + if de.bestAddr.epAddr == asEpAddr { de.debugUpdates.Add(EndpointChange{ When: time.Now(), What: "deleteEndpointLocked-bestAddr-" + why, @@ -468,11 +500,12 @@ func (de *endpoint) initFakeUDPAddr() { } // noteRecvActivity records receive activity on de, and invokes -// Conn.noteRecvActivity no more than once every 10s. -func (de *endpoint) noteRecvActivity(ipp netip.AddrPort, now mono.Time) { +// Conn.noteRecvActivity no more than once every 10s, returning true if it +// was called, otherwise false. +func (de *endpoint) noteRecvActivity(src epAddr, now mono.Time) bool { if de.isWireguardOnly { de.mu.Lock() - de.bestAddr.AddrPort = ipp + de.bestAddr.ap = src.ap de.bestAddrAt = now de.trustBestAddrUntil = now.Add(5 * time.Second) de.mu.Unlock() @@ -482,7 +515,7 @@ func (de *endpoint) noteRecvActivity(ipp netip.AddrPort, now mono.Time) { // kick off discovery disco pings every trustUDPAddrDuration and mirror // to DERP. de.mu.Lock() - if de.heartbeatDisabled && de.bestAddr.AddrPort == ipp { + if de.heartbeatDisabled && de.bestAddr.epAddr == src { de.trustBestAddrUntil = now.Add(trustUDPAddrDuration) } de.mu.Unlock() @@ -493,10 +526,12 @@ func (de *endpoint) noteRecvActivity(ipp netip.AddrPort, now mono.Time) { de.lastRecvWG.StoreAtomic(now) if de.c.noteRecvActivity == nil { - return + return false } de.c.noteRecvActivity(de.publicKey) + return true } + return false } func (de *endpoint) discoShort() string { @@ -530,10 +565,10 @@ func (de *endpoint) DstToBytes() []byte { return packIPPort(de.fakeWGAddr) } // de.mu must be held. // // TODO(val): Rewrite the addrFor*Locked() variations to share code. -func (de *endpoint) addrForSendLocked(now mono.Time) (udpAddr, derpAddr netip.AddrPort, sendWGPing bool) { - udpAddr = de.bestAddr.AddrPort +func (de *endpoint) addrForSendLocked(now mono.Time) (udpAddr epAddr, derpAddr netip.AddrPort, sendWGPing bool) { + udpAddr = de.bestAddr.epAddr - if udpAddr.IsValid() && !now.After(de.trustBestAddrUntil) { + if udpAddr.ap.IsValid() && !now.After(de.trustBestAddrUntil) { return udpAddr, netip.AddrPort{}, false } @@ -557,7 +592,7 @@ func (de *endpoint) addrForSendLocked(now mono.Time) (udpAddr, derpAddr netip.Ad // best latency is used. // // de.mu must be held. -func (de *endpoint) addrForWireGuardSendLocked(now mono.Time) (udpAddr netip.AddrPort, shouldPing bool) { +func (de *endpoint) addrForWireGuardSendLocked(now mono.Time) (udpAddr epAddr, shouldPing bool) { if len(de.endpointState) == 0 { de.c.logf("magicsock: addrForSendWireguardLocked: [unexpected] no candidates available for endpoint") return udpAddr, false @@ -581,22 +616,22 @@ func (de *endpoint) addrForWireGuardSendLocked(now mono.Time) (udpAddr netip.Add // TODO(catzkorn): Consider a small increase in latency to use // IPv6 in comparison to IPv4, when possible. lowestLatency = latency - udpAddr = ipp + udpAddr.ap = ipp } } } needPing := len(de.endpointState) > 1 && now.Sub(oldestPing) > wireguardPingInterval - if !udpAddr.IsValid() { + if !udpAddr.ap.IsValid() { candidates := slicesx.MapKeys(de.endpointState) // Randomly select an address to use until we retrieve latency information // and give it a short trustBestAddrUntil time so we avoid flapping between // addresses while waiting on latency information to be populated. - udpAddr = candidates[rand.IntN(len(candidates))] + udpAddr.ap = candidates[rand.IntN(len(candidates))] } - de.bestAddr.AddrPort = udpAddr + de.bestAddr.epAddr = epAddr{ap: udpAddr.ap} // Only extend trustBestAddrUntil by one second to avoid packet // reordering and/or CPU usage from random selection during the first // second. We should receive a response due to a WireGuard handshake in @@ -614,18 +649,18 @@ func (de *endpoint) addrForWireGuardSendLocked(now mono.Time) (udpAddr netip.Add // both of the returned UDP address and DERP address may be non-zero. // // de.mu must be held. -func (de *endpoint) addrForPingSizeLocked(now mono.Time, size int) (udpAddr, derpAddr netip.AddrPort) { +func (de *endpoint) addrForPingSizeLocked(now mono.Time, size int) (udpAddr epAddr, derpAddr netip.AddrPort) { if size == 0 { udpAddr, derpAddr, _ = de.addrForSendLocked(now) return } - udpAddr = de.bestAddr.AddrPort + udpAddr = de.bestAddr.epAddr pathMTU := de.bestAddr.wireMTU - requestedMTU := pingSizeToPktLen(size, udpAddr.Addr().Is6()) + requestedMTU := pingSizeToPktLen(size, udpAddr) mtuOk := requestedMTU <= pathMTU - if udpAddr.IsValid() && mtuOk { + if udpAddr.ap.IsValid() && mtuOk { if !now.After(de.trustBestAddrUntil) { return udpAddr, netip.AddrPort{} } @@ -638,7 +673,7 @@ func (de *endpoint) addrForPingSizeLocked(now mono.Time, size int) (udpAddr, der // for the packet. Return a zero-value udpAddr to signal that we should // keep probing the path MTU to all addresses for this endpoint, and a // valid DERP addr to signal that we should also send via DERP. - return netip.AddrPort{}, de.derpAddr + return epAddr{}, de.derpAddr } // maybeProbeUDPLifetimeLocked returns an afterInactivityFor duration and true @@ -649,7 +684,7 @@ func (de *endpoint) maybeProbeUDPLifetimeLocked() (afterInactivityFor time.Durat if p == nil { return afterInactivityFor, false } - if !de.bestAddr.IsValid() { + if !de.bestAddr.ap.IsValid() { return afterInactivityFor, false } epDisco := de.disco.Load() @@ -701,7 +736,7 @@ func (de *endpoint) scheduleHeartbeatForLifetimeLocked(after time.Duration, via } de.c.dlogf("[v1] magicsock: disco: scheduling UDP lifetime probe for cliff=%v via=%v to %v (%v)", p.currentCliffDurationEndpointLocked(), via, de.publicKey.ShortString(), de.discoShort()) - p.bestAddr = de.bestAddr.AddrPort + p.bestAddr = de.bestAddr.epAddr p.timer = time.AfterFunc(after, de.heartbeatForLifetime) if via == heartbeatForLifetimeViaSelf { metricUDPLifetimeCliffsRescheduled.Add(1) @@ -729,7 +764,7 @@ func (de *endpoint) heartbeatForLifetime() { return } p.timer = nil - if !p.bestAddr.IsValid() || de.bestAddr.AddrPort != p.bestAddr { + if !p.bestAddr.ap.IsValid() || de.bestAddr.epAddr != p.bestAddr { // best path changed p.resetCycleEndpointLocked() return @@ -761,7 +796,7 @@ func (de *endpoint) heartbeatForLifetime() { } de.c.dlogf("[v1] magicsock: disco: sending disco ping for UDP lifetime probe cliff=%v to %v (%v)", p.currentCliffDurationEndpointLocked(), de.publicKey.ShortString(), de.discoShort()) - de.startDiscoPingLocked(de.bestAddr.AddrPort, mono.Now(), pingHeartbeatForUDPLifetime, 0, nil) + de.startDiscoPingLocked(de.bestAddr.epAddr, mono.Now(), pingHeartbeatForUDPLifetime, 0, nil) } // heartbeat is called every heartbeatInterval to keep the best UDP path alive, @@ -819,7 +854,7 @@ func (de *endpoint) heartbeat() { } udpAddr, _, _ := de.addrForSendLocked(now) - if udpAddr.IsValid() { + if udpAddr.ap.IsValid() { // We have a preferred path. Ping that every 'heartbeatInterval'. de.startDiscoPingLocked(udpAddr, now, pingHeartbeat, 0, nil) } @@ -828,6 +863,10 @@ func (de *endpoint) heartbeat() { de.sendDiscoPingsLocked(now, true) } + if de.wantUDPRelayPathDiscoveryLocked(now) { + de.discoverUDPRelayPathsLocked(now) + } + de.heartBeatTimer = time.AfterFunc(heartbeatInterval, de.heartbeat) } @@ -838,6 +877,53 @@ func (de *endpoint) setHeartbeatDisabled(v bool) { de.heartbeatDisabled = v } +// discoverUDPRelayPathsLocked starts UDP relay path discovery. +func (de *endpoint) discoverUDPRelayPathsLocked(now mono.Time) { + if !de.c.hasPeerRelayServers.Load() { + // Changes in this value between its access and the logic following + // are fine, we will eventually do the "right" thing during future path + // discovery. The worst case is we suppress path discovery for the + // current cycle, or we unnecessarily call into [relayManager] and do + // some wasted work. + return + } + de.lastUDPRelayPathDiscovery = now + lastBest := de.bestAddr + lastBestIsTrusted := mono.Now().Before(de.trustBestAddrUntil) + de.c.relayManager.startUDPRelayPathDiscoveryFor(de, lastBest, lastBestIsTrusted) +} + +// wantUDPRelayPathDiscoveryLocked reports whether we should kick off UDP relay +// path discovery. +func (de *endpoint) wantUDPRelayPathDiscoveryLocked(now mono.Time) bool { + if runtime.GOOS == "js" { + return false + } + if !de.relayCapable { + return false + } + if de.bestAddr.isDirect() && now.Before(de.trustBestAddrUntil) { + return false + } + if !de.lastUDPRelayPathDiscovery.IsZero() && now.Sub(de.lastUDPRelayPathDiscovery) < discoverUDPRelayPathsInterval { + return false + } + // TODO(jwhited): consider applying 'goodEnoughLatency' suppression here, + // but not until we have a strategy for triggering CallMeMaybeVia regularly + // and/or enabling inbound packets to act as a UDP relay path discovery + // trigger, otherwise clients without relay servers may fall off a UDP + // relay path and never come back. They are dependent on the remote side + // regularly TX'ing CallMeMaybeVia, which currently only happens as part + // of full UDP relay path discovery. + if now.After(de.trustBestAddrUntil) { + return true + } + if !de.lastUDPRelayPathDiscovery.IsZero() && now.Sub(de.lastUDPRelayPathDiscovery) >= upgradeUDPRelayInterval { + return true + } + return false +} + // wantFullPingLocked reports whether we should ping to all our peers looking for // a better path. // @@ -846,7 +932,7 @@ func (de *endpoint) wantFullPingLocked(now mono.Time) bool { if runtime.GOOS == "js" { return false } - if !de.bestAddr.IsValid() || de.lastFullPing.IsZero() { + if !de.bestAddr.isDirect() || de.lastFullPing.IsZero() { return true } if now.After(de.trustBestAddrUntil) { @@ -855,7 +941,7 @@ func (de *endpoint) wantFullPingLocked(now mono.Time) bool { if de.bestAddr.latency <= goodEnoughLatency { return false } - if now.Sub(de.lastFullPing) >= upgradeInterval { + if now.Sub(de.lastFullPing) >= upgradeUDPDirectInterval { return true } return false @@ -906,9 +992,9 @@ func (de *endpoint) discoPing(res *ipnstate.PingResult, size int, cb func(*ipnst udpAddr, derpAddr := de.addrForPingSizeLocked(now, size) if derpAddr.IsValid() { - de.startDiscoPingLocked(derpAddr, now, pingCLI, size, resCB) + de.startDiscoPingLocked(epAddr{ap: derpAddr}, now, pingCLI, size, resCB) } - if udpAddr.IsValid() && now.Before(de.trustBestAddrUntil) { + if udpAddr.ap.IsValid() && now.Before(de.trustBestAddrUntil) { // Already have an active session, so just ping the address we're using. // Otherwise "tailscale ping" results to a node on the local network // can look like they're bouncing between, say 10.0.0.0/9 and the peer's @@ -916,7 +1002,7 @@ func (de *endpoint) discoPing(res *ipnstate.PingResult, size int, cb func(*ipnst de.startDiscoPingLocked(udpAddr, now, pingCLI, size, resCB) } else { for ep := range de.endpointState { - de.startDiscoPingLocked(ep, now, pingCLI, size, resCB) + de.startDiscoPingLocked(epAddr{ap: ep}, now, pingCLI, size, resCB) } } } @@ -927,7 +1013,7 @@ var ( errPingTooBig = errors.New("ping size too big") ) -func (de *endpoint) send(buffs [][]byte) error { +func (de *endpoint) send(buffs [][]byte, offset int) error { de.mu.Lock() if de.expired { de.mu.Unlock() @@ -941,14 +1027,21 @@ func (de *endpoint) send(buffs [][]byte) error { if startWGPing { de.sendWireGuardOnlyPingsLocked(now) } - } else if !udpAddr.IsValid() || now.After(de.trustBestAddrUntil) { + } else if !udpAddr.isDirect() || now.After(de.trustBestAddrUntil) { de.sendDiscoPingsLocked(now, true) } + // TODO(jwhited): consider triggering UDP relay path discovery here under + // certain conditions. We currently only trigger it in heartbeat(), which + // is both good and bad. It's good because the first heartbeat() tick is 3s + // after the first packet, which gives us time to discover a UDP direct + // path and potentially avoid what would be wasted UDP relay path discovery + // work. It's bad because we might not discover a UDP direct path, and we + // incur a 3s delay before we try to discover a UDP relay path. de.noteTxActivityExtTriggerLocked(now) de.lastSendAny = now de.mu.Unlock() - if !udpAddr.IsValid() && !derpAddr.IsValid() { + if !udpAddr.ap.IsValid() && !derpAddr.IsValid() { // Make a last ditch effort to see if we have a DERP route for them. If // they contacted us over DERP and we don't know their UDP endpoints or // their DERP home, we can at least assume they're reachable over the @@ -960,8 +1053,8 @@ func (de *endpoint) send(buffs [][]byte) error { } } var err error - if udpAddr.IsValid() { - _, err = de.c.sendUDPBatch(udpAddr, buffs) + if udpAddr.ap.IsValid() { + _, err = de.c.sendUDPBatch(udpAddr, buffs, offset) // If the error is known to indicate that the endpoint is no longer // usable, clear the endpoint statistics so that the next send will @@ -972,29 +1065,41 @@ func (de *endpoint) send(buffs [][]byte) error { var txBytes int for _, b := range buffs { - txBytes += len(b) + txBytes += len(b[offset:]) } switch { - case udpAddr.Addr().Is4(): - de.c.metrics.outboundPacketsIPv4Total.Add(int64(len(buffs))) - de.c.metrics.outboundBytesIPv4Total.Add(int64(txBytes)) - case udpAddr.Addr().Is6(): - de.c.metrics.outboundPacketsIPv6Total.Add(int64(len(buffs))) - de.c.metrics.outboundBytesIPv6Total.Add(int64(txBytes)) + case udpAddr.ap.Addr().Is4(): + if udpAddr.vni.isSet() { + de.c.metrics.outboundPacketsPeerRelayIPv4Total.Add(int64(len(buffs))) + de.c.metrics.outboundBytesPeerRelayIPv4Total.Add(int64(txBytes)) + } else { + de.c.metrics.outboundPacketsIPv4Total.Add(int64(len(buffs))) + de.c.metrics.outboundBytesIPv4Total.Add(int64(txBytes)) + } + case udpAddr.ap.Addr().Is6(): + if udpAddr.vni.isSet() { + de.c.metrics.outboundPacketsPeerRelayIPv6Total.Add(int64(len(buffs))) + de.c.metrics.outboundBytesPeerRelayIPv6Total.Add(int64(txBytes)) + } else { + de.c.metrics.outboundPacketsIPv6Total.Add(int64(len(buffs))) + de.c.metrics.outboundBytesIPv6Total.Add(int64(txBytes)) + } } // TODO(raggi): needs updating for accuracy, as in error conditions we may have partial sends. if stats := de.c.stats.Load(); err == nil && stats != nil { - stats.UpdateTxPhysical(de.nodeAddr, udpAddr, len(buffs), txBytes) + stats.UpdateTxPhysical(de.nodeAddr, udpAddr.ap, len(buffs), txBytes) } } if derpAddr.IsValid() { allOk := true var txBytes int for _, buff := range buffs { + buff = buff[offset:] const isDisco = false - ok, _ := de.c.sendAddr(derpAddr, de.publicKey, buff, isDisco) + const isGeneveEncap = false + ok, _ := de.c.sendAddr(derpAddr, de.publicKey, buff, isDisco, isGeneveEncap) txBytes += len(buff) if !ok { allOk = false @@ -1054,7 +1159,12 @@ func (de *endpoint) discoPingTimeout(txid stun.TxID) { if !ok { return } - if debugDisco() || !de.bestAddr.IsValid() || mono.Now().After(de.trustBestAddrUntil) { + bestUntrusted := mono.Now().After(de.trustBestAddrUntil) + if sp.to == de.bestAddr.epAddr && sp.to.vni.isSet() && bestUntrusted { + // TODO(jwhited): consider applying this to direct UDP paths as well + de.clearBestAddrLocked() + } + if debugDisco() || !de.bestAddr.ap.IsValid() || bestUntrusted { de.c.dlogf("[v1] magicsock: disco: timeout waiting for pong %x from %v (%v, %v)", txid[:6], sp.to, de.publicKey.ShortString(), de.discoShort()) } de.removeSentDiscoPingLocked(txid, sp, discoPingTimedOut) @@ -1108,11 +1218,11 @@ const discoPingSize = len(disco.Magic) + key.DiscoPublicRawLen + disco.NonceLen // // The caller should use de.discoKey as the discoKey argument. // It is passed in so that sendDiscoPing doesn't need to lock de.mu. -func (de *endpoint) sendDiscoPing(ep netip.AddrPort, discoKey key.DiscoPublic, txid stun.TxID, size int, logLevel discoLogLevel) { +func (de *endpoint) sendDiscoPing(ep epAddr, discoKey key.DiscoPublic, txid stun.TxID, size int, logLevel discoLogLevel) { size = min(size, MaxDiscoPingSize) padding := max(size-discoPingSize, 0) - sent, _ := de.c.sendDiscoMessage(ep, virtualNetworkID{}, de.publicKey, discoKey, &disco.Ping{ + sent, _ := de.c.sendDiscoMessage(ep, de.publicKey, discoKey, &disco.Ping{ TxID: [12]byte(txid), NodeKey: de.c.publicKeyAtomic.Load(), Padding: padding, @@ -1124,7 +1234,7 @@ func (de *endpoint) sendDiscoPing(ep netip.AddrPort, discoKey key.DiscoPublic, t if size != 0 { metricSentDiscoPeerMTUProbes.Add(1) - metricSentDiscoPeerMTUProbeBytes.Add(int64(pingSizeToPktLen(size, ep.Addr().Is6()))) + metricSentDiscoPeerMTUProbeBytes.Add(int64(pingSizeToPktLen(size, ep))) } } @@ -1155,7 +1265,7 @@ const ( // if non-nil, means that a caller external to the magicsock package internals // is interested in the result (such as a CLI "tailscale ping" or a c2n ping // request, etc) -func (de *endpoint) startDiscoPingLocked(ep netip.AddrPort, now mono.Time, purpose discoPingPurpose, size int, resCB *pingResultAndCallback) { +func (de *endpoint) startDiscoPingLocked(ep epAddr, now mono.Time, purpose discoPingPurpose, size int, resCB *pingResultAndCallback) { if runtime.GOOS == "js" { return } @@ -1163,8 +1273,9 @@ func (de *endpoint) startDiscoPingLocked(ep netip.AddrPort, now mono.Time, purpo if epDisco == nil { return } - if purpose != pingCLI { - st, ok := de.endpointState[ep] + if purpose != pingCLI && + !ep.vni.isSet() { // de.endpointState is only relevant for direct/non-vni epAddr's + st, ok := de.endpointState[ep.ap] if !ok { // Shouldn't happen. But don't ping an endpoint that's // not active for us. @@ -1181,11 +1292,11 @@ func (de *endpoint) startDiscoPingLocked(ep netip.AddrPort, now mono.Time, purpo // Default to sending a single ping of the specified size sizes := []int{size} if de.c.PeerMTUEnabled() { - isDerp := ep.Addr() == tailcfg.DerpMagicIPAddr + isDerp := ep.ap.Addr() == tailcfg.DerpMagicIPAddr if !isDerp && ((purpose == pingDiscovery) || (purpose == pingCLI && size == 0)) { de.c.dlogf("[v1] magicsock: starting MTU probe") sizes = mtuProbePingSizesV4 - if ep.Addr().Is6() { + if ep.ap.Addr().Is6() { sizes = mtuProbePingSizesV6 } } @@ -1240,7 +1351,7 @@ func (de *endpoint) sendDiscoPingsLocked(now mono.Time, sendCallMeMaybe bool) { de.c.dlogf("[v1] magicsock: disco: send, starting discovery for %v (%v)", de.publicKey.ShortString(), de.discoShort()) } - de.startDiscoPingLocked(ep, now, pingDiscovery, 0, nil) + de.startDiscoPingLocked(epAddr{ap: ep}, now, pingDiscovery, 0, nil) } derpAddr := de.derpAddr if sentAny && sendCallMeMaybe && derpAddr.IsValid() { @@ -1250,13 +1361,6 @@ func (de *endpoint) sendDiscoPingsLocked(now mono.Time, sendCallMeMaybe bool) { // sent so our firewall ports are probably open and now // would be a good time for them to connect. go de.c.enqueueCallMeMaybe(derpAddr, de) - - // Schedule allocation of relay endpoints. We make no considerations for - // current relay endpoints or best UDP path state for now, keep it - // simple. - if de.relayCapable { - go de.c.relayManager.allocateAndHandshakeAllServers(de) - } } } @@ -1398,6 +1502,8 @@ func (de *endpoint) updateFromNode(n tailcfg.NodeView, heartbeatDisabled bool, p } de.setEndpointsLocked(n.Endpoints()) + + de.relayCapable = capVerIsRelayCapable(n.Cap()) } func (de *endpoint) setEndpointsLocked(eps interface { @@ -1495,17 +1601,19 @@ func (de *endpoint) clearBestAddrLocked() { de.trustBestAddrUntil = 0 } -// noteBadEndpoint marks ipp as a bad endpoint that would need to be +// noteBadEndpoint marks udpAddr as a bad endpoint that would need to be // re-evaluated before future use, this should be called for example if a send -// to ipp fails due to a host unreachable error or similar. -func (de *endpoint) noteBadEndpoint(ipp netip.AddrPort) { +// to udpAddr fails due to a host unreachable error or similar. +func (de *endpoint) noteBadEndpoint(udpAddr epAddr) { de.mu.Lock() defer de.mu.Unlock() de.clearBestAddrLocked() - if st, ok := de.endpointState[ipp]; ok { - st.clear() + if !udpAddr.vni.isSet() { + if st, ok := de.endpointState[udpAddr.ap]; ok { + st.clear() + } } } @@ -1525,17 +1633,20 @@ func (de *endpoint) noteConnectivityChange() { // pingSizeToPktLen calculates the minimum path MTU that would permit // a disco ping message of length size to reach its target at -// addr. size is the length of the entire disco message including +// udpAddr. size is the length of the entire disco message including // disco headers. If size is zero, assume it is the safe wire MTU. -func pingSizeToPktLen(size int, is6 bool) tstun.WireMTU { +func pingSizeToPktLen(size int, udpAddr epAddr) tstun.WireMTU { if size == 0 { return tstun.SafeWireMTU() } headerLen := ipv4.HeaderLen - if is6 { + if udpAddr.ap.Addr().Is6() { headerLen = ipv6.HeaderLen } headerLen += 8 // UDP header length + if udpAddr.vni.isSet() { + headerLen += packet.GeneveFixedHeaderLength + } return tstun.WireMTU(size + headerLen) } @@ -1562,11 +1673,11 @@ func pktLenToPingSize(mtu tstun.WireMTU, is6 bool) int { // It should be called with the Conn.mu held. // // It reports whether m.TxID corresponds to a ping that this endpoint sent. -func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip.AddrPort) (knownTxID bool) { +func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src epAddr) (knownTxID bool) { de.mu.Lock() defer de.mu.Unlock() - isDerp := src.Addr() == tailcfg.DerpMagicIPAddr + isDerp := src.ap.Addr() == tailcfg.DerpMagicIPAddr sp, ok := de.sentPing[m.TxID] if !ok { @@ -1576,7 +1687,7 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip knownTxID = true // for naked returns below de.removeSentDiscoPingLocked(m.TxID, sp, discoPongReceived) - pktLen := int(pingSizeToPktLen(sp.size, sp.to.Addr().Is6())) + pktLen := int(pingSizeToPktLen(sp.size, src)) if sp.size != 0 { m := getPeerMTUsProbedMetric(tstun.WireMTU(pktLen)) m.Add(1) @@ -1588,19 +1699,21 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip now := mono.Now() latency := now.Sub(sp.at) - if !isDerp { - st, ok := de.endpointState[sp.to] + if !isDerp && !src.vni.isSet() { + // Note: we check vni.isSet() as relay [epAddr]'s are not stored in + // endpointState, they are either de.bestAddr or not. + st, ok := de.endpointState[sp.to.ap] if !ok { // This is no longer an endpoint we care about. return } - de.c.peerMap.setNodeKeyForIPPort(src, de.publicKey) + de.c.peerMap.setNodeKeyForEpAddr(src, de.publicKey) st.addPongReplyLocked(pongReply{ latency: latency, pongAt: now, - from: src, + from: src.ap, pongSrc: m.Src, }) } @@ -1624,8 +1737,22 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip // Promote this pong response to our current best address if it's lower latency. // TODO(bradfitz): decide how latency vs. preference order affects decision if !isDerp { - thisPong := addrQuality{sp.to, latency, tstun.WireMTU(pingSizeToPktLen(sp.size, sp.to.Addr().Is6()))} + thisPong := addrQuality{ + epAddr: sp.to, + latency: latency, + wireMTU: pingSizeToPktLen(sp.size, sp.to), + } + // TODO(jwhited): consider checking de.trustBestAddrUntil as well. If + // de.bestAddr is untrusted we may want to clear it, otherwise we could + // get stuck with a forever untrusted bestAddr that blackholes, since + // we don't clear direct UDP paths on disco ping timeout (see + // discoPingTimeout). if betterAddr(thisPong, de.bestAddr) { + if src.vni.isSet() { + // This would be unexpected. Switching to a Geneve-encapsulated + // path should only happen in de.relayEndpointReady(). + de.c.logf("[unexpected] switching to Geneve-encapsulated path %v from %v", thisPong, de.bestAddr) + } de.c.logf("magicsock: disco: node %v %v now using %v mtu=%v tx=%x", de.publicKey.ShortString(), de.discoShort(), sp.to, thisPong.wireMTU, m.TxID[:6]) de.debugUpdates.Add(EndpointChange{ When: time.Now(), @@ -1635,7 +1762,7 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip }) de.setBestAddrLocked(thisPong) } - if de.bestAddr.AddrPort == thisPong.AddrPort { + if de.bestAddr.epAddr == thisPong.epAddr { de.debugUpdates.Add(EndpointChange{ When: time.Now(), What: "handlePongConnLocked-bestAddr-latency", @@ -1650,20 +1777,43 @@ func (de *endpoint) handlePongConnLocked(m *disco.Pong, di *discoInfo, src netip return } -// addrQuality is an IPPort with an associated latency and path mtu. +// epAddr is a [netip.AddrPort] with an optional Geneve header (RFC8926) +// [virtualNetworkID]. +type epAddr struct { + ap netip.AddrPort // if ap == tailcfg.DerpMagicIPAddr then vni is never set + vni virtualNetworkID // vni.isSet() indicates if this [epAddr] involves a Geneve header +} + +// isDirect returns true if e.ap is valid and not tailcfg.DerpMagicIPAddr, +// and a VNI is not set. +func (e epAddr) isDirect() bool { + return e.ap.IsValid() && e.ap.Addr() != tailcfg.DerpMagicIPAddr && !e.vni.isSet() +} + +func (e epAddr) String() string { + if !e.vni.isSet() { + return e.ap.String() + } + return fmt.Sprintf("%v:vni:%d", e.ap.String(), e.vni.get()) +} + +// addrQuality is an [epAddr], an optional [key.DiscoPublic] if a relay server +// is associated, a round-trip latency measurement, and path mtu. type addrQuality struct { - netip.AddrPort - latency time.Duration - wireMTU tstun.WireMTU + epAddr + relayServerDisco key.DiscoPublic // only relevant if epAddr.vni.isSet(), otherwise zero value + latency time.Duration + wireMTU tstun.WireMTU } func (a addrQuality) String() string { - return fmt.Sprintf("%v@%v+%v", a.AddrPort, a.latency, a.wireMTU) + // TODO(jwhited): consider including relayServerDisco + return fmt.Sprintf("%v@%v+%v", a.epAddr, a.latency, a.wireMTU) } // betterAddr reports whether a is a better addr to use than b. func betterAddr(a, b addrQuality) bool { - if a.AddrPort == b.AddrPort { + if a.epAddr == b.epAddr { if a.wireMTU > b.wireMTU { // TODO(val): Think harder about the case of lower // latency and smaller or unknown MTU, and higher @@ -1674,10 +1824,19 @@ func betterAddr(a, b addrQuality) bool { } return false } - if !b.IsValid() { + if !b.ap.IsValid() { return true } - if !a.IsValid() { + if !a.ap.IsValid() { + return false + } + + // Geneve-encapsulated paths (UDP relay servers) are lower preference in + // relation to non. + if !a.vni.isSet() && b.vni.isSet() { + return true + } + if a.vni.isSet() && !b.vni.isSet() { return false } @@ -1701,27 +1860,27 @@ func betterAddr(a, b addrQuality) bool { // addresses, and prefer link-local unicast addresses over other types // of private IP addresses since it's definitionally more likely that // they'll be on the same network segment than a general private IP. - if a.Addr().IsLoopback() { + if a.ap.Addr().IsLoopback() { aPoints += 50 - } else if a.Addr().IsLinkLocalUnicast() { + } else if a.ap.Addr().IsLinkLocalUnicast() { aPoints += 30 - } else if a.Addr().IsPrivate() { + } else if a.ap.Addr().IsPrivate() { aPoints += 20 } - if b.Addr().IsLoopback() { + if b.ap.Addr().IsLoopback() { bPoints += 50 - } else if b.Addr().IsLinkLocalUnicast() { + } else if b.ap.Addr().IsLinkLocalUnicast() { bPoints += 30 - } else if b.Addr().IsPrivate() { + } else if b.ap.Addr().IsPrivate() { bPoints += 20 } // Prefer IPv6 for being a bit more robust, as long as // the latencies are roughly equivalent. - if a.Addr().Is6() { + if a.ap.Addr().Is6() { aPoints += 10 } - if b.Addr().Is6() { + if b.ap.Addr().Is6() { bPoints += 10 } @@ -1822,8 +1981,12 @@ func (de *endpoint) populatePeerStatus(ps *ipnstate.PeerStatus) { ps.LastWrite = de.lastSendExt.WallTime() ps.Active = now.Sub(de.lastSendExt) < sessionActiveTimeout - if udpAddr, derpAddr, _ := de.addrForSendLocked(now); udpAddr.IsValid() && !derpAddr.IsValid() { - ps.CurAddr = udpAddr.String() + if udpAddr, derpAddr, _ := de.addrForSendLocked(now); udpAddr.ap.IsValid() && !derpAddr.IsValid() { + if udpAddr.vni.isSet() { + ps.PeerRelay = udpAddr.String() + } else { + ps.CurAddr = udpAddr.String() + } } } @@ -1878,8 +2041,15 @@ func (de *endpoint) numStopAndReset() int64 { return atomic.LoadInt64(&de.numStopAndResetAtomic) } +// setDERPHome sets the provided regionID as home for de. Calls to setDERPHome +// must never run concurrent to [Conn.updateRelayServersSet], otherwise +// [candidatePeerRelay] DERP home changes may be missed from the perspective of +// [relayManager]. func (de *endpoint) setDERPHome(regionID uint16) { de.mu.Lock() defer de.mu.Unlock() de.derpAddr = netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, uint16(regionID)) + if de.c.hasPeerRelayServers.Load() { + de.c.relayManager.handleDERPHomeChange(de.publicKey, regionID) + } } diff --git a/wgengine/magicsock/endpoint_test.go b/wgengine/magicsock/endpoint_test.go index 1e2de8967511c..92f4ef1d3aac1 100644 --- a/wgengine/magicsock/endpoint_test.go +++ b/wgengine/magicsock/endpoint_test.go @@ -8,7 +8,8 @@ import ( "testing" "time" - "github.com/dsnet/try" + "tailscale.com/tailcfg" + "tailscale.com/tstime/mono" "tailscale.com/types/key" ) @@ -154,7 +155,7 @@ func Test_endpoint_maybeProbeUDPLifetimeLocked(t *testing.T) { lower = b higher = a } - addr := addrQuality{AddrPort: try.E1[netip.AddrPort](netip.ParseAddrPort("1.1.1.1:1"))} + addr := addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort("1.1.1.1:1")}} newProbeUDPLifetime := func() *probeUDPLifetime { return &probeUDPLifetime{ config: *defaultProbeUDPLifetimeConfig, @@ -324,3 +325,132 @@ func Test_endpoint_maybeProbeUDPLifetimeLocked(t *testing.T) { }) } } + +func Test_epAddr_isDirectUDP(t *testing.T) { + vni := virtualNetworkID{} + vni.set(7) + tests := []struct { + name string + ap netip.AddrPort + vni virtualNetworkID + want bool + }{ + { + name: "true", + ap: netip.MustParseAddrPort("192.0.2.1:7"), + vni: virtualNetworkID{}, + want: true, + }, + { + name: "false derp magic addr", + ap: netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, 0), + vni: virtualNetworkID{}, + want: false, + }, + { + name: "false vni set", + ap: netip.MustParseAddrPort("192.0.2.1:7"), + vni: vni, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := epAddr{ + ap: tt.ap, + vni: tt.vni, + } + if got := e.isDirect(); got != tt.want { + t.Errorf("isDirect() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_endpoint_udpRelayEndpointReady(t *testing.T) { + directAddrQuality := addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.1:7")}} + peerRelayAddrQuality := addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.2:77")}, latency: time.Second} + peerRelayAddrQuality.vni.set(1) + peerRelayAddrQualityHigherLatencySameServer := addrQuality{ + epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.3:77"), vni: peerRelayAddrQuality.vni}, + latency: peerRelayAddrQuality.latency * 10, + } + peerRelayAddrQualityHigherLatencyDiffServer := addrQuality{ + epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.3:77"), vni: peerRelayAddrQuality.vni}, + latency: peerRelayAddrQuality.latency * 10, + relayServerDisco: key.NewDisco().Public(), + } + peerRelayAddrQualityLowerLatencyDiffServer := addrQuality{ + epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.4:77"), vni: peerRelayAddrQuality.vni}, + latency: peerRelayAddrQuality.latency / 10, + relayServerDisco: key.NewDisco().Public(), + } + peerRelayAddrQualityEqualLatencyDiffServer := addrQuality{ + epAddr: epAddr{ap: netip.MustParseAddrPort("192.0.2.4:77"), vni: peerRelayAddrQuality.vni}, + latency: peerRelayAddrQuality.latency, + relayServerDisco: key.NewDisco().Public(), + } + tests := []struct { + name string + curBestAddr addrQuality + trustBestAddrUntil mono.Time + maybeBest addrQuality + wantBestAddr addrQuality + }{ + { + name: "bestAddr trusted direct", + curBestAddr: directAddrQuality, + trustBestAddrUntil: mono.Now().Add(1 * time.Hour), + maybeBest: peerRelayAddrQuality, + wantBestAddr: directAddrQuality, + }, + { + name: "bestAddr untrusted direct", + curBestAddr: directAddrQuality, + trustBestAddrUntil: mono.Now().Add(-1 * time.Hour), + maybeBest: peerRelayAddrQuality, + wantBestAddr: peerRelayAddrQuality, + }, + { + name: "maybeBest same relay server higher latency bestAddr trusted", + curBestAddr: peerRelayAddrQuality, + trustBestAddrUntil: mono.Now().Add(1 * time.Hour), + maybeBest: peerRelayAddrQualityHigherLatencySameServer, + wantBestAddr: peerRelayAddrQualityHigherLatencySameServer, + }, + { + name: "maybeBest diff relay server higher latency bestAddr trusted", + curBestAddr: peerRelayAddrQuality, + trustBestAddrUntil: mono.Now().Add(1 * time.Hour), + maybeBest: peerRelayAddrQualityHigherLatencyDiffServer, + wantBestAddr: peerRelayAddrQuality, + }, + { + name: "maybeBest diff relay server lower latency bestAddr trusted", + curBestAddr: peerRelayAddrQuality, + trustBestAddrUntil: mono.Now().Add(1 * time.Hour), + maybeBest: peerRelayAddrQualityLowerLatencyDiffServer, + wantBestAddr: peerRelayAddrQualityLowerLatencyDiffServer, + }, + { + name: "maybeBest diff relay server equal latency bestAddr trusted", + curBestAddr: peerRelayAddrQuality, + trustBestAddrUntil: mono.Now().Add(1 * time.Hour), + maybeBest: peerRelayAddrQualityEqualLatencyDiffServer, + wantBestAddr: peerRelayAddrQuality, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + de := &endpoint{ + c: &Conn{logf: func(msg string, args ...any) { return }}, + bestAddr: tt.curBestAddr, + trustBestAddrUntil: tt.trustBestAddrUntil, + } + de.udpRelayEndpointReady(tt.maybeBest) + if de.bestAddr != tt.wantBestAddr { + t.Errorf("de.bestAddr = %v, want %v", de.bestAddr, tt.wantBestAddr) + } + }) + } +} diff --git a/wgengine/magicsock/magicsock.go b/wgengine/magicsock/magicsock.go index 61cdf49543493..d2835aed34547 100644 --- a/wgengine/magicsock/magicsock.go +++ b/wgengine/magicsock/magicsock.go @@ -26,6 +26,7 @@ import ( "time" "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/device" "go4.org/mem" "golang.org/x/net/ipv6" @@ -63,6 +64,7 @@ import ( "tailscale.com/util/set" "tailscale.com/util/testenv" "tailscale.com/util/usermetric" + "tailscale.com/wgengine/filter" "tailscale.com/wgengine/wgint" ) @@ -85,9 +87,11 @@ const ( type Path string const ( - PathDirectIPv4 Path = "direct_ipv4" - PathDirectIPv6 Path = "direct_ipv6" - PathDERP Path = "derp" + PathDirectIPv4 Path = "direct_ipv4" + PathDirectIPv6 Path = "direct_ipv6" + PathDERP Path = "derp" + PathPeerRelayIPv4 Path = "peer_relay_ipv4" + PathPeerRelayIPv6 Path = "peer_relay_ipv6" ) type pathLabel struct { @@ -95,6 +99,8 @@ type pathLabel struct { // - direct_ipv4 // - direct_ipv6 // - derp + // - peer_relay_ipv4 + // - peer_relay_ipv6 Path Path } @@ -106,27 +112,35 @@ type pathLabel struct { type metrics struct { // inboundPacketsTotal is the total number of inbound packets received, // labeled by the path the packet took. - inboundPacketsIPv4Total expvar.Int - inboundPacketsIPv6Total expvar.Int - inboundPacketsDERPTotal expvar.Int + inboundPacketsIPv4Total expvar.Int + inboundPacketsIPv6Total expvar.Int + inboundPacketsDERPTotal expvar.Int + inboundPacketsPeerRelayIPv4Total expvar.Int + inboundPacketsPeerRelayIPv6Total expvar.Int // inboundBytesTotal is the total number of inbound bytes received, // labeled by the path the packet took. - inboundBytesIPv4Total expvar.Int - inboundBytesIPv6Total expvar.Int - inboundBytesDERPTotal expvar.Int + inboundBytesIPv4Total expvar.Int + inboundBytesIPv6Total expvar.Int + inboundBytesDERPTotal expvar.Int + inboundBytesPeerRelayIPv4Total expvar.Int + inboundBytesPeerRelayIPv6Total expvar.Int // outboundPacketsTotal is the total number of outbound packets sent, // labeled by the path the packet took. - outboundPacketsIPv4Total expvar.Int - outboundPacketsIPv6Total expvar.Int - outboundPacketsDERPTotal expvar.Int + outboundPacketsIPv4Total expvar.Int + outboundPacketsIPv6Total expvar.Int + outboundPacketsDERPTotal expvar.Int + outboundPacketsPeerRelayIPv4Total expvar.Int + outboundPacketsPeerRelayIPv6Total expvar.Int // outboundBytesTotal is the total number of outbound bytes sent, // labeled by the path the packet took. - outboundBytesIPv4Total expvar.Int - outboundBytesIPv6Total expvar.Int - outboundBytesDERPTotal expvar.Int + outboundBytesIPv4Total expvar.Int + outboundBytesIPv6Total expvar.Int + outboundBytesDERPTotal expvar.Int + outboundBytesPeerRelayIPv4Total expvar.Int + outboundBytesPeerRelayIPv6Total expvar.Int // outboundPacketsDroppedErrors is the total number of outbound packets // dropped due to errors. @@ -159,6 +173,18 @@ type Conn struct { connCtxCancel func() // closes connCtx donec <-chan struct{} // connCtx.Done()'s to avoid context.cancelCtx.Done()'s mutex per call + // These [eventbus.Subscriber] fields are solely accessed by + // consumeEventbusTopics once initialized. + pmSub *eventbus.Subscriber[portmapper.Mapping] + filterSub *eventbus.Subscriber[FilterUpdate] + nodeViewsSub *eventbus.Subscriber[NodeViewsUpdate] + nodeMutsSub *eventbus.Subscriber[NodeMutationsUpdate] + syncSub *eventbus.Subscriber[syncPoint] + syncPub *eventbus.Publisher[syncPoint] + allocRelayEndpointPub *eventbus.Publisher[UDPRelayAllocReq] + allocRelayEndpointSub *eventbus.Subscriber[UDPRelayAllocResp] + subsDoneCh chan struct{} // closed when consumeEventbusTopics returns + // pconn4 and pconn6 are the underlying UDP sockets used to // send/receive packets for wireguard and other magicsock // protocols. @@ -247,6 +273,14 @@ type Conn struct { // captureHook, if non-nil, is the pcap logging callback when capturing. captureHook syncs.AtomicValue[packet.CaptureCallback] + // hasPeerRelayServers is whether [relayManager] is configured with at least + // one peer relay server via [relayManager.handleRelayServersSet]. It is + // only accessed by [Conn.updateRelayServersSet], [endpoint.setDERPHome], + // and [endpoint.discoverUDPRelayPathsLocked]. It exists to suppress + // calls into [relayManager] leading to wasted work involving channel + // operations and goroutine creation. + hasPeerRelayServers atomic.Bool + // discoPrivate is the private naclbox key used for active // discovery traffic. It is always present, and immutable. discoPrivate key.DiscoPrivate @@ -339,17 +373,19 @@ type Conn struct { // magicsock could do with any complexity reduction it can get. netInfoLast *tailcfg.NetInfo - derpMap *tailcfg.DERPMap // nil (or zero regions/nodes) means DERP is disabled - peers views.Slice[tailcfg.NodeView] // from last SetNetworkMap update - lastFlags debugFlags // at time of last SetNetworkMap - firstAddrForTest netip.Addr // from last SetNetworkMap update; for tests only - privateKey key.NodePrivate // WireGuard private key for this node - everHadKey bool // whether we ever had a non-zero private key - myDerp int // nearest DERP region ID; 0 means none/unknown - homeless bool // if true, don't try to find & stay conneted to a DERP home (myDerp will stay 0) - derpStarted chan struct{} // closed on first connection to DERP; for tests & cleaner Close - activeDerp map[int]activeDerp // DERP regionID -> connection to a node in that region - prevDerp map[int]*syncs.WaitGroupChan + derpMap *tailcfg.DERPMap // nil (or zero regions/nodes) means DERP is disabled + self tailcfg.NodeView // from last onNodeViewsUpdate + peers views.Slice[tailcfg.NodeView] // from last onNodeViewsUpdate, sorted by Node.ID; Note: [netmap.NodeMutation]'s rx'd in onNodeMutationsUpdate are never applied + filt *filter.Filter // from last onFilterUpdate + relayClientEnabled bool // whether we can allocate UDP relay endpoints on UDP relay servers or receive CallMeMaybeVia messages from peers + lastFlags debugFlags // at time of last onNodeViewsUpdate + privateKey key.NodePrivate // WireGuard private key for this node + everHadKey bool // whether we ever had a non-zero private key + myDerp int // nearest DERP region ID; 0 means none/unknown + homeless bool // if true, don't try to find & stay conneted to a DERP home (myDerp will stay 0) + derpStarted chan struct{} // closed on first connection to DERP; for tests & cleaner Close + activeDerp map[int]activeDerp // DERP regionID -> connection to a node in that region + prevDerp map[int]*syncs.WaitGroupChan // derpRoute contains optional alternate routes to use as an // optimization instead of contacting a peer via their home @@ -410,10 +446,8 @@ func (c *Conn) dlogf(format string, a ...any) { // Options contains options for Listen. type Options struct { // EventBus, if non-nil, is used for event publication and subscription by - // each Conn created from these Options. - // - // TODO(creachadair): As of 2025-03-19 this is optional, but is intended to - // become required non-nil. + // each Conn created from these Options. It must not be nil outside of + // tests. EventBus *eventbus.Bus // Logf provides a log function to use. It must not be nil. @@ -502,6 +536,77 @@ func (o *Options) derpActiveFunc() func() { return o.DERPActiveFunc } +// NodeViewsUpdate represents an update event of [tailcfg.NodeView] for all +// nodes. This event is published over an [eventbus.Bus]. It may be published +// with an invalid SelfNode, and/or zero/nil Peers. [magicsock.Conn] is the sole +// subscriber as of 2025-06. If you are adding more subscribers consider moving +// this type out of magicsock. +type NodeViewsUpdate struct { + SelfNode tailcfg.NodeView + Peers []tailcfg.NodeView // sorted by Node.ID +} + +// NodeMutationsUpdate represents an update event of one or more +// [netmap.NodeMutation]. This event is published over an [eventbus.Bus]. +// [magicsock.Conn] is the sole subscriber as of 2025-06. If you are adding more +// subscribers consider moving this type out of magicsock. +type NodeMutationsUpdate struct { + Mutations []netmap.NodeMutation +} + +// FilterUpdate represents an update event for a [*filter.Filter]. This event is +// signaled over an [eventbus.Bus]. [magicsock.Conn] is the sole subscriber as +// of 2025-06. If you are adding more subscribers consider moving this type out +// of magicsock. +type FilterUpdate struct { + *filter.Filter +} + +// syncPoint is an event published over an [eventbus.Bus] by [Conn.Synchronize]. +// It serves as a synchronization point, allowing to wait until magicsock +// has processed all pending events. +type syncPoint chan struct{} + +// Wait blocks until [syncPoint.Signal] is called. +func (s syncPoint) Wait() { + <-s +} + +// Signal signals the sync point, unblocking the [syncPoint.Wait] call. +func (s syncPoint) Signal() { + close(s) +} + +// UDPRelayAllocReq represents a [*disco.AllocateUDPRelayEndpointRequest] +// reception event. This is signaled over an [eventbus.Bus] from +// [magicsock.Conn] towards [relayserver.extension]. +type UDPRelayAllocReq struct { + // RxFromNodeKey is the unauthenticated (DERP server claimed src) node key + // of the transmitting party, noted at disco message reception time over + // DERP. This node key is unambiguously-aligned with RxFromDiscoKey being + // that the disco message is received over DERP. + RxFromNodeKey key.NodePublic + // RxFromDiscoKey is the disco key of the transmitting party, noted and + // authenticated at reception time. + RxFromDiscoKey key.DiscoPublic + // Message is the disco message. + Message *disco.AllocateUDPRelayEndpointRequest +} + +// UDPRelayAllocResp represents a [*disco.AllocateUDPRelayEndpointResponse] +// that is yet to be transmitted over DERP (or delivered locally if +// ReqRxFromNodeKey is self). This is signaled over an [eventbus.Bus] from +// [relayserver.extension] towards [magicsock.Conn]. +type UDPRelayAllocResp struct { + // ReqRxFromNodeKey is copied from [UDPRelayAllocReq.RxFromNodeKey]. It + // enables peer lookup leading up to transmission over DERP. + ReqRxFromNodeKey key.NodePublic + // ReqRxFromDiscoKey is copied from [UDPRelayAllocReq.RxFromDiscoKey]. + ReqRxFromDiscoKey key.DiscoPublic + // Message is the disco message. + Message *disco.AllocateUDPRelayEndpointResponse +} + // newConn is the error-free, network-listening-side-effect-free based // of NewConn. Mostly for tests. func newConn(logf logger.Logf) *Conn { @@ -535,6 +640,77 @@ func newConn(logf logger.Logf) *Conn { return c } +// consumeEventbusTopics consumes events from all [Conn]-relevant +// [eventbus.Subscriber]'s and passes them to their related handler. Events are +// always handled in the order they are received, i.e. the next event is not +// read until the previous event's handler has returned. It returns when the +// [portmapper.Mapping] subscriber is closed, which is interpreted to be the +// same as the [eventbus.Client] closing ([eventbus.Subscribers] are either +// all open or all closed). +func (c *Conn) consumeEventbusTopics() { + defer close(c.subsDoneCh) + + for { + select { + case <-c.pmSub.Done(): + return + case <-c.pmSub.Events(): + c.onPortMapChanged() + case filterUpdate := <-c.filterSub.Events(): + c.onFilterUpdate(filterUpdate) + case nodeViews := <-c.nodeViewsSub.Events(): + c.onNodeViewsUpdate(nodeViews) + case nodeMuts := <-c.nodeMutsSub.Events(): + c.onNodeMutationsUpdate(nodeMuts) + case syncPoint := <-c.syncSub.Events(): + c.dlogf("magicsock: received sync point after reconfig") + syncPoint.Signal() + case allocResp := <-c.allocRelayEndpointSub.Events(): + c.onUDPRelayAllocResp(allocResp) + } + } +} + +func (c *Conn) onUDPRelayAllocResp(allocResp UDPRelayAllocResp) { + c.mu.Lock() + defer c.mu.Unlock() + ep, ok := c.peerMap.endpointForNodeKey(allocResp.ReqRxFromNodeKey) + if !ok { + // If it's not a peer, it might be for self (we can peer relay through + // ourselves), in which case we want to hand it down to [relayManager] + // now versus taking a network round-trip through DERP. + selfNodeKey := c.publicKeyAtomic.Load() + if selfNodeKey.Compare(allocResp.ReqRxFromNodeKey) == 0 && + allocResp.ReqRxFromDiscoKey.Compare(c.discoPublic) == 0 { + c.relayManager.handleRxDiscoMsg(c, allocResp.Message, selfNodeKey, allocResp.ReqRxFromDiscoKey, epAddr{}) + } + return + } + disco := ep.disco.Load() + if disco == nil { + return + } + if disco.key.Compare(allocResp.ReqRxFromDiscoKey) != 0 { + return + } + ep.mu.Lock() + defer ep.mu.Unlock() + derpAddr := ep.derpAddr + go c.sendDiscoMessage(epAddr{ap: derpAddr}, ep.publicKey, disco.key, allocResp.Message, discoVerboseLog) +} + +// Synchronize waits for all [eventbus] events published +// prior to this call to be processed by the receiver. +func (c *Conn) Synchronize() { + if c.syncPub == nil { + // Eventbus is not used; no need to synchronize (in certain tests). + return + } + sp := syncPoint(make(chan struct{})) + c.syncPub.Publish(sp) + sp.Wait() +} + // NewConn creates a magic Conn listening on opts.Port. // As the set of possible endpoints for a Conn changes, the // callback opts.EndpointsFunc is called. @@ -553,29 +729,21 @@ func NewConn(opts Options) (*Conn, error) { c.testOnlyPacketListener = opts.TestOnlyPacketListener c.noteRecvActivity = opts.NoteRecvActivity - // If an event bus is enabled, subscribe to portmapping changes; otherwise - // use the callback mechanism of portmapper.Client. - // - // TODO(creachadair): Remove the switch once the event bus is mandatory. - onPortMapChanged := c.onPortMapChanged if c.eventBus != nil { c.eventClient = c.eventBus.Client("magicsock.Conn") - pmSub := eventbus.Subscribe[portmapper.Mapping](c.eventClient) - go func() { - defer pmSub.Close() - for { - select { - case <-pmSub.Events(): - c.onPortMapChanged() - case <-pmSub.Done(): - return - } - } - }() - - // Disable the explicit callback from the portmapper, the subscriber handles it. - onPortMapChanged = nil + // Subscribe calls must return before NewConn otherwise published + // events can be missed. + c.pmSub = eventbus.Subscribe[portmapper.Mapping](c.eventClient) + c.filterSub = eventbus.Subscribe[FilterUpdate](c.eventClient) + c.nodeViewsSub = eventbus.Subscribe[NodeViewsUpdate](c.eventClient) + c.nodeMutsSub = eventbus.Subscribe[NodeMutationsUpdate](c.eventClient) + c.syncSub = eventbus.Subscribe[syncPoint](c.eventClient) + c.syncPub = eventbus.Publish[syncPoint](c.eventClient) + c.allocRelayEndpointPub = eventbus.Publish[UDPRelayAllocReq](c.eventClient) + c.allocRelayEndpointSub = eventbus.Subscribe[UDPRelayAllocResp](c.eventClient) + c.subsDoneCh = make(chan struct{}) + go c.consumeEventbusTopics() } // Don't log the same log messages possibly every few seconds in our @@ -591,7 +759,6 @@ func NewConn(opts Options) (*Conn, error) { NetMon: opts.NetMon, DebugKnobs: portMapOpts, ControlKnobs: opts.ControlKnobs, - OnChange: onPortMapChanged, }) c.portMapper.SetGatewayLookupFunc(opts.NetMon.GatewayAndSelfIP) c.netMon = opts.NetMon @@ -640,6 +807,8 @@ func registerMetrics(reg *usermetric.Registry) *metrics { pathDirectV4 := pathLabel{Path: PathDirectIPv4} pathDirectV6 := pathLabel{Path: PathDirectIPv6} pathDERP := pathLabel{Path: PathDERP} + pathPeerRelayV4 := pathLabel{Path: PathPeerRelayIPv4} + pathPeerRelayV6 := pathLabel{Path: PathPeerRelayIPv6} inboundPacketsTotal := usermetric.NewMultiLabelMapWithRegistry[pathLabel]( reg, "tailscaled_inbound_packets_total", @@ -672,25 +841,37 @@ func registerMetrics(reg *usermetric.Registry) *metrics { metricRecvDataPacketsIPv4.Register(&m.inboundPacketsIPv4Total) metricRecvDataPacketsIPv6.Register(&m.inboundPacketsIPv6Total) metricRecvDataPacketsDERP.Register(&m.inboundPacketsDERPTotal) + metricRecvDataPacketsPeerRelayIPv4.Register(&m.inboundPacketsPeerRelayIPv4Total) + metricRecvDataPacketsPeerRelayIPv6.Register(&m.inboundPacketsPeerRelayIPv6Total) metricSendUDP.Register(&m.outboundPacketsIPv4Total) metricSendUDP.Register(&m.outboundPacketsIPv6Total) metricSendDERP.Register(&m.outboundPacketsDERPTotal) + metricSendPeerRelay.Register(&m.outboundPacketsPeerRelayIPv4Total) + metricSendPeerRelay.Register(&m.outboundPacketsPeerRelayIPv6Total) inboundPacketsTotal.Set(pathDirectV4, &m.inboundPacketsIPv4Total) inboundPacketsTotal.Set(pathDirectV6, &m.inboundPacketsIPv6Total) inboundPacketsTotal.Set(pathDERP, &m.inboundPacketsDERPTotal) + inboundPacketsTotal.Set(pathPeerRelayV4, &m.inboundPacketsPeerRelayIPv4Total) + inboundPacketsTotal.Set(pathPeerRelayV6, &m.inboundPacketsPeerRelayIPv6Total) inboundBytesTotal.Set(pathDirectV4, &m.inboundBytesIPv4Total) inboundBytesTotal.Set(pathDirectV6, &m.inboundBytesIPv6Total) inboundBytesTotal.Set(pathDERP, &m.inboundBytesDERPTotal) + inboundBytesTotal.Set(pathPeerRelayV4, &m.inboundBytesPeerRelayIPv4Total) + inboundBytesTotal.Set(pathPeerRelayV6, &m.inboundBytesPeerRelayIPv6Total) outboundPacketsTotal.Set(pathDirectV4, &m.outboundPacketsIPv4Total) outboundPacketsTotal.Set(pathDirectV6, &m.outboundPacketsIPv6Total) outboundPacketsTotal.Set(pathDERP, &m.outboundPacketsDERPTotal) + outboundPacketsTotal.Set(pathPeerRelayV4, &m.outboundPacketsPeerRelayIPv4Total) + outboundPacketsTotal.Set(pathPeerRelayV6, &m.outboundPacketsPeerRelayIPv6Total) outboundBytesTotal.Set(pathDirectV4, &m.outboundBytesIPv4Total) outboundBytesTotal.Set(pathDirectV6, &m.outboundBytesIPv6Total) outboundBytesTotal.Set(pathDERP, &m.outboundBytesDERPTotal) + outboundBytesTotal.Set(pathPeerRelayV4, &m.outboundBytesPeerRelayIPv4Total) + outboundBytesTotal.Set(pathPeerRelayV6, &m.outboundBytesPeerRelayIPv6Total) outboundPacketsDroppedErrors.Set(usermetric.DropLabels{Reason: usermetric.ReasonError}, &m.outboundPacketsDroppedErrors) @@ -703,8 +884,11 @@ func deregisterMetrics(m *metrics) { metricRecvDataPacketsIPv4.UnregisterAll() metricRecvDataPacketsIPv6.UnregisterAll() metricRecvDataPacketsDERP.UnregisterAll() + metricRecvDataPacketsPeerRelayIPv4.UnregisterAll() + metricRecvDataPacketsPeerRelayIPv6.UnregisterAll() metricSendUDP.UnregisterAll() metricSendDERP.UnregisterAll() + metricSendPeerRelay.UnregisterAll() } // InstallCaptureHook installs a callback which is called to @@ -950,7 +1134,7 @@ func (c *Conn) callNetInfoCallbackLocked(ni *tailcfg.NetInfo) { func (c *Conn) addValidDiscoPathForTest(nodeKey key.NodePublic, addr netip.AddrPort) { c.mu.Lock() defer c.mu.Unlock() - c.peerMap.setNodeKeyForIPPort(addr, nodeKey) + c.peerMap.setNodeKeyForEpAddr(epAddr{ap: addr}, nodeKey) } // SetNetInfoCallback sets the func to be called whenever the network conditions @@ -1019,13 +1203,17 @@ func (c *Conn) Ping(peer tailcfg.NodeView, res *ipnstate.PingResult, size int, c } // c.mu must be held -func (c *Conn) populateCLIPingResponseLocked(res *ipnstate.PingResult, latency time.Duration, ep netip.AddrPort) { +func (c *Conn) populateCLIPingResponseLocked(res *ipnstate.PingResult, latency time.Duration, ep epAddr) { res.LatencySeconds = latency.Seconds() - if ep.Addr() != tailcfg.DerpMagicIPAddr { - res.Endpoint = ep.String() + if ep.ap.Addr() != tailcfg.DerpMagicIPAddr { + if ep.vni.isSet() { + res.PeerRelay = ep.String() + } else { + res.Endpoint = ep.String() + } return } - regionID := int(ep.Port()) + regionID := int(ep.ap.Port()) res.DERPRegionID = regionID res.DERPRegionCode = c.derpRegionCodeLocked(regionID) } @@ -1264,8 +1452,8 @@ func (c *Conn) networkDown() bool { return !c.networkUp.Load() } // Send implements conn.Bind. // -// See https://pkg.go.dev/golang.zx2c4.com/wireguard/conn#Bind.Send -func (c *Conn) Send(buffs [][]byte, ep conn.Endpoint) (err error) { +// See https://pkg.go.dev/github.com/tailscale/wireguard-go/conn#Bind.Send +func (c *Conn) Send(buffs [][]byte, ep conn.Endpoint, offset int) (err error) { n := int64(len(buffs)) defer func() { if err != nil { @@ -1277,12 +1465,18 @@ func (c *Conn) Send(buffs [][]byte, ep conn.Endpoint) (err error) { metricSendDataNetworkDown.Add(n) return errNetworkDown } - if ep, ok := ep.(*endpoint); ok { - return ep.send(buffs) + switch ep := ep.(type) { + case *endpoint: + return ep.send(buffs, offset) + case *lazyEndpoint: + // A [*lazyEndpoint] may end up on this TX codepath when wireguard-go is + // deemed "under handshake load" and ends up transmitting a cookie reply + // using the received [conn.Endpoint] in [device.SendHandshakeCookie]. + if ep.src.ap.Addr().Is6() { + return c.pconn6.WriteBatchTo(buffs, ep.src, offset) + } + return c.pconn4.WriteBatchTo(buffs, ep.src, offset) } - // If it's not of type *endpoint, it's probably *lazyEndpoint, which means - // we don't actually know who the peer is and we're waiting for wireguard-go - // to switch the endpoint. See go/corp/20732. return nil } @@ -1294,19 +1488,19 @@ var errNoUDP = errors.New("no UDP available on platform") var errUnsupportedConnType = errors.New("unsupported connection type") -func (c *Conn) sendUDPBatch(addr netip.AddrPort, buffs [][]byte) (sent bool, err error) { +func (c *Conn) sendUDPBatch(addr epAddr, buffs [][]byte, offset int) (sent bool, err error) { isIPv6 := false switch { - case addr.Addr().Is4(): - case addr.Addr().Is6(): + case addr.ap.Addr().Is4(): + case addr.ap.Addr().Is6(): isIPv6 = true default: panic("bogus sendUDPBatch addr type") } if isIPv6 { - err = c.pconn6.WriteBatchTo(buffs, addr) + err = c.pconn6.WriteBatchTo(buffs, addr, offset) } else { - err = c.pconn4.WriteBatchTo(buffs, addr) + err = c.pconn4.WriteBatchTo(buffs, addr, offset) } if err != nil { var errGSO neterror.ErrUDPGSODisabled @@ -1322,23 +1516,37 @@ func (c *Conn) sendUDPBatch(addr netip.AddrPort, buffs [][]byte) (sent bool, err // sendUDP sends UDP packet b to ipp. // See sendAddr's docs on the return value meanings. -func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool) (sent bool, err error) { +func (c *Conn) sendUDP(ipp netip.AddrPort, b []byte, isDisco bool, isGeneveEncap bool) (sent bool, err error) { if runtime.GOOS == "js" { return false, errNoUDP } sent, err = c.sendUDPStd(ipp, b) if err != nil { - metricSendUDPError.Add(1) + if isGeneveEncap { + metricSendPeerRelayError.Add(1) + } else { + metricSendUDPError.Add(1) + } c.maybeRebindOnError(err) } else { if sent && !isDisco { switch { case ipp.Addr().Is4(): - c.metrics.outboundPacketsIPv4Total.Add(1) - c.metrics.outboundBytesIPv4Total.Add(int64(len(b))) + if isGeneveEncap { + c.metrics.outboundPacketsPeerRelayIPv4Total.Add(1) + c.metrics.outboundBytesPeerRelayIPv4Total.Add(int64(len(b))) + } else { + c.metrics.outboundPacketsIPv4Total.Add(1) + c.metrics.outboundBytesIPv4Total.Add(int64(len(b))) + } case ipp.Addr().Is6(): - c.metrics.outboundPacketsIPv6Total.Add(1) - c.metrics.outboundBytesIPv6Total.Add(int64(len(b))) + if isGeneveEncap { + c.metrics.outboundPacketsPeerRelayIPv6Total.Add(1) + c.metrics.outboundBytesPeerRelayIPv6Total.Add(int64(len(b))) + } else { + c.metrics.outboundPacketsIPv6Total.Add(1) + c.metrics.outboundBytesIPv6Total.Add(int64(len(b))) + } } } } @@ -1413,9 +1621,9 @@ func (c *Conn) sendUDPStd(addr netip.AddrPort, b []byte) (sent bool, err error) // An example of when they might be different: sending to an // IPv6 address when the local machine doesn't have IPv6 support // returns (false, nil); it's not an error, but nothing was sent. -func (c *Conn) sendAddr(addr netip.AddrPort, pubKey key.NodePublic, b []byte, isDisco bool) (sent bool, err error) { +func (c *Conn) sendAddr(addr netip.AddrPort, pubKey key.NodePublic, b []byte, isDisco bool, isGeneveEncap bool) (sent bool, err error) { if addr.Addr() != tailcfg.DerpMagicIPAddr { - return c.sendUDP(addr, b, isDisco) + return c.sendUDP(addr, b, isDisco, isGeneveEncap) } regionID := int(addr.Port()) @@ -1469,7 +1677,9 @@ func (c *Conn) putReceiveBatch(batch *receiveBatch) { func (c *Conn) receiveIPv4() conn.ReceiveFunc { return c.mkReceiveFunc(&c.pconn4, c.health.ReceiveFuncStats(health.ReceiveIPv4), &c.metrics.inboundPacketsIPv4Total, + &c.metrics.inboundPacketsPeerRelayIPv4Total, &c.metrics.inboundBytesIPv4Total, + &c.metrics.inboundBytesPeerRelayIPv4Total, ) } @@ -1477,15 +1687,17 @@ func (c *Conn) receiveIPv4() conn.ReceiveFunc { func (c *Conn) receiveIPv6() conn.ReceiveFunc { return c.mkReceiveFunc(&c.pconn6, c.health.ReceiveFuncStats(health.ReceiveIPv6), &c.metrics.inboundPacketsIPv6Total, + &c.metrics.inboundPacketsPeerRelayIPv6Total, &c.metrics.inboundBytesIPv6Total, + &c.metrics.inboundBytesPeerRelayIPv6Total, ) } // mkReceiveFunc creates a ReceiveFunc reading from ruc. // The provided healthItem and metrics are updated if non-nil. -func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFuncStats, packetMetric, bytesMetric *expvar.Int) conn.ReceiveFunc { - // epCache caches an IPPort->endpoint for hot flows. - var epCache ippEndpointCache +func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFuncStats, directPacketMetric, peerRelayPacketMetric, directBytesMetric, peerRelayBytesMetric *expvar.Int) conn.ReceiveFunc { + // epCache caches an epAddr->endpoint for hot flows. + var epCache epAddrEndpointCache return func(buffs [][]byte, sizes []int, eps []conn.Endpoint) (_ int, retErr error) { if healthItem != nil { @@ -1519,15 +1731,24 @@ func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFu continue } ipp := msg.Addr.(*net.UDPAddr).AddrPort() - if ep, ok := c.receiveIP(msg.Buffers[0][:msg.N], ipp, &epCache); ok { - if packetMetric != nil { - packetMetric.Add(1) - } - if bytesMetric != nil { - bytesMetric.Add(int64(msg.N)) + if ep, size, isGeneveEncap, ok := c.receiveIP(msg.Buffers[0][:msg.N], ipp, &epCache); ok { + if isGeneveEncap { + if peerRelayPacketMetric != nil { + peerRelayPacketMetric.Add(1) + } + if peerRelayBytesMetric != nil { + peerRelayBytesMetric.Add(int64(msg.N)) + } + } else { + if directPacketMetric != nil { + directPacketMetric.Add(1) + } + if directBytesMetric != nil { + directBytesMetric.Add(int64(msg.N)) + } } eps[i] = ep - sizes[i] = msg.N + sizes[i] = size reportToCaller = true } else { sizes[i] = 0 @@ -1540,49 +1761,117 @@ func (c *Conn) mkReceiveFunc(ruc *RebindingUDPConn, healthItem *health.ReceiveFu } } +// looksLikeInitiationMsg returns true if b looks like a WireGuard initiation +// message, otherwise it returns false. +func looksLikeInitiationMsg(b []byte) bool { + if len(b) == device.MessageInitiationSize && + binary.BigEndian.Uint32(b) == device.MessageInitiationType { + return true + } + return false +} + // receiveIP is the shared bits of ReceiveIPv4 and ReceiveIPv6. // +// size is the length of 'b' to report up to wireguard-go (only relevant if +// 'ok' is true). +// +// isGeneveEncap is whether 'b' is encapsulated by a Geneve header (only +// relevant if 'ok' is true). +// // ok is whether this read should be reported up to wireguard-go (our // caller). -func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *ippEndpointCache) (_ conn.Endpoint, ok bool) { +func (c *Conn) receiveIP(b []byte, ipp netip.AddrPort, cache *epAddrEndpointCache) (_ conn.Endpoint, size int, isGeneveEncap bool, ok bool) { var ep *endpoint - if stun.Is(b) { - c.netChecker.ReceiveSTUNPacket(b, ipp) - return nil, false + size = len(b) + + var geneve packet.GeneveHeader + pt, isGeneveEncap := packetLooksLike(b) + src := epAddr{ap: ipp} + if isGeneveEncap { + err := geneve.Decode(b) + if err != nil { + // Decode only returns an error when 'b' is too short, and + // 'isGeneveEncap' indicates it's a sufficient length. + c.logf("[unexpected] geneve header decoding error: %v", err) + return nil, 0, false, false + } + src.vni.set(geneve.VNI) } - if c.handleDiscoMessage(b, ipp, key.NodePublic{}, discoRXPathUDP) { - return nil, false + switch pt { + case packetLooksLikeDisco: + if isGeneveEncap { + b = b[packet.GeneveFixedHeaderLength:] + } + // The Geneve header control bit should only be set for relay handshake + // messages terminating on or originating from a UDP relay server. We + // have yet to open the encrypted disco payload to determine the + // [disco.MessageType], but we assert it should be handshake-related. + shouldByRelayHandshakeMsg := geneve.Control == true + c.handleDiscoMessage(b, src, shouldByRelayHandshakeMsg, key.NodePublic{}, discoRXPathUDP) + return nil, 0, false, false + case packetLooksLikeSTUNBinding: + c.netChecker.ReceiveSTUNPacket(b, ipp) + return nil, 0, false, false + default: + // Fall through for all other packet types as they are assumed to + // be potentially WireGuard. } + if !c.havePrivateKey.Load() { // If we have no private key, we're logged out or // stopped. Don't try to pass these wireguard packets // up to wireguard-go; it'll just complain (issue 1167). - return nil, false + return nil, 0, false, false } - if cache.ipp == ipp && cache.de != nil && cache.gen == cache.de.numStopAndReset() { + + if src.vni.isSet() { + // Strip away the Geneve header before returning the packet to + // wireguard-go. + // + // TODO(jwhited): update [github.com/tailscale/wireguard-go/conn.ReceiveFunc] + // to support returning start offset in order to get rid of this memmove perf + // penalty. + size = copy(b, b[packet.GeneveFixedHeaderLength:]) + } + + if cache.epAddr == src && cache.de != nil && cache.gen == cache.de.numStopAndReset() { ep = cache.de } else { c.mu.Lock() - de, ok := c.peerMap.endpointForIPPort(ipp) + de, ok := c.peerMap.endpointForEpAddr(src) c.mu.Unlock() if !ok { if c.controlKnobs != nil && c.controlKnobs.DisableCryptorouting.Load() { - return nil, false + // Note: UDP relay is dependent on cryptorouting enablement. We + // only update Geneve-encapsulated [epAddr]s in the [peerMap] + // via [lazyEndpoint]. + return nil, 0, false, false } - return &lazyEndpoint{c: c, src: ipp}, true + // TODO(jwhited): reuse [lazyEndpoint] across calls to receiveIP() + // for the same batch & [epAddr] src. + return &lazyEndpoint{c: c, src: src}, size, isGeneveEncap, true } - cache.ipp = ipp + cache.epAddr = src cache.de = de cache.gen = de.numStopAndReset() ep = de } now := mono.Now() ep.lastRecvUDPAny.StoreAtomic(now) - ep.noteRecvActivity(ipp, now) + connNoted := ep.noteRecvActivity(src, now) if stats := c.stats.Load(); stats != nil { stats.UpdateRxPhysical(ep.nodeAddr, ipp, 1, len(b)) } - return ep, true + if src.vni.isSet() && (connNoted || looksLikeInitiationMsg(b)) { + // connNoted is periodic, but we also want to verify if the peer is who + // we believe for all initiation messages, otherwise we could get + // unlucky and fail to JIT configure the "correct" peer. + // TODO(jwhited): relax this to include direct connections + // See http://go/corp/29422 & http://go/corp/30042 + return &lazyEndpoint{c: c, maybeEP: ep, src: src}, size, isGeneveEncap, true + } + return ep, size, isGeneveEncap, true } // discoLogLevel controls the verbosity of discovery log messages. @@ -1630,18 +1919,36 @@ func (v *virtualNetworkID) get() uint32 { return v._vni & vniGetMask } +// sendDiscoAllocateUDPRelayEndpointRequest is primarily an alias for +// sendDiscoMessage, but it will alternatively send m over the eventbus if dst +// is a DERP IP:port, and dstKey is self. This saves a round-trip through DERP +// when we are attempting to allocate on a self (in-process) peer relay server. +func (c *Conn) sendDiscoAllocateUDPRelayEndpointRequest(dst epAddr, dstKey key.NodePublic, dstDisco key.DiscoPublic, allocReq *disco.AllocateUDPRelayEndpointRequest, logLevel discoLogLevel) (sent bool, err error) { + isDERP := dst.ap.Addr() == tailcfg.DerpMagicIPAddr + selfNodeKey := c.publicKeyAtomic.Load() + if isDERP && dstKey.Compare(selfNodeKey) == 0 { + c.allocRelayEndpointPub.Publish(UDPRelayAllocReq{ + RxFromNodeKey: selfNodeKey, + RxFromDiscoKey: c.discoPublic, + Message: allocReq, + }) + return true, nil + } + return c.sendDiscoMessage(dst, dstKey, dstDisco, allocReq, logLevel) +} + // sendDiscoMessage sends discovery message m to dstDisco at dst. // -// If dst is a DERP IP:port, then dstKey must be non-zero. +// If dst.ap is a DERP IP:port, then dstKey must be non-zero. // -// If vni.isSet(), the [disco.Message] will be preceded by a Geneve header with -// the VNI field set to the value returned by vni.get(). +// If dst.vni.isSet(), the [disco.Message] will be preceded by a Geneve header +// with the VNI field set to the value returned by vni.get(). // // The dstKey should only be non-zero if the dstDisco key // unambiguously maps to exactly one peer. -func (c *Conn) sendDiscoMessage(dst netip.AddrPort, vni virtualNetworkID, dstKey key.NodePublic, dstDisco key.DiscoPublic, m disco.Message, logLevel discoLogLevel) (sent bool, err error) { - isDERP := dst.Addr() == tailcfg.DerpMagicIPAddr - if _, isPong := m.(*disco.Pong); isPong && !isDERP && dst.Addr().Is4() { +func (c *Conn) sendDiscoMessage(dst epAddr, dstKey key.NodePublic, dstDisco key.DiscoPublic, m disco.Message, logLevel discoLogLevel) (sent bool, err error) { + isDERP := dst.ap.Addr() == tailcfg.DerpMagicIPAddr + if _, isPong := m.(*disco.Pong); isPong && !isDERP && dst.ap.Addr().Is4() { time.Sleep(debugIPv4DiscoPingPenalty()) } @@ -1678,11 +1985,11 @@ func (c *Conn) sendDiscoMessage(dst netip.AddrPort, vni virtualNetworkID, dstKey c.mu.Unlock() pkt := make([]byte, 0, 512) // TODO: size it correctly? pool? if it matters. - if vni.isSet() { + if dst.vni.isSet() { gh := packet.GeneveHeader{ Version: 0, Protocol: packet.GeneveProtocolDisco, - VNI: vni.get(), + VNI: dst.vni.get(), Control: isRelayHandshakeMsg, } pkt = append(pkt, make([]byte, packet.GeneveFixedHeaderLength)...) @@ -1703,7 +2010,7 @@ func (c *Conn) sendDiscoMessage(dst netip.AddrPort, vni virtualNetworkID, dstKey box := di.sharedKey.Seal(m.AppendMarshal(nil)) pkt = append(pkt, box...) const isDisco = true - sent, err = c.sendAddr(dst, dstKey, pkt, isDisco) + sent, err = c.sendAddr(dst.ap, dstKey, pkt, isDisco, dst.vni.isSet()) if sent { if logLevel == discoLog || (logLevel == discoVerboseLog && debugDisco()) { node := "?" @@ -1745,45 +2052,96 @@ const ( const discoHeaderLen = len(disco.Magic) + key.DiscoPublicRawLen -// isDiscoMaybeGeneve reports whether msg is a Tailscale Disco protocol -// message, and if true, whether it is encapsulated by a Geneve header. +type packetLooksLikeType int + +const ( + packetLooksLikeWireGuard packetLooksLikeType = iota + packetLooksLikeSTUNBinding + packetLooksLikeDisco +) + +// packetLooksLike reports a [packetsLooksLikeType] for 'msg', and whether +// 'msg' is encapsulated by a Geneve header (or naked). // -// isGeneveEncap is only relevant when isDiscoMsg is true. +// [packetLooksLikeSTUNBinding] is never Geneve-encapsulated. // -// Naked Disco, Geneve followed by Disco, and naked WireGuard can be confidently -// distinguished based on the following: -// 1. [disco.Magic] is sufficiently non-overlapping with a Geneve protocol -// field value of [packet.GeneveProtocolDisco]. -// 2. [disco.Magic] is sufficiently non-overlapping with the first 4 bytes of -// a WireGuard packet. -// 3. [packet.GeneveHeader] with a Geneve protocol field value of -// [packet.GeneveProtocolDisco] is sufficiently non-overlapping with the -// first 4 bytes of a WireGuard packet. -func isDiscoMaybeGeneve(msg []byte) (isDiscoMsg bool, isGeneveEncap bool) { - if len(msg) < discoHeaderLen { - return false, false - } - if string(msg[:len(disco.Magic)]) == disco.Magic { - return true, false - } - if len(msg) < packet.GeneveFixedHeaderLength+discoHeaderLen { - return false, false - } - if msg[0]&0xC0 != 0 || // version bits that we always transmit as 0s - msg[1]&0x3F != 0 || // reserved bits that we always transmit as 0s - binary.BigEndian.Uint16(msg[2:4]) != packet.GeneveProtocolDisco || - msg[7] != 0 { // reserved byte that we always transmit as 0 - return false, false - } - msg = msg[packet.GeneveFixedHeaderLength:] - if string(msg[:len(disco.Magic)]) == disco.Magic { - return true, true - } - return false, false -} - -// handleDiscoMessage handles a discovery message and reports whether -// msg was a Tailscale inter-node discovery message. +// Naked STUN binding, Naked Disco, Geneve followed by Disco, naked WireGuard, +// and Geneve followed by WireGuard can be confidently distinguished based on +// the following: +// +// 1. STUN binding @ msg[1] (0x01) is sufficiently non-overlapping with the +// Geneve header where the LSB is always 0 (part of 6 "reserved" bits). +// +// 2. STUN binding @ msg[1] (0x01) is sufficiently non-overlapping with naked +// WireGuard, which is always a 0 byte value (WireGuard message type +// occupies msg[0:4], and msg[1:4] are always 0). +// +// 3. STUN binding @ msg[1] (0x01) is sufficiently non-overlapping with the +// second byte of [disco.Magic] (0x53). +// +// 4. [disco.Magic] @ msg[2:4] (0xf09f) is sufficiently non-overlapping with a +// Geneve protocol field value of [packet.GeneveProtocolDisco] or +// [packet.GeneveProtocolWireGuard] . +// +// 5. [disco.Magic] @ msg[0] (0x54) is sufficiently non-overlapping with the +// first byte of a WireGuard packet (0x01-0x04). +// +// 6. [packet.GeneveHeader] with a Geneve protocol field value of +// [packet.GeneveProtocolDisco] or [packet.GeneveProtocolWireGuard] +// (msg[2:4]) is sufficiently non-overlapping with the second 2 bytes of a +// WireGuard packet which are always 0x0000. +func packetLooksLike(msg []byte) (t packetLooksLikeType, isGeneveEncap bool) { + if stun.Is(msg) && + msg[1] == 0x01 { // method binding + return packetLooksLikeSTUNBinding, false + } + + // TODO(jwhited): potentially collapse into disco.LooksLikeDiscoWrapper() + // if safe to do so. + looksLikeDisco := func(msg []byte) bool { + if len(msg) >= discoHeaderLen && string(msg[:len(disco.Magic)]) == disco.Magic { + return true + } + return false + } + + // Do we have a Geneve header? + if len(msg) >= packet.GeneveFixedHeaderLength && + msg[0]&0xC0 == 0 && // version bits that we always transmit as 0s + msg[1]&0x3F == 0 && // reserved bits that we always transmit as 0s + msg[7] == 0 { // reserved byte that we always transmit as 0 + switch binary.BigEndian.Uint16(msg[2:4]) { + case packet.GeneveProtocolDisco: + if looksLikeDisco(msg[packet.GeneveFixedHeaderLength:]) { + return packetLooksLikeDisco, true + } else { + // The Geneve header is well-formed, and it indicated this + // was disco, but it's not. The evaluated bytes at this point + // are always distinct from naked WireGuard (msg[2:4] are always + // 0x0000) and naked Disco (msg[2:4] are always 0xf09f), but + // maintain pre-Geneve behavior and fall back to assuming it's + // naked WireGuard. + return packetLooksLikeWireGuard, false + } + case packet.GeneveProtocolWireGuard: + return packetLooksLikeWireGuard, true + default: + // The Geneve header is well-formed, but the protocol field value is + // unknown to us. The evaluated bytes at this point are not + // necessarily distinct from naked WireGuard or naked Disco, fall + // through. + } + } + + if looksLikeDisco(msg) { + return packetLooksLikeDisco, false + } else { + return packetLooksLikeWireGuard, false + } +} + +// handleDiscoMessage handles a discovery message. The caller is assumed to have +// verified 'msg' returns [packetLooksLikeDisco] from packetLooksLike(). // // A discovery message has the form: // @@ -1792,32 +2150,17 @@ func isDiscoMaybeGeneve(msg []byte) (isDiscoMsg bool, isGeneveEncap bool) { // - nonce [24]byte // - naclbox of payload (see tailscale.com/disco package for inner payload format) // -// For messages received over DERP, the src.Addr() will be derpMagicIP (with -// src.Port() being the region ID) and the derpNodeSrc will be the node key +// For messages received over DERP, the src.ap.Addr() will be derpMagicIP (with +// src.ap.Port() being the region ID) and the derpNodeSrc will be the node key // it was received from at the DERP layer. derpNodeSrc is zero when received // over UDP. -func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc key.NodePublic, via discoRXPath) (isDiscoMsg bool) { - isDiscoMsg, isGeneveEncap := isDiscoMaybeGeneve(msg) - if !isDiscoMsg { - return - } - var geneve packet.GeneveHeader - if isGeneveEncap { - err := geneve.Decode(msg) - if err != nil { - // Decode only returns an error when 'msg' is too short, and - // 'isGeneveEncap' indicates it's a sufficient length. - c.logf("[unexpected] geneve header decoding error: %v", err) - return - } - msg = msg[packet.GeneveFixedHeaderLength:] - } - // The control bit should only be set for relay handshake messages - // terminating on or originating from a UDP relay server. We have yet to - // open the encrypted payload to determine the [disco.MessageType], but - // we assert it should be handshake-related. - shouldBeRelayHandshakeMsg := isGeneveEncap && geneve.Control - +// +// If 'msg' was encapsulated by a Geneve header it is assumed to have already +// been stripped. +// +// 'shouldBeRelayHandshakeMsg' will be true if 'msg' was encapsulated +// by a Geneve header with the control bit set. +func (c *Conn) handleDiscoMessage(msg []byte, src epAddr, shouldBeRelayHandshakeMsg bool, derpNodeSrc key.NodePublic, via discoRXPath) { sender := key.DiscoPublicFromRaw32(mem.B(msg[len(disco.Magic):discoHeaderLen])) c.mu.Lock() @@ -1831,7 +2174,6 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke } if c.privateKey.IsZero() { // Ignore disco messages when we're stopped. - // Still return true, to not pass it down to wireguard. return } @@ -1842,7 +2184,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke di, ok = c.relayManager.discoInfo(sender) if !ok { if debugDisco() { - c.logf("magicsock: disco: ignoring disco-looking relay handshake frame, no active handshakes with key %v over VNI %d", sender.ShortString(), geneve.VNI) + c.logf("magicsock: disco: ignoring disco-looking relay handshake frame, no active handshakes with key %v over %v", sender.ShortString(), src) } return } @@ -1856,10 +2198,10 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke return } - isDERP := src.Addr() == tailcfg.DerpMagicIPAddr + isDERP := src.ap.Addr() == tailcfg.DerpMagicIPAddr if !isDERP && !shouldBeRelayHandshakeMsg { // Record receive time for UDP transport packets. - pi, ok := c.peerMap.byIPPort[src] + pi, ok := c.peerMap.byEpAddr[src] if ok { pi.ep.lastRecvUDPAny.StoreAtomic(mono.Now()) } @@ -1891,7 +2233,8 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke // Emit information about the disco frame into the pcap stream // if a capture hook is installed. if cb := c.captureHook.Load(); cb != nil { - cb(packet.PathDisco, time.Now(), disco.ToPCAPFrame(src, derpNodeSrc, payload), packet.CaptureMeta{}) + // TODO(jwhited): include VNI context? + cb(packet.PathDisco, time.Now(), disco.ToPCAPFrame(src.ap, derpNodeSrc, payload), packet.CaptureMeta{}) } dm, err := disco.Parse(payload) @@ -1923,45 +2266,47 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke c.logf("[unexpected] %T packets should not come from a relay server with Geneve control bit set", dm) return } - c.relayManager.handleBindUDPRelayEndpointChallenge(challenge, di, src, geneve.VNI) + c.relayManager.handleRxDiscoMsg(c, challenge, key.NodePublic{}, di.discoKey, src) return } switch dm := dm.(type) { case *disco.Ping: metricRecvDiscoPing.Add(1) - if isGeneveEncap { - // TODO(jwhited): handle Geneve-encapsulated disco ping. - return - } c.handlePingLocked(dm, src, di, derpNodeSrc) case *disco.Pong: metricRecvDiscoPong.Add(1) - if isGeneveEncap { - // TODO(jwhited): handle Geneve-encapsulated disco pong. - return - } // There might be multiple nodes for the sender's DiscoKey. // Ask each to handle it, stopping once one reports that // the Pong's TxID was theirs. + knownTxID := false c.peerMap.forEachEndpointWithDiscoKey(sender, func(ep *endpoint) (keepGoing bool) { if ep.handlePongConnLocked(dm, di, src) { + knownTxID = true return false } return true }) + if !knownTxID && src.vni.isSet() { + // If it's an unknown TxID, and it's Geneve-encapsulated, then + // make [relayManager] aware. It might be in the middle of probing + // src. + c.relayManager.handleRxDiscoMsg(c, dm, key.NodePublic{}, di.discoKey, src) + } case *disco.CallMeMaybe, *disco.CallMeMaybeVia: var via *disco.CallMeMaybeVia isVia := false msgType := "CallMeMaybe" cmm, ok := dm.(*disco.CallMeMaybe) - if !ok { + if ok { + metricRecvDiscoCallMeMaybe.Add(1) + } else { + metricRecvDiscoCallMeMaybeVia.Add(1) via = dm.(*disco.CallMeMaybeVia) msgType = "CallMeMaybeVia" isVia = true } - metricRecvDiscoCallMeMaybe.Add(1) if !isDERP || derpNodeSrc.IsZero() { // CallMeMaybe{Via} messages should only come via DERP. c.logf("[unexpected] %s packets should only come via DERP", msgType) @@ -1970,12 +2315,26 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke nodeKey := derpNodeSrc ep, ok := c.peerMap.endpointForNodeKey(nodeKey) if !ok { - metricRecvDiscoCallMeMaybeBadNode.Add(1) + if isVia { + metricRecvDiscoCallMeMaybeViaBadNode.Add(1) + } else { + metricRecvDiscoCallMeMaybeBadNode.Add(1) + } c.logf("magicsock: disco: ignoring %s from %v; %v is unknown", msgType, sender.ShortString(), derpNodeSrc.ShortString()) return } + // If the "disable-relay-client" node attr is set for this node, it + // can't be a UDP relay client, so drop any CallMeMaybeVia messages it + // receives. + if isVia && !c.relayClientEnabled { + c.logf("magicsock: disco: ignoring %s from %v; disable-relay-client node attr is set", msgType, sender.ShortString()) + return + } + ep.mu.Lock() relayCapable := ep.relayCapable + lastBest := ep.bestAddr + lastBestIsTrusted := mono.Now().Before(ep.trustBestAddrUntil) ep.mu.Unlock() if isVia && !relayCapable { c.logf("magicsock: disco: ignoring %s from %v; %v is not known to be relay capable", msgType, sender.ShortString(), sender.ShortString()) @@ -1986,7 +2345,11 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke return } if epDisco.key != di.discoKey { - metricRecvDiscoCallMeMaybeBadDisco.Add(1) + if isVia { + metricRecvDiscoCallMeMaybeViaBadDisco.Add(1) + } else { + metricRecvDiscoCallMeMaybeBadDisco.Add(1) + } c.logf("[unexpected] %s from peer via DERP whose netmap discokey != disco source", msgType) return } @@ -1995,7 +2358,7 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke c.discoShort, epDisco.short, via.ServerDisco.ShortString(), ep.publicKey.ShortString(), derpStr(src.String()), len(via.AddrPorts)) - c.relayManager.handleCallMeMaybeVia(ep, via) + c.relayManager.handleCallMeMaybeVia(ep, lastBest, lastBestIsTrusted, via) } else { c.dlogf("[v1] magicsock: disco: %v<-%v (%v, %v) got call-me-maybe, %d endpoints", c.discoShort, epDisco.short, @@ -2003,7 +2366,95 @@ func (c *Conn) handleDiscoMessage(msg []byte, src netip.AddrPort, derpNodeSrc ke len(cmm.MyNumber)) go ep.handleCallMeMaybe(cmm) } + case *disco.AllocateUDPRelayEndpointRequest, *disco.AllocateUDPRelayEndpointResponse: + var resp *disco.AllocateUDPRelayEndpointResponse + isResp := false + msgType := "AllocateUDPRelayEndpointRequest" + req, ok := dm.(*disco.AllocateUDPRelayEndpointRequest) + if ok { + metricRecvDiscoAllocUDPRelayEndpointRequest.Add(1) + } else { + metricRecvDiscoAllocUDPRelayEndpointResponse.Add(1) + resp = dm.(*disco.AllocateUDPRelayEndpointResponse) + msgType = "AllocateUDPRelayEndpointResponse" + isResp = true + } + if !isDERP { + // These messages should only come via DERP. + c.logf("[unexpected] %s packets should only come via DERP", msgType) + return + } + nodeKey := derpNodeSrc + ep, ok := c.peerMap.endpointForNodeKey(nodeKey) + if !ok { + c.logf("magicsock: disco: ignoring %s from %v; %v is unknown", msgType, sender.ShortString(), derpNodeSrc.ShortString()) + return + } + epDisco := ep.disco.Load() + if epDisco == nil { + return + } + if epDisco.key != di.discoKey { + if isResp { + metricRecvDiscoAllocUDPRelayEndpointResponseBadDisco.Add(1) + } else { + metricRecvDiscoAllocUDPRelayEndpointRequestBadDisco.Add(1) + } + c.logf("[unexpected] %s from peer via DERP whose netmap discokey != disco source", msgType) + return + } + + if isResp { + c.dlogf("[v1] magicsock: disco: %v<-%v (%v, %v) got %s, %d endpoints", + c.discoShort, epDisco.short, + ep.publicKey.ShortString(), derpStr(src.String()), + msgType, + len(resp.AddrPorts)) + c.relayManager.handleRxDiscoMsg(c, resp, nodeKey, di.discoKey, src) + return + } else if sender.Compare(req.ClientDisco[0]) != 0 && sender.Compare(req.ClientDisco[1]) != 0 { + // An allocation request must contain the sender's disco key in + // ClientDisco. One of the relay participants must be the sender. + c.logf("magicsock: disco: %s from %v; %v does not contain sender's disco key", + msgType, sender.ShortString(), derpNodeSrc.ShortString()) + return + } else { + c.dlogf("[v1] magicsock: disco: %v<-%v (%v, %v) got %s, for %d<->%d", + c.discoShort, epDisco.short, + ep.publicKey.ShortString(), derpStr(src.String()), + msgType, + req.ClientDisco[0], req.ClientDisco[1]) + } + + if c.filt == nil { + return + } + // Binary search of peers is O(log n) while c.mu is held. + // TODO: We might be able to use ep.nodeAddr instead of all addresses, + // or we might be able to release c.mu before doing this work. Keep it + // simple and slow for now. c.peers.AsSlice is a copy. We may need to + // write our own binary search for a [views.Slice]. + peerI, ok := slices.BinarySearchFunc(c.peers.AsSlice(), ep.nodeID, func(peer tailcfg.NodeView, target tailcfg.NodeID) int { + if peer.ID() < target { + return -1 + } else if peer.ID() > target { + return 1 + } + return 0 + }) + if !ok { + // unexpected + return + } + if !nodeHasCap(c.filt, c.peers.At(peerI), c.self, tailcfg.PeerCapabilityRelay) { + return + } + c.allocRelayEndpointPub.Publish(UDPRelayAllocReq{ + RxFromDiscoKey: sender, + RxFromNodeKey: nodeKey, + Message: req, + }) } return } @@ -2048,14 +2499,30 @@ func (c *Conn) unambiguousNodeKeyOfPingLocked(dm *disco.Ping, dk key.DiscoPublic // di is the discoInfo of the source of the ping. // derpNodeSrc is non-zero if the ping arrived via DERP. -func (c *Conn) handlePingLocked(dm *disco.Ping, src netip.AddrPort, di *discoInfo, derpNodeSrc key.NodePublic) { +func (c *Conn) handlePingLocked(dm *disco.Ping, src epAddr, di *discoInfo, derpNodeSrc key.NodePublic) { likelyHeartBeat := src == di.lastPingFrom && time.Since(di.lastPingTime) < 5*time.Second di.lastPingFrom = src di.lastPingTime = time.Now() - isDerp := src.Addr() == tailcfg.DerpMagicIPAddr + isDerp := src.ap.Addr() == tailcfg.DerpMagicIPAddr + + if src.vni.isSet() { + if isDerp { + c.logf("[unexpected] got Geneve-encapsulated disco ping from %v/%v over DERP", src, derpNodeSrc) + return + } + + // [relayManager] is always responsible for handling (replying) to + // Geneve-encapsulated [disco.Ping] messages in the interest of + // simplicity. It might be in the middle of probing src, so it must be + // made aware. + c.relayManager.handleRxDiscoMsg(c, dm, key.NodePublic{}, di.discoKey, src) + return + } + + // This is a naked [disco.Ping] without a VNI. // If we can figure out with certainty which node key this disco - // message is for, eagerly update our IP:port<>node and disco<>node + // message is for, eagerly update our [epAddr]<>node and disco<>node // mappings to make p2p path discovery faster in simple // cases. Without this, disco would still work, but would be // reliant on DERP call-me-maybe to establish the disco<>node @@ -2063,10 +2530,14 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src netip.AddrPort, di *discoInf // the IP:port<>disco mapping. if nk, ok := c.unambiguousNodeKeyOfPingLocked(dm, di.discoKey, derpNodeSrc); ok { if !isDerp { - c.peerMap.setNodeKeyForIPPort(src, nk) + c.peerMap.setNodeKeyForEpAddr(src, nk) } } + // numNodes tracks how many nodes (node keys) are associated with the disco + // key tied to this inbound ping. Multiple nodes may share the same disco + // key in the case of node sharing and users switching accounts. + var numNodes int // If we got a ping over DERP, then derpNodeSrc is non-zero and we reply // over DERP (in which case ipDst is also a DERP address). // But if the ping was over UDP (ipDst is not a DERP address), then dstKey @@ -2075,18 +2546,17 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src netip.AddrPort, di *discoInf dstKey := derpNodeSrc // Remember this route if not present. - var numNodes int var dup bool if isDerp { if ep, ok := c.peerMap.endpointForNodeKey(derpNodeSrc); ok { - if ep.addCandidateEndpoint(src, dm.TxID) { + if ep.addCandidateEndpoint(src.ap, dm.TxID) { return } numNodes = 1 } } else { c.peerMap.forEachEndpointWithDiscoKey(di.discoKey, func(ep *endpoint) (keepGoing bool) { - if ep.addCandidateEndpoint(src, dm.TxID) { + if ep.addCandidateEndpoint(src.ap, dm.TxID) { dup = true return false } @@ -2121,9 +2591,9 @@ func (c *Conn) handlePingLocked(dm *disco.Ping, src netip.AddrPort, di *discoInf ipDst := src discoDest := di.discoKey - go c.sendDiscoMessage(ipDst, virtualNetworkID{}, dstKey, discoDest, &disco.Pong{ + go c.sendDiscoMessage(ipDst, dstKey, discoDest, &disco.Pong{ TxID: dm.TxID, - Src: src, + Src: src.ap, }, discoVerboseLog) } @@ -2166,12 +2636,12 @@ func (c *Conn) enqueueCallMeMaybe(derpAddr netip.AddrPort, de *endpoint) { for _, ep := range c.lastEndpoints { eps = append(eps, ep.Addr) } - go de.c.sendDiscoMessage(derpAddr, virtualNetworkID{}, de.publicKey, epDisco.key, &disco.CallMeMaybe{MyNumber: eps}, discoLog) + go de.c.sendDiscoMessage(epAddr{ap: derpAddr}, de.publicKey, epDisco.key, &disco.CallMeMaybe{MyNumber: eps}, discoLog) if debugSendCallMeUnknownPeer() { // Send a callMeMaybe packet to a non-existent peer unknownKey := key.NewNode().Public() c.logf("magicsock: sending CallMeMaybe to unknown peer per TS_DEBUG_SEND_CALLME_UNKNOWN_PEER") - go de.c.sendDiscoMessage(derpAddr, virtualNetworkID{}, unknownKey, epDisco.key, &disco.CallMeMaybe{MyNumber: eps}, discoLog) + go de.c.sendDiscoMessage(epAddr{ap: derpAddr}, unknownKey, epDisco.key, &disco.CallMeMaybe{MyNumber: eps}, discoLog) } } @@ -2392,34 +2862,178 @@ func (c *Conn) SetProbeUDPLifetime(v bool) { }) } -// SetNetworkMap is called when the control client gets a new network -// map from the control server. It must always be non-nil. +// capVerIsRelayCapable returns true if version is relay client and server +// capable, otherwise it returns false. +func capVerIsRelayCapable(version tailcfg.CapabilityVersion) bool { + return version >= 121 +} + +// onFilterUpdate is called when a [FilterUpdate] is received over the +// [eventbus.Bus]. +func (c *Conn) onFilterUpdate(f FilterUpdate) { + c.mu.Lock() + c.filt = f.Filter + self := c.self + peers := c.peers + relayClientEnabled := c.relayClientEnabled + c.mu.Unlock() // release c.mu before potentially calling c.updateRelayServersSet which is O(m * n) + + if !relayClientEnabled { + // Early return if we cannot operate as a relay client. + return + } + + // The filter has changed, and we are operating as a relay server client. + // Re-evaluate it in order to produce an updated relay server set. + c.updateRelayServersSet(f.Filter, self, peers) +} + +// updateRelayServersSet iterates all peers and self, evaluating filt for each +// one in order to determine which are relay server candidates. filt, self, and +// peers are passed as args (vs c.mu-guarded fields) to enable callers to +// release c.mu before calling as this is O(m * n) (we iterate all cap rules 'm' +// in filt for every peer 'n'). +// +// Calls to updateRelayServersSet must never run concurrent to +// [endpoint.setDERPHome], otherwise [candidatePeerRelay] DERP home changes may +// be missed from the perspective of [relayManager]. // -// It should not use the DERPMap field of NetworkMap; that's -// conditionally sent to SetDERPMap instead. -func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) { +// TODO: Optimize this so that it's not O(m * n). This might involve: +// 1. Changes to [filter.Filter], e.g. adding a CapsWithValues() to check for +// a given capability instead of building and returning a map of all of +// them. +// 2. Moving this work upstream into [nodeBackend] or similar, and publishing +// the computed result over the eventbus instead. +func (c *Conn) updateRelayServersSet(filt *filter.Filter, self tailcfg.NodeView, peers views.Slice[tailcfg.NodeView]) { + relayServers := make(set.Set[candidatePeerRelay]) + nodes := append(peers.AsSlice(), self) + for _, maybeCandidate := range nodes { + if maybeCandidate.ID() != self.ID() && !capVerIsRelayCapable(maybeCandidate.Cap()) { + // If maybeCandidate's [tailcfg.CapabilityVersion] is not relay-capable, + // we skip it. If maybeCandidate happens to be self, then this check is + // unnecessary as self is always capable from this point (the statically + // compiled [tailcfg.CurrentCapabilityVersion]) forward. + continue + } + if !nodeHasCap(filt, maybeCandidate, self, tailcfg.PeerCapabilityRelayTarget) { + continue + } + relayServers.Add(candidatePeerRelay{ + nodeKey: maybeCandidate.Key(), + discoKey: maybeCandidate.DiscoKey(), + derpHomeRegionID: uint16(maybeCandidate.HomeDERP()), + }) + } + c.relayManager.handleRelayServersSet(relayServers) + if len(relayServers) > 0 { + c.hasPeerRelayServers.Store(true) + } else { + c.hasPeerRelayServers.Store(false) + } +} + +// nodeHasCap returns true if src has cap on dst, otherwise it returns false. +func nodeHasCap(filt *filter.Filter, src, dst tailcfg.NodeView, cap tailcfg.PeerCapability) bool { + if filt == nil || + !src.Valid() || + !dst.Valid() { + return false + } + for _, srcPrefix := range src.Addresses().All() { + if !srcPrefix.IsSingleIP() { + continue + } + srcAddr := srcPrefix.Addr() + for _, dstPrefix := range dst.Addresses().All() { + if !dstPrefix.IsSingleIP() { + continue + } + dstAddr := dstPrefix.Addr() + if dstAddr.BitLen() == srcAddr.BitLen() { // same address family + // [nodeBackend.peerCapsLocked] only returns/considers the + // [tailcfg.PeerCapMap] between the passed src and the _first_ + // host (/32 or /128) address for self. We are consistent with + // that behavior here. If src and dst host addresses are of the + // same address family they either have the capability or not. + // We do not check against additional host addresses of the same + // address family. + return filt.CapsWithValues(srcAddr, dstAddr).HasCapability(cap) + } + } + } + return false +} + +// candidatePeerRelay represents the identifiers and DERP home region ID for a +// peer relay server. +type candidatePeerRelay struct { + nodeKey key.NodePublic + discoKey key.DiscoPublic + derpHomeRegionID uint16 +} + +func (c *candidatePeerRelay) isValid() bool { + return !c.nodeKey.IsZero() && !c.discoKey.IsZero() +} + +// onNodeViewsUpdate is called when a [NodeViewsUpdate] is received over the +// [eventbus.Bus]. +func (c *Conn) onNodeViewsUpdate(update NodeViewsUpdate) { + peersChanged := c.updateNodes(update) + + relayClientEnabled := update.SelfNode.Valid() && + !update.SelfNode.HasCap(tailcfg.NodeAttrDisableRelayClient) && + !update.SelfNode.HasCap(tailcfg.NodeAttrOnlyTCP443) + + c.mu.Lock() + relayClientChanged := c.relayClientEnabled != relayClientEnabled + c.relayClientEnabled = relayClientEnabled + filt := c.filt + self := c.self + peers := c.peers + c.mu.Unlock() // release c.mu before potentially calling c.updateRelayServersSet which is O(m * n) + + if peersChanged || relayClientChanged { + if !relayClientEnabled { + c.relayManager.handleRelayServersSet(nil) + } else { + c.updateRelayServersSet(filt, self, peers) + } + } +} + +// updateNodes updates [Conn] to reflect the [tailcfg.NodeView]'s contained +// in update. It returns true if update.Peers was unequal to c.peers, otherwise +// false. +func (c *Conn) updateNodes(update NodeViewsUpdate) (peersChanged bool) { c.mu.Lock() defer c.mu.Unlock() if c.closed { - return + return false } priorPeers := c.peers - metricNumPeers.Set(int64(len(nm.Peers))) + metricNumPeers.Set(int64(len(update.Peers))) - // Update c.netMap regardless, before the following early return. - curPeers := views.SliceOf(nm.Peers) + // Update c.self & c.peers regardless, before the following early return. + c.self = update.SelfNode + curPeers := views.SliceOf(update.Peers) c.peers = curPeers + // [debugFlags] are mutable in [Conn.SetSilentDisco] & + // [Conn.SetProbeUDPLifetime]. These setters are passed [controlknobs.Knobs] + // values by [ipnlocal.LocalBackend] around netmap reception. + // [controlknobs.Knobs] are simply self [tailcfg.NodeCapability]'s. They are + // useful as a global view of notable feature toggles, but the magicsock + // setters are completely unnecessary as we have the same values right here + // (update.SelfNode.Capabilities) at a time they are considered most + // up-to-date. + // TODO: mutate [debugFlags] here instead of in various [Conn] setters. flags := c.debugFlagsLocked() - if addrs := nm.GetAddresses(); addrs.Len() > 0 { - c.firstAddrForTest = addrs.At(0).Addr() - } else { - c.firstAddrForTest = netip.Addr{} - } - if nodesEqual(priorPeers, curPeers) && c.lastFlags == flags { + peersChanged = !nodesEqual(priorPeers, curPeers) + if !peersChanged && c.lastFlags == flags { // The rest of this function is all adjusting state for peers that have // changed. But if the set of peers is equal and the debug flags (for // silent disco and probe UDP lifetime) haven't changed, there is no @@ -2429,16 +3043,16 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) { c.lastFlags = flags - c.logf("[v1] magicsock: got updated network map; %d peers", len(nm.Peers)) + c.logf("[v1] magicsock: got updated network map; %d peers", len(update.Peers)) - entriesPerBuffer := debugRingBufferSize(len(nm.Peers)) + entriesPerBuffer := debugRingBufferSize(len(update.Peers)) // Try a pass of just upserting nodes and creating missing // endpoints. If the set of nodes is the same, this is an // efficient alloc-free update. If the set of nodes is different, // we'll fall through to the next pass, which allocates but can // handle full set updates. - for _, n := range nm.Peers { + for _, n := range update.Peers { if n.ID() == 0 { devPanicf("node with zero ID") continue @@ -2538,14 +3152,14 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) { c.peerMap.upsertEndpoint(ep, key.DiscoPublic{}) } - // If the set of nodes changed since the last SetNetworkMap, the + // If the set of nodes changed since the last onNodeViewsUpdate, the // upsert loop just above made c.peerMap contain the union of the // old and new peers - which will be larger than the set from the // current netmap. If that happens, go through the allocful // deletion path to clean up moribund nodes. - if c.peerMap.nodeCount() != len(nm.Peers) { + if c.peerMap.nodeCount() != len(update.Peers) { keep := set.Set[key.NodePublic]{} - for _, n := range nm.Peers { + for _, n := range update.Peers { keep.Add(n.Key()) } c.peerMap.forEachEndpoint(func(ep *endpoint) { @@ -2561,6 +3175,8 @@ func (c *Conn) SetNetworkMap(nm *netmap.NetworkMap) { delete(c.discoInfo, dk) } } + + return peersChanged } func devPanicf(format string, a ...any) { @@ -2687,9 +3303,6 @@ func (c *connBind) Close() error { if c.closeDisco6 != nil { c.closeDisco6.Close() } - if c.eventClient != nil { - c.eventClient.Close() - } // Send an empty read result to unblock receiveDERP, // which will then check connBind.Closed. // connBind.Closed takes c.mu, but c.derpRecvCh is buffered. @@ -2708,6 +3321,17 @@ func (c *connBind) isClosed() bool { // // Only the first close does anything. Any later closes return nil. func (c *Conn) Close() error { + // Close the [eventbus.Client] and wait for Conn.consumeEventbusTopics to + // return. Do this before acquiring c.mu: + // 1. Conn.consumeEventbusTopics event handlers also acquire c.mu, they can + // deadlock with c.Close(). + // 2. Conn.consumeEventbusTopics event handlers may not guard against + // undesirable post/in-progress Conn.Close() behaviors. + if c.eventClient != nil { + c.eventClient.Close() + <-c.subsDoneCh + } + c.mu.Lock() defer c.mu.Unlock() if c.closed { @@ -2738,7 +3362,6 @@ func (c *Conn) Close() error { if c.closeDisco6 != nil { c.closeDisco6.Close() } - // Wait on goroutines updating right at the end, once everything is // already closed. We want everything else in the Conn to be // consistently in the closed state before we release mu to wait @@ -2860,9 +3483,9 @@ func (c *Conn) listenPacket(network string, port uint16) (nettype.PacketConn, er return nettype.MakePacketListenerWithNetIP(netns.Listener(c.logf, c.netMon)).ListenPacket(ctx, network, addr) } -// bindSocket initializes rucPtr if necessary and binds a UDP socket to it. +// bindSocket binds a UDP socket to ruc. // Network indicates the UDP socket type; it must be "udp4" or "udp6". -// If rucPtr had an existing UDP socket bound, it closes that socket. +// If ruc had an existing UDP socket bound, it closes that socket. // The caller is responsible for informing the portMapper of any changes. // If curPortFate is set to dropCurrentPort, no attempt is made to reuse // the current port. @@ -3070,12 +3693,13 @@ func simpleDur(d time.Duration) time.Duration { return d.Round(time.Minute) } -// UpdateNetmapDelta implements controlclient.NetmapDeltaUpdater. -func (c *Conn) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) { +// onNodeMutationsUpdate is called when a [NodeMutationsUpdate] is received over +// the [eventbus.Bus]. Note: It does not apply these mutations to c.peers. +func (c *Conn) onNodeMutationsUpdate(update NodeMutationsUpdate) { c.mu.Lock() defer c.mu.Unlock() - for _, m := range muts { + for _, m := range update.Mutations { nodeID := m.NodeIDBeingMutated() ep, ok := c.peerMap.endpointForNodeID(nodeID) if !ok { @@ -3090,10 +3714,9 @@ func (c *Conn) UpdateNetmapDelta(muts []netmap.NodeMutation) (handled bool) { ep.mu.Unlock() } } - return true } -// UpdateStatus implements the interface nede by ipnstate.StatusBuilder. +// UpdateStatus implements the interface needed by ipnstate.StatusBuilder. // // This method adds in the magicsock-specific information only. Most // of the status is otherwise populated by LocalBackend. @@ -3159,9 +3782,17 @@ const ( // keep NAT mappings alive. sessionActiveTimeout = 45 * time.Second - // upgradeInterval is how often we try to upgrade to a better path - // even if we have some non-DERP route that works. - upgradeInterval = 1 * time.Minute + // upgradeUDPDirectInterval is how often we try to upgrade to a better, + // direct UDP path even if we have some direct UDP path that works. + upgradeUDPDirectInterval = 1 * time.Minute + + // upgradeUDPRelayInterval is how often we try to discover UDP relay paths + // even if we have a UDP relay path that works. + upgradeUDPRelayInterval = 1 * time.Minute + + // discoverUDPRelayPathsInterval is the minimum time between UDP relay path + // discovery. + discoverUDPRelayPathsInterval = 30 * time.Second // heartbeatInterval is how often pings to the best UDP address // are sent. @@ -3267,23 +3898,27 @@ func portableTrySetSocketBuffer(pconn nettype.PacketConn, logf logger.Logf) { // derpStr replaces DERP IPs in s with "derp-". func derpStr(s string) string { return strings.ReplaceAll(s, "127.3.3.40:", "derp-") } -// ippEndpointCache is a mutex-free single-element cache, mapping from -// a single netip.AddrPort to a single endpoint. -type ippEndpointCache struct { - ipp netip.AddrPort - gen int64 - de *endpoint +// epAddrEndpointCache is a mutex-free single-element cache, mapping from +// a single [epAddr] to a single [*endpoint]. +type epAddrEndpointCache struct { + epAddr epAddr + gen int64 + de *endpoint } // discoInfo is the info and state for the DiscoKey -// in the Conn.discoInfo map key. +// in the [Conn.discoInfo] and [relayManager.discoInfoByServerDisco] map keys. +// +// When the disco protocol is used to handshake with a peer relay server, the +// corresponding discoInfo is held in [relayManager.discoInfoByServerDisco] +// instead of [Conn.discoInfo]. // // Note that a DiscoKey does not necessarily map to exactly one // node. In the case of shared nodes and users switching accounts, two // nodes in the NetMap may legitimately have the same DiscoKey. As // such, no fields in here should be considered node-specific. type discoInfo struct { - // discoKey is the same as the Conn.discoInfo map key, + // discoKey is the same as the corresponding map key, // just so you can pass around a *discoInfo alone. // Not modified once initialized. discoKey key.DiscoPublic @@ -3294,14 +3929,16 @@ type discoInfo struct { // sharedKey is the precomputed key for communication with the // peer that has the DiscoKey used to look up this *discoInfo in - // Conn.discoInfo. + // the corresponding map. // Not modified once initialized. sharedKey key.DiscoShared - // Mutable fields follow, owned by Conn.mu: + // Mutable fields follow, owned by [Conn.mu]. These are irrelevant when + // discoInfo is a peer relay server disco key in the + // [relayManager.discoInfoByServerDisco] map: // lastPingFrom is the src of a ping for discoKey. - lastPingFrom netip.AddrPort + lastPingFrom epAddr // lastPingTime is the last time of a ping for discoKey. lastPingTime time.Time @@ -3322,15 +3959,19 @@ var ( metricSendDERPErrorQueue = clientmetric.NewCounter("magicsock_send_derp_error_queue") metricSendUDP = clientmetric.NewAggregateCounter("magicsock_send_udp") metricSendUDPError = clientmetric.NewCounter("magicsock_send_udp_error") + metricSendPeerRelay = clientmetric.NewAggregateCounter("magicsock_send_peer_relay") + metricSendPeerRelayError = clientmetric.NewCounter("magicsock_send_peer_relay_error") metricSendDERP = clientmetric.NewAggregateCounter("magicsock_send_derp") metricSendDERPError = clientmetric.NewCounter("magicsock_send_derp_error") // Data packets (non-disco) - metricSendData = clientmetric.NewCounter("magicsock_send_data") - metricSendDataNetworkDown = clientmetric.NewCounter("magicsock_send_data_network_down") - metricRecvDataPacketsDERP = clientmetric.NewAggregateCounter("magicsock_recv_data_derp") - metricRecvDataPacketsIPv4 = clientmetric.NewAggregateCounter("magicsock_recv_data_ipv4") - metricRecvDataPacketsIPv6 = clientmetric.NewAggregateCounter("magicsock_recv_data_ipv6") + metricSendData = clientmetric.NewCounter("magicsock_send_data") + metricSendDataNetworkDown = clientmetric.NewCounter("magicsock_send_data_network_down") + metricRecvDataPacketsDERP = clientmetric.NewAggregateCounter("magicsock_recv_data_derp") + metricRecvDataPacketsIPv4 = clientmetric.NewAggregateCounter("magicsock_recv_data_ipv4") + metricRecvDataPacketsIPv6 = clientmetric.NewAggregateCounter("magicsock_recv_data_ipv6") + metricRecvDataPacketsPeerRelayIPv4 = clientmetric.NewAggregateCounter("magicsock_recv_data_peer_relay_ipv4") + metricRecvDataPacketsPeerRelayIPv6 = clientmetric.NewAggregateCounter("magicsock_recv_data_peer_relay_ipv6") // Disco packets metricSendDiscoUDP = clientmetric.NewCounter("magicsock_disco_send_udp") @@ -3346,15 +3987,22 @@ var ( metricRecvDiscoBadKey = clientmetric.NewCounter("magicsock_disco_recv_bad_key") metricRecvDiscoBadParse = clientmetric.NewCounter("magicsock_disco_recv_bad_parse") - metricRecvDiscoUDP = clientmetric.NewCounter("magicsock_disco_recv_udp") - metricRecvDiscoDERP = clientmetric.NewCounter("magicsock_disco_recv_derp") - metricRecvDiscoPing = clientmetric.NewCounter("magicsock_disco_recv_ping") - metricRecvDiscoPong = clientmetric.NewCounter("magicsock_disco_recv_pong") - metricRecvDiscoCallMeMaybe = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe") - metricRecvDiscoCallMeMaybeBadNode = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_node") - metricRecvDiscoCallMeMaybeBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_disco") - metricRecvDiscoDERPPeerNotHere = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_not_here") - metricRecvDiscoDERPPeerGoneUnknown = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_gone_unknown") + metricRecvDiscoUDP = clientmetric.NewCounter("magicsock_disco_recv_udp") + metricRecvDiscoDERP = clientmetric.NewCounter("magicsock_disco_recv_derp") + metricRecvDiscoPing = clientmetric.NewCounter("magicsock_disco_recv_ping") + metricRecvDiscoPong = clientmetric.NewCounter("magicsock_disco_recv_pong") + metricRecvDiscoCallMeMaybe = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe") + metricRecvDiscoCallMeMaybeVia = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia") + metricRecvDiscoCallMeMaybeBadNode = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_node") + metricRecvDiscoCallMeMaybeViaBadNode = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia_bad_node") + metricRecvDiscoCallMeMaybeBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybe_bad_disco") + metricRecvDiscoCallMeMaybeViaBadDisco = clientmetric.NewCounter("magicsock_disco_recv_callmemaybevia_bad_disco") + metricRecvDiscoAllocUDPRelayEndpointRequest = clientmetric.NewCounter("magicsock_disco_recv_alloc_udp_relay_endpoint_request") + metricRecvDiscoAllocUDPRelayEndpointRequestBadDisco = clientmetric.NewCounter("magicsock_disco_recv_alloc_udp_relay_endpoint_request_bad_disco") + metricRecvDiscoAllocUDPRelayEndpointResponseBadDisco = clientmetric.NewCounter("magicsock_disco_recv_alloc_udp_relay_endpoint_response_bad_disco") + metricRecvDiscoAllocUDPRelayEndpointResponse = clientmetric.NewCounter("magicsock_disco_recv_alloc_udp_relay_endpoint_response") + metricRecvDiscoDERPPeerNotHere = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_not_here") + metricRecvDiscoDERPPeerGoneUnknown = clientmetric.NewCounter("magicsock_disco_recv_derp_peer_gone_unknown") // metricDERPHomeChange is how many times our DERP home region DI has // changed from non-zero to a different non-zero. metricDERPHomeChange = clientmetric.NewCounter("derp_home_change") @@ -3428,34 +4076,133 @@ func (c *Conn) SetLastNetcheckReportForTest(ctx context.Context, report *netchec c.lastNetCheckReport.Store(report) } -// lazyEndpoint is a wireguard conn.Endpoint for when magicsock received a +// lazyEndpoint is a wireguard [conn.Endpoint] for when magicsock received a // non-disco (presumably WireGuard) packet from a UDP address from which we -// can't map to a Tailscale peer. But Wireguard most likely can, once it -// decrypts it. So we implement the conn.PeerAwareEndpoint interface -// from https://github.com/tailscale/wireguard-go/pull/27 to allow WireGuard -// to tell us who it is later and get the correct conn.Endpoint. +// can't map to a Tailscale peer. But WireGuard most likely can, once it +// decrypts it. So we implement the [conn.InitiationAwareEndpoint] and +// [conn.PeerAwareEndpoint] interfaces, to allow WireGuard to tell us who it is +// later, just-in-time to configure the peer, and set the associated [epAddr] +// in the [peerMap]. Future receives on the associated [epAddr] will then +// resolve directly to an [*endpoint]. +// +// We also sometimes (see [Conn.receiveIP]) return a [*lazyEndpoint] to +// wireguard-go to verify an [epAddr] resolves to the [*endpoint] (maybeEP) we +// believe it to be, to resolve [epAddr] collisions across peers. [epAddr] +// collisions have a higher chance of occurrence for packets received over peer +// relays versus direct connections, as peer relay connections do not upsert +// into [peerMap] around disco packet reception, but direct connections do. type lazyEndpoint struct { - c *Conn - src netip.AddrPort + c *Conn + maybeEP *endpoint // or nil if unknown + src epAddr } +var _ conn.InitiationAwareEndpoint = (*lazyEndpoint)(nil) var _ conn.PeerAwareEndpoint = (*lazyEndpoint)(nil) var _ conn.Endpoint = (*lazyEndpoint)(nil) -func (le *lazyEndpoint) ClearSrc() {} -func (le *lazyEndpoint) SrcIP() netip.Addr { return le.src.Addr() } -func (le *lazyEndpoint) DstIP() netip.Addr { return netip.Addr{} } -func (le *lazyEndpoint) SrcToString() string { return le.src.String() } -func (le *lazyEndpoint) DstToString() string { return "dst" } -func (le *lazyEndpoint) DstToBytes() []byte { return nil } -func (le *lazyEndpoint) GetPeerEndpoint(peerPublicKey [32]byte) conn.Endpoint { +// InitiationMessagePublicKey implements [conn.InitiationAwareEndpoint]. +// wireguard-go calls us here if we passed it a [*lazyEndpoint] for an +// initiation message, for which it might not have the relevant peer configured, +// enabling us to just-in-time configure it and note its activity via +// [*endpoint.noteRecvActivity], before it performs peer lookup and attempts +// decryption. +// +// Reception of all other WireGuard message types implies pre-existing knowledge +// of the peer by wireguard-go for it to do useful work. See +// [userspaceEngine.maybeReconfigWireguardLocked] & +// [userspaceEngine.noteRecvActivity] for more details around just-in-time +// wireguard-go peer (de)configuration. +func (le *lazyEndpoint) InitiationMessagePublicKey(peerPublicKey [32]byte) { + pubKey := key.NodePublicFromRaw32(mem.B(peerPublicKey[:])) + if le.maybeEP != nil && pubKey.Compare(le.maybeEP.publicKey) == 0 { + return + } + le.c.mu.Lock() + ep, ok := le.c.peerMap.endpointForNodeKey(pubKey) + // [Conn.mu] must not be held while [Conn.noteRecvActivity] is called, which + // [endpoint.noteRecvActivity] can end up calling. See + // [Options.NoteRecvActivity] docs. + le.c.mu.Unlock() + if !ok { + return + } + now := mono.Now() + ep.lastRecvUDPAny.StoreAtomic(now) + ep.noteRecvActivity(le.src, now) + // [ep.noteRecvActivity] may end up JIT configuring the peer, but we don't + // update [peerMap] as wireguard-go hasn't decrypted the initiation + // message yet. wireguard-go will call us below in [lazyEndpoint.FromPeer] + // if it successfully decrypts the message, at which point it's safe to + // insert le.src into the [peerMap] for ep. +} + +func (le *lazyEndpoint) ClearSrc() {} +func (le *lazyEndpoint) SrcIP() netip.Addr { return netip.Addr{} } + +// DstIP returns the remote address of the peer. +// +// Note: DstIP is used internally by wireguard-go as part of handshake DoS +// mitigation. +func (le *lazyEndpoint) DstIP() netip.Addr { return le.src.ap.Addr() } + +func (le *lazyEndpoint) SrcToString() string { return "" } +func (le *lazyEndpoint) DstToString() string { return le.src.String() } + +// DstToBytes returns a binary representation of the remote address of the peer. +// +// Note: DstToBytes is used internally by wireguard-go as part of handshake DoS +// mitigation. +func (le *lazyEndpoint) DstToBytes() []byte { + b, _ := le.src.ap.MarshalBinary() + return b +} + +// FromPeer implements [conn.PeerAwareEndpoint]. We return a [*lazyEndpoint] in +// [Conn.receiveIP] when we are unable to identify the peer at WireGuard +// packet reception time, pre-decryption, or we want wireguard-go to verify who +// we believe it to be (le.maybeEP). If wireguard-go successfully decrypts the +// packet it calls us here, and we update our [peerMap] to associate le.src with +// peerPublicKey. +func (le *lazyEndpoint) FromPeer(peerPublicKey [32]byte) { pubKey := key.NodePublicFromRaw32(mem.B(peerPublicKey[:])) + if le.maybeEP != nil && pubKey.Compare(le.maybeEP.publicKey) == 0 { + return + } le.c.mu.Lock() defer le.c.mu.Unlock() ep, ok := le.c.peerMap.endpointForNodeKey(pubKey) if !ok { - return nil + return + } + // TODO(jwhited): Consider [lazyEndpoint] effectiveness as a means to make + // this the sole call site for setNodeKeyForEpAddr. If this is the sole + // call site, and we always update the mapping based on successful + // Cryptokey Routing identification events, then we can go ahead and make + // [epAddr]s singular per peer (like they are for Geneve-encapsulated ones + // already). + // See http://go/corp/29422 & http://go/corp/30042 + le.c.peerMap.setNodeKeyForEpAddr(le.src, pubKey) + le.c.logf("magicsock: lazyEndpoint.FromPeer(%v) setting epAddr(%v) in peerMap for node(%v)", pubKey.ShortString(), le.src, ep.nodeAddr) +} + +// PeerRelays returns the current set of candidate peer relays. +func (c *Conn) PeerRelays() set.Set[netip.Addr] { + candidatePeerRelays := c.relayManager.getServers() + servers := make(set.Set[netip.Addr], len(candidatePeerRelays)) + c.mu.Lock() + defer c.mu.Unlock() + for relay := range candidatePeerRelays { + pi, ok := c.peerMap.byNodeKey[relay.nodeKey] + if !ok { + if c.self.Key().Compare(relay.nodeKey) == 0 { + if c.self.Addresses().Len() > 0 { + servers.Add(c.self.Addresses().At(0).Addr()) + } + } + continue + } + servers.Add(pi.ep.nodeAddr) } - le.c.logf("magicsock: lazyEndpoint.GetPeerEndpoint(%v) found: %v", pubKey.ShortString(), ep.nodeAddr) - return ep + return servers } diff --git a/wgengine/magicsock/magicsock_linux.go b/wgengine/magicsock/magicsock_linux.go index c5df555cd4ecd..07038002912f7 100644 --- a/wgengine/magicsock/magicsock_linux.go +++ b/wgengine/magicsock/magicsock_linux.go @@ -66,10 +66,10 @@ var ( // fragmented, and we don't want to handle reassembly. bpf.LoadAbsolute{Off: 6, Size: 2}, // More Fragments bit set means this is part of a fragmented packet. - bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x2000, SkipTrue: 7, SkipFalse: 0}, + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x2000, SkipTrue: 8, SkipFalse: 0}, // Non-zero fragment offset with MF=0 means this is the last // fragment of packet. - bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x1fff, SkipTrue: 6, SkipFalse: 0}, + bpf.JumpIf{Cond: bpf.JumpBitsSet, Val: 0x1fff, SkipTrue: 7, SkipFalse: 0}, // Load IP header length into X register. bpf.LoadMemShift{Off: 0}, @@ -453,7 +453,13 @@ func (c *Conn) receiveDisco(pc *socket.Conn, isIPV6 bool) { metricRecvDiscoPacketIPv4.Add(1) } - c.handleDiscoMessage(payload, srcAddr, key.NodePublic{}, discoRXPathRawSocket) + pt, isGeneveEncap := packetLooksLike(payload) + if pt == packetLooksLikeDisco && !isGeneveEncap { + // The BPF program matching on disco does not currently support + // Geneve encapsulation. isGeneveEncap should not return true if + // payload is disco. + c.handleDiscoMessage(payload, epAddr{ap: srcAddr}, false, key.NodePublic{}, discoRXPathRawSocket) + } } } diff --git a/wgengine/magicsock/magicsock_linux_test.go b/wgengine/magicsock/magicsock_linux_test.go index 6b86b04f2c8d4..28ccd220ee784 100644 --- a/wgengine/magicsock/magicsock_linux_test.go +++ b/wgengine/magicsock/magicsock_linux_test.go @@ -9,6 +9,7 @@ import ( "net/netip" "testing" + "golang.org/x/net/bpf" "golang.org/x/sys/cpu" "golang.org/x/sys/unix" "tailscale.com/disco" @@ -146,3 +147,78 @@ func TestEthernetProto(t *testing.T) { } } } + +func TestBpfDiscardV4(t *testing.T) { + // Good packet as a reference for what should not be rejected + udp4Packet := []byte{ + // IPv4 header + 0x45, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x11, 0x00, 0x00, + 0x7f, 0x00, 0x00, 0x01, // source ip + 0x7f, 0x00, 0x00, 0x02, // dest ip + + // UDP header + 0x30, 0x39, // src port + 0xd4, 0x31, // dest port + 0x00, 0x12, // length; 8 bytes header + 10 bytes payload = 18 bytes + 0x00, 0x00, // checksum; unused + + // Payload: disco magic plus 32 bytes for key and 24 bytes for nonce + 0x54, 0x53, 0xf0, 0x9f, 0x92, 0xac, 0x00, 0x01, 0x02, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, + } + + vm, err := bpf.NewVM(magicsockFilterV4) + if err != nil { + t.Fatalf("failed creating BPF VM: %v", err) + } + + tests := []struct { + name string + replace map[int]byte + accept bool + }{ + { + name: "base accepted datagram", + replace: map[int]byte{}, + accept: true, + }, + { + name: "more fragments", + replace: map[int]byte{ + 6: 0x20, + }, + accept: false, + }, + { + name: "some fragment", + replace: map[int]byte{ + 7: 0x01, + }, + accept: false, + }, + } + + udp4PacketChanged := make([]byte, len(udp4Packet)) + copy(udp4PacketChanged, udp4Packet) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.replace { + udp4PacketChanged[k] = v + } + ret, err := vm.Run(udp4PacketChanged) + if err != nil { + t.Fatalf("BPF VM error: %v", err) + } + + if (ret != 0) != tt.accept { + t.Errorf("expected accept=%v, got ret=%v", tt.accept, ret) + } + }) + } +} diff --git a/wgengine/magicsock/magicsock_test.go b/wgengine/magicsock/magicsock_test.go index ddbf3e3940efe..8a09df27d2ce7 100644 --- a/wgengine/magicsock/magicsock_test.go +++ b/wgengine/magicsock/magicsock_test.go @@ -50,6 +50,7 @@ import ( "tailscale.com/net/netmon" "tailscale.com/net/packet" "tailscale.com/net/ping" + "tailscale.com/net/stun" "tailscale.com/net/stun/stuntest" "tailscale.com/net/tstun" "tailscale.com/tailcfg" @@ -62,6 +63,7 @@ import ( "tailscale.com/types/netmap" "tailscale.com/types/nettype" "tailscale.com/types/ptr" + "tailscale.com/types/views" "tailscale.com/util/cibuild" "tailscale.com/util/eventbus" "tailscale.com/util/must" @@ -70,6 +72,7 @@ import ( "tailscale.com/util/slicesx" "tailscale.com/util/usermetric" "tailscale.com/wgengine/filter" + "tailscale.com/wgengine/filter/filtertype" "tailscale.com/wgengine/wgcfg" "tailscale.com/wgengine/wgcfg/nmcfg" "tailscale.com/wgengine/wglog" @@ -165,7 +168,7 @@ type magicStack struct { } // newMagicStack builds and initializes an idle magicsock and -// friends. You need to call conn.SetNetworkMap and dev.Reconfig +// friends. You need to call conn.onNodeViewsUpdate and dev.Reconfig // before anything interesting happens. func newMagicStack(t testing.TB, logf logger.Logf, l nettype.PacketListener, derpMap *tailcfg.DERPMap) *magicStack { privateKey := key.NewNode() @@ -274,7 +277,10 @@ func (s *magicStack) Status() *ipnstate.Status { func (s *magicStack) IP() netip.Addr { for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { s.conn.mu.Lock() - addr := s.conn.firstAddrForTest + var addr netip.Addr + if s.conn.self.Valid() && s.conn.self.Addresses().Len() > 0 { + addr = s.conn.self.Addresses().At(0).Addr() + } s.conn.mu.Unlock() if addr.IsValid() { return addr @@ -338,9 +344,13 @@ func meshStacks(logf logger.Logf, mutateNetmap func(idx int, nm *netmap.NetworkM for i, m := range ms { nm := buildNetmapLocked(i) - m.conn.SetNetworkMap(nm) - peerSet := make(set.Set[key.NodePublic], len(nm.Peers)) - for _, peer := range nm.Peers { + nv := NodeViewsUpdate{ + SelfNode: nm.SelfNode, + Peers: nm.Peers, + } + m.conn.onNodeViewsUpdate(nv) + peerSet := make(set.Set[key.NodePublic], len(nv.Peers)) + for _, peer := range nv.Peers { peerSet.Add(peer.Key()) } m.conn.UpdatePeers(peerSet) @@ -1290,41 +1300,6 @@ func assertConnStatsAndUserMetricsEqual(t *testing.T, ms *magicStack) { c.Assert(metricRecvDataPacketsDERP.Value(), qt.Equals, metricDERPRxPackets*2) } -func TestDiscoMessage(t *testing.T) { - c := newConn(t.Logf) - c.privateKey = key.NewNode() - - peer1Pub := c.DiscoPublicKey() - peer1Priv := c.discoPrivate - n := &tailcfg.Node{ - Key: key.NewNode().Public(), - DiscoKey: peer1Pub, - } - ep := &endpoint{ - nodeID: 1, - publicKey: n.Key, - } - ep.disco.Store(&endpointDisco{ - key: n.DiscoKey, - short: n.DiscoKey.ShortString(), - }) - c.peerMap.upsertEndpoint(ep, key.DiscoPublic{}) - - const payload = "why hello" - - var nonce [24]byte - crand.Read(nonce[:]) - - pkt := peer1Pub.AppendTo([]byte("TS💬")) - - box := peer1Priv.Shared(c.discoPrivate.Public()).Seal([]byte(payload)) - pkt = append(pkt, box...) - got := c.handleDiscoMessage(pkt, netip.AddrPort{}, key.NodePublic{}, discoRXPathUDP) - if !got { - t.Error("failed to open it") - } -} - // tests that having a endpoint.String prevents wireguard-go's // log.Printf("%v") of its conn.Endpoint values from using reflect to // walk into read mutex while they're being used and then causing data @@ -1358,11 +1333,11 @@ func Test32bitAlignment(t *testing.T) { t.Fatalf("endpoint.lastRecvWG is not 8-byte aligned") } - de.noteRecvActivity(netip.AddrPort{}, mono.Now()) // verify this doesn't panic on 32-bit + de.noteRecvActivity(epAddr{}, mono.Now()) // verify this doesn't panic on 32-bit if called != 1 { t.Fatal("expected call to noteRecvActivity") } - de.noteRecvActivity(netip.AddrPort{}, mono.Now()) + de.noteRecvActivity(epAddr{}, mono.Now()) if called != 1 { t.Error("expected no second call to noteRecvActivity") } @@ -1400,7 +1375,7 @@ func newTestConn(t testing.TB) *Conn { return conn } -// addTestEndpoint sets conn's network map to a single peer expected +// addTestEndpoint sets conn's node views to a single peer expected // to receive packets from sendConn (or DERP), and returns that peer's // nodekey and discokey. func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (key.NodePublic, key.DiscoPublic) { @@ -1409,7 +1384,7 @@ func addTestEndpoint(tb testing.TB, conn *Conn, sendConn net.PacketConn) (key.No // codepath. discoKey := key.DiscoPublicFromRaw32(mem.B([]byte{31: 1})) nodeKey := key.NodePublicFromRaw32(mem.B([]byte{0: 'N', 1: 'K', 31: 0})) - conn.SetNetworkMap(&netmap.NetworkMap{ + conn.onNodeViewsUpdate(NodeViewsUpdate{ Peers: nodeViews([]*tailcfg.Node{ { ID: 1, @@ -1598,11 +1573,11 @@ func nodeViews(v []*tailcfg.Node) []tailcfg.NodeView { return nv } -// Test that a netmap update where node changes its node key but +// Test that a node views update where node changes its node key but // doesn't change its disco key doesn't result in a broken state. // // https://github.com/tailscale/tailscale/issues/1391 -func TestSetNetworkMapChangingNodeKey(t *testing.T) { +func TestOnNodeViewsUpdateChangingNodeKey(t *testing.T) { conn := newTestConn(t) t.Cleanup(func() { conn.Close() }) var buf tstest.MemLogger @@ -1614,7 +1589,7 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) { nodeKey1 := key.NodePublicFromRaw32(mem.B([]byte{0: 'N', 1: 'K', 2: '1', 31: 0})) nodeKey2 := key.NodePublicFromRaw32(mem.B([]byte{0: 'N', 1: 'K', 2: '2', 31: 0})) - conn.SetNetworkMap(&netmap.NetworkMap{ + conn.onNodeViewsUpdate(NodeViewsUpdate{ Peers: nodeViews([]*tailcfg.Node{ { ID: 1, @@ -1630,7 +1605,7 @@ func TestSetNetworkMapChangingNodeKey(t *testing.T) { } for range 3 { - conn.SetNetworkMap(&netmap.NetworkMap{ + conn.onNodeViewsUpdate(NodeViewsUpdate{ Peers: nodeViews([]*tailcfg.Node{ { ID: 2, @@ -1799,10 +1774,15 @@ func TestEndpointSetsEqual(t *testing.T) { func TestBetterAddr(t *testing.T) { const ms = time.Millisecond al := func(ipps string, d time.Duration) addrQuality { - return addrQuality{AddrPort: netip.MustParseAddrPort(ipps), latency: d} + return addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort(ipps)}, latency: d} } almtu := func(ipps string, d time.Duration, mtu tstun.WireMTU) addrQuality { - return addrQuality{AddrPort: netip.MustParseAddrPort(ipps), latency: d, wireMTU: mtu} + return addrQuality{epAddr: epAddr{ap: netip.MustParseAddrPort(ipps)}, latency: d, wireMTU: mtu} + } + avl := func(ipps string, vni uint32, d time.Duration) addrQuality { + q := al(ipps, d) + q.vni.set(vni) + return q } zero := addrQuality{} @@ -1908,6 +1888,18 @@ func TestBetterAddr(t *testing.T) { b: al("[::1]:555", 100*ms), want: false, }, + + // Prefer non-Geneve over Geneve-encapsulated + { + a: al(publicV4, 100*ms), + b: avl(publicV4, 1, 100*ms), + want: true, + }, + { + a: avl(publicV4, 1, 100*ms), + b: al(publicV4, 100*ms), + want: false, + }, } for i, tt := range tests { got := betterAddr(tt.a, tt.b) @@ -1938,7 +1930,7 @@ func eps(s ...string) []netip.AddrPort { return eps } -func TestStressSetNetworkMap(t *testing.T) { +func TestStressOnNodeViewsUpdate(t *testing.T) { t.Parallel() conn := newTestConn(t) @@ -1986,15 +1978,15 @@ func TestStressSetNetworkMap(t *testing.T) { allPeers[j].Key = randNodeKey() } } - // Clone existing peers into a new netmap. + // Clone existing peers. peers := make([]*tailcfg.Node, 0, len(allPeers)) for peerIdx, p := range allPeers { if present[peerIdx] { peers = append(peers, p.Clone()) } } - // Set the netmap. - conn.SetNetworkMap(&netmap.NetworkMap{ + // Set the node views. + conn.onNodeViewsUpdate(NodeViewsUpdate{ Peers: nodeViews(peers), }) // Check invariants. @@ -2019,9 +2011,9 @@ func (m *peerMap) validate() error { return fmt.Errorf("duplicate endpoint present: %v", pi.ep.publicKey) } seenEps[pi.ep] = true - for ipp := range pi.ipPorts { - if got := m.byIPPort[ipp]; got != pi { - return fmt.Errorf("m.byIPPort[%v] = %v, want %v", ipp, got, pi) + for addr := range pi.epAddrs { + if got := m.byEpAddr[addr]; got != pi { + return fmt.Errorf("m.byEpAddr[%v] = %v, want %v", addr, got, pi) } } } @@ -2037,13 +2029,13 @@ func (m *peerMap) validate() error { } } - for ipp, pi := range m.byIPPort { - if !pi.ipPorts.Contains(ipp) { - return fmt.Errorf("ipPorts[%v] for %v is false", ipp, pi.ep.publicKey) + for addr, pi := range m.byEpAddr { + if !pi.epAddrs.Contains(addr) { + return fmt.Errorf("epAddrs[%v] for %v is false", addr, pi.ep.publicKey) } pi2 := m.byNodeKey[pi.ep.publicKey] if pi != pi2 { - return fmt.Errorf("byNodeKey[%v]=%p doesn't match byIPPort[%v]=%p", pi, pi, pi.ep.publicKey, pi2) + return fmt.Errorf("byNodeKey[%v]=%p doesn't match byEpAddr[%v]=%p", pi, pi, pi.ep.publicKey, pi2) } } @@ -2119,10 +2111,10 @@ func TestRebindingUDPConn(t *testing.T) { } // https://github.com/tailscale/tailscale/issues/6680: don't ignore -// SetNetworkMap calls when there are no peers. (A too aggressive fast path was +// onNodeViewsUpdate calls when there are no peers. (A too aggressive fast path was // previously bailing out early, thinking there were no changes since all zero -// peers didn't change, but the netmap has non-peer info in it too we shouldn't discard) -func TestSetNetworkMapWithNoPeers(t *testing.T) { +// peers didn't change, but the node views has non-peer info in it too we shouldn't discard) +func TestOnNodeViewsUpdateWithNoPeers(t *testing.T) { var c Conn knobs := &controlknobs.Knobs{} c.logf = logger.Discard @@ -2131,9 +2123,9 @@ func TestSetNetworkMapWithNoPeers(t *testing.T) { for i := 1; i <= 3; i++ { v := !debugEnableSilentDisco() envknob.Setenv("TS_DEBUG_ENABLE_SILENT_DISCO", fmt.Sprint(v)) - nm := &netmap.NetworkMap{} - c.SetNetworkMap(nm) - t.Logf("ptr %d: %p", i, nm) + nv := NodeViewsUpdate{} + c.onNodeViewsUpdate(nv) + t.Logf("ptr %d: %p", i, nv) if c.lastFlags.heartbeatDisabled != v { t.Fatalf("call %d: didn't store netmap", i) } @@ -2230,7 +2222,11 @@ func TestIsWireGuardOnlyPeer(t *testing.T) { }, }), } - m.conn.SetNetworkMap(nm) + nv := NodeViewsUpdate{ + SelfNode: nm.SelfNode, + Peers: nm.Peers, + } + m.conn.onNodeViewsUpdate(nv) cfg, err := nmcfg.WGCfg(nm, t.Logf, netmap.AllowSubnetRoutes, "") if err != nil { @@ -2292,7 +2288,11 @@ func TestIsWireGuardOnlyPeerWithMasquerade(t *testing.T) { }, }), } - m.conn.SetNetworkMap(nm) + nv := NodeViewsUpdate{ + SelfNode: nm.SelfNode, + Peers: nm.Peers, + } + m.conn.onNodeViewsUpdate(nv) cfg, err := nmcfg.WGCfg(nm, t.Logf, netmap.AllowSubnetRoutes, "") if err != nil { @@ -2329,7 +2329,11 @@ func TestIsWireGuardOnlyPeerWithMasquerade(t *testing.T) { // configures WG. func applyNetworkMap(t *testing.T, m *magicStack, nm *netmap.NetworkMap) { t.Helper() - m.conn.SetNetworkMap(nm) + nv := NodeViewsUpdate{ + SelfNode: nm.SelfNode, + Peers: nm.Peers, + } + m.conn.onNodeViewsUpdate(nv) // Make sure we can't use v6 to avoid test failures. m.conn.noV6.Store(true) @@ -2444,7 +2448,7 @@ func TestIsWireGuardOnlyPickEndpointByPing(t *testing.T) { // Check that we got a valid address set on the first send - this // will be randomly selected, but because we have noV6 set to true, // it will be the IPv4 address. - if !pi.ep.bestAddr.Addr().IsValid() { + if !pi.ep.bestAddr.ap.Addr().IsValid() { t.Fatal("bestaddr was nil") } @@ -2504,12 +2508,12 @@ func TestIsWireGuardOnlyPickEndpointByPing(t *testing.T) { t.Fatal("wgkey doesn't exist in peer map") } - if !pi.ep.bestAddr.Addr().IsValid() { + if !pi.ep.bestAddr.ap.Addr().IsValid() { t.Error("no bestAddr address was set") } - if pi.ep.bestAddr.Addr() != wgEp.Addr() { - t.Errorf("bestAddr was not set to the expected IPv4 address: got %v, want %v", pi.ep.bestAddr.Addr().String(), wgEp.Addr()) + if pi.ep.bestAddr.ap.Addr() != wgEp.Addr() { + t.Errorf("bestAddr was not set to the expected IPv4 address: got %v, want %v", pi.ep.bestAddr.ap.Addr().String(), wgEp.Addr()) } if pi.ep.trustBestAddrUntil.IsZero() { @@ -2670,7 +2674,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { sendFollowUpPing bool pingTime mono.Time ep []endpointDetails - want netip.AddrPort + want epAddr }{ { name: "no endpoints", @@ -2679,7 +2683,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { sendFollowUpPing: false, pingTime: testTime, ep: []endpointDetails{}, - want: netip.AddrPort{}, + want: epAddr{}, }, { name: "singular endpoint does not request ping", @@ -2693,7 +2697,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { latency: 100 * time.Millisecond, }, }, - want: netip.MustParseAddrPort("1.1.1.1:111"), + want: epAddr{ap: netip.MustParseAddrPort("1.1.1.1:111")}, }, { name: "ping sent within wireguardPingInterval should not request ping", @@ -2711,7 +2715,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { latency: 2000 * time.Millisecond, }, }, - want: netip.MustParseAddrPort("1.1.1.1:111"), + want: epAddr{ap: netip.MustParseAddrPort("1.1.1.1:111")}, }, { name: "ping sent outside of wireguardPingInterval should request ping", @@ -2729,7 +2733,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { latency: 150 * time.Millisecond, }, }, - want: netip.MustParseAddrPort("1.1.1.1:111"), + want: epAddr{ap: netip.MustParseAddrPort("1.1.1.1:111")}, }, { name: "choose lowest latency for useable IPv4 and IPv6", @@ -2747,7 +2751,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { latency: 10 * time.Millisecond, }, }, - want: netip.MustParseAddrPort("[2345:0425:2CA1:0000:0000:0567:5673:23b5]:222"), + want: epAddr{ap: netip.MustParseAddrPort("[2345:0425:2CA1:0000:0000:0567:5673:23b5]:222")}, }, { name: "choose IPv6 address when latency is the same for v4 and v6", @@ -2765,7 +2769,7 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { latency: 100 * time.Millisecond, }, }, - want: netip.MustParseAddrPort("[1::1]:567"), + want: epAddr{ap: netip.MustParseAddrPort("[1::1]:567")}, }, } @@ -2785,8 +2789,8 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { endpoint.endpointState[epd.addrPort] = &endpointState{} } udpAddr, _, shouldPing := endpoint.addrForSendLocked(testTime) - if udpAddr.IsValid() != test.validAddr { - t.Errorf("udpAddr validity is incorrect; got %v, want %v", udpAddr.IsValid(), test.validAddr) + if udpAddr.ap.IsValid() != test.validAddr { + t.Errorf("udpAddr validity is incorrect; got %v, want %v", udpAddr.ap.IsValid(), test.validAddr) } if shouldPing != test.sendInitialPing { t.Errorf("addrForSendLocked did not indiciate correct ping state; got %v, want %v", shouldPing, test.sendInitialPing) @@ -2818,8 +2822,8 @@ func TestAddrForSendLockedForWireGuardOnly(t *testing.T) { if shouldPing != test.sendFollowUpPing { t.Errorf("addrForSendLocked did not indiciate correct ping state; got %v, want %v", shouldPing, test.sendFollowUpPing) } - if endpoint.bestAddr.AddrPort != test.want { - t.Errorf("bestAddr.AddrPort is not as expected: got %v, want %v", endpoint.bestAddr.AddrPort, test.want) + if endpoint.bestAddr.epAddr != test.want { + t.Errorf("bestAddr.epAddr is not as expected: got %v, want %v", endpoint.bestAddr.epAddr, test.want) } }) } @@ -2906,7 +2910,7 @@ func TestAddrForPingSizeLocked(t *testing.T) { t.Run(test.desc, func(t *testing.T) { bestAddr := addrQuality{wireMTU: test.mtu} if test.bestAddr { - bestAddr.AddrPort = validUdpAddr + bestAddr.epAddr.ap = validUdpAddr } ep := &endpoint{ derpAddr: validDerpAddr, @@ -2918,10 +2922,10 @@ func TestAddrForPingSizeLocked(t *testing.T) { udpAddr, derpAddr := ep.addrForPingSizeLocked(testTime, test.size) - if test.wantUDP && !udpAddr.IsValid() { + if test.wantUDP && !udpAddr.ap.IsValid() { t.Errorf("%s: udpAddr returned is not valid, won't be sent to UDP address", test.desc) } - if !test.wantUDP && udpAddr.IsValid() { + if !test.wantUDP && udpAddr.ap.IsValid() { t.Errorf("%s: udpAddr returned is valid, discovery will not start", test.desc) } if test.wantDERP && !derpAddr.IsValid() { @@ -3147,7 +3151,7 @@ func TestNetworkDownSendErrors(t *testing.T) { defer conn.Close() conn.SetNetworkUp(false) - if err := conn.Send([][]byte{{00}}, &lazyEndpoint{}); err == nil { + if err := conn.Send([][]byte{{00}}, &lazyEndpoint{}, 0); err == nil { t.Error("expected error, got nil") } resp := httptest.NewRecorder() @@ -3157,7 +3161,7 @@ func TestNetworkDownSendErrors(t *testing.T) { } } -func Test_isDiscoMaybeGeneve(t *testing.T) { +func Test_packetLooksLike(t *testing.T) { discoPub := key.DiscoPublicFromRaw32(mem.B([]byte{1: 1, 30: 30, 31: 31})) nakedDisco := make([]byte, 0, 512) nakedDisco = append(nakedDisco, disco.Magic...) @@ -3240,80 +3244,92 @@ func Test_isDiscoMaybeGeneve(t *testing.T) { copy(geneveEncapDiscoNonZeroGeneveVNILSB[packet.GeneveFixedHeaderLength:], nakedDisco) tests := []struct { - name string - msg []byte - wantIsDiscoMsg bool - wantIsGeneveEncap bool + name string + msg []byte + wantPacketLooksLikeType packetLooksLikeType + wantIsGeneveEncap bool }{ { - name: "naked disco", - msg: nakedDisco, - wantIsDiscoMsg: true, - wantIsGeneveEncap: false, + name: "STUN binding success response", + msg: stun.Response(stun.NewTxID(), netip.MustParseAddrPort("127.0.0.1:1")), + wantPacketLooksLikeType: packetLooksLikeSTUNBinding, + wantIsGeneveEncap: false, + }, + { + name: "naked disco", + msg: nakedDisco, + wantPacketLooksLikeType: packetLooksLikeDisco, + wantIsGeneveEncap: false, + }, + { + name: "geneve encap disco", + msg: geneveEncapDisco, + wantPacketLooksLikeType: packetLooksLikeDisco, + wantIsGeneveEncap: true, }, { - name: "geneve encap disco", - msg: geneveEncapDisco, - wantIsDiscoMsg: true, - wantIsGeneveEncap: true, + name: "geneve encap too short disco", + msg: geneveEncapDisco[:len(geneveEncapDisco)-key.DiscoPublicRawLen], + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "geneve encap disco nonzero geneve version", - msg: geneveEncapDiscoNonZeroGeneveVersion, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "geneve encap disco nonzero geneve version", + msg: geneveEncapDiscoNonZeroGeneveVersion, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "geneve encap disco nonzero geneve reserved bits", - msg: geneveEncapDiscoNonZeroGeneveReservedBits, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "geneve encap disco nonzero geneve reserved bits", + msg: geneveEncapDiscoNonZeroGeneveReservedBits, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "geneve encap disco nonzero geneve vni lsb", - msg: geneveEncapDiscoNonZeroGeneveVNILSB, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "geneve encap disco nonzero geneve vni lsb", + msg: geneveEncapDiscoNonZeroGeneveVNILSB, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "geneve encap wireguard", - msg: geneveEncapWireGuard, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "geneve encap wireguard", + msg: geneveEncapWireGuard, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: true, }, { - name: "naked WireGuard Initiation type", - msg: nakedWireGuardInitiation, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "naked WireGuard Initiation type", + msg: nakedWireGuardInitiation, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "naked WireGuard Response type", - msg: nakedWireGuardResponse, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "naked WireGuard Response type", + msg: nakedWireGuardResponse, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "naked WireGuard Cookie Reply type", - msg: nakedWireGuardCookieReply, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "naked WireGuard Cookie Reply type", + msg: nakedWireGuardCookieReply, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, { - name: "naked WireGuard Transport type", - msg: nakedWireGuardTransport, - wantIsDiscoMsg: false, - wantIsGeneveEncap: false, + name: "naked WireGuard Transport type", + msg: nakedWireGuardTransport, + wantPacketLooksLikeType: packetLooksLikeWireGuard, + wantIsGeneveEncap: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotIsDiscoMsg, gotIsGeneveEncap := isDiscoMaybeGeneve(tt.msg) - if gotIsDiscoMsg != tt.wantIsDiscoMsg { - t.Errorf("isDiscoMaybeGeneve() gotIsDiscoMsg = %v, want %v", gotIsDiscoMsg, tt.wantIsDiscoMsg) + gotPacketLooksLikeType, gotIsGeneveEncap := packetLooksLike(tt.msg) + if gotPacketLooksLikeType != tt.wantPacketLooksLikeType { + t.Errorf("packetLooksLike() gotPacketLooksLikeType = %v, want %v", gotPacketLooksLikeType, tt.wantPacketLooksLikeType) } if gotIsGeneveEncap != tt.wantIsGeneveEncap { - t.Errorf("isDiscoMaybeGeneve() gotIsGeneveEncap = %v, want %v", gotIsGeneveEncap, tt.wantIsGeneveEncap) + t.Errorf("packetLooksLike() gotIsGeneveEncap = %v, want %v", gotIsGeneveEncap, tt.wantIsGeneveEncap) } }) } @@ -3367,3 +3383,308 @@ func Test_virtualNetworkID(t *testing.T) { }) } } + +func Test_looksLikeInitiationMsg(t *testing.T) { + initMsg := make([]byte, device.MessageInitiationSize) + binary.BigEndian.PutUint32(initMsg, device.MessageInitiationType) + initMsgSizeTransportType := make([]byte, device.MessageInitiationSize) + binary.BigEndian.PutUint32(initMsgSizeTransportType, device.MessageTransportType) + tests := []struct { + name string + b []byte + want bool + }{ + { + name: "valid initiation", + b: initMsg, + want: true, + }, + { + name: "invalid message type field", + b: initMsgSizeTransportType, + want: false, + }, + { + name: "too small", + b: initMsg[:device.MessageInitiationSize-1], + want: false, + }, + { + name: "too big", + b: append(initMsg, 0), + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := looksLikeInitiationMsg(tt.b); got != tt.want { + t.Errorf("looksLikeInitiationMsg() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_nodeHasCap(t *testing.T) { + nodeAOnlyIPv4 := &tailcfg.Node{ + ID: 1, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("1.1.1.1/32"), + }, + } + nodeBOnlyIPv6 := nodeAOnlyIPv4.Clone() + nodeBOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::1/128") + + nodeCOnlyIPv4 := &tailcfg.Node{ + ID: 2, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("2.2.2.2/32"), + }, + } + nodeDOnlyIPv6 := nodeCOnlyIPv4.Clone() + nodeDOnlyIPv6.Addresses[0] = netip.MustParsePrefix("::2/128") + + tests := []struct { + name string + filt *filter.Filter + src tailcfg.NodeView + dst tailcfg.NodeView + cap tailcfg.PeerCapability + want bool + }{ + { + name: "match v4", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("1.1.1.1/32"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + src: nodeCOnlyIPv4.View(), + dst: nodeAOnlyIPv4.View(), + cap: tailcfg.PeerCapabilityRelayTarget, + want: true, + }, + { + name: "match v6", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("::2/128")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("::1/128"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + src: nodeDOnlyIPv6.View(), + dst: nodeBOnlyIPv6.View(), + cap: tailcfg.PeerCapabilityRelayTarget, + want: true, + }, + { + name: "no match CapMatch Dst", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("::2/128")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("::3/128"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + src: nodeDOnlyIPv6.View(), + dst: nodeBOnlyIPv6.View(), + cap: tailcfg.PeerCapabilityRelayTarget, + want: false, + }, + { + name: "no match peer cap", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("::2/128")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("::1/128"), + Cap: tailcfg.PeerCapabilityIngress, + }, + }, + }, + }, nil, nil, nil, nil, nil), + src: nodeDOnlyIPv6.View(), + dst: nodeBOnlyIPv6.View(), + cap: tailcfg.PeerCapabilityRelayTarget, + want: false, + }, + { + name: "nil src", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("1.1.1.1/32"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + src: tailcfg.NodeView{}, + dst: nodeAOnlyIPv4.View(), + cap: tailcfg.PeerCapabilityRelayTarget, + want: false, + }, + { + name: "nil dst", + filt: filter.New([]filtertype.Match{ + { + Srcs: []netip.Prefix{netip.MustParsePrefix("2.2.2.2/32")}, + Caps: []filtertype.CapMatch{ + { + Dst: netip.MustParsePrefix("1.1.1.1/32"), + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + src: nodeCOnlyIPv4.View(), + dst: tailcfg.NodeView{}, + cap: tailcfg.PeerCapabilityRelayTarget, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := nodeHasCap(tt.filt, tt.src, tt.dst, tt.cap); got != tt.want { + t.Errorf("nodeHasCap() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestConn_updateRelayServersSet(t *testing.T) { + peerNodeCandidateRelay := &tailcfg.Node{ + Cap: 121, + ID: 1, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("1.1.1.1/32"), + }, + HomeDERP: 1, + Key: key.NewNode().Public(), + DiscoKey: key.NewDisco().Public(), + } + + peerNodeNotCandidateRelayCapVer := &tailcfg.Node{ + Cap: 120, // intentionally lower to fail capVer check + ID: 1, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("1.1.1.1/32"), + }, + HomeDERP: 1, + Key: key.NewNode().Public(), + DiscoKey: key.NewDisco().Public(), + } + + selfNode := &tailcfg.Node{ + Cap: 120, // intentionally lower than capVerIsRelayCapable to verify self check + ID: 2, + Addresses: []netip.Prefix{ + netip.MustParsePrefix("2.2.2.2/32"), + }, + HomeDERP: 2, + Key: key.NewNode().Public(), + DiscoKey: key.NewDisco().Public(), + } + + tests := []struct { + name string + filt *filter.Filter + self tailcfg.NodeView + peers views.Slice[tailcfg.NodeView] + wantRelayServers set.Set[candidatePeerRelay] + }{ + { + name: "candidate relay server", + filt: filter.New([]filtertype.Match{ + { + Srcs: peerNodeCandidateRelay.Addresses, + Caps: []filtertype.CapMatch{ + { + Dst: selfNode.Addresses[0], + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfNode.View(), + peers: views.SliceOf([]tailcfg.NodeView{peerNodeCandidateRelay.View()}), + wantRelayServers: set.SetOf([]candidatePeerRelay{ + { + nodeKey: peerNodeCandidateRelay.Key, + discoKey: peerNodeCandidateRelay.DiscoKey, + derpHomeRegionID: 1, + }, + }), + }, + { + name: "self candidate relay server", + filt: filter.New([]filtertype.Match{ + { + Srcs: selfNode.Addresses, + Caps: []filtertype.CapMatch{ + { + Dst: selfNode.Addresses[0], + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfNode.View(), + peers: views.SliceOf([]tailcfg.NodeView{selfNode.View()}), + wantRelayServers: set.SetOf([]candidatePeerRelay{ + { + nodeKey: selfNode.Key, + discoKey: selfNode.DiscoKey, + derpHomeRegionID: 2, + }, + }), + }, + { + name: "no candidate relay server", + filt: filter.New([]filtertype.Match{ + { + Srcs: peerNodeNotCandidateRelayCapVer.Addresses, + Caps: []filtertype.CapMatch{ + { + Dst: selfNode.Addresses[0], + Cap: tailcfg.PeerCapabilityRelayTarget, + }, + }, + }, + }, nil, nil, nil, nil, nil), + self: selfNode.View(), + peers: views.SliceOf([]tailcfg.NodeView{peerNodeNotCandidateRelayCapVer.View()}), + wantRelayServers: make(set.Set[candidatePeerRelay]), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Conn{} + c.updateRelayServersSet(tt.filt, tt.self, tt.peers) + got := c.relayManager.getServers() + if !got.Equal(tt.wantRelayServers) { + t.Fatalf("got: %v != want: %v", got, tt.wantRelayServers) + } + if len(tt.wantRelayServers) > 0 != c.hasPeerRelayServers.Load() { + t.Fatalf("c.hasPeerRelayServers: %v != wantRelayServers: %v", c.hasPeerRelayServers.Load(), tt.wantRelayServers) + } + }) + } +} diff --git a/wgengine/magicsock/peermap.go b/wgengine/magicsock/peermap.go index e1c7db1f6c632..838905396002d 100644 --- a/wgengine/magicsock/peermap.go +++ b/wgengine/magicsock/peermap.go @@ -4,8 +4,6 @@ package magicsock import ( - "net/netip" - "tailscale.com/tailcfg" "tailscale.com/types/key" "tailscale.com/util/set" @@ -15,17 +13,17 @@ import ( // peer. type peerInfo struct { ep *endpoint // always non-nil. - // ipPorts is an inverted version of peerMap.byIPPort (below), so + // epAddrs is an inverted version of peerMap.byEpAddr (below), so // that when we're deleting this node, we can rapidly find out the - // keys that need deleting from peerMap.byIPPort without having to - // iterate over every IPPort known for any peer. - ipPorts set.Set[netip.AddrPort] + // keys that need deleting from peerMap.byEpAddr without having to + // iterate over every epAddr known for any peer. + epAddrs set.Set[epAddr] } func newPeerInfo(ep *endpoint) *peerInfo { return &peerInfo{ ep: ep, - ipPorts: set.Set[netip.AddrPort]{}, + epAddrs: set.Set[epAddr]{}, } } @@ -35,9 +33,21 @@ func newPeerInfo(ep *endpoint) *peerInfo { // It doesn't do any locking; all access must be done with Conn.mu held. type peerMap struct { byNodeKey map[key.NodePublic]*peerInfo - byIPPort map[netip.AddrPort]*peerInfo + byEpAddr map[epAddr]*peerInfo byNodeID map[tailcfg.NodeID]*peerInfo + // relayEpAddrByNodeKey ensures we only hold a single relay + // [epAddr] (vni.isSet()) for a given node key in byEpAddr, vs letting them + // grow unbounded. Relay [epAddr]'s are dynamically created by + // [relayManager] during path discovery, and are only useful to track in + // peerMap so long as they are the endpoint.bestAddr. [relayManager] handles + // all creation and initial probing responsibilities otherwise, and it does + // not depend on [peerMap]. + // + // Note: This doesn't address unbounded growth of non-relay epAddr's in + // byEpAddr. That issue is being tracked in http://go/corp/29422. + relayEpAddrByNodeKey map[key.NodePublic]epAddr + // nodesOfDisco contains the set of nodes that are using a // DiscoKey. Usually those sets will be just one node. nodesOfDisco map[key.DiscoPublic]set.Set[key.NodePublic] @@ -45,10 +55,11 @@ type peerMap struct { func newPeerMap() peerMap { return peerMap{ - byNodeKey: map[key.NodePublic]*peerInfo{}, - byIPPort: map[netip.AddrPort]*peerInfo{}, - byNodeID: map[tailcfg.NodeID]*peerInfo{}, - nodesOfDisco: map[key.DiscoPublic]set.Set[key.NodePublic]{}, + byNodeKey: map[key.NodePublic]*peerInfo{}, + byEpAddr: map[epAddr]*peerInfo{}, + byNodeID: map[tailcfg.NodeID]*peerInfo{}, + relayEpAddrByNodeKey: map[key.NodePublic]epAddr{}, + nodesOfDisco: map[key.DiscoPublic]set.Set[key.NodePublic]{}, } } @@ -88,10 +99,10 @@ func (m *peerMap) endpointForNodeID(nodeID tailcfg.NodeID) (ep *endpoint, ok boo return nil, false } -// endpointForIPPort returns the endpoint for the peer we -// believe to be at ipp, or nil if we don't know of any such peer. -func (m *peerMap) endpointForIPPort(ipp netip.AddrPort) (ep *endpoint, ok bool) { - if info, ok := m.byIPPort[ipp]; ok { +// endpointForEpAddr returns the endpoint for the peer we +// believe to be at addr, or nil if we don't know of any such peer. +func (m *peerMap) endpointForEpAddr(addr epAddr) (ep *endpoint, ok bool) { + if info, ok := m.byEpAddr[addr]; ok { return info.ep, true } return nil, false @@ -148,10 +159,10 @@ func (m *peerMap) upsertEndpoint(ep *endpoint, oldDiscoKey key.DiscoPublic) { // TODO(raggi,catzkorn): this could mean that if a "isWireguardOnly" // peer has, say, 192.168.0.2 and so does a tailscale peer, the // wireguard one will win. That may not be the outcome that we want - - // perhaps we should prefer bestAddr.AddrPort if it is set? + // perhaps we should prefer bestAddr.epAddr.ap if it is set? // see tailscale/tailscale#7994 for ipp := range ep.endpointState { - m.setNodeKeyForIPPort(ipp, ep.publicKey) + m.setNodeKeyForEpAddr(epAddr{ap: ipp}, ep.publicKey) } return } @@ -163,20 +174,31 @@ func (m *peerMap) upsertEndpoint(ep *endpoint, oldDiscoKey key.DiscoPublic) { discoSet.Add(ep.publicKey) } -// setNodeKeyForIPPort makes future peer lookups by ipp return the +// setNodeKeyForEpAddr makes future peer lookups by addr return the // same endpoint as a lookup by nk. // -// This should only be called with a fully verified mapping of ipp to +// This should only be called with a fully verified mapping of addr to // nk, because calling this function defines the endpoint we hand to -// WireGuard for packets received from ipp. -func (m *peerMap) setNodeKeyForIPPort(ipp netip.AddrPort, nk key.NodePublic) { - if pi := m.byIPPort[ipp]; pi != nil { - delete(pi.ipPorts, ipp) - delete(m.byIPPort, ipp) +// WireGuard for packets received from addr. +func (m *peerMap) setNodeKeyForEpAddr(addr epAddr, nk key.NodePublic) { + if pi := m.byEpAddr[addr]; pi != nil { + delete(pi.epAddrs, addr) + delete(m.byEpAddr, addr) + if addr.vni.isSet() { + delete(m.relayEpAddrByNodeKey, pi.ep.publicKey) + } } if pi, ok := m.byNodeKey[nk]; ok { - pi.ipPorts.Add(ipp) - m.byIPPort[ipp] = pi + if addr.vni.isSet() { + relay, ok := m.relayEpAddrByNodeKey[nk] + if ok { + delete(pi.epAddrs, relay) + delete(m.byEpAddr, relay) + } + m.relayEpAddrByNodeKey[nk] = addr + } + pi.epAddrs.Add(addr) + m.byEpAddr[addr] = pi } } @@ -203,7 +225,8 @@ func (m *peerMap) deleteEndpoint(ep *endpoint) { // Unexpected. But no logger plumbed here to log so. return } - for ip := range pi.ipPorts { - delete(m.byIPPort, ip) + for ip := range pi.epAddrs { + delete(m.byEpAddr, ip) } + delete(m.relayEpAddrByNodeKey, ep.publicKey) } diff --git a/wgengine/magicsock/peermap_test.go b/wgengine/magicsock/peermap_test.go new file mode 100644 index 0000000000000..52504272ff8e2 --- /dev/null +++ b/wgengine/magicsock/peermap_test.go @@ -0,0 +1,36 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package magicsock + +import ( + "net/netip" + "testing" + + "tailscale.com/types/key" +) + +func Test_peerMap_oneRelayEpAddrPerNK(t *testing.T) { + pm := newPeerMap() + nk := key.NewNode().Public() + ep := &endpoint{ + nodeID: 1, + publicKey: nk, + } + ed := &endpointDisco{key: key.NewDisco().Public()} + ep.disco.Store(ed) + pm.upsertEndpoint(ep, key.DiscoPublic{}) + vni := virtualNetworkID{} + vni.set(1) + relayEpAddrA := epAddr{ap: netip.MustParseAddrPort("127.0.0.1:1"), vni: vni} + relayEpAddrB := epAddr{ap: netip.MustParseAddrPort("127.0.0.1:2"), vni: vni} + pm.setNodeKeyForEpAddr(relayEpAddrA, nk) + pm.setNodeKeyForEpAddr(relayEpAddrB, nk) + if len(pm.byEpAddr) != 1 { + t.Fatalf("expected 1 epAddr in byEpAddr, got: %d", len(pm.byEpAddr)) + } + got := pm.relayEpAddrByNodeKey[nk] + if got != relayEpAddrB { + t.Fatalf("expected relay epAddr %v, got: %v", relayEpAddrB, got) + } +} diff --git a/wgengine/magicsock/rebinding_conn.go b/wgengine/magicsock/rebinding_conn.go index c27abbadc9ced..8b9ad4bb0bead 100644 --- a/wgengine/magicsock/rebinding_conn.go +++ b/wgengine/magicsock/rebinding_conn.go @@ -5,6 +5,7 @@ package magicsock import ( "errors" + "fmt" "net" "net/netip" "sync" @@ -13,6 +14,7 @@ import ( "golang.org/x/net/ipv6" "tailscale.com/net/netaddr" + "tailscale.com/net/packet" "tailscale.com/types/nettype" ) @@ -71,20 +73,36 @@ func (c *RebindingUDPConn) ReadFromUDPAddrPort(b []byte) (int, netip.AddrPort, e } // WriteBatchTo writes buffs to addr. -func (c *RebindingUDPConn) WriteBatchTo(buffs [][]byte, addr netip.AddrPort) error { +func (c *RebindingUDPConn) WriteBatchTo(buffs [][]byte, addr epAddr, offset int) error { + if offset != packet.GeneveFixedHeaderLength { + return fmt.Errorf("RebindingUDPConn.WriteBatchTo: [unexpected] offset (%d) != Geneve header length (%d)", offset, packet.GeneveFixedHeaderLength) + } for { pconn := *c.pconnAtomic.Load() b, ok := pconn.(batchingConn) if !ok { + vniIsSet := addr.vni.isSet() + var gh packet.GeneveHeader + if vniIsSet { + gh = packet.GeneveHeader{ + Protocol: packet.GeneveProtocolWireGuard, + VNI: addr.vni.get(), + } + } for _, buf := range buffs { - _, err := c.writeToUDPAddrPortWithInitPconn(pconn, buf, addr) + if vniIsSet { + gh.Encode(buf) + } else { + buf = buf[offset:] + } + _, err := c.writeToUDPAddrPortWithInitPconn(pconn, buf, addr.ap) if err != nil { return err } } return nil } - err := b.WriteBatchTo(buffs, addr) + err := b.WriteBatchTo(buffs, addr, offset) if err != nil { if pconn != c.currentConn() { continue diff --git a/wgengine/magicsock/relaymanager.go b/wgengine/magicsock/relaymanager.go index 0b19bb83fcc1a..ad8c5fc763adb 100644 --- a/wgengine/magicsock/relaymanager.go +++ b/wgengine/magicsock/relaymanager.go @@ -4,45 +4,57 @@ package magicsock import ( - "bytes" "context" - "encoding/json" - "io" - "net/http" + "errors" "net/netip" "sync" "time" "tailscale.com/disco" + "tailscale.com/net/stun" udprelay "tailscale.com/net/udprelay/endpoint" + "tailscale.com/tailcfg" + "tailscale.com/tstime" "tailscale.com/types/key" - "tailscale.com/util/httpm" "tailscale.com/util/set" ) -// relayManager manages allocation and handshaking of -// [tailscale.com/net/udprelay.Server] endpoints. The zero value is ready for -// use. +// relayManager manages allocation, handshaking, and initial probing (disco +// ping/pong) of [tailscale.com/net/udprelay.Server] endpoints. The zero value +// is ready for use. +// +// [relayManager] methods can be called by [Conn] and [endpoint] while their .mu +// mutexes are held. Therefore, in order to avoid deadlocks, [relayManager] must +// never attempt to acquire those mutexes synchronously from its runLoop(), +// including synchronous calls back towards [Conn] or [endpoint] methods that +// acquire them. type relayManager struct { initOnce sync.Once // =================================================================== // The following fields are owned by a single goroutine, runLoop(). - serversByAddrPort map[netip.AddrPort]key.DiscoPublic - serversByDisco map[key.DiscoPublic]netip.AddrPort - allocWorkByEndpoint map[*endpoint]*relayEndpointAllocWork - handshakeWorkByEndpointByServerDisco map[*endpoint]map[key.DiscoPublic]*relayHandshakeWork - handshakeWorkByServerDiscoVNI map[serverDiscoVNI]*relayHandshakeWork + serversByNodeKey map[key.NodePublic]candidatePeerRelay + allocWorkByCandidatePeerRelayByEndpoint map[*endpoint]map[candidatePeerRelay]*relayEndpointAllocWork + allocWorkByDiscoKeysByServerNodeKey map[key.NodePublic]map[key.SortedPairOfDiscoPublic]*relayEndpointAllocWork + handshakeWorkByServerDiscoByEndpoint map[*endpoint]map[key.DiscoPublic]*relayHandshakeWork + handshakeWorkByServerDiscoVNI map[serverDiscoVNI]*relayHandshakeWork + handshakeWorkAwaitingPong map[*relayHandshakeWork]addrPortVNI + addrPortVNIToHandshakeWork map[addrPortVNI]*relayHandshakeWork + handshakeGeneration uint32 + allocGeneration uint32 // =================================================================== // The following chan fields serve event inputs to a single goroutine, // runLoop(). - allocateHandshakeCh chan *endpoint + startDiscoveryCh chan endpointWithLastBest allocateWorkDoneCh chan relayEndpointAllocWorkDoneEvent handshakeWorkDoneCh chan relayEndpointHandshakeWorkDoneEvent cancelWorkCh chan *endpoint newServerEndpointCh chan newRelayServerEndpointEvent - rxChallengeCh chan relayHandshakeChallengeEvent + rxDiscoMsgCh chan relayDiscoMsgEvent + serversCh chan set.Set[candidatePeerRelay] + getServersCh chan chan set.Set[candidatePeerRelay] + derpHomeChangeCh chan derpHomeChangeEvent discoInfoMu sync.Mutex // guards the following field discoInfoByServerDisco map[key.DiscoPublic]*relayHandshakeDiscoInfo @@ -63,19 +75,19 @@ type serverDiscoVNI struct { // relayHandshakeWork serves to track in-progress relay handshake work for a // [udprelay.ServerEndpoint]. This structure is immutable once initialized. type relayHandshakeWork struct { - ep *endpoint - se udprelay.ServerEndpoint + wlb endpointWithLastBest + se udprelay.ServerEndpoint - // In order to not deadlock, runLoop() must select{} read doneCh when - // attempting to write into rxChallengeCh, and the handshake work goroutine - // must close(doneCh) before attempting to write to - // relayManager.handshakeWorkDoneCh. - rxChallengeCh chan relayHandshakeChallengeEvent - doneCh chan struct{} + // handshakeServerEndpoint() always writes to doneCh (len 1) when it + // returns. It may end up writing the same event afterward to + // relayManager.handshakeWorkDoneCh if runLoop() can receive it. runLoop() + // must select{} read on doneCh to prevent deadlock when attempting to write + // to rxDiscoMsgCh. + rxDiscoMsgCh chan relayDiscoMsgEvent + doneCh chan relayEndpointHandshakeWorkDoneEvent ctx context.Context cancel context.CancelFunc - wg *sync.WaitGroup } // newRelayServerEndpointEvent indicates a new [udprelay.ServerEndpoint] has @@ -83,30 +95,65 @@ type relayHandshakeWork struct { // [disco.CallMeMaybeVia] reception. This structure is immutable once // initialized. type newRelayServerEndpointEvent struct { - ep *endpoint + wlb endpointWithLastBest se udprelay.ServerEndpoint - server netip.AddrPort // zero value if learned via [disco.CallMeMaybeVia] + server candidatePeerRelay // zero value if learned via [disco.CallMeMaybeVia] } // relayEndpointAllocWorkDoneEvent indicates relay server endpoint allocation // work for an [*endpoint] has completed. This structure is immutable once // initialized. type relayEndpointAllocWorkDoneEvent struct { - work *relayEndpointAllocWork + work *relayEndpointAllocWork + allocated udprelay.ServerEndpoint // !allocated.ServerDisco.IsZero() if allocation succeeded } // relayEndpointHandshakeWorkDoneEvent indicates relay server endpoint handshake // work for an [*endpoint] has completed. This structure is immutable once // initialized. type relayEndpointHandshakeWorkDoneEvent struct { - work *relayHandshakeWork - answerSentTo netip.AddrPort // zero value if answer was not transmitted + work *relayHandshakeWork + pongReceivedFrom netip.AddrPort // or zero value if handshake or ping/pong did not complete + latency time.Duration // only relevant if pongReceivedFrom.IsValid() +} + +// hasActiveWorkRunLoop returns true if there is outstanding allocation or +// handshaking work for any endpoint, otherwise it returns false. +func (r *relayManager) hasActiveWorkRunLoop() bool { + return len(r.allocWorkByCandidatePeerRelayByEndpoint) > 0 || len(r.handshakeWorkByServerDiscoByEndpoint) > 0 } -// activeWorkRunLoop returns true if there is outstanding allocation or -// handshaking work, otherwise it returns false. -func (r *relayManager) activeWorkRunLoop() bool { - return len(r.allocWorkByEndpoint) > 0 || len(r.handshakeWorkByEndpointByServerDisco) > 0 +// hasActiveWorkForEndpointRunLoop returns true if there is outstanding +// allocation or handshaking work for the provided endpoint, otherwise it +// returns false. +func (r *relayManager) hasActiveWorkForEndpointRunLoop(ep *endpoint) bool { + _, handshakeWork := r.handshakeWorkByServerDiscoByEndpoint[ep] + _, allocWork := r.allocWorkByCandidatePeerRelayByEndpoint[ep] + return handshakeWork || allocWork +} + +// derpHomeChangeEvent represents a change in the DERP home region for the +// node identified by nodeKey. This structure is immutable once initialized. +type derpHomeChangeEvent struct { + nodeKey key.NodePublic + regionID uint16 +} + +// handleDERPHomeChange handles a DERP home change event for nodeKey and +// regionID. +func (r *relayManager) handleDERPHomeChange(nodeKey key.NodePublic, regionID uint16) { + relayManagerInputEvent(r, nil, &r.derpHomeChangeCh, derpHomeChangeEvent{ + nodeKey: nodeKey, + regionID: regionID, + }) +} + +func (r *relayManager) handleDERPHomeChangeRunLoop(event derpHomeChangeEvent) { + c, ok := r.serversByNodeKey[event.nodeKey] + if ok { + c.derpHomeRegionID = event.regionID + r.serversByNodeKey[event.nodeKey] = c + } } // runLoop is a form of event loop. It ensures exclusive access to most of @@ -118,83 +165,133 @@ func (r *relayManager) runLoop() { for { select { - case ep := <-r.allocateHandshakeCh: - r.stopWorkRunLoop(ep, stopHandshakeWorkOnlyKnownServers) - r.allocateAllServersRunLoop(ep) - if !r.activeWorkRunLoop() { + case startDiscovery := <-r.startDiscoveryCh: + if !r.hasActiveWorkForEndpointRunLoop(startDiscovery.ep) { + r.allocateAllServersRunLoop(startDiscovery) + } + if !r.hasActiveWorkRunLoop() { return } case done := <-r.allocateWorkDoneCh: - work, ok := r.allocWorkByEndpoint[done.work.ep] - if ok && work == done.work { - // Verify the work in the map is the same as the one that we're - // cleaning up. New events on r.allocateHandshakeCh can - // overwrite pre-existing keys. - delete(r.allocWorkByEndpoint, done.work.ep) - } - if !r.activeWorkRunLoop() { + r.handleAllocWorkDoneRunLoop(done) + if !r.hasActiveWorkRunLoop() { return } case ep := <-r.cancelWorkCh: - r.stopWorkRunLoop(ep, stopHandshakeWorkAllServers) - if !r.activeWorkRunLoop() { + r.stopWorkRunLoop(ep) + if !r.hasActiveWorkRunLoop() { return } case newServerEndpoint := <-r.newServerEndpointCh: r.handleNewServerEndpointRunLoop(newServerEndpoint) - if !r.activeWorkRunLoop() { + if !r.hasActiveWorkRunLoop() { return } case done := <-r.handshakeWorkDoneCh: r.handleHandshakeWorkDoneRunLoop(done) - if !r.activeWorkRunLoop() { + if !r.hasActiveWorkRunLoop() { return } - case challenge := <-r.rxChallengeCh: - r.handleRxChallengeRunLoop(challenge) - if !r.activeWorkRunLoop() { + case discoMsgEvent := <-r.rxDiscoMsgCh: + r.handleRxDiscoMsgRunLoop(discoMsgEvent) + if !r.hasActiveWorkRunLoop() { + return + } + case serversUpdate := <-r.serversCh: + r.handleServersUpdateRunLoop(serversUpdate) + if !r.hasActiveWorkRunLoop() { + return + } + case getServersCh := <-r.getServersCh: + r.handleGetServersRunLoop(getServersCh) + if !r.hasActiveWorkRunLoop() { + return + } + case derpHomeChange := <-r.derpHomeChangeCh: + r.handleDERPHomeChangeRunLoop(derpHomeChange) + if !r.hasActiveWorkRunLoop() { return } } } } -type relayHandshakeChallengeEvent struct { - challenge [32]byte - disco key.DiscoPublic - from netip.AddrPort - vni uint32 - at time.Time +func (r *relayManager) handleGetServersRunLoop(getServersCh chan set.Set[candidatePeerRelay]) { + servers := make(set.Set[candidatePeerRelay], len(r.serversByNodeKey)) + for _, v := range r.serversByNodeKey { + servers.Add(v) + } + getServersCh <- servers +} + +func (r *relayManager) getServers() set.Set[candidatePeerRelay] { + ch := make(chan set.Set[candidatePeerRelay]) + relayManagerInputEvent(r, nil, &r.getServersCh, ch) + return <-ch +} + +func (r *relayManager) handleServersUpdateRunLoop(update set.Set[candidatePeerRelay]) { + for _, v := range r.serversByNodeKey { + if !update.Contains(v) { + delete(r.serversByNodeKey, v.nodeKey) + } + } + for _, v := range update.Slice() { + r.serversByNodeKey[v.nodeKey] = v + } +} + +type relayDiscoMsgEvent struct { + conn *Conn // for access to [Conn] if there is no associated [relayHandshakeWork] + msg disco.Message + relayServerNodeKey key.NodePublic // nonzero if msg is a [*disco.AllocateUDPRelayEndpointResponse] + disco key.DiscoPublic + from netip.AddrPort + vni uint32 + at time.Time } // relayEndpointAllocWork serves to track in-progress relay endpoint allocation // for an [*endpoint]. This structure is immutable once initialized. type relayEndpointAllocWork struct { - // ep is the [*endpoint] associated with the work - ep *endpoint - // cancel() will signal all associated goroutines to return + wlb endpointWithLastBest + discoKeys key.SortedPairOfDiscoPublic + candidatePeerRelay candidatePeerRelay + + // allocateServerEndpoint() always writes to doneCh (len 1) when it + // returns. It may end up writing the same event afterward to + // [relayManager.allocateWorkDoneCh] if runLoop() can receive it. runLoop() + // must select{} read on doneCh to prevent deadlock when attempting to write + // to rxDiscoMsgCh. + rxDiscoMsgCh chan *disco.AllocateUDPRelayEndpointResponse + doneCh chan relayEndpointAllocWorkDoneEvent + + ctx context.Context cancel context.CancelFunc - // wg.Wait() will return once all associated goroutines have returned - wg *sync.WaitGroup } // init initializes [relayManager] if it is not already initialized. func (r *relayManager) init() { r.initOnce.Do(func() { r.discoInfoByServerDisco = make(map[key.DiscoPublic]*relayHandshakeDiscoInfo) - r.serversByDisco = make(map[key.DiscoPublic]netip.AddrPort) - r.serversByAddrPort = make(map[netip.AddrPort]key.DiscoPublic) - r.allocWorkByEndpoint = make(map[*endpoint]*relayEndpointAllocWork) - r.handshakeWorkByEndpointByServerDisco = make(map[*endpoint]map[key.DiscoPublic]*relayHandshakeWork) + r.serversByNodeKey = make(map[key.NodePublic]candidatePeerRelay) + r.allocWorkByCandidatePeerRelayByEndpoint = make(map[*endpoint]map[candidatePeerRelay]*relayEndpointAllocWork) + r.allocWorkByDiscoKeysByServerNodeKey = make(map[key.NodePublic]map[key.SortedPairOfDiscoPublic]*relayEndpointAllocWork) + r.handshakeWorkByServerDiscoByEndpoint = make(map[*endpoint]map[key.DiscoPublic]*relayHandshakeWork) r.handshakeWorkByServerDiscoVNI = make(map[serverDiscoVNI]*relayHandshakeWork) - r.allocateHandshakeCh = make(chan *endpoint) + r.handshakeWorkAwaitingPong = make(map[*relayHandshakeWork]addrPortVNI) + r.addrPortVNIToHandshakeWork = make(map[addrPortVNI]*relayHandshakeWork) + r.startDiscoveryCh = make(chan endpointWithLastBest) r.allocateWorkDoneCh = make(chan relayEndpointAllocWorkDoneEvent) r.handshakeWorkDoneCh = make(chan relayEndpointHandshakeWorkDoneEvent) r.cancelWorkCh = make(chan *endpoint) r.newServerEndpointCh = make(chan newRelayServerEndpointEvent) - r.rxChallengeCh = make(chan relayHandshakeChallengeEvent) + r.rxDiscoMsgCh = make(chan relayDiscoMsgEvent) + r.serversCh = make(chan set.Set[candidatePeerRelay]) + r.getServersCh = make(chan chan set.Set[candidatePeerRelay]) + r.derpHomeChangeCh = make(chan derpHomeChangeEvent) r.runLoopStoppedCh = make(chan struct{}, 1) - go r.runLoop() + r.runLoopStoppedCh <- struct{}{} }) } @@ -222,7 +319,7 @@ func (r *relayManager) ensureDiscoInfoFor(work *relayHandshakeWork) { di.di = &discoInfo{ discoKey: work.se.ServerDisco, discoShort: work.se.ServerDisco.ShortString(), - sharedKey: work.ep.c.discoPrivate.Shared(work.se.ServerDisco), + sharedKey: work.wlb.ep.c.discoPrivate.Shared(work.se.ServerDisco), } } } @@ -255,9 +352,10 @@ func (r *relayManager) discoInfo(serverDisco key.DiscoPublic) (_ *discoInfo, ok return nil, false } -func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, dm *disco.CallMeMaybeVia) { +func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, lastBest addrQuality, lastBestIsTrusted bool, dm *disco.CallMeMaybeVia) { se := udprelay.ServerEndpoint{ ServerDisco: dm.ServerDisco, + ClientDisco: dm.ClientDisco, LamportID: dm.LamportID, AddrPorts: dm.AddrPorts, VNI: dm.VNI, @@ -265,13 +363,35 @@ func (r *relayManager) handleCallMeMaybeVia(ep *endpoint, dm *disco.CallMeMaybeV se.BindLifetime.Duration = dm.BindLifetime se.SteadyStateLifetime.Duration = dm.SteadyStateLifetime relayManagerInputEvent(r, nil, &r.newServerEndpointCh, newRelayServerEndpointEvent{ - ep: ep, + wlb: endpointWithLastBest{ + ep: ep, + lastBest: lastBest, + lastBestIsTrusted: lastBestIsTrusted, + }, se: se, }) } -func (r *relayManager) handleBindUDPRelayEndpointChallenge(dm *disco.BindUDPRelayEndpointChallenge, di *discoInfo, src netip.AddrPort, vni uint32) { - relayManagerInputEvent(r, nil, &r.rxChallengeCh, relayHandshakeChallengeEvent{challenge: dm.Challenge, disco: di.discoKey, from: src, vni: vni, at: time.Now()}) +// handleRxDiscoMsg handles reception of disco messages that [relayManager] +// may be interested in. This includes all Geneve-encapsulated disco messages +// and [*disco.AllocateUDPRelayEndpointResponse]. If dm is a +// [*disco.AllocateUDPRelayEndpointResponse] then relayServerNodeKey must be +// nonzero. +func (r *relayManager) handleRxDiscoMsg(conn *Conn, dm disco.Message, relayServerNodeKey key.NodePublic, discoKey key.DiscoPublic, src epAddr) { + relayManagerInputEvent(r, nil, &r.rxDiscoMsgCh, relayDiscoMsgEvent{ + conn: conn, + msg: dm, + relayServerNodeKey: relayServerNodeKey, + disco: discoKey, + from: src.ap, + vni: src.vni.get(), + at: time.Now(), + }) +} + +// handleRelayServersSet handles an update of the complete relay server set. +func (r *relayManager) handleRelayServersSet(servers set.Set[candidatePeerRelay]) { + relayManagerInputEvent(r, nil, &r.serversCh, servers) } // relayManagerInputEvent initializes [relayManager] if necessary, starts @@ -301,11 +421,23 @@ func relayManagerInputEvent[T any](r *relayManager, ctx context.Context, eventCh } } -// allocateAndHandshakeAllServers kicks off allocation and handshaking of relay -// endpoints for 'ep' on all known relay servers, canceling any existing -// in-progress work. -func (r *relayManager) allocateAndHandshakeAllServers(ep *endpoint) { - relayManagerInputEvent(r, nil, &r.allocateHandshakeCh, ep) +// endpointWithLastBest represents an [*endpoint], its last bestAddr, and if +// the last bestAddr was trusted (see endpoint.trustBestAddrUntil) at the time +// of init. This structure is immutable once initialized. +type endpointWithLastBest struct { + ep *endpoint + lastBest addrQuality + lastBestIsTrusted bool +} + +// startUDPRelayPathDiscoveryFor starts UDP relay path discovery for ep on all +// known relay servers if ep has no in-progress work. +func (r *relayManager) startUDPRelayPathDiscoveryFor(ep *endpoint, lastBest addrQuality, lastBestIsTrusted bool) { + relayManagerInputEvent(r, nil, &r.startDiscoveryCh, endpointWithLastBest{ + ep: ep, + lastBest: lastBest, + lastBestIsTrusted: lastBestIsTrusted, + }) } // stopWork stops all outstanding allocation & handshaking work for 'ep'. @@ -313,56 +445,161 @@ func (r *relayManager) stopWork(ep *endpoint) { relayManagerInputEvent(r, nil, &r.cancelWorkCh, ep) } -// stopHandshakeWorkFilter represents filters for handshake work cancellation -type stopHandshakeWorkFilter bool - -const ( - stopHandshakeWorkAllServers stopHandshakeWorkFilter = false - stopHandshakeWorkOnlyKnownServers = true -) - // stopWorkRunLoop cancels & clears outstanding allocation and handshaking -// work for 'ep'. Handshake work cancellation is subject to the filter supplied -// in 'f'. -func (r *relayManager) stopWorkRunLoop(ep *endpoint, f stopHandshakeWorkFilter) { - allocWork, ok := r.allocWorkByEndpoint[ep] +// work for 'ep'. +func (r *relayManager) stopWorkRunLoop(ep *endpoint) { + byDiscoKeys, ok := r.allocWorkByCandidatePeerRelayByEndpoint[ep] if ok { - allocWork.cancel() - allocWork.wg.Wait() - delete(r.allocWorkByEndpoint, ep) + for _, work := range byDiscoKeys { + work.cancel() + done := <-work.doneCh + r.handleAllocWorkDoneRunLoop(done) + } } - byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[ep] + byServerDisco, ok := r.handshakeWorkByServerDiscoByEndpoint[ep] if ok { - for disco, handshakeWork := range byServerDisco { - _, knownServer := r.serversByDisco[disco] - if knownServer || f == stopHandshakeWorkAllServers { - handshakeWork.cancel() - handshakeWork.wg.Wait() - delete(byServerDisco, disco) - delete(r.handshakeWorkByServerDiscoVNI, serverDiscoVNI{handshakeWork.se.ServerDisco, handshakeWork.se.VNI}) - } + for _, handshakeWork := range byServerDisco { + handshakeWork.cancel() + done := <-handshakeWork.doneCh + r.handleHandshakeWorkDoneRunLoop(done) + } + } +} + +// addrPortVNI represents a combined netip.AddrPort and Geneve header virtual +// network identifier. +type addrPortVNI struct { + addrPort netip.AddrPort + vni uint32 +} + +func (r *relayManager) handleRxDiscoMsgRunLoop(event relayDiscoMsgEvent) { + var ( + work *relayHandshakeWork + ok bool + ) + apv := addrPortVNI{event.from, event.vni} + switch msg := event.msg.(type) { + case *disco.AllocateUDPRelayEndpointResponse: + sorted := key.NewSortedPairOfDiscoPublic(msg.ClientDisco[0], msg.ClientDisco[1]) + byDiscoKeys, ok := r.allocWorkByDiscoKeysByServerNodeKey[event.relayServerNodeKey] + if !ok { + // No outstanding work tied to this relay sever, discard. + return + } + allocWork, ok := byDiscoKeys[sorted] + if !ok { + // No outstanding work tied to these disco keys, discard. + return + } + select { + case done := <-allocWork.doneCh: + // allocateServerEndpoint returned, clean up its state + r.handleAllocWorkDoneRunLoop(done) + return + case allocWork.rxDiscoMsgCh <- msg: + return } - if len(byServerDisco) == 0 { - delete(r.handshakeWorkByEndpointByServerDisco, ep) + case *disco.BindUDPRelayEndpointChallenge: + work, ok = r.handshakeWorkByServerDiscoVNI[serverDiscoVNI{event.disco, event.vni}] + if !ok { + // No outstanding work tied to this challenge, discard. + return + } + _, ok = r.handshakeWorkAwaitingPong[work] + if ok { + // We've seen a challenge for this relay endpoint previously, + // discard. Servers only respond to the first src ip:port they see + // binds from. + return } + _, ok = r.addrPortVNIToHandshakeWork[apv] + if ok { + // There is existing work for the same [addrPortVNI] that is not + // 'work'. If both instances happen to be on the same server we + // could attempt to resolve event order using LamportID. For now + // just leave both work instances alone and take no action other + // than to discard this challenge msg. + return + } + // Update state so that future ping/pong will route to 'work'. + r.handshakeWorkAwaitingPong[work] = apv + r.addrPortVNIToHandshakeWork[apv] = work + case *disco.Ping: + // Always TX a pong. We might not have any associated work if ping + // reception raced with our call to [endpoint.udpRelayEndpointReady()], so + // err on the side of enabling the remote side to use this path. + // + // Conn.handlePingLocked() makes efforts to suppress duplicate pongs + // where the same ping can be received both via raw socket and UDP + // socket on Linux. We make no such efforts here as the raw socket BPF + // program does not support Geneve-encapsulated disco, and is also + // disabled by default. + vni := virtualNetworkID{} + vni.set(event.vni) + go event.conn.sendDiscoMessage(epAddr{ap: event.from, vni: vni}, key.NodePublic{}, event.disco, &disco.Pong{ + TxID: msg.TxID, + Src: event.from, + }, discoVerboseLog) + + work, ok = r.addrPortVNIToHandshakeWork[apv] + if !ok { + // No outstanding work tied to this [addrPortVNI], return early. + return + } + case *disco.Pong: + work, ok = r.addrPortVNIToHandshakeWork[apv] + if !ok { + // No outstanding work tied to this [addrPortVNI], discard. + return + } + default: + // Unexpected message type, discard. + return + } + select { + case done := <-work.doneCh: + // handshakeServerEndpoint() returned, clean up its state. + r.handleHandshakeWorkDoneRunLoop(done) + return + case work.rxDiscoMsgCh <- event: + return } } -func (r *relayManager) handleRxChallengeRunLoop(challenge relayHandshakeChallengeEvent) { - work, ok := r.handshakeWorkByServerDiscoVNI[serverDiscoVNI{challenge.disco, challenge.vni}] +func (r *relayManager) handleAllocWorkDoneRunLoop(done relayEndpointAllocWorkDoneEvent) { + byCandidatePeerRelay, ok := r.allocWorkByCandidatePeerRelayByEndpoint[done.work.wlb.ep] if !ok { return } - select { - case <-work.doneCh: + work, ok := byCandidatePeerRelay[done.work.candidatePeerRelay] + if !ok || work != done.work { return - case work.rxChallengeCh <- challenge: + } + delete(byCandidatePeerRelay, done.work.candidatePeerRelay) + if len(byCandidatePeerRelay) == 0 { + delete(r.allocWorkByCandidatePeerRelayByEndpoint, done.work.wlb.ep) + } + byDiscoKeys, ok := r.allocWorkByDiscoKeysByServerNodeKey[done.work.candidatePeerRelay.nodeKey] + if !ok { + // unexpected return } + delete(byDiscoKeys, done.work.discoKeys) + if len(byDiscoKeys) == 0 { + delete(r.allocWorkByDiscoKeysByServerNodeKey, done.work.candidatePeerRelay.nodeKey) + } + if !done.allocated.ServerDisco.IsZero() { + r.handleNewServerEndpointRunLoop(newRelayServerEndpointEvent{ + wlb: done.work.wlb, + se: done.allocated, + server: done.work.candidatePeerRelay, + }) + } } func (r *relayManager) handleHandshakeWorkDoneRunLoop(done relayEndpointHandshakeWorkDoneEvent) { - byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[done.work.ep] + byServerDisco, ok := r.handshakeWorkByServerDiscoByEndpoint[done.work.wlb.ep] if !ok { return } @@ -372,17 +609,31 @@ func (r *relayManager) handleHandshakeWorkDoneRunLoop(done relayEndpointHandshak } delete(byServerDisco, done.work.se.ServerDisco) if len(byServerDisco) == 0 { - delete(r.handshakeWorkByEndpointByServerDisco, done.work.ep) + delete(r.handshakeWorkByServerDiscoByEndpoint, done.work.wlb.ep) } delete(r.handshakeWorkByServerDiscoVNI, serverDiscoVNI{done.work.se.ServerDisco, done.work.se.VNI}) - if !done.answerSentTo.IsValid() { - // The handshake timed out. + apv, ok := r.handshakeWorkAwaitingPong[work] + if ok { + delete(r.handshakeWorkAwaitingPong, work) + delete(r.addrPortVNIToHandshakeWork, apv) + } + if !done.pongReceivedFrom.IsValid() { + // The handshake or ping/pong probing timed out. return } - // We received a challenge from and transmitted an answer towards the relay - // server. - // TODO(jwhited): Make the associated [*endpoint] aware of this - // [tailscale.com/net/udprelay.ServerEndpoint]. + // This relay endpoint is functional. + vni := virtualNetworkID{} + vni.set(done.work.se.VNI) + addr := epAddr{ap: done.pongReceivedFrom, vni: vni} + // ep.udpRelayEndpointReady() must be called in a new goroutine to prevent + // deadlocks as it acquires [endpoint] & [Conn] mutexes. See [relayManager] + // docs for details. + go done.work.wlb.ep.udpRelayEndpointReady(addrQuality{ + epAddr: addr, + relayServerDisco: done.work.se.ServerDisco, + latency: done.latency, + wireMTU: pingSizeToPktLen(0, addr), + }) } func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelayServerEndpointEvent) { @@ -398,23 +649,14 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay return } - // The existing work is no longer valid, clean it up. Be sure to lookup - // by the existing work's [*endpoint], not the incoming "new" work as - // they are not necessarily matching. + // The existing work is no longer valid, clean it up. existingWork.cancel() - existingWork.wg.Wait() - delete(r.handshakeWorkByServerDiscoVNI, sdv) - byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[existingWork.ep] - if ok { - delete(byServerDisco, sdv.serverDisco) - if len(byServerDisco) == 0 { - delete(r.handshakeWorkByEndpointByServerDisco, existingWork.ep) - } - } + done := <-existingWork.doneCh + r.handleHandshakeWorkDoneRunLoop(done) } // Check for duplicate work by [*endpoint] + server disco. - byServerDisco, ok := r.handshakeWorkByEndpointByServerDisco[newServerEndpoint.ep] + byServerDisco, ok := r.handshakeWorkByServerDiscoByEndpoint[newServerEndpoint.wlb.ep] if ok { existingWork, ok := byServerDisco[newServerEndpoint.se.ServerDisco] if ok { @@ -425,86 +667,129 @@ func (r *relayManager) handleNewServerEndpointRunLoop(newServerEndpoint newRelay } // Cancel existing handshake that has a lower lamport ID. existingWork.cancel() - existingWork.wg.Wait() - delete(r.handshakeWorkByServerDiscoVNI, sdv) - delete(byServerDisco, sdv.serverDisco) - if len(byServerDisco) == 0 { - delete(r.handshakeWorkByEndpointByServerDisco, existingWork.ep) - } + done := <-existingWork.doneCh + r.handleHandshakeWorkDoneRunLoop(done) } } // We're now reasonably sure we're dealing with the latest // [udprelay.ServerEndpoint] from a server event order perspective - // (LamportID). Update server disco key tracking if appropriate. - if newServerEndpoint.server.IsValid() { - serverDisco, ok := r.serversByAddrPort[newServerEndpoint.server] - if !ok { - // Allocation raced with an update to our known servers set. This - // server is no longer known. Return early. - return - } - if serverDisco.Compare(newServerEndpoint.se.ServerDisco) != 0 { - // The server's disco key has either changed, or simply become - // known for the first time. In the former case we end up detaching - // any in-progress handshake work from a "known" relay server. - // Practically speaking we expect the detached work to fail - // if the server key did in fact change (server restart) while we - // were attempting to handshake with it. It is possible, though - // unlikely, for a server addr:port to effectively move between - // nodes. Either way, there is no harm in detaching existing work, - // and we explicitly let that happen for the rare case the detached - // handshake would complete and remain functional. - delete(r.serversByDisco, serverDisco) - delete(r.serversByAddrPort, newServerEndpoint.server) - r.serversByDisco[serverDisco] = newServerEndpoint.server - r.serversByAddrPort[newServerEndpoint.server] = serverDisco - } + // (LamportID). + + if newServerEndpoint.server.isValid() { + // Send a [disco.CallMeMaybeVia] to the remote peer if we allocated this + // endpoint, regardless of if we start a handshake below. + go r.sendCallMeMaybeVia(newServerEndpoint.wlb.ep, newServerEndpoint.se) + } + + lastBestMatchingServer := newServerEndpoint.se.ServerDisco.Compare(newServerEndpoint.wlb.lastBest.relayServerDisco) == 0 + if lastBestMatchingServer && newServerEndpoint.wlb.lastBestIsTrusted { + // This relay server endpoint is the same as [endpoint]'s bestAddr at + // the time UDP relay path discovery was started, and it was also a + // trusted path (see endpoint.trustBestAddrUntil), so return early. + // + // If we were to start a new handshake, there is a chance that we + // cause [endpoint] to blackhole some packets on its bestAddr if we end + // up shifting to a new address family or src, e.g. IPv4 to IPv6, due to + // the window of time between the handshake completing, and our call to + // udpRelayEndpointReady(). The relay server can only forward packets + // from us on a single [epAddr]. + return } + // TODO(jwhited): if lastBest is untrusted, consider some strategies + // to reduce the chance we blackhole if it were to transition to + // trusted during/before the new handshake: + // 1. Start by attempting a handshake with only lastBest.epAddr. If + // that fails then try the remaining [epAddr]s. + // 2. Signal bestAddr trust transitions between [endpoint] and + // [relayManager] in order to prevent a handshake from starting + // and/or stop one that is running. + // We're ready to start a new handshake. ctx, cancel := context.WithCancel(context.Background()) - wg := &sync.WaitGroup{} work := &relayHandshakeWork{ - ep: newServerEndpoint.ep, - se: newServerEndpoint.se, - doneCh: make(chan struct{}), - ctx: ctx, - cancel: cancel, - wg: wg, + wlb: newServerEndpoint.wlb, + se: newServerEndpoint.se, + rxDiscoMsgCh: make(chan relayDiscoMsgEvent), + doneCh: make(chan relayEndpointHandshakeWorkDoneEvent, 1), + ctx: ctx, + cancel: cancel, } if byServerDisco == nil { byServerDisco = make(map[key.DiscoPublic]*relayHandshakeWork) - r.handshakeWorkByEndpointByServerDisco[newServerEndpoint.ep] = byServerDisco + r.handshakeWorkByServerDiscoByEndpoint[newServerEndpoint.wlb.ep] = byServerDisco } byServerDisco[newServerEndpoint.se.ServerDisco] = work r.handshakeWorkByServerDiscoVNI[sdv] = work - wg.Add(1) - go r.handshakeServerEndpoint(work) + r.handshakeGeneration++ + if r.handshakeGeneration == 0 { // generation must be nonzero + r.handshakeGeneration++ + } + + go r.handshakeServerEndpoint(work, r.handshakeGeneration) } -func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { - defer work.wg.Done() +// sendCallMeMaybeVia sends a [disco.CallMeMaybeVia] to ep over DERP. It must be +// called as part of a goroutine independent from runLoop(), for 2 reasons: +// 1. it acquires ep.mu (refer to [relayManager] docs for reasoning) +// 2. it makes a networking syscall, which can introduce unwanted backpressure +func (r *relayManager) sendCallMeMaybeVia(ep *endpoint, se udprelay.ServerEndpoint) { + ep.mu.Lock() + derpAddr := ep.derpAddr + ep.mu.Unlock() + epDisco := ep.disco.Load() + if epDisco == nil || !derpAddr.IsValid() { + return + } + callMeMaybeVia := &disco.CallMeMaybeVia{ + UDPRelayEndpoint: disco.UDPRelayEndpoint{ + ServerDisco: se.ServerDisco, + ClientDisco: se.ClientDisco, + LamportID: se.LamportID, + VNI: se.VNI, + BindLifetime: se.BindLifetime.Duration, + SteadyStateLifetime: se.SteadyStateLifetime.Duration, + AddrPorts: se.AddrPorts, + }, + } + ep.c.sendDiscoMessage(epAddr{ap: derpAddr}, ep.publicKey, epDisco.key, callMeMaybeVia, discoVerboseLog) +} +func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork, generation uint32) { done := relayEndpointHandshakeWorkDoneEvent{work: work} r.ensureDiscoInfoFor(work) defer func() { r.derefDiscoInfoFor(work) - close(work.doneCh) + work.doneCh <- done relayManagerInputEvent(r, work.ctx, &r.handshakeWorkDoneCh, done) work.cancel() }() + ep := work.wlb.ep + epDisco := ep.disco.Load() + if epDisco == nil { + return + } + + common := disco.BindUDPRelayEndpointCommon{ + VNI: work.se.VNI, + Generation: generation, + RemoteKey: epDisco.key, + } + sentBindAny := false - bind := &disco.BindUDPRelayEndpoint{} + bind := &disco.BindUDPRelayEndpoint{ + BindUDPRelayEndpointCommon: common, + } vni := virtualNetworkID{} vni.set(work.se.VNI) for _, addrPort := range work.se.AddrPorts { if addrPort.IsValid() { sentBindAny = true - go work.ep.c.sendDiscoMessage(addrPort, vni, key.NodePublic{}, work.se.ServerDisco, bind, discoLog) + go ep.c.sendDiscoMessage(epAddr{ap: addrPort, vni: vni}, key.NodePublic{}, work.se.ServerDisco, bind, discoVerboseLog) } } if !sentBindAny { @@ -518,107 +803,206 @@ func (r *relayManager) handshakeServerEndpoint(work *relayHandshakeWork) { timer := time.NewTimer(min(work.se.BindLifetime.Duration, maxHandshakeLifetime)) defer timer.Stop() - // Wait for cancellation, a challenge to be rx'd, or handshake lifetime to - // expire. Our initial implementation values simplicity over other aspects, - // e.g. it is not resilient to any packet loss. - // - // We may want to eventually consider [disc.BindUDPRelayEndpoint] - // retransmission lacking challenge rx, and - // [disco.BindUDPRelayEndpointAnswer] duplication in front of - // [disco.Ping] until [disco.Ping] or [disco.Pong] is received. - select { - case <-work.ctx.Done(): - return - case challenge := <-work.rxChallengeCh: - answer := &disco.BindUDPRelayEndpointAnswer{Answer: challenge.challenge} - done.answerSentTo = challenge.from - // Send answer back to relay server. Typically sendDiscoMessage() calls - // are invoked via a new goroutine in attempt to limit crypto+syscall - // time contributing to system backpressure, and to fire roundtrip - // latency-relevant messages as closely together as possible. We - // intentionally don't do that here, because: - // 1. The primary backpressure concern is around the work.rxChallengeCh - // writer on the [Conn] packet rx path, who is already unblocked - // since we read from the channel. Relay servers only ever tx one - // challenge per rx'd bind message for a given (the first seen) src. - // 2. runLoop() may be waiting for this 'work' to complete if - // explicitly canceled for some reason elsewhere, but this is - // typically only around [*endpoint] and/or [Conn] shutdown. - // 3. It complicates the defer()'d [*discoInfo] deref and 'work' - // completion event order. sendDiscoMessage() assumes the related - // [*discoInfo] is still available. We also don't want the - // [*endpoint] to send a [disco.Ping] before the - // [disco.BindUDPRelayEndpointAnswer] has gone out, otherwise the - // remote side will never see the ping, delaying/preventing the - // [udprelay.ServerEndpoint] from becoming fully operational. - // 4. This is a singular tx with no roundtrip latency measurements - // involved. - work.ep.c.sendDiscoMessage(challenge.from, vni, key.NodePublic{}, work.se.ServerDisco, answer, discoLog) - return - case <-timer.C: - // The handshake timed out. - return - } -} + // Limit the number of pings we will transmit. Inbound pings trigger + // outbound pings, so we want to be a little defensive. + const limitPings = 10 -func (r *relayManager) allocateAllServersRunLoop(ep *endpoint) { - if len(r.serversByAddrPort) == 0 { - return - } - ctx, cancel := context.WithCancel(context.Background()) - started := &relayEndpointAllocWork{ep: ep, cancel: cancel, wg: &sync.WaitGroup{}} - for k := range r.serversByAddrPort { - started.wg.Add(1) - go r.allocateSingleServer(ctx, started.wg, k, ep) - } - r.allocWorkByEndpoint[ep] = started - go func() { - started.wg.Wait() - started.cancel() - relayManagerInputEvent(r, ctx, &r.allocateWorkDoneCh, relayEndpointAllocWorkDoneEvent{work: started}) - }() -} + var ( + handshakeState disco.BindUDPRelayHandshakeState = disco.BindUDPRelayHandshakeStateBindSent + sentPingAt = make(map[stun.TxID]time.Time) + ) -func (r *relayManager) allocateSingleServer(ctx context.Context, wg *sync.WaitGroup, server netip.AddrPort, ep *endpoint) { - // TODO(jwhited): introduce client metrics counters for notable failures - defer wg.Done() - var b bytes.Buffer - remoteDisco := ep.disco.Load() - if remoteDisco == nil { - return - } - type allocateRelayEndpointReq struct { - DiscoKeys []key.DiscoPublic + txPing := func(to netip.AddrPort, withAnswer *[32]byte) { + if len(sentPingAt) == limitPings { + return + } + txid := stun.NewTxID() + sentPingAt[txid] = time.Now() + ping := &disco.Ping{ + TxID: txid, + NodeKey: ep.c.publicKeyAtomic.Load(), + } + go func() { + if withAnswer != nil { + answer := &disco.BindUDPRelayEndpointAnswer{BindUDPRelayEndpointCommon: common} + answer.Challenge = *withAnswer + ep.c.sendDiscoMessage(epAddr{ap: to, vni: vni}, key.NodePublic{}, work.se.ServerDisco, answer, discoVerboseLog) + } + ep.c.sendDiscoMessage(epAddr{ap: to, vni: vni}, key.NodePublic{}, epDisco.key, ping, discoVerboseLog) + }() } - a := &allocateRelayEndpointReq{ - DiscoKeys: []key.DiscoPublic{ep.c.discoPublic, remoteDisco.key}, + + validateVNIAndRemoteKey := func(common disco.BindUDPRelayEndpointCommon) error { + if common.VNI != work.se.VNI { + return errors.New("mismatching VNI") + } + if common.RemoteKey.Compare(epDisco.key) != 0 { + return errors.New("mismatching RemoteKey") + } + return nil } - err := json.NewEncoder(&b).Encode(a) - if err != nil { - return + + // This for{select{}} is responsible for handshaking and tx'ing ping/pong + // when the handshake is complete. + for { + select { + case <-work.ctx.Done(): + return + case msgEvent := <-work.rxDiscoMsgCh: + switch msg := msgEvent.msg.(type) { + case *disco.BindUDPRelayEndpointChallenge: + err := validateVNIAndRemoteKey(msg.BindUDPRelayEndpointCommon) + if err != nil { + continue + } + if handshakeState >= disco.BindUDPRelayHandshakeStateAnswerSent { + continue + } + txPing(msgEvent.from, &msg.Challenge) + handshakeState = disco.BindUDPRelayHandshakeStateAnswerSent + case *disco.Ping: + if handshakeState < disco.BindUDPRelayHandshakeStateAnswerSent { + continue + } + // An inbound ping from the remote peer indicates we completed a + // handshake with the relay server (our answer msg was + // received). Chances are our ping was dropped before the remote + // handshake was complete. We need to rx a pong to determine + // latency, so send another ping. Since the handshake is + // complete we do not need to send an answer in front of this + // one. + // + // We don't need to TX a pong, that was already handled for us + // in handleRxDiscoMsgRunLoop(). + txPing(msgEvent.from, nil) + case *disco.Pong: + at, ok := sentPingAt[msg.TxID] + if !ok { + continue + } + // The relay server endpoint is functional! Record the + // round-trip latency and return. + done.pongReceivedFrom = msgEvent.from + done.latency = time.Since(at) + return + default: + // unexpected message type, silently discard + continue + } + case <-timer.C: + // The handshake timed out. + return + } } - const reqTimeout = time.Second * 10 - reqCtx, cancel := context.WithTimeout(ctx, reqTimeout) - defer cancel() - req, err := http.NewRequestWithContext(reqCtx, httpm.POST, "http://"+server.String()+"/relay/endpoint", &b) - if err != nil { - return +} + +const allocateUDPRelayEndpointRequestTimeout = time.Second * 10 + +func (r *relayManager) allocateServerEndpoint(work *relayEndpointAllocWork, generation uint32) { + done := relayEndpointAllocWorkDoneEvent{work: work} + + defer func() { + work.doneCh <- done + relayManagerInputEvent(r, work.ctx, &r.allocateWorkDoneCh, done) + work.cancel() + }() + + dm := &disco.AllocateUDPRelayEndpointRequest{ + ClientDisco: work.discoKeys.Get(), + Generation: generation, + } + + sendAllocReq := func() { + work.wlb.ep.c.sendDiscoAllocateUDPRelayEndpointRequest( + epAddr{ + ap: netip.AddrPortFrom(tailcfg.DerpMagicIPAddr, work.candidatePeerRelay.derpHomeRegionID), + }, + work.candidatePeerRelay.nodeKey, + work.candidatePeerRelay.discoKey, + dm, + discoVerboseLog, + ) + } + go sendAllocReq() + + returnAfterTimer := time.NewTimer(allocateUDPRelayEndpointRequestTimeout) + defer returnAfterTimer.Stop() + // While connections to DERP are over TCP, they can be lossy on the DERP + // server when data moves between the two independent streams. Also, the + // peer relay server may not be "ready" (see [tailscale.com/net/udprelay.ErrServerNotReady]). + // So, start a timer to retry once if needed. + retryAfterTimer := time.NewTimer(udprelay.ServerRetryAfter) + defer retryAfterTimer.Stop() + + for { + select { + case <-work.ctx.Done(): + return + case <-returnAfterTimer.C: + return + case <-retryAfterTimer.C: + go sendAllocReq() + case resp := <-work.rxDiscoMsgCh: + if resp.Generation != generation || + !work.discoKeys.Equal(key.NewSortedPairOfDiscoPublic(resp.ClientDisco[0], resp.ClientDisco[1])) { + continue + } + done.allocated = udprelay.ServerEndpoint{ + ServerDisco: resp.ServerDisco, + ClientDisco: resp.ClientDisco, + LamportID: resp.LamportID, + AddrPorts: resp.AddrPorts, + VNI: resp.VNI, + BindLifetime: tstime.GoDuration{Duration: resp.BindLifetime}, + SteadyStateLifetime: tstime.GoDuration{Duration: resp.SteadyStateLifetime}, + } + return + } } - resp, err := http.DefaultClient.Do(req) - if err != nil { +} + +func (r *relayManager) allocateAllServersRunLoop(wlb endpointWithLastBest) { + if len(r.serversByNodeKey) == 0 { return } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { + remoteDisco := wlb.ep.disco.Load() + if remoteDisco == nil { return } - var se udprelay.ServerEndpoint - err = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&se) - if err != nil { - return + discoKeys := key.NewSortedPairOfDiscoPublic(wlb.ep.c.discoPublic, remoteDisco.key) + for _, v := range r.serversByNodeKey { + byDiscoKeys, ok := r.allocWorkByDiscoKeysByServerNodeKey[v.nodeKey] + if !ok { + byDiscoKeys = make(map[key.SortedPairOfDiscoPublic]*relayEndpointAllocWork) + r.allocWorkByDiscoKeysByServerNodeKey[v.nodeKey] = byDiscoKeys + } else { + _, ok = byDiscoKeys[discoKeys] + if ok { + // If there is an existing key, a disco key collision may have + // occurred across peers ([*endpoint]). Do not overwrite the + // existing work, let it finish. + wlb.ep.c.logf("[unexpected] magicsock: relayManager: suspected disco key collision on server %v for keys: %v", v.nodeKey.ShortString(), discoKeys) + continue + } + } + ctx, cancel := context.WithCancel(context.Background()) + started := &relayEndpointAllocWork{ + wlb: wlb, + discoKeys: discoKeys, + candidatePeerRelay: v, + rxDiscoMsgCh: make(chan *disco.AllocateUDPRelayEndpointResponse), + doneCh: make(chan relayEndpointAllocWorkDoneEvent, 1), + ctx: ctx, + cancel: cancel, + } + byDiscoKeys[discoKeys] = started + byCandidatePeerRelay, ok := r.allocWorkByCandidatePeerRelayByEndpoint[wlb.ep] + if !ok { + byCandidatePeerRelay = make(map[candidatePeerRelay]*relayEndpointAllocWork) + r.allocWorkByCandidatePeerRelayByEndpoint[wlb.ep] = byCandidatePeerRelay + } + byCandidatePeerRelay[v] = started + r.allocGeneration++ + go r.allocateServerEndpoint(started, r.allocGeneration) } - relayManagerInputEvent(r, ctx, &r.newServerEndpointCh, newRelayServerEndpointEvent{ - ep: ep, - se: se, - }) } diff --git a/wgengine/magicsock/relaymanager_test.go b/wgengine/magicsock/relaymanager_test.go index 3b75db9f6e1f9..e4891f5678a24 100644 --- a/wgengine/magicsock/relaymanager_test.go +++ b/wgengine/magicsock/relaymanager_test.go @@ -4,16 +4,16 @@ package magicsock import ( - "net/netip" "testing" "tailscale.com/disco" "tailscale.com/types/key" + "tailscale.com/util/set" ) func TestRelayManagerInitAndIdle(t *testing.T) { rm := relayManager{} - rm.allocateAndHandshakeAllServers(&endpoint{}) + rm.startUDPRelayPathDiscoveryFor(&endpoint{}, addrQuality{}, false) <-rm.runLoopStoppedCh rm = relayManager{} @@ -21,10 +21,60 @@ func TestRelayManagerInitAndIdle(t *testing.T) { <-rm.runLoopStoppedCh rm = relayManager{} - rm.handleCallMeMaybeVia(&endpoint{c: &Conn{discoPrivate: key.NewDisco()}}, &disco.CallMeMaybeVia{ServerDisco: key.NewDisco().Public()}) + rm.handleCallMeMaybeVia(&endpoint{c: &Conn{discoPrivate: key.NewDisco()}}, addrQuality{}, false, &disco.CallMeMaybeVia{UDPRelayEndpoint: disco.UDPRelayEndpoint{ServerDisco: key.NewDisco().Public()}}) <-rm.runLoopStoppedCh rm = relayManager{} - rm.handleBindUDPRelayEndpointChallenge(&disco.BindUDPRelayEndpointChallenge{}, &discoInfo{}, netip.AddrPort{}, 0) + rm.handleRxDiscoMsg(&Conn{discoPrivate: key.NewDisco()}, &disco.BindUDPRelayEndpointChallenge{}, key.NodePublic{}, key.DiscoPublic{}, epAddr{}) <-rm.runLoopStoppedCh + + rm = relayManager{} + rm.handleRelayServersSet(make(set.Set[candidatePeerRelay])) + <-rm.runLoopStoppedCh + + rm = relayManager{} + rm.getServers() + <-rm.runLoopStoppedCh + + rm = relayManager{} + rm.handleDERPHomeChange(key.NodePublic{}, 1) + <-rm.runLoopStoppedCh +} + +func TestRelayManagerHandleDERPHomeChange(t *testing.T) { + rm := relayManager{} + servers := make(set.Set[candidatePeerRelay], 1) + c := candidatePeerRelay{ + nodeKey: key.NewNode().Public(), + discoKey: key.NewDisco().Public(), + derpHomeRegionID: 1, + } + servers.Add(c) + rm.handleRelayServersSet(servers) + want := c + want.derpHomeRegionID = 2 + rm.handleDERPHomeChange(c.nodeKey, 2) + got := rm.getServers() + if len(got) != 1 { + t.Fatalf("got %d servers, want 1", len(got)) + } + _, ok := got[want] + if !ok { + t.Fatal("DERP home change failed to propagate") + } +} + +func TestRelayManagerGetServers(t *testing.T) { + rm := relayManager{} + servers := make(set.Set[candidatePeerRelay], 1) + c := candidatePeerRelay{ + nodeKey: key.NewNode().Public(), + discoKey: key.NewDisco().Public(), + } + servers.Add(c) + rm.handleRelayServersSet(servers) + got := rm.getServers() + if !servers.Equal(got) { + t.Errorf("got %v != want %v", got, servers) + } } diff --git a/wgengine/netstack/netstack.go b/wgengine/netstack/netstack.go index dab692ead4aa7..d97c669463d78 100644 --- a/wgengine/netstack/netstack.go +++ b/wgengine/netstack/netstack.go @@ -1435,6 +1435,13 @@ func (ns *Impl) acceptTCP(r *tcp.ForwarderRequest) { } } +// tcpCloser is an interface to abstract around various TCPConn types that +// allow closing of the read and write streams independently of each other. +type tcpCloser interface { + CloseRead() error + CloseWrite() error +} + func (ns *Impl) forwardTCP(getClient func(...tcpip.SettableSocketOption) *gonet.TCPConn, clientRemoteIP netip.Addr, wq *waiter.Queue, dialAddr netip.AddrPort) (handled bool) { dialAddrStr := dialAddr.String() if debugNetstack() { @@ -1501,18 +1508,48 @@ func (ns *Impl) forwardTCP(getClient func(...tcpip.SettableSocketOption) *gonet. } defer client.Close() + // As of 2025-07-03, backend is always either a net.TCPConn + // from stdDialer.DialContext (which has the requisite functions), + // or nil from hangDialer in tests (in which case we would have + // errored out by now), so this conversion should always succeed. + backendTCPCloser, backendIsTCPCloser := backend.(tcpCloser) connClosed := make(chan error, 2) go func() { _, err := io.Copy(backend, client) + if err != nil { + err = fmt.Errorf("client -> backend: %w", err) + } connClosed <- err + err = nil + if backendIsTCPCloser { + err = backendTCPCloser.CloseWrite() + } + err = errors.Join(err, client.CloseRead()) + if err != nil { + ns.logf("client -> backend close connection: %v", err) + } }() go func() { _, err := io.Copy(client, backend) + if err != nil { + err = fmt.Errorf("backend -> client: %w", err) + } connClosed <- err + err = nil + if backendIsTCPCloser { + err = backendTCPCloser.CloseRead() + } + err = errors.Join(err, client.CloseWrite()) + if err != nil { + ns.logf("backend -> client close connection: %v", err) + } }() - err = <-connClosed - if err != nil { - ns.logf("proxy connection closed with error: %v", err) + // Wait for both ends of the connection to close. + for range 2 { + err = <-connClosed + if err != nil { + ns.logf("proxy connection closed with error: %v", err) + } } ns.logf("[v2] netstack: forwarder connection to %s closed", dialAddrStr) return diff --git a/wgengine/userspace.go b/wgengine/userspace.go index b1b82032b2a6c..4a9f321430c12 100644 --- a/wgengine/userspace.go +++ b/wgengine/userspace.go @@ -1300,7 +1300,6 @@ func (e *userspaceEngine) linkChange(delta *netmon.ChangeDelta) { } func (e *userspaceEngine) SetNetworkMap(nm *netmap.NetworkMap) { - e.magicConn.SetNetworkMap(nm) e.mu.Lock() e.netMap = nm e.mu.Unlock() diff --git a/wgengine/wgcfg/device_test.go b/wgengine/wgcfg/device_test.go index d54282e4bdf04..9138d6e5a0f47 100644 --- a/wgengine/wgcfg/device_test.go +++ b/wgengine/wgcfg/device_test.go @@ -242,9 +242,9 @@ type noopBind struct{} func (noopBind) Open(port uint16) (fns []conn.ReceiveFunc, actualPort uint16, err error) { return nil, 1, nil } -func (noopBind) Close() error { return nil } -func (noopBind) SetMark(mark uint32) error { return nil } -func (noopBind) Send(b [][]byte, ep conn.Endpoint) error { return nil } +func (noopBind) Close() error { return nil } +func (noopBind) SetMark(mark uint32) error { return nil } +func (noopBind) Send(b [][]byte, ep conn.Endpoint, offset int) error { return nil } func (noopBind) ParseEndpoint(s string) (conn.Endpoint, error) { return dummyEndpoint(s), nil }