diff --git a/README.md b/README.md index cf252b708..1088524de 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,22 @@ curl :/metrics The admission server is an optional component aiming at reducing bad config changes for DO managed objects (LBs, etc). If you want to know more about it, read the [docs](./docs/admission-server.md). +#### Configure Node IP Address Families + +By default, the CCM discovers both IPv4 and IPv6 addresses for nodes that have +public IPv6 networking enabled. Use the `DO_IP_ADDR_FAMILIES` environment variable +to control which IP address families are included in the node status: + +- `ipv4` — Only IPv4 addresses (private + public) +- `ipv6` — Only public IPv6 address +- `ipv4,ipv6` — Both IPv4 and IPv6 addresses (same as default) + +When unset, all available addresses are included. Example: + +```bash +DO_IP_ADDR_FAMILIES=ipv4 digitalocean-cloud-controller-manager ... +``` + ### DO API rate limiting DO API usage is subject to [certain rate limits](https://docs.digitalocean.com/reference/api/api-reference/#section/Introduction/Rate-Limit). In order to protect against running out of quota for extremely heavy regular usage or pathological cases (e.g., bugs or API thrashing due to an interfering third-party controller), a custom rate limit can be configured via the `DO_API_RATE_LIMIT_QPS` environment variable. It accepts a float value, e.g., `DO_API_RATE_LIMIT_QPS=3.5` to restrict API usage to 3.5 queries per second. diff --git a/cloud-controller-manager/do/cloud.go b/cloud-controller-manager/do/cloud.go index 2a8990b9a..df94edde2 100644 --- a/cloud-controller-manager/do/cloud.go +++ b/cloud-controller-manager/do/cloud.go @@ -164,6 +164,10 @@ func newCloud() (cloudprovider.Interface, error) { defaultLBType = lbType } + if err := setIPFamiliesFromEnv(); err != nil { + return nil, fmt.Errorf("failed to parse %s: %v", doIPAddrFamiliesEnv, err) + } + return &cloud{ client: doClient, instances: newInstances(resources, region), diff --git a/cloud-controller-manager/do/common.go b/cloud-controller-manager/do/common.go index bd84fe137..23650aba5 100644 --- a/cloud-controller-manager/do/common.go +++ b/cloud-controller-manager/do/common.go @@ -20,11 +20,74 @@ import ( "context" "errors" "fmt" + "os" + "strings" "github.com/digitalocean/godo" v1 "k8s.io/api/core/v1" ) +// IPFamily represents an IP address family for node address discovery. +type IPFamily string + +const ( + ipv4Family IPFamily = "ipv4" + ipv6Family IPFamily = "ipv6" + + doIPAddrFamiliesEnv string = "DO_IP_ADDR_FAMILIES" +) + +var ipFamilies []IPFamily + +// getIPFamilies returns the configured IP address families. +// When DO_IP_ADDR_FAMILIES is not set or empty, returns the backward-compatible +// default (IPv4 and IPv6 if available). +func getIPFamilies() []IPFamily { + if ipFamilies != nil { + return ipFamilies + } + return []IPFamily{ipv4Family, ipv6Family} +} + +// setIPFamiliesFromEnv parses DO_IP_ADDR_FAMILIES and configures the families. +// Returns an error if any value is invalid. +func setIPFamiliesFromEnv() error { + v := os.Getenv(doIPAddrFamiliesEnv) + if v == "" { + return nil + } + var families []IPFamily + for _, f := range strings.Split(v, ",") { + f = strings.TrimSpace(f) + lower := strings.ToLower(f) + switch lower { + case string(ipv4Family): + families = append(families, ipv4Family) + case string(ipv6Family): + families = append(families, ipv6Family) + default: + return fmt.Errorf("invalid IP family %q in %s (expected ipv4 or ipv6)", f, doIPAddrFamiliesEnv) + } + } + if len(families) == 0 { + return fmt.Errorf("must specify at least one IP family in %s", doIPAddrFamiliesEnv) + } + ipFamilies = families + return nil +} + +// describeIPFamily returns a human-readable description of an IP family. +func describeIPFamily(family IPFamily) string { + switch family { + case ipv4Family: + return "IPv4" + case ipv6Family: + return "IPv6" + default: + return string(family) + } +} + // max page size is 200, but choose a smaller value b/c sometimes objects being listed // are very large and the response gets too big with 200 objects const apiResultsPerPage = 50 @@ -127,32 +190,54 @@ func allLoadBalancerList(ctx context.Context, client *godo.Client) ([]godo.LoadB } // nodeAddresses returns a []v1.NodeAddress from droplet. +// The address families are controlled by the DO_IP_ADDR_FAMILIES environment +// variable. When unset, the default is to include all available addresses. func nodeAddresses(droplet *godo.Droplet) ([]v1.NodeAddress, error) { var addresses []v1.NodeAddress addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: droplet.Name}) - privateIP, err := droplet.PrivateIPv4() - if err != nil || privateIP == "" { - return nil, fmt.Errorf("could not get private ip: %v", err) + for _, family := range getIPFamilies() { + addr, err := discoverAddress(droplet, family) + if err != nil { + return nil, fmt.Errorf("could not get %s addresses: %v", describeIPFamily(family), err) + } + addresses = append(addresses, addr...) } - addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: privateIP}) - publicIPv4, err := droplet.PublicIPv4() - if err != nil { - return nil, fmt.Errorf("could not get public ipv4: %v", err) - } - if publicIPv4 != "" { - addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIPv4}) - } + return addresses, nil +} - publicIPv6, err := droplet.PublicIPv6() - if err != nil { - return nil, fmt.Errorf("could not get public ipv6: %v", err) - } - if publicIPv6 != "" { - // not all instances come with ipv6 addresses - addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIPv6}) +// discoverAddress discovers node addresses for a given IP family from a droplet. +func discoverAddress(droplet *godo.Droplet, family IPFamily) ([]v1.NodeAddress, error) { + switch family { + case ipv4Family: + privateIP, err := droplet.PrivateIPv4() + if err != nil || privateIP == "" { + return nil, fmt.Errorf("could not get private ip: %v", err) + } + addrs := []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: privateIP}, + } + publicIPv4, err := droplet.PublicIPv4() + if err != nil { + return nil, fmt.Errorf("could not get public ipv4: %v", err) + } + if publicIPv4 != "" { + addrs = append(addrs, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIPv4}) + } + return addrs, nil + case ipv6Family: + publicIPv6, err := droplet.PublicIPv6() + if err != nil { + return nil, fmt.Errorf("could not get public ipv6: %v", err) + } + if publicIPv6 == "" { + return nil, nil + } + return []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: publicIPv6}, + }, nil + default: + return nil, fmt.Errorf("unknown IP family: %s", family) } - - return addresses, nil } diff --git a/cloud-controller-manager/do/droplets_test.go b/cloud-controller-manager/do/droplets_test.go index 4ee3c6d71..554b66776 100644 --- a/cloud-controller-manager/do/droplets_test.go +++ b/cloud-controller-manager/do/droplets_test.go @@ -678,6 +678,131 @@ func TestInstanceMetadata(t *testing.T) { } } +func TestNodeAddressesIPFamilies(t *testing.T) { + tt := []struct { + name string + envVar string + droplet *godo.Droplet + expectedAddresses []v1.NodeAddress + expectedErr string + }{ + { + name: "default (no env var) - IPv4 only droplet", + envVar: "", + droplet: newFakeDroplet(), + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: "test-droplet"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.0"}, + {Type: v1.NodeExternalIP, Address: "99.99.99.99"}, + }, + }, + { + name: "private IPv4 only", + envVar: "", + droplet: newFakeDropletWithoutPublicIPv4(), + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: "test-droplet"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.0"}, + }, + }, + { + envVar: "", + droplet: newFakeDropletWithIPv6(), + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: "test-droplet"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.0"}, + {Type: v1.NodeExternalIP, Address: "99.99.99.99"}, + {Type: v1.NodeExternalIP, Address: "2604:a880:800:10::1"}, + }, + }, + { + name: "ipv4 only", + envVar: "ipv4", + droplet: newFakeDropletWithIPv6(), + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: "test-droplet"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.0"}, + {Type: v1.NodeExternalIP, Address: "99.99.99.99"}, + }, + }, + { + name: "ipv6 only", + envVar: "ipv6", + droplet: newFakeDropletWithIPv6(), + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: "test-droplet"}, + {Type: v1.NodeExternalIP, Address: "2604:a880:800:10::1"}, + }, + }, + { + name: "ipv4,ipv6 dual stack", + envVar: "ipv4,ipv6", + droplet: newFakeDropletWithIPv6(), + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: "test-droplet"}, + {Type: v1.NodeInternalIP, Address: "10.0.0.0"}, + {Type: v1.NodeExternalIP, Address: "99.99.99.99"}, + {Type: v1.NodeExternalIP, Address: "2604:a880:800:10::1"}, + }, + }, + { + name: "ipv6 only on IPv4-only droplet - no IPv6 addresses", + envVar: "ipv6", + droplet: newFakeDroplet(), + expectedAddresses: []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: "test-droplet"}, + }, + }, + { + name: "invalid family value", + envVar: "ipv5", + droplet: newFakeDroplet(), + expectedErr: "invalid IP family", + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + ipFamilies = nil + t.Cleanup(func() { ipFamilies = nil }) + if tc.envVar != "" { + t.Setenv(doIPAddrFamiliesEnv, tc.envVar) + } else { + t.Setenv(doIPAddrFamiliesEnv, "") + } + if err := setIPFamiliesFromEnv(); err != nil { + if tc.expectedErr == "" { + t.Fatalf("unexpected error from setIPFamiliesFromEnv: %v", err) + } + if !strings.Contains(err.Error(), tc.expectedErr) { + t.Fatalf("got error %v, expected %s", err, tc.expectedErr) + } + return + } + if tc.expectedErr != "" { + t.Fatalf("expected error containing %q, got none", tc.expectedErr) + } + + addrs, err := nodeAddresses(tc.droplet) + if err != nil { + if tc.expectedErr == "" { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(err.Error(), tc.expectedErr) { + t.Fatalf("got error %v, expected %s", err, tc.expectedErr) + } + return + } + if tc.expectedErr != "" { + t.Fatalf("expected error containing %q, got none", tc.expectedErr) + } + if !reflect.DeepEqual(addrs, tc.expectedAddresses) { + t.Errorf("addresses mismatch\ngot: %v\nwant: %v", addrs, tc.expectedAddresses) + } + }) + } +} + func Test_dropletIDFromProviderID(t *testing.T) { testcases := []struct { name string diff --git a/docs/controllers/node/examples/README.md b/docs/controllers/node/examples/README.md index aaf5cc30f..59b28278c 100644 --- a/docs/controllers/node/examples/README.md +++ b/docs/controllers/node/examples/README.md @@ -82,6 +82,26 @@ DigitalOcean cloud controller manager has made the cluster aware of the size of a failure domain which the scheduler can use for region failovers. Note also that the correct addresses were assigned to the node. The `InternalIP` now represents the private IP of the droplet, and the `ExternalIP` is it's public IP. +## IP Address Family Configuration + +By default, the CCM discovers both IPv4 and IPv6 node addresses from droplets that +have public IPv6 networking enabled. Use the `DO_IP_ADDR_FAMILIES` environment +variable to control which address families are included in the node status: + +| Value | Behavior | +|-------|----------| +| (unset) | Include all available addresses (IPv4 and IPv6 if present) | +| `ipv4` | Only IPv4 addresses (private + public) | +| `ipv6` | Only public IPv6 address | +| `ipv4,ipv6` | Both IPv4 and IPv6 addresses (same as default) | + +For example, to restrict to IPv4 only: +```yaml +env: + - name: DO_IP_ADDR_FAMILIES + value: "ipv4" +``` + ## Node clean up When deleting a node in a Kubernetes cluster, deleting droplets would leave the corresponding Kubernetes node in a `NotReady` state. It was the responsibility