Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,22 @@ curl <host>:<port>/metrics
The admission server is an optional component aiming at reducing bad config changes for DO managed objects (LBs, etc).
If you want to know more about it, read the [docs](./docs/admission-server.md).

#### Configure Node IP Address Families

By default, the CCM discovers both IPv4 and IPv6 addresses for nodes that have
public IPv6 networking enabled. Use the `DO_IP_ADDR_FAMILIES` environment variable
to control which IP address families are included in the node status:

- `ipv4` — Only IPv4 addresses (private + public)
- `ipv6` — Only public IPv6 address
- `ipv4,ipv6` — Both IPv4 and IPv6 addresses (same as default)

When unset, all available addresses are included. Example:

```bash
DO_IP_ADDR_FAMILIES=ipv4 digitalocean-cloud-controller-manager ...
```

### DO API rate limiting

DO API usage is subject to [certain rate limits](https://docs.digitalocean.com/reference/api/api-reference/#section/Introduction/Rate-Limit). In order to protect against running out of quota for extremely heavy regular usage or pathological cases (e.g., bugs or API thrashing due to an interfering third-party controller), a custom rate limit can be configured via the `DO_API_RATE_LIMIT_QPS` environment variable. It accepts a float value, e.g., `DO_API_RATE_LIMIT_QPS=3.5` to restrict API usage to 3.5 queries per second.
Expand Down
4 changes: 4 additions & 0 deletions cloud-controller-manager/do/cloud.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,10 @@ func newCloud() (cloudprovider.Interface, error) {
defaultLBType = lbType
}

if err := setIPFamiliesFromEnv(); err != nil {
return nil, fmt.Errorf("failed to parse %s: %v", doIPAddrFamiliesEnv, err)
}

return &cloud{
client: doClient,
instances: newInstances(resources, region),
Expand Down
125 changes: 105 additions & 20 deletions cloud-controller-manager/do/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,74 @@ import (
"context"
"errors"
"fmt"
"os"
"strings"

"github.com/digitalocean/godo"
v1 "k8s.io/api/core/v1"
)

// IPFamily represents an IP address family for node address discovery.
type IPFamily string

const (
ipv4Family IPFamily = "ipv4"
ipv6Family IPFamily = "ipv6"

doIPAddrFamiliesEnv string = "DO_IP_ADDR_FAMILIES"
)

var ipFamilies []IPFamily

// getIPFamilies returns the configured IP address families.
// When DO_IP_ADDR_FAMILIES is not set or empty, returns the backward-compatible
// default (IPv4 and IPv6 if available).
func getIPFamilies() []IPFamily {
if ipFamilies != nil {
return ipFamilies
}
return []IPFamily{ipv4Family, ipv6Family}
}

// setIPFamiliesFromEnv parses DO_IP_ADDR_FAMILIES and configures the families.
// Returns an error if any value is invalid.
func setIPFamiliesFromEnv() error {
v := os.Getenv(doIPAddrFamiliesEnv)
if v == "" {
return nil
}
var families []IPFamily
for _, f := range strings.Split(v, ",") {
f = strings.TrimSpace(f)
lower := strings.ToLower(f)
switch lower {
case string(ipv4Family):
families = append(families, ipv4Family)
case string(ipv6Family):
families = append(families, ipv6Family)
default:
return fmt.Errorf("invalid IP family %q in %s (expected ipv4 or ipv6)", f, doIPAddrFamiliesEnv)
}
}
if len(families) == 0 {
return fmt.Errorf("must specify at least one IP family in %s", doIPAddrFamiliesEnv)
}
ipFamilies = families
return nil
}

// describeIPFamily returns a human-readable description of an IP family.
func describeIPFamily(family IPFamily) string {
switch family {
case ipv4Family:
return "IPv4"
case ipv6Family:
return "IPv6"
default:
return string(family)
}
}

// max page size is 200, but choose a smaller value b/c sometimes objects being listed
// are very large and the response gets too big with 200 objects
const apiResultsPerPage = 50
Expand Down Expand Up @@ -127,32 +190,54 @@ func allLoadBalancerList(ctx context.Context, client *godo.Client) ([]godo.LoadB
}

// nodeAddresses returns a []v1.NodeAddress from droplet.
// The address families are controlled by the DO_IP_ADDR_FAMILIES environment
// variable. When unset, the default is to include all available addresses.
func nodeAddresses(droplet *godo.Droplet) ([]v1.NodeAddress, error) {
var addresses []v1.NodeAddress
addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: droplet.Name})

privateIP, err := droplet.PrivateIPv4()
if err != nil || privateIP == "" {
return nil, fmt.Errorf("could not get private ip: %v", err)
for _, family := range getIPFamilies() {
addr, err := discoverAddress(droplet, family)
if err != nil {
return nil, fmt.Errorf("could not get %s addresses: %v", describeIPFamily(family), err)
}
addresses = append(addresses, addr...)
}
addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: privateIP})

publicIPv4, err := droplet.PublicIPv4()
if err != nil {
return nil, fmt.Errorf("could not get public ipv4: %v", err)
}
if publicIPv4 != "" {
addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIPv4})
}
return addresses, nil
}

publicIPv6, err := droplet.PublicIPv6()
if err != nil {
return nil, fmt.Errorf("could not get public ipv6: %v", err)
}
if publicIPv6 != "" {
// not all instances come with ipv6 addresses
addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIPv6})
// discoverAddress discovers node addresses for a given IP family from a droplet.
func discoverAddress(droplet *godo.Droplet, family IPFamily) ([]v1.NodeAddress, error) {
switch family {
case ipv4Family:
privateIP, err := droplet.PrivateIPv4()
if err != nil || privateIP == "" {
return nil, fmt.Errorf("could not get private ip: %v", err)
}
addrs := []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: privateIP},
}
publicIPv4, err := droplet.PublicIPv4()
if err != nil {
return nil, fmt.Errorf("could not get public ipv4: %v", err)
}
if publicIPv4 != "" {
addrs = append(addrs, v1.NodeAddress{Type: v1.NodeExternalIP, Address: publicIPv4})
}
return addrs, nil
case ipv6Family:
publicIPv6, err := droplet.PublicIPv6()
if err != nil {
return nil, fmt.Errorf("could not get public ipv6: %v", err)
}
if publicIPv6 == "" {
return nil, nil
}
return []v1.NodeAddress{
{Type: v1.NodeExternalIP, Address: publicIPv6},
}, nil
default:
return nil, fmt.Errorf("unknown IP family: %s", family)
}

return addresses, nil
}
125 changes: 125 additions & 0 deletions cloud-controller-manager/do/droplets_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -678,6 +678,131 @@ func TestInstanceMetadata(t *testing.T) {
}
}

func TestNodeAddressesIPFamilies(t *testing.T) {
tt := []struct {
name string
envVar string
droplet *godo.Droplet
expectedAddresses []v1.NodeAddress
expectedErr string
}{
{
name: "default (no env var) - IPv4 only droplet",
envVar: "",
droplet: newFakeDroplet(),
expectedAddresses: []v1.NodeAddress{
{Type: v1.NodeHostName, Address: "test-droplet"},
{Type: v1.NodeInternalIP, Address: "10.0.0.0"},
{Type: v1.NodeExternalIP, Address: "99.99.99.99"},
},
},
{
name: "private IPv4 only",
envVar: "",
droplet: newFakeDropletWithoutPublicIPv4(),
expectedAddresses: []v1.NodeAddress{
{Type: v1.NodeHostName, Address: "test-droplet"},
{Type: v1.NodeInternalIP, Address: "10.0.0.0"},
},
},
{
envVar: "",
droplet: newFakeDropletWithIPv6(),
expectedAddresses: []v1.NodeAddress{
{Type: v1.NodeHostName, Address: "test-droplet"},
{Type: v1.NodeInternalIP, Address: "10.0.0.0"},
{Type: v1.NodeExternalIP, Address: "99.99.99.99"},
{Type: v1.NodeExternalIP, Address: "2604:a880:800:10::1"},
},
},
{
name: "ipv4 only",
envVar: "ipv4",
droplet: newFakeDropletWithIPv6(),
expectedAddresses: []v1.NodeAddress{
{Type: v1.NodeHostName, Address: "test-droplet"},
{Type: v1.NodeInternalIP, Address: "10.0.0.0"},
{Type: v1.NodeExternalIP, Address: "99.99.99.99"},
},
},
{
name: "ipv6 only",
envVar: "ipv6",
droplet: newFakeDropletWithIPv6(),
expectedAddresses: []v1.NodeAddress{
{Type: v1.NodeHostName, Address: "test-droplet"},
{Type: v1.NodeExternalIP, Address: "2604:a880:800:10::1"},
},
},
{
name: "ipv4,ipv6 dual stack",
envVar: "ipv4,ipv6",
droplet: newFakeDropletWithIPv6(),
expectedAddresses: []v1.NodeAddress{
{Type: v1.NodeHostName, Address: "test-droplet"},
{Type: v1.NodeInternalIP, Address: "10.0.0.0"},
{Type: v1.NodeExternalIP, Address: "99.99.99.99"},
{Type: v1.NodeExternalIP, Address: "2604:a880:800:10::1"},
},
},
{
name: "ipv6 only on IPv4-only droplet - no IPv6 addresses",
envVar: "ipv6",
droplet: newFakeDroplet(),
expectedAddresses: []v1.NodeAddress{
{Type: v1.NodeHostName, Address: "test-droplet"},
},
},
{
name: "invalid family value",
envVar: "ipv5",
droplet: newFakeDroplet(),
expectedErr: "invalid IP family",
},
}

for _, tc := range tt {
t.Run(tc.name, func(t *testing.T) {
ipFamilies = nil
t.Cleanup(func() { ipFamilies = nil })
if tc.envVar != "" {
t.Setenv(doIPAddrFamiliesEnv, tc.envVar)
} else {
t.Setenv(doIPAddrFamiliesEnv, "")
}
if err := setIPFamiliesFromEnv(); err != nil {
if tc.expectedErr == "" {
t.Fatalf("unexpected error from setIPFamiliesFromEnv: %v", err)
}
if !strings.Contains(err.Error(), tc.expectedErr) {
t.Fatalf("got error %v, expected %s", err, tc.expectedErr)
}
return
}
if tc.expectedErr != "" {
t.Fatalf("expected error containing %q, got none", tc.expectedErr)
}

addrs, err := nodeAddresses(tc.droplet)
if err != nil {
if tc.expectedErr == "" {
t.Fatalf("unexpected error: %v", err)
}
if !strings.Contains(err.Error(), tc.expectedErr) {
t.Fatalf("got error %v, expected %s", err, tc.expectedErr)
}
return
}
if tc.expectedErr != "" {
t.Fatalf("expected error containing %q, got none", tc.expectedErr)
}
if !reflect.DeepEqual(addrs, tc.expectedAddresses) {
t.Errorf("addresses mismatch\ngot: %v\nwant: %v", addrs, tc.expectedAddresses)
}
})
}
}

func Test_dropletIDFromProviderID(t *testing.T) {
testcases := []struct {
name string
Expand Down
20 changes: 20 additions & 0 deletions docs/controllers/node/examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,26 @@ DigitalOcean cloud controller manager has made the cluster aware of the size of
a failure domain which the scheduler can use for region failovers. Note also that the correct addresses were assigned to the node. The `InternalIP` now represents
the private IP of the droplet, and the `ExternalIP` is it's public IP.

## IP Address Family Configuration

By default, the CCM discovers both IPv4 and IPv6 node addresses from droplets that
have public IPv6 networking enabled. Use the `DO_IP_ADDR_FAMILIES` environment
variable to control which address families are included in the node status:

| Value | Behavior |
|-------|----------|
| (unset) | Include all available addresses (IPv4 and IPv6 if present) |
| `ipv4` | Only IPv4 addresses (private + public) |
| `ipv6` | Only public IPv6 address |
| `ipv4,ipv6` | Both IPv4 and IPv6 addresses (same as default) |

For example, to restrict to IPv4 only:
```yaml
env:
- name: DO_IP_ADDR_FAMILIES
value: "ipv4"
```

## Node clean up

When deleting a node in a Kubernetes cluster, deleting droplets would leave the corresponding Kubernetes node in a `NotReady` state. It was the responsibility
Expand Down