Skip to content

Commit b6e521f

Browse files
komer3komer3
committed
fix ipv6 backend for vpcs
1 parent 791b6a8 commit b6e521f

6 files changed

Lines changed: 167 additions & 65 deletions

File tree

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,15 +168,15 @@ mgmt-and-capl-cluster: docker-setup mgmt-cluster capl-cluster
168168
capl-cluster: generate-capl-cluster-manifests create-capl-cluster patch-linode-ccm
169169

170170
.PHONY: generate-capl-cluster-manifests
171-
generate-capl-cluster-manifests:
171+
generate-capl-cluster-manifests: clusterctl
172172
# Create the CAPL cluster manifests without any CSI driver stuff
173173
LINODE_FIREWALL_ENABLED=$(LINODE_FIREWALL_ENABLED) LINODE_OS=$(LINODE_OS) VPC_NAME=$(VPC_NAME) $(CLUSTERCTL) generate cluster $(CLUSTER_NAME) \
174174
--kubernetes-version $(K8S_VERSION) --infrastructure linode-linode:$(CAPL_VERSION) \
175175
--control-plane-machine-count $(CONTROLPLANE_NODES) --worker-machine-count $(WORKER_NODES) --flavor kubeadm-dual-stack > $(MANIFEST_NAME).yaml
176176
yq -i e 'select(.kind == "LinodeVPC").spec.ipv6Range = [{"range": "auto"}] | select(.kind == "LinodeVPC").spec.subnets = [{"ipv4": "10.0.0.0/8", "label": "default", "ipv6Range": [{"range": "auto"}]}, {"ipv4": "172.16.0.0/16", "label": "testing", "ipv6Range": [{"range": "auto"}]}]' $(MANIFEST_NAME).yaml
177177

178178
.PHONY: create-capl-cluster
179-
create-capl-cluster:
179+
create-capl-cluster: clusterctl
180180
# Create a CAPL cluster with updated CCM and wait for it to be ready
181181
kubectl apply -f $(MANIFEST_NAME).yaml
182182
kubectl wait --for=condition=ControlPlaneReady cluster/$(CLUSTER_NAME) --timeout=600s || (kubectl get cluster -o yaml; kubectl get linodecluster -o yaml; kubectl get linodemachines -o yaml; kubectl logs -n capl-system deployments/capl-controller-manager --tail=50)

‎cloud/linode/loadbalancers.go‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -505,10 +505,7 @@ func (l *loadbalancers) updateNodeBalancer(
505505
subnetID = id
506506
}
507507

508-
useIPv6Backends, ignoreVPCBackends := resolveIPv6NodeBalancerBackendState(service, currentNBNodes)
509-
if ignoreVPCBackends {
510-
subnetID = 0
511-
}
508+
useIPv6Backends := resolveIPv6NodeBalancerBackendState(service)
512509
newNBNodes, err := l.buildNodeBalancerConfigNodes(service, nodes, port.NodePort, subnetID, useIPv6Backends, newNBCfg.Protocol, oldNBNodeIDs)
513510
if err != nil {
514511
return err
@@ -952,8 +949,7 @@ func (l *loadbalancers) createNodeBalancer(ctx context.Context, clusterName stri
952949
Type: nbType,
953950
}
954951

955-
_, ignoreVPCBackends := resolveIPv6NodeBalancerBackendState(service, nil)
956-
if len(options.Options.VPCNames) > 0 && !options.Options.DisableNodeBalancerVPCBackends && !ignoreVPCBackends {
952+
if len(options.Options.VPCNames) > 0 && !options.Options.DisableNodeBalancerVPCBackends {
957953
createOpts.VPCs, err = l.getVPCCreateOptions(ctx, service)
958954
if err != nil {
959955
return nil, err
@@ -1152,7 +1148,7 @@ func (l *loadbalancers) buildLoadBalancerRequest(ctx context.Context, clusterNam
11521148
}
11531149
ports := service.Spec.Ports
11541150
configs := make([]*linodego.NodeBalancerConfigCreateOptions, 0, len(ports))
1155-
useIPv6Backends, ignoreVPCBackends := resolveIPv6NodeBalancerBackendState(service, nil)
1151+
useIPv6Backends := resolveIPv6NodeBalancerBackendState(service)
11561152

11571153
subnetID := 0
11581154
if options.Options.NodeBalancerBackendIPv4SubnetID != 0 {
@@ -1165,7 +1161,7 @@ func (l *loadbalancers) buildLoadBalancerRequest(ctx context.Context, clusterNam
11651161
return nil, err
11661162
}
11671163
}
1168-
if len(options.Options.VPCNames) > 0 && !options.Options.DisableNodeBalancerVPCBackends && !ignoreVPCBackends {
1164+
if len(options.Options.VPCNames) > 0 && !options.Options.DisableNodeBalancerVPCBackends {
11691165
id, err := l.getSubnetIDForSVC(ctx, service)
11701166
if err != nil {
11711167
return nil, err
@@ -1230,20 +1226,13 @@ func (l *loadbalancers) buildNodeBalancerNodeConfigRebuildOptions(service *v1.Se
12301226
return nodeOptions, nil
12311227
}
12321228

1233-
func resolveIPv6NodeBalancerBackendState(service *v1.Service, currentNBNodes []linodego.NodeBalancerNode) (useIPv6Backends bool, ignoreVPCBackends bool) {
1229+
func resolveIPv6NodeBalancerBackendState(service *v1.Service) bool {
12341230
useIPv6 := getServiceBoolAnnotation(service, annotations.AnnLinodeEnableIPv6Backends)
12351231
if useIPv6 != nil {
1236-
return *useIPv6, *useIPv6
1237-
}
1238-
1239-
if len(currentNBNodes) > 0 {
1240-
if isNodeBalancerBackendIPv6(currentNBNodes[0].Address) {
1241-
return true, true
1242-
}
1243-
return false, false
1232+
return *useIPv6
12441233
}
12451234

1246-
return options.Options.EnableIPv6ForNodeBalancerBackends, false
1235+
return options.Options.EnableIPv6ForNodeBalancerBackends
12471236
}
12481237

12491238
func formatNodeBalancerBackendAddress(ip string, nodePort int32) string {
@@ -1438,7 +1427,7 @@ func getNodePrivateIP(node *v1.Node, subnetID int) (string, error) {
14381427
}
14391428

14401429
func getNodeBackendIP(service *v1.Service, node *v1.Node, subnetID int, useIPv6Backends bool) (string, error) {
1441-
if subnetID != 0 || !useIPv6Backends {
1430+
if !useIPv6Backends {
14421431
return getNodePrivateIP(node, subnetID)
14431432
}
14441433

‎cloud/linode/loadbalancers_test.go‎

Lines changed: 149 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -4125,7 +4125,7 @@ func Test_getNodeBackendIP(t *testing.T) {
41254125
expectedIP: "2600:3c06::1",
41264126
},
41274127
{
4128-
name: "preserves VPC backend behavior even when IPv6 is requested",
4128+
name: "uses IPv6 backend address even when VPC backends are enabled",
41294129
node: &v1.Node{
41304130
ObjectMeta: metav1.ObjectMeta{
41314131
Name: "node-1",
@@ -4142,7 +4142,7 @@ func Test_getNodeBackendIP(t *testing.T) {
41424142
},
41434143
subnetID: 100,
41444144
useIPv6Backends: true,
4145-
expectedIP: "10.0.0.2",
4145+
expectedIP: "2600:3c06::1",
41464146
},
41474147
{
41484148
name: "errors when IPv6 backends are requested and node lacks public IPv6",
@@ -4187,15 +4187,13 @@ func Test_resolveIPv6NodeBalancerBackendState(t *testing.T) {
41874187
}()
41884188

41894189
testcases := []struct {
4190-
name string
4191-
globalFlag bool
4192-
service *v1.Service
4193-
currentNBNodes []linodego.NodeBalancerNode
4194-
expectedUseIPv6 bool
4195-
expectedIgnoreVPC bool
4190+
name string
4191+
globalFlag bool
4192+
service *v1.Service
4193+
expectedUseIPv6 bool
41964194
}{
41974195
{
4198-
name: "service annotation enables IPv6 and overrides VPC behavior",
4196+
name: "service annotation enables IPv6",
41994197
globalFlag: false,
42004198
service: &v1.Service{
42014199
ObjectMeta: metav1.ObjectMeta{
@@ -4204,47 +4202,163 @@ func Test_resolveIPv6NodeBalancerBackendState(t *testing.T) {
42044202
},
42054203
},
42064204
},
4207-
expectedUseIPv6: true,
4208-
expectedIgnoreVPC: true,
4205+
expectedUseIPv6: true,
42094206
},
42104207
{
4211-
name: "new services respect global IPv6 backend flag without ignoring VPC",
4212-
globalFlag: true,
4213-
service: &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}},
4214-
expectedUseIPv6: true,
4215-
expectedIgnoreVPC: false,
4208+
name: "service annotation disables IPv6 even when global flag is enabled",
4209+
globalFlag: true,
4210+
service: &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{annotations.AnnLinodeEnableIPv6Backends: "false"}}},
4211+
expectedUseIPv6: false,
42164212
},
42174213
{
4218-
name: "existing IPv4 nodebalancer backends stay IPv4 by default",
4219-
globalFlag: true,
4220-
service: &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}},
4221-
currentNBNodes: []linodego.NodeBalancerNode{
4222-
{Address: "10.0.0.10:30000"},
4223-
},
4224-
expectedUseIPv6: false,
4225-
expectedIgnoreVPC: false,
4214+
name: "services use global IPv6 backend flag when annotation is absent",
4215+
globalFlag: true,
4216+
service: &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}},
4217+
expectedUseIPv6: true,
42264218
},
42274219
{
4228-
name: "existing IPv6 nodebalancer backends stay IPv6 and ignore VPC",
4229-
globalFlag: false,
4230-
service: &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}},
4231-
currentNBNodes: []linodego.NodeBalancerNode{
4232-
{Address: "[2600:3c06::1]:30000"},
4233-
},
4234-
expectedUseIPv6: true,
4235-
expectedIgnoreVPC: true,
4220+
name: "services do not use IPv6 when annotation is absent and global flag is disabled",
4221+
globalFlag: false,
4222+
service: &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}},
4223+
expectedUseIPv6: false,
42364224
},
42374225
}
42384226

42394227
for _, test := range testcases {
42404228
t.Run(test.name, func(t *testing.T) {
42414229
options.Options.EnableIPv6ForNodeBalancerBackends = test.globalFlag
4242-
useIPv6Backends, ignoreVPCBackends := resolveIPv6NodeBalancerBackendState(test.service, test.currentNBNodes)
4230+
useIPv6Backends := resolveIPv6NodeBalancerBackendState(test.service)
42434231
if useIPv6Backends != test.expectedUseIPv6 {
42444232
t.Fatalf("expected useIPv6Backends=%t, got %t", test.expectedUseIPv6, useIPv6Backends)
42454233
}
4246-
if ignoreVPCBackends != test.expectedIgnoreVPC {
4247-
t.Fatalf("expected ignoreVPCBackends=%t, got %t", test.expectedIgnoreVPC, ignoreVPCBackends)
4234+
})
4235+
}
4236+
}
4237+
4238+
func Test_buildLoadBalancerRequestPreservesVPCConfigForIPv6Backends(t *testing.T) {
4239+
prevVPCNames := options.Options.VPCNames
4240+
prevSubnetNames := options.Options.SubnetNames
4241+
prevDisableVPC := options.Options.DisableNodeBalancerVPCBackends
4242+
prevEnableIPv6Backends := options.Options.EnableIPv6ForNodeBalancerBackends
4243+
defer func() {
4244+
options.Options.VPCNames = prevVPCNames
4245+
options.Options.SubnetNames = prevSubnetNames
4246+
options.Options.DisableNodeBalancerVPCBackends = prevDisableVPC
4247+
options.Options.EnableIPv6ForNodeBalancerBackends = prevEnableIPv6Backends
4248+
}()
4249+
4250+
options.Options.VPCNames = []string{"test-vpc"}
4251+
options.Options.SubnetNames = []string{"default"}
4252+
options.Options.DisableNodeBalancerVPCBackends = false
4253+
4254+
testcases := []struct {
4255+
name string
4256+
globalFlag bool
4257+
annotations map[string]string
4258+
}{
4259+
{
4260+
name: "service annotation preserves VPC config for IPv6 backends",
4261+
globalFlag: false,
4262+
annotations: map[string]string{
4263+
annotations.AnnLinodeDefaultProtocol: "tcp",
4264+
annotations.AnnLinodeEnableIPv6Backends: "true",
4265+
},
4266+
},
4267+
{
4268+
name: "global flag preserves VPC config for IPv6 backends",
4269+
globalFlag: true,
4270+
annotations: map[string]string{
4271+
annotations.AnnLinodeDefaultProtocol: "tcp",
4272+
},
4273+
},
4274+
}
4275+
4276+
for _, test := range testcases {
4277+
t.Run(test.name, func(t *testing.T) {
4278+
options.Options.EnableIPv6ForNodeBalancerBackends = test.globalFlag
4279+
4280+
fake := newFake(t)
4281+
ts := httptest.NewServer(fake)
4282+
defer ts.Close()
4283+
4284+
client := linodego.NewClient(http.DefaultClient)
4285+
client.SetBaseURL(ts.URL)
4286+
lb := newLoadbalancers(&client, "us-west").(*loadbalancers)
4287+
4288+
fake.vpc[1] = &linodego.VPC{
4289+
ID: 1,
4290+
Label: "test-vpc",
4291+
Subnets: []linodego.VPCSubnet{
4292+
{
4293+
ID: 101,
4294+
Label: "default",
4295+
IPv4: "10.0.0.0/8",
4296+
},
4297+
},
4298+
}
4299+
fake.subnet[101] = &linodego.VPCSubnet{
4300+
ID: 101,
4301+
Label: "default",
4302+
IPv4: "10.0.0.0/8",
4303+
}
4304+
4305+
svc := &v1.Service{
4306+
ObjectMeta: metav1.ObjectMeta{
4307+
Name: "test",
4308+
UID: "foobar123",
4309+
Annotations: test.annotations,
4310+
},
4311+
Spec: v1.ServiceSpec{
4312+
Ports: []v1.ServicePort{
4313+
{
4314+
Name: "test",
4315+
Protocol: "TCP",
4316+
Port: int32(80),
4317+
NodePort: int32(30000),
4318+
},
4319+
},
4320+
},
4321+
}
4322+
nodes := []*v1.Node{
4323+
{
4324+
ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
4325+
Status: v1.NodeStatus{
4326+
Addresses: []v1.NodeAddress{
4327+
{Type: v1.NodeInternalIP, Address: "10.0.0.2"},
4328+
{Type: v1.NodeExternalIP, Address: "2600:3c06:e727:1::1"},
4329+
},
4330+
},
4331+
},
4332+
}
4333+
4334+
_, err := lb.buildLoadBalancerRequest(t.Context(), "linodelb", svc, nodes)
4335+
if err != nil {
4336+
t.Fatal(err)
4337+
}
4338+
4339+
var req *fakeRequest
4340+
for request := range fake.requests {
4341+
if request.Method == http.MethodPost && request.Path == "/nodebalancers" {
4342+
req = &request
4343+
break
4344+
}
4345+
}
4346+
if req == nil {
4347+
t.Fatal("expected nodebalancer create request")
4348+
}
4349+
4350+
var createOpts linodego.NodeBalancerCreateOptions
4351+
if err := json.Unmarshal([]byte(req.Body), &createOpts); err != nil {
4352+
t.Fatalf("unable to unmarshal create request body %#v, error: %#v", req.Body, err)
4353+
}
4354+
if len(createOpts.VPCs) != 1 || createOpts.VPCs[0].SubnetID == 0 {
4355+
t.Fatalf("expected nodebalancer create request to preserve VPC config, got %#v", createOpts.VPCs)
4356+
}
4357+
if len(createOpts.Configs) != 1 || len(createOpts.Configs[0].Nodes) != 1 {
4358+
t.Fatalf("expected a single nodebalancer config with one backend node, got %#v", createOpts.Configs)
4359+
}
4360+
if !isNodeBalancerBackendIPv6(createOpts.Configs[0].Nodes[0].Address) {
4361+
t.Fatalf("expected IPv6 backend node address, got %q", createOpts.Configs[0].Nodes[0].Address)
42484362
}
42494363
})
42504364
}

‎docs/configuration/annotations.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ The keys and the values in [annotations must be strings](https://kubernetes.io/d
4040
| `firewall-acl` | string | | The Firewall rules to be applied to the NodeBalancer. See [Firewall Configuration](#firewall-configuration) |
4141
| `nodebalancer-type` | string | | The type of NodeBalancer to create (options: common, premium, premium_40gb). See [NodeBalancer Types](#nodebalancer-type). Note: NodeBalancer types should always be specified in lowercase. |
4242
| `enable-ipv6-ingress` | bool | `false` | When `true`, both IPv4 and IPv6 addresses will be included in the LoadBalancerStatus ingress |
43-
| `enable-ipv6-backends` | bool | `false` | When `true`, non-VPC NodeBalancer services use public IPv6 backend nodes. This requires a dual-stack cluster and a dual-stack Service configuration. Reconciliation fails if a selected backend node does not have public IPv6. |
43+
| `enable-ipv6-backends` | bool | `false` | When `true`, NodeBalancer services use IPv6 backend nodes. If VPC-backed NodeBalancers are enabled, CCM preserves the NodeBalancer VPC configuration. This requires a dual-stack cluster and a dual-stack Service configuration. Reconciliation fails if a selected backend node does not have the required IPv6 address. |
4444
| `backend-ipv4-range` | string | | The IPv4 range from VPC subnet to be applied to the NodeBalancer backend. See [Nodebalancer VPC Configuration](#nodebalancer-vpc-configuration) |
4545
| `backend-vpc-name` | string | | VPC which is connected to the NodeBalancer backend. See [Nodebalancer VPC Configuration](#nodebalancer-vpc-configuration) |
4646
| `backend-subnet-name` | string | | Subnet within VPC which is connected to the NodeBalancer backend. See [Nodebalancer VPC Configuration](#nodebalancer-vpc-configuration) |

‎docs/configuration/environment.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ The CCM supports the following flags:
5353
| `--nodebalancer-backend-ipv4-subnet-name` | String | `""` | ipv4 subnet name to use for NodeBalancer backends |
5454
| `--disable-nodebalancer-vpc-backends` | Boolean | `false` | don't use VPC specific ip-addresses for nodebalancer backend ips when running in VPC (set to `true` for backward compatibility if needed) |
5555
| `--enable-ipv6-for-loadbalancers` | Boolean | `false` | Set both IPv4 and IPv6 addresses for all LoadBalancer services (when disabled, only IPv4 is used). This can also be configured per-service using the `service.beta.kubernetes.io/linode-loadbalancer-enable-ipv6-ingress` annotation. |
56-
| `--enable-ipv6-for-nodebalancer-backends` | Boolean | `false` | Use public IPv6 addresses for non-VPC NodeBalancer service backends. This requires a dual-stack cluster and dual-stack Service configuration. If enabled, every selected backend node must have public IPv6 or reconciliation will fail. This can also be configured per-service using the `service.beta.kubernetes.io/linode-loadbalancer-enable-ipv6-backends` annotation. |
56+
| `--enable-ipv6-for-nodebalancer-backends` | Boolean | `false` | Use IPv6 addresses for NodeBalancer service backends. If VPC-backed NodeBalancers are enabled, CCM preserves the NodeBalancer VPC configuration. Enabling this flag can migrate existing eligible NodeBalancer services from IPv4 to IPv6 backends during reconcile. This requires a dual-stack cluster and dual-stack Service configuration. If enabled, every selected backend node must have the required IPv6 address or reconciliation will fail. This can also be configured per-service using the `service.beta.kubernetes.io/linode-loadbalancer-enable-ipv6-backends` annotation. |
5757
| `--node-cidr-mask-size-ipv4` | Int | `24` | ipv4 cidr mask size for pod cidrs allocated to nodes |
5858
| `--node-cidr-mask-size-ipv6` | Int | `64` | ipv6 cidr mask size for pod cidrs allocated to nodes |
5959
| `--nodebalancer-prefix` | String | `ccm` | Name prefix for NoadBalancers. |

‎docs/configuration/loadbalancer.md‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ IPv6 frontends and IPv6 backends are configured independently. Frontend IPv6 con
5050

5151
IPv6 backends require a dual-stack workload cluster. In practice, the cluster networking stack must support IPv6 NodePort traffic, and the Service itself should be created as dual-stack. A single-stack IPv4 `LoadBalancer` Service can still be annotated for IPv6 backends, but the NodeBalancer health checks and traffic path may fail because the backend NodePort is not exposed over IPv6.
5252

53-
For newly created non-VPC NodeBalancer services, you can enable public IPv6 backends globally:
53+
You can enable IPv6 backends globally for NodeBalancer services:
5454

5555
```yaml
5656
spec:
@@ -71,13 +71,12 @@ metadata:
7171
```
7272

7373
When IPv6 backends are enabled:
74-
- only non-VPC NodeBalancer services are affected
75-
- existing services are not migrated automatically
76-
- every selected backend node must have public IPv6
74+
- both VPC-backed and non-VPC-backed NodeBalancer services are affected
75+
- when VPC-backed NodeBalancers are enabled, CCM preserves the NodeBalancer VPC configuration instead of dropping it
76+
- enabling the global `--enable-ipv6-for-nodebalancer-backends` flag can migrate existing eligible NodeBalancer services from IPv4 to IPv6 backends during reconcile
77+
- every selected backend node must have an IPv6 address in the currently selected backend path
7778
- the workload cluster and Service must be configured for dual-stack networking
78-
- reconciliation fails and CCM logs an error if a selected backend node does not have public IPv6
79-
80-
VPC backend behavior is unchanged and continues to use the existing IPv4/VPC backend flow.
79+
- reconciliation fails and CCM logs an error if a selected backend node does not have the required IPv6 address
8180

8281
Recommended Service configuration for IPv6 backends:
8382

0 commit comments

Comments
 (0)