fix(lb): prevent empty backend pools on IPAM failures

When IPAM queries fail with 5xx errors during node initialization,
InternalIPs are never populated, causing SetBackendServers to be
called with an empty list.

This adds two layers of defense:
- Fallback to direct IPAM query if node addresses are empty
- Refuse to clear existing backends when no new IPs are found

Signed-off-by: Andreas Wachs <awa@corti.ai>
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: andreaswachs

scaleway/loadbalancers.go

@@ -608,8 +608,28 @@ func (l *loadbalancers) updateLoadBalancer(ctx context.Context, loadbalancer *sc
 	}
 
 	var targetIPs []string
-	if getForceInternalIP(service) || l.pnID != "" {
+	pnIDs := getPrivateNetworkIDs(service)
+	if getForceInternalIP(service) || l.pnID != "" || len(pnIDs) > 0 {
 		targetIPs = extractNodesInternalIps(nodes)
+
+		// Fallback: if no internal IPs and we're in private network mode, try IPAM directly
+		if len(targetIPs) == 0 && (l.pnID != "" || len(pnIDs) > 0) {
+			klog.Warningf("no internal IPs in node status for service %s/%s, attempting IPAM fallback",
+				service.Namespace, service.Name)
+
+			fallbackIPs, fallbackErr := l.getNodeIPsFromIPAM(nodes, loadbalancer.Zone)
+			if fallbackErr != nil {
+				klog.Errorf("IPAM fallback failed for service %s/%s: %v", service.Namespace, service.Name, fallbackErr)
+			} else {
+				targetIPs = fallbackIPs
+			}
+		}
+
+		if len(targetIPs) == 0 {
+			klog.Errorf("CRITICAL: no backend IPs found for service %s/%s with pnID %s",
+				service.Namespace, service.Name, l.pnID)
+		}
+
 		klog.V(3).Infof("using internal nodes ips: %s on loadbalancer %s", strings.Join(targetIPs, ","), loadbalancer.ID)
 	} else {
 		targetIPs = extractNodesExternalIps(nodes)
@@ -700,6 +720,13 @@ func (l *loadbalancers) updateLoadBalancer(ctx context.Context, loadbalancer *sc
 
 		// Update backend servers
 		if !stringArrayEqual(backend.Pool, targetIPs) {
+			// Safety: don't clear existing backends if no new IPs and pool isn't empty
+			if len(targetIPs) == 0 && len(backend.Pool) > 0 {
+				klog.Warningf("refusing to clear backend pool for backend %s on loadbalancer %s - keeping %d servers",
+					backend.ID, loadbalancer.ID, len(backend.Pool))
+				continue
+			}
+
 			klog.V(3).Infof("update server list for backend: %s port: %d loadbalancer: %s", backend.ID, port.NodePort, loadbalancer.ID)
 			if _, err := l.api.SetBackendServers(&scwlb.ZonedAPISetBackendServersRequest{
 				Zone:      loadbalancer.Zone,
@@ -882,6 +909,57 @@ func (l *loadbalancers) attachPrivateNetworks(loadbalancer *scwlb.LB, service *v
 	return nil
 }
 
+// getNodeIPsFromIPAM queries IPAM directly to get private network IPs for nodes.
+// This is used as a fallback when node.Status.Addresses doesn't contain InternalIPs.
+func (l *loadbalancers) getNodeIPsFromIPAM(nodes []*v1.Node, zone scw.Zone) ([]string, error) {
+	if l.pnID == "" {
+		return nil, fmt.Errorf("cannot query IPAM without private network ID")
+	}
+
+	region, err := zone.Region()
+	if err != nil {
+		return nil, fmt.Errorf("unable to get region from zone %s: %v", zone, err)
+	}
+
+	var ips []string
+	for _, node := range nodes {
+		if node.Spec.ProviderID == "" {
+			klog.Warningf("node %s has no provider ID, skipping IPAM lookup", node.Name)
+			continue
+		}
+
+		_, _, serverID, err := ServerInfoFromProviderID(node.Spec.ProviderID)
+		if err != nil {
+			klog.Warningf("failed to parse provider ID for node %s: %v", node.Name, err)
+			continue
+		}
+
+		// Query IPAM for IPs on the configured private network for this server
+		ipamRes, err := l.ipam.ListIPs(&scwipam.ListIPsRequest{
+			ResourceType:     scwipam.ResourceTypeInstancePrivateNic,
+			IsIPv6:           scw.BoolPtr(false),
+			Region:           region,
+			PrivateNetworkID: &l.pnID,
+			ResourceName:     &serverID,
+		})
+		if err != nil {
+			klog.Warningf("IPAM query failed for node %s (server %s): %v", node.Name, serverID, err)
+			continue
+		}
+
+		for _, ip := range ipamRes.IPs {
+			ips = append(ips, ip.Address.IP.String())
+		}
+	}
+
+	if len(ips) == 0 {
+		return nil, fmt.Errorf("no IPs found via IPAM for any nodes")
+	}
+
+	klog.V(3).Infof("IPAM fallback found %d IPs: %v", len(ips), ips)
+	return ips, nil
+}
+
 // createPrivateServiceStatus creates a LoadBalancer status for services with private load balancers
 func (l *loadbalancers) createPrivateServiceStatus(service *v1.Service, lb *scwlb.LB, ipMode *v1.LoadBalancerIPMode) (*v1.LoadBalancerStatus, error) {
 	if l.pnID == "" {

scaleway/loadbalancers_test.go

@@ -2561,3 +2561,109 @@ func TestServicePortToBackendWithHealthCheckFromService(t *testing.T) {
 		})
 	}
 }
+
+func TestBackendPreservation(t *testing.T) {
+	// Test that stringArrayEqual correctly identifies when pools differ
+	t.Run("EmptyTargetIPsVsNonEmptyPool", func(t *testing.T) {
+		existingPool := []string{"10.0.0.1", "10.0.0.2"}
+		targetIPs := []string{}
+
+		// These are not equal, so the update logic would trigger
+		equal := stringArrayEqual(existingPool, targetIPs)
+		if equal {
+			t.Error("Expected pools to be unequal")
+		}
+
+		// The safety check should prevent clearing: len(targetIPs) == 0 && len(existingPool) > 0
+		shouldPreserve := len(targetIPs) == 0 && len(existingPool) > 0
+		if !shouldPreserve {
+			t.Error("Expected preservation logic to trigger")
+		}
+	})
+
+	t.Run("BothEmpty", func(t *testing.T) {
+		existingPool := []string{}
+		targetIPs := []string{}
+
+		equal := stringArrayEqual(existingPool, targetIPs)
+		if !equal {
+			t.Error("Expected empty pools to be equal")
+		}
+
+		// No preservation needed - both are empty
+		shouldPreserve := len(targetIPs) == 0 && len(existingPool) > 0
+		if shouldPreserve {
+			t.Error("Should not preserve when both pools are empty")
+		}
+	})
+
+	t.Run("NonEmptyTargetIPs", func(t *testing.T) {
+		existingPool := []string{"10.0.0.1", "10.0.0.2"}
+		targetIPs := []string{"10.0.0.3"}
+
+		equal := stringArrayEqual(existingPool, targetIPs)
+		if equal {
+			t.Error("Expected pools to be unequal")
+		}
+
+		// Should NOT preserve - we have new IPs to set
+		shouldPreserve := len(targetIPs) == 0 && len(existingPool) > 0
+		if shouldPreserve {
+			t.Error("Should not preserve when we have new target IPs")
+		}
+	})
+}
+
+func TestExtractNodesInternalIps(t *testing.T) {
+	t.Run("NodesWithInternalIPs", func(t *testing.T) {
+		nodes := []*v1.Node{
+			{
+				Status: v1.NodeStatus{
+					Addresses: []v1.NodeAddress{
+						{Type: v1.NodeInternalIP, Address: "10.0.0.1"},
+						{Type: v1.NodeExternalIP, Address: "1.2.3.4"},
+					},
+				},
+			},
+			{
+				Status: v1.NodeStatus{
+					Addresses: []v1.NodeAddress{
+						{Type: v1.NodeInternalIP, Address: "10.0.0.2"},
+					},
+				},
+			},
+		}
+
+		ips := extractNodesInternalIps(nodes)
+		if len(ips) != 2 {
+			t.Errorf("Expected 2 IPs, got %d", len(ips))
+		}
+	})
+
+	t.Run("NodesWithoutInternalIPs", func(t *testing.T) {
+		nodes := []*v1.Node{
+			{
+				Status: v1.NodeStatus{
+					Addresses: []v1.NodeAddress{
+						{Type: v1.NodeExternalIP, Address: "1.2.3.4"},
+						{Type: v1.NodeHostName, Address: "node1"},
+					},
+				},
+			},
+		}
+
+		ips := extractNodesInternalIps(nodes)
+		if len(ips) != 0 {
+			t.Errorf("Expected 0 IPs when no internal IPs, got %d", len(ips))
+		}
+	})
+
+	t.Run("EmptyNodes", func(t *testing.T) {
+		nodes := []*v1.Node{}
+
+		ips := extractNodesInternalIps(nodes)
+		if len(ips) != 0 {
+			t.Errorf("Expected 0 IPs for empty nodes, got %d", len(ips))
+		}
+	})
+}