Merge pull request #18262 from hakman/azure-cloud-routes

azure: enable CCM cloud routes for kubenet and kindnet

Author: k8s-ci-robot

nodeup/pkg/model/cloudconfig.go

@@ -50,6 +50,7 @@ type azureCloudConfig struct {
 	Location                    string `json:"location,omitempty"`
 	VnetName                    string `json:"vnetName,omitempty"`
 	SubnetName                  string `json:"subnetName,omitempty"`
+	RouteTableName              string `json:"routeTableName,omitempty"`
 	SecurityGroupName           string `json:"securityGroupName,omitempty"`
 	UseInstanceMetadata         bool   `json:"useInstanceMetadata,omitempty"`
 	DisableAvailabilitySetNodes bool   `json:"disableAvailabilitySetNodes,omitempty"`
@@ -133,6 +134,7 @@ func (b *CloudConfigBuilder) build(c *fi.NodeupModelBuilderContext, inTree bool)
 			Location:                    b.NodeupConfig.AzureLocation,
 			VnetName:                    vnetName,
 			SubnetName:                  b.NodeupConfig.AzureSubnetName,
+			RouteTableName:              b.NodeupConfig.AzureRouteTableName,
 			SecurityGroupName:           b.NodeupConfig.AzureSecurityGroupName,
 			UseInstanceMetadata:         true,
 			DisableAvailabilitySetNodes: true,

nodeup/pkg/model/cloudconfig_test.go

@@ -105,6 +105,7 @@ func TestBuildAzure(t *testing.T) {
 		ResourceGroup:               resourceGroupName,
 		VnetName:                    vnetName,
 		SubnetName:                  "test-subnet",
+		RouteTableName:              cluster.Name,
 		SecurityGroupName:           vnetName,
 		UseInstanceMetadata:         true,
 		UseManagedIdentityExtension: true,

pkg/model/azuremodel/network.go

@@ -180,6 +180,22 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 		DestinationApplicationSecurityGroupNames: []*string{fi.PtrTo(b.NameForApplicationSecurityGroupNodes())},
 		DestinationPortRange:                     fi.PtrTo("*"),
 	})
+	// Kindnet preserves pod-CIDR source IPs to host services; Azure NSG ASG
+	// matching needs an explicit allow since pod IPs aren't NIC-assigned.
+	// Scoped to nodes to keep DenyAllToControlPlane and the etcd denies effective.
+	if b.Cluster.Spec.Networking.Kindnet != nil && b.Cluster.Spec.Networking.PodCIDR != "" {
+		nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
+			Name:                                     fi.PtrTo("AllowPodCIDRToNodes"),
+			Priority:                                 fi.PtrTo[int32](1006),
+			Access:                                   network.SecurityRuleAccessAllow,
+			Direction:                                network.SecurityRuleDirectionInbound,
+			Protocol:                                 network.SecurityRuleProtocolAsterisk,
+			SourceAddressPrefix:                      fi.PtrTo(b.Cluster.Spec.Networking.PodCIDR),
+			SourcePortRange:                          fi.PtrTo("*"),
+			DestinationApplicationSecurityGroupNames: []*string{fi.PtrTo(b.NameForApplicationSecurityGroupNodes())},
+			DestinationPortRange:                     fi.PtrTo("*"),
+		})
+	}
 	etcdPeerMax := wellknownports.EtcdEventsPeerPort
 	for _, c := range b.Cluster.Spec.EtcdClusters {
 		if c.Name == "leases" {

pkg/model/components/azurecloudcontrollermanager.go

@@ -55,5 +55,16 @@ func (b *AzureCloudControllerManagerOptionsBuilder) BuildOptions(cluster *kops.C
 		eccm.LogLevel = 2
 	}
 
+	// Kubenet and kindnet rely on cloud routes to deliver pod-to-pod traffic across nodes:
+	// the Azure VNet is L3 and drops packets whose destination IP is not a known VNet IP
+	// (pod CIDRs are not), so the Azure CCM must populate UDRs in the route table that
+	// kOps associates with the node subnet. EnableIPForwarding is already set on every NIC.
+	networking := cluster.Spec.Networking
+	if networking.Kubenet != nil || networking.Kindnet != nil {
+		eccm.ConfigureCloudRoutes = fi.PtrTo(true)
+	} else {
+		eccm.ConfigureCloudRoutes = fi.PtrTo(false)
+	}
+
 	return nil
 }

tests/integration/update_cluster/gossip-azure/data/azurerm_storage_blob_cluster-completed.spec_source

@@ -19,6 +19,7 @@ spec:
     manageStorageClasses: true
   cloudControllerManager:
     azureNodeManagerImage: mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.34.3
+    configureCloudRoutes: false
     image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.34.3
     leaderElection:
       leaderElect: true

tests/integration/update_cluster/minimal_azure/data/azurerm_storage_blob_cluster-completed.spec_source

@@ -20,6 +20,7 @@ spec:
     manageStorageClasses: true
   cloudControllerManager:
     azureNodeManagerImage: mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.34.3
+    configureCloudRoutes: false
     image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.34.3
     leaderElection:
       leaderElect: true

upup/models/cloudup/resources/addons/azure-cloud-controller.addons.k8s.io/helm-values.yaml

@@ -10,10 +10,10 @@ infra:
 cloudControllerManager:
   # Overridden by kustomize patch.
   imageTag: "v0.0.0"
-  # No CNI on Azure uses node CIDR allocation.
+  # KCM allocates pod CIDRs; CCM only writes UDRs into the route table.
   allocateNodeCidrs: "false"
-  # No CNI on Azure uses CCM-managed routes.
-  configureCloudRoutes: "false"
+  # Enabled for kubenet/kindnet, which require UDRs to forward pod-CIDR traffic across nodes.
+  configureCloudRoutes: "{{ WithDefaultBool .ExternalCloudControllerManager.ConfigureCloudRoutes false }}"
   clusterCIDR: "{{ .Networking.PodCIDR }}"
   leaderElect: "{{ .ExternalCloudControllerManager.LeaderElection.LeaderElect }}"
   logVerbosity: "{{ .ExternalCloudControllerManager.LogLevel }}"

upup/models/cloudup/resources/addons/azure-cloud-controller.addons.k8s.io/k8s-1.31.yaml.template

@@ -213,7 +213,7 @@ spec:
         - --cloud-provider=azure
         - --cluster-cidr={{ .Networking.PodCIDR }}
         - --cluster-name={{ ClusterName }}
-        - --configure-cloud-routes=false
+        - --configure-cloud-routes={{ WithDefaultBool .ExternalCloudControllerManager.ConfigureCloudRoutes false }}
         - --controllers=*,-cloud-node
         - --leader-elect={{ .ExternalCloudControllerManager.LeaderElection.LeaderElect
           }}