Skip to content

Commit 57b1803

Browse files
jumpinthefireawesomenix
jumpinthefire
and
awesomenix
authored
fix(linux): allow secondary nics to be configured on boot (#8642)
Signed-off-by: Hossam Rabbouh <horabbouh@microsoft.com> Co-authored-by: Nishchay <awesomenix@users.noreply.github.com>
1 parent 36a7b27 commit 57b1803

17 files changed

Lines changed: 1041 additions & 15 deletions

File tree

‎aks-node-controller/parser/parser.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string {
196196
"CSE_TIMEOUT": getCSETimeout(config),
197197
"SKIP_WAAGENT_HOLD": "true",
198198
"NETWORK_ISOLATED_CLUSTER_TEST_MODE": "false", // temp: needs to be added to config
199+
"STANDARD_SECONDARY_NIC_COUNT": fmt.Sprintf("%d", config.GetNetworkConfig().GetStandardSecondaryNicCount()),
199200
}
200201

201202
for i, cert := range config.CustomCaCerts {

‎e2e/aks_model.go‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -109,19 +109,14 @@ func getAzureOverlayNetworkDualStackClusterModel(name, location, k8sSystemPoolSK
109109
networkProfile.PodCidr = to.Ptr("10.244.0.0/16")
110110
networkProfile.PodCidrs = []*string{
111111
networkProfile.PodCidr,
112-
to.Ptr("fd12:3456:789a::/64 "),
112+
to.Ptr("fd12:3456:789a::/64"),
113113
}
114114
networkProfile.ServiceCidr = to.Ptr("172.16.0.0/16")
115115
networkProfile.ServiceCidrs = []*string{
116116
networkProfile.ServiceCidr,
117117
to.Ptr("fd12:3456:789a:1::/108"),
118118
}
119119

120-
networkProfile.PodCidr = nil
121-
networkProfile.PodCidrs = nil
122-
networkProfile.ServiceCidr = nil
123-
networkProfile.ServiceCidrs = nil
124-
125120
return model
126121
}
127122

‎e2e/cache.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ var ClusterAzureOverlayNetworkDualStack = cachedFunc(clusterAzureOverlayNetworkD
198198

199199
// clusterAzureOverlayNetworkDualStack creates a dual-stack (IPv4+IPv6) Azure CNI Overlay cluster
200200
func clusterAzureOverlayNetworkDualStack(ctx context.Context, request ClusterRequest) (*Cluster, error) {
201-
model := getAzureOverlayNetworkDualStackClusterModel("abe2e-azure-overlay-dualstack-v4", request.Location, request.K8sSystemPoolSKU)
201+
model := getAzureOverlayNetworkDualStackClusterModel("abe2e-azure-overlay-dualstack-v6", request.Location, request.K8sSystemPoolSKU)
202202
return prepareCluster(ctx, model, false, false)
203203
}
204204

‎e2e/cluster.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,20 @@ func prepareCluster(ctx context.Context, clusterModel *armcontainerservice.Manag
8888
clusterModel.Name = to.Ptr(fmt.Sprintf("%s-%s", *clusterModel.Name, hash(clusterModel)))
8989
// If configureSharedVNet marked this model, create the per-cluster subnet
9090
// now that we have the final hashed name, with an auto-allocated CIDR.
91+
// Dual-stack clusters need subnets with both IPv4 and IPv6 address prefixes.
92+
isDualStack := false
93+
if clusterModel.Properties.NetworkProfile != nil {
94+
for _, family := range clusterModel.Properties.NetworkProfile.IPFamilies {
95+
if family != nil && *family == armcontainerservice.IPFamilyIPv6 {
96+
isDualStack = true
97+
break
98+
}
99+
}
100+
}
91101
subnetID, err := CachedEnsureClusterSubnet(ctx, ClusterSubnetRequest{
92102
Location: *clusterModel.Location,
93103
ClusterName: *clusterModel.Name,
104+
DualStack: isDualStack,
94105
})
95106
if err != nil {
96107
return nil, fmt.Errorf("ensuring cluster subnet: %w", err)

‎e2e/scenario_test.go‎

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3020,3 +3020,259 @@ func Test_Ubuntu2204Gen2_ImagePullIdentityBinding_Disabled_Scriptless(t *testing
30203020
},
30213021
})
30223022
}
3023+
3024+
func Test_Ubuntu2404_SecondaryNIC(t *testing.T) {
3025+
RunScenario(t, &Scenario{
3026+
Description: "Tests that a secondary NIC is properly configured via configureSecondaryNICs on Ubuntu",
3027+
Config: Config{
3028+
Cluster: ClusterKubenet,
3029+
VHD: config.VHDUbuntu2404Gen2Containerd,
3030+
// configureSecondaryNICs is new and not yet baked into released VHDs.
3031+
// The scriptless_nbc path always uses VHD scripts (DisableCustomData=true),
3032+
// so it can't pick up the new function until the next VHD release.
3033+
SkipScriptlessNBC: true,
3034+
BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
3035+
// Embed scripts in customData instead of using VHD scripts.
3036+
nbc.EnableScriptlessCSECmd = false
3037+
nbc.StandardSecondaryNICCount = 1
3038+
},
3039+
VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
3040+
addSecondaryNIC(vmss)
3041+
},
3042+
Validator: func(ctx context.Context, s *Scenario) {
3043+
ValidateFileExists(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml")
3044+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4: true")
3045+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "route-metric: 200")
3046+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "use-dns: false")
3047+
ValidateSecondaryNICUp(ctx, s, resolveSecondaryNICName(ctx, s))
3048+
},
3049+
},
3050+
})
3051+
}
3052+
3053+
func Test_AzureLinuxV3_SecondaryNIC(t *testing.T) {
3054+
RunScenario(t, &Scenario{
3055+
Description: "Tests that a secondary NIC is properly configured via configureSecondaryNICs on Azure Linux",
3056+
Config: Config{
3057+
Cluster: ClusterKubenet,
3058+
VHD: config.VHDAzureLinuxV3Gen2,
3059+
SkipScriptlessNBC: true,
3060+
BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
3061+
nbc.EnableScriptlessCSECmd = false
3062+
nbc.StandardSecondaryNICCount = 1
3063+
},
3064+
VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
3065+
addSecondaryNIC(vmss)
3066+
},
3067+
Validator: func(ctx context.Context, s *Scenario) {
3068+
ValidateFileExists(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network")
3069+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "DHCP=ipv4")
3070+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "RouteMetric=2100")
3071+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "UseDNS=false")
3072+
ValidateSecondaryNICUp(ctx, s, resolveSecondaryNICName(ctx, s))
3073+
},
3074+
},
3075+
})
3076+
}
3077+
3078+
func Test_Ubuntu2204_SecondaryNIC(t *testing.T) {
3079+
RunScenario(t, &Scenario{
3080+
Description: "Tests that a secondary NIC is properly configured via configureSecondaryNICs on Ubuntu 22.04",
3081+
Config: Config{
3082+
Cluster: ClusterKubenet,
3083+
VHD: config.VHDUbuntu2204Gen2Containerd,
3084+
SkipScriptlessNBC: true,
3085+
BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
3086+
nbc.EnableScriptlessCSECmd = false
3087+
nbc.StandardSecondaryNICCount = 1
3088+
},
3089+
VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
3090+
addSecondaryNIC(vmss)
3091+
},
3092+
Validator: func(ctx context.Context, s *Scenario) {
3093+
ValidateFileExists(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml")
3094+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4: true")
3095+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "route-metric: 200")
3096+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "use-dns: false")
3097+
ValidateSecondaryNICUp(ctx, s, resolveSecondaryNICName(ctx, s))
3098+
},
3099+
},
3100+
})
3101+
}
3102+
3103+
func Test_ACL_SecondaryNIC(t *testing.T) {
3104+
RunScenario(t, &Scenario{
3105+
Description: "Tests that a secondary NIC is properly configured via configureSecondaryNICs on ACL",
3106+
Config: Config{
3107+
Cluster: ClusterKubenet,
3108+
VHD: config.VHDACLGen2TL,
3109+
SkipScriptlessNBC: true,
3110+
BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
3111+
nbc.EnableScriptlessCSECmd = false
3112+
nbc.StandardSecondaryNICCount = 1
3113+
},
3114+
VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
3115+
vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties)
3116+
addSecondaryNIC(vmss)
3117+
},
3118+
Validator: func(ctx context.Context, s *Scenario) {
3119+
ValidateFileExists(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network")
3120+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "DHCP=ipv4")
3121+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "RouteMetric=2100")
3122+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "UseDNS=false")
3123+
ValidateSecondaryNICUp(ctx, s, resolveSecondaryNICName(ctx, s))
3124+
},
3125+
},
3126+
})
3127+
}
3128+
3129+
func Test_Ubuntu2404_SecondaryNIC_DualStack(t *testing.T) {
3130+
RunScenario(t, &Scenario{
3131+
Description: "Tests that a dual-stack secondary NIC is properly configured on Ubuntu 24.04",
3132+
Config: Config{
3133+
Cluster: ClusterAzureOverlayNetworkDualStack,
3134+
VHD: config.VHDUbuntu2404Gen2Containerd,
3135+
SkipScriptlessNBC: true,
3136+
BootstrapConfigMutator: func(c *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
3137+
nbc.EnableScriptlessCSECmd = false
3138+
nbc.StandardSecondaryNICCount = 1
3139+
if nbc.ContainerService.Properties.FeatureFlags == nil {
3140+
nbc.ContainerService.Properties.FeatureFlags = &datamodel.FeatureFlags{}
3141+
}
3142+
nbc.ContainerService.Properties.FeatureFlags.EnableIPv6DualStack = true
3143+
nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone)
3144+
nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone)
3145+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/podnetwork-type"] = "overlay"
3146+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/nodenetwork-vnetguid"] = c.VNetResourceGUID
3147+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/azure-cni-overlay"] = "true"
3148+
},
3149+
VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
3150+
DualStackVMConfigMutator(vmss)
3151+
addDualStackSecondaryNIC(vmss)
3152+
},
3153+
Validator: func(ctx context.Context, s *Scenario) {
3154+
ValidateFileExists(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml")
3155+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4: true")
3156+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp6: true")
3157+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4-overrides:")
3158+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp6-overrides:")
3159+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "route-metric: 200")
3160+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "use-dns: false")
3161+
ValidateSecondaryNICDualStack(ctx, s, resolveSecondaryNICName(ctx, s))
3162+
},
3163+
},
3164+
})
3165+
}
3166+
3167+
func Test_Ubuntu2204_SecondaryNIC_DualStack(t *testing.T) {
3168+
RunScenario(t, &Scenario{
3169+
Description: "Tests that a dual-stack secondary NIC is properly configured on Ubuntu 22.04",
3170+
Config: Config{
3171+
Cluster: ClusterAzureOverlayNetworkDualStack,
3172+
VHD: config.VHDUbuntu2204Gen2Containerd,
3173+
SkipScriptlessNBC: true,
3174+
BootstrapConfigMutator: func(c *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
3175+
nbc.EnableScriptlessCSECmd = false
3176+
nbc.StandardSecondaryNICCount = 1
3177+
if nbc.ContainerService.Properties.FeatureFlags == nil {
3178+
nbc.ContainerService.Properties.FeatureFlags = &datamodel.FeatureFlags{}
3179+
}
3180+
nbc.ContainerService.Properties.FeatureFlags.EnableIPv6DualStack = true
3181+
nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone)
3182+
nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone)
3183+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/podnetwork-type"] = "overlay"
3184+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/nodenetwork-vnetguid"] = c.VNetResourceGUID
3185+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/azure-cni-overlay"] = "true"
3186+
},
3187+
VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
3188+
DualStackVMConfigMutator(vmss)
3189+
addDualStackSecondaryNIC(vmss)
3190+
},
3191+
Validator: func(ctx context.Context, s *Scenario) {
3192+
ValidateFileExists(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml")
3193+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4: true")
3194+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp6: true")
3195+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp4-overrides:")
3196+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "dhcp6-overrides:")
3197+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "route-metric: 200")
3198+
ValidateFileHasContent(ctx, s, "/etc/netplan/60-secondary-nic-1.yaml", "use-dns: false")
3199+
ValidateSecondaryNICDualStack(ctx, s, resolveSecondaryNICName(ctx, s))
3200+
},
3201+
},
3202+
})
3203+
}
3204+
3205+
func Test_AzureLinuxV3_SecondaryNIC_DualStack(t *testing.T) {
3206+
RunScenario(t, &Scenario{
3207+
Description: "Tests that a dual-stack secondary NIC is properly configured on Azure Linux",
3208+
Config: Config{
3209+
Cluster: ClusterAzureOverlayNetworkDualStack,
3210+
VHD: config.VHDAzureLinuxV3Gen2,
3211+
SkipScriptlessNBC: true,
3212+
BootstrapConfigMutator: func(c *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
3213+
nbc.EnableScriptlessCSECmd = false
3214+
nbc.StandardSecondaryNICCount = 1
3215+
if nbc.ContainerService.Properties.FeatureFlags == nil {
3216+
nbc.ContainerService.Properties.FeatureFlags = &datamodel.FeatureFlags{}
3217+
}
3218+
nbc.ContainerService.Properties.FeatureFlags.EnableIPv6DualStack = true
3219+
nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone)
3220+
nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone)
3221+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/podnetwork-type"] = "overlay"
3222+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/nodenetwork-vnetguid"] = c.VNetResourceGUID
3223+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/azure-cni-overlay"] = "true"
3224+
},
3225+
VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
3226+
DualStackVMConfigMutator(vmss)
3227+
addDualStackSecondaryNIC(vmss)
3228+
},
3229+
Validator: func(ctx context.Context, s *Scenario) {
3230+
ValidateFileExists(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network")
3231+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "DHCP=yes")
3232+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "IPv6AcceptRA=yes")
3233+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "[DHCPv6]")
3234+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "RouteMetric=2100")
3235+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "UseDNS=false")
3236+
ValidateSecondaryNICDualStack(ctx, s, resolveSecondaryNICName(ctx, s))
3237+
},
3238+
},
3239+
})
3240+
}
3241+
3242+
func Test_ACL_SecondaryNIC_DualStack(t *testing.T) {
3243+
RunScenario(t, &Scenario{
3244+
Description: "Tests that a dual-stack secondary NIC is properly configured on ACL",
3245+
Config: Config{
3246+
Cluster: ClusterAzureOverlayNetworkDualStack,
3247+
VHD: config.VHDACLGen2TL,
3248+
SkipScriptlessNBC: true,
3249+
BootstrapConfigMutator: func(c *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
3250+
nbc.EnableScriptlessCSECmd = false
3251+
nbc.StandardSecondaryNICCount = 1
3252+
if nbc.ContainerService.Properties.FeatureFlags == nil {
3253+
nbc.ContainerService.Properties.FeatureFlags = &datamodel.FeatureFlags{}
3254+
}
3255+
nbc.ContainerService.Properties.FeatureFlags.EnableIPv6DualStack = true
3256+
nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone)
3257+
nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginNone)
3258+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/podnetwork-type"] = "overlay"
3259+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/nodenetwork-vnetguid"] = c.VNetResourceGUID
3260+
nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/azure-cni-overlay"] = "true"
3261+
},
3262+
VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
3263+
vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties)
3264+
DualStackVMConfigMutator(vmss)
3265+
addDualStackSecondaryNIC(vmss)
3266+
},
3267+
Validator: func(ctx context.Context, s *Scenario) {
3268+
ValidateFileExists(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network")
3269+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "DHCP=yes")
3270+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "IPv6AcceptRA=yes")
3271+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "[DHCPv6]")
3272+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "RouteMetric=2100")
3273+
ValidateFileHasContent(ctx, s, "/etc/systemd/network/10-secondary-nic-1.network", "UseDNS=false")
3274+
ValidateSecondaryNICDualStack(ctx, s, resolveSecondaryNICName(ctx, s))
3275+
},
3276+
},
3277+
})
3278+
}

0 commit comments

Comments
 (0)