Skip to content

Commit d08a773

Browse files
prasannajsnsprasannajsns
authored
Merge pull request #21 from tomaandreisacuiu/tscuiu/microsoft.virtualnodes-ext-cli-customization
Add CLI customization for the microsoft.virtualnodes partner extension type for AKS
2 parents 612e5b0 + 89bcab5 commit d08a773

3 files changed

Lines changed: 344 additions & 0 deletions

File tree

‎src/k8s-extension/azext_k8s_extension/custom.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
from .partner_extensions.AzureMLKubernetes import AzureMLKubernetes
4545
from .partner_extensions.DataProtectionKubernetes import DataProtectionKubernetes
4646
from .partner_extensions.Dapr import Dapr
47+
from .partner_extensions.VirtualNodes import VirtualNodes
4748
from .partner_extensions.DefaultExtension import (
4849
DefaultExtension,
4950
user_confirmation_factory,
@@ -77,6 +78,7 @@ def ExtensionFactory(extension_name):
7778
"microsoft.azureml.kubernetes": AzureMLKubernetes,
7879
"microsoft.dapr": Dapr,
7980
"microsoft.dataprotection.kubernetes": DataProtectionKubernetes,
81+
"microsoft.virtualnodes": VirtualNodes,
8082
}
8183

8284
# Return the extension if we find it in the map, else return the default

‎src/k8s-extension/azext_k8s_extension/partner_extensions/VirtualNodes.py‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
# --------------------------------------------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the MIT License. See License.txt in the project root for license information.
4+
# --------------------------------------------------------------------------------------------
5+
6+
# pylint: disable=unused-argument
7+
# pylint: disable=too-many-locals
8+
9+
import ipaddress
10+
11+
from azure.cli.core.azclierror import InvalidArgumentValueError, ResourceNotFoundError
12+
from azure.cli.core.commands.client_factory import get_mgmt_service_client
13+
from azure.core.exceptions import HttpResponseError, ResourceNotFoundError as SdkResourceNotFoundError
14+
from azure.mgmt.compute import ComputeManagementClient
15+
from azure.mgmt.containerservice import ContainerServiceClient
16+
from azure.mgmt.core.tools import parse_resource_id
17+
from azure.mgmt.network import NetworkManagementClient
18+
from knack.log import get_logger
19+
20+
from ..vendored_sdks.models import Extension, PatchExtension, Scope, ScopeCluster
21+
from .DefaultExtension import DefaultExtension, user_confirmation_factory
22+
23+
logger = get_logger(__name__)
24+
25+
MIN_CPU_CORES = 4
26+
MIN_MEM_GB = 16
27+
ACI_SUBNET_PREFIX = 16
28+
RELEASE_NAMESPACE = "vn-system" # release unique namespace
29+
ACI_SUBNET_NAME = "virtualnodes-aci-subnet"
30+
ACI_DELEGATION_SERVICE_NAME = "Microsoft.ContainerInstance/containerGroups"
31+
ALLOWED_CONFIG_SETTINGS_KEYS = [
32+
"replicaCount",
33+
"admissionControllerReplicaCount",
34+
"podAnnotations",
35+
"nodeSelector",
36+
"tolerations",
37+
"affinity",
38+
"zones",
39+
"nodeLabels",
40+
]
41+
42+
43+
class VirtualNodes(DefaultExtension):
44+
def __init__(self):
45+
pass
46+
47+
def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp,
48+
extension_type, scope, auto_upgrade_minor_version, auto_upgrade_mode, release_train,
49+
version, target_namespace, release_namespace, configuration_settings,
50+
configuration_protected_settings, configuration_settings_file,
51+
configuration_protected_settings_file, plan_name, plan_publisher, plan_product):
52+
logger.info("Validating parameters and AKS cluster configuration for Microsoft.virtualnodes extension...")
53+
54+
if scope is not None and scope.lower() == "namespace":
55+
raise InvalidArgumentValueError(
56+
"Invalid scope '{}'. This extension can be installed only at 'cluster' scope.".format(scope)
57+
)
58+
59+
if cluster_type is not None and cluster_type.lower() != "managedclusters":
60+
raise InvalidArgumentValueError(
61+
"Invalid cluster type '{}'. This extension can be installed only on AKS clusters "
62+
"(cluster type 'managedClusters').".format(cluster_type)
63+
)
64+
65+
if release_namespace is not None and release_namespace != RELEASE_NAMESPACE:
66+
raise InvalidArgumentValueError(
67+
f"The '--release-namespace' argument is not configurable for Microsoft.virtualnodes extension. "
68+
f"The extension must be installed in the '{RELEASE_NAMESPACE}' namespace."
69+
)
70+
71+
aks_client = get_mgmt_service_client(cmd.cli_ctx, ContainerServiceClient)
72+
try:
73+
cluster = aks_client.managed_clusters.get(resource_group_name, cluster_name)
74+
except SdkResourceNotFoundError as e:
75+
raise ResourceNotFoundError(
76+
f"AKS cluster '{cluster_name}' was not found in resource group '{resource_group_name}'."
77+
) from e
78+
except HttpResponseError as e:
79+
raise InvalidArgumentValueError(
80+
f"Failed to get AKS cluster '{cluster_name}' in resource group '{resource_group_name}': {e.message}"
81+
) from e
82+
83+
if configuration_settings is None:
84+
configuration_settings = {}
85+
validate_configuration(configuration_settings, configuration_protected_settings, extension_type)
86+
validate_node_pools(cmd, cluster)
87+
check_aks_cluster_network_config(cluster)
88+
89+
logger.info("Validation successful. Proceeding with creating subnet and delegating it for the ACI CGs.")
90+
91+
add_and_delegate_aci_subnet(cmd, cluster, resource_group_name)
92+
93+
logger.info("Subnet created and delegated successfully. Proceeding with extension creation.")
94+
95+
scope_cluster = ScopeCluster(release_namespace=RELEASE_NAMESPACE)
96+
ext_scope = Scope(cluster=scope_cluster, namespace=None)
97+
98+
create_identity = True
99+
extension = Extension(
100+
extension_type=extension_type,
101+
auto_upgrade_minor_version=auto_upgrade_minor_version,
102+
auto_upgrade_mode=auto_upgrade_mode,
103+
release_train=release_train,
104+
version=version,
105+
scope=ext_scope,
106+
configuration_settings=configuration_settings,
107+
configuration_protected_settings=configuration_protected_settings,
108+
)
109+
return extension, name, create_identity
110+
111+
def Update(self, cmd, resource_group_name, cluster_name, auto_upgrade_minor_version, auto_upgrade_mode,
112+
release_train, version, configuration_settings, configuration_protected_settings,
113+
original_extension, yes=False):
114+
validate_allowed_keys(configuration_settings, original_extension.extension_type)
115+
validate_allowed_keys(configuration_protected_settings, original_extension.extension_type)
116+
117+
return PatchExtension(
118+
auto_upgrade_minor_version=auto_upgrade_minor_version,
119+
auto_upgrade_mode=auto_upgrade_mode,
120+
release_train=release_train,
121+
version=version,
122+
configuration_settings=configuration_settings,
123+
configuration_protected_settings=configuration_protected_settings,
124+
)
125+
126+
def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp, yes):
127+
user_confirmation_factory(cmd, yes)
128+
129+
130+
def validate_allowed_keys(config_dict, extension_type):
131+
if not config_dict:
132+
return
133+
for key in config_dict:
134+
if key not in ALLOWED_CONFIG_SETTINGS_KEYS:
135+
raise InvalidArgumentValueError(
136+
f"Unsupported configuration setting: '{key}'. "
137+
f"Only {ALLOWED_CONFIG_SETTINGS_KEYS} are allowed for extensions of type {extension_type}."
138+
)
139+
140+
141+
def validate_configuration(configuration_settings, configuration_protected_settings, extension_type):
142+
validate_allowed_keys(configuration_settings, extension_type)
143+
validate_allowed_keys(configuration_protected_settings, extension_type)
144+
145+
configuration_settings["aciSubnetName"] = ACI_SUBNET_NAME
146+
147+
148+
def validate_node_pools(cmd, cluster):
149+
compute_client = get_mgmt_service_client(cmd.cli_ctx, ComputeManagementClient)
150+
location = cluster.location
151+
vm_sizes = compute_client.virtual_machine_sizes.list(location)
152+
vm_size_dict = {vm.name: vm for vm in vm_sizes}
153+
154+
for pool in cluster.agent_pool_profiles:
155+
vm_size = pool.vm_size
156+
vm_info = vm_size_dict.get(vm_size)
157+
if vm_info and vm_info.number_of_cores >= MIN_CPU_CORES and vm_info.memory_in_mb / 1024 >= MIN_MEM_GB:
158+
return
159+
160+
raise InvalidArgumentValueError(
161+
f"Nodes selected for AKS must be at least {MIN_CPU_CORES} CPU and {MIN_MEM_GB} GB RAM "
162+
f"to accommodate virtual nodes being run on them, though they can be larger. "
163+
f"No node pool in cluster '{cluster.name}' meets the minimum requirements."
164+
)
165+
166+
167+
def check_aks_cluster_network_config(cluster):
168+
network_profile = cluster.network_profile
169+
plugin = (network_profile.network_plugin or "").lower()
170+
plugin_mode = (network_profile.network_plugin_mode or "").lower()
171+
policy = (network_profile.network_policy or "").lower()
172+
173+
if plugin != "azure" or plugin_mode not in ("", "none"):
174+
raise InvalidArgumentValueError(
175+
"VirtualNode requires Azure CNI Node Subnet as network configuration. "
176+
"This cluster does not meet the requirements."
177+
)
178+
179+
if policy != "calico":
180+
raise InvalidArgumentValueError(
181+
f"Calico network policy is not enabled for this AKS cluster. "
182+
f"It is instead: {network_profile.network_policy}."
183+
)
184+
185+
node_rg = (cluster.node_resource_group or "").lower()
186+
for pool in cluster.agent_pool_profiles or []:
187+
vnet_subnet_id = getattr(pool, "vnet_subnet_id", None)
188+
if not vnet_subnet_id:
189+
continue
190+
parsed = parse_resource_id(vnet_subnet_id)
191+
subnet_rg = (parsed.get("resource_group") or "").lower()
192+
if subnet_rg != node_rg:
193+
raise InvalidArgumentValueError(
194+
f"Microsoft.Virtualnodes extension type requires the cluster VNET to be in the node resource group "
195+
f"('{cluster.node_resource_group}'), but agent pool '{pool.name}' uses a subnet in resource group "
196+
f"'{parsed.get('resource_group')}'. BYO VNETs outside the node resource group are not supported."
197+
)
198+
199+
200+
def add_and_delegate_aci_subnet(cmd, cluster, resource_group_name):
201+
vnet_rg = cluster.node_resource_group
202+
network_client = get_mgmt_service_client(cmd.cli_ctx, NetworkManagementClient)
203+
204+
vnets = list(network_client.virtual_networks.list(vnet_rg))
205+
if not vnets:
206+
raise ResourceNotFoundError(f"No VNET found in node resource group '{vnet_rg}'.")
207+
if len(vnets) > 1:
208+
raise InvalidArgumentValueError(
209+
f"Expected exactly one VNET in node resource group '{vnet_rg}', found {len(vnets)}."
210+
)
211+
vnet = vnets[0]
212+
vnet_name = vnet.name
213+
214+
existing = next((s for s in (vnet.subnets or []) if s.name == ACI_SUBNET_NAME), None)
215+
if existing is not None:
216+
delegated = any(
217+
(d.service_name or "").lower() == ACI_DELEGATION_SERVICE_NAME.lower()
218+
for d in (existing.delegations or [])
219+
)
220+
if not delegated:
221+
raise InvalidArgumentValueError(
222+
f"Subnet '{ACI_SUBNET_NAME}' already exists in VNET '{vnet_name}' but is not delegated to "
223+
f"'{ACI_DELEGATION_SERVICE_NAME}'."
224+
)
225+
logger.info(
226+
"Reusing existing subnet '%s' in VNET '%s' (resource group '%s') that is already delegated to ACI.",
227+
ACI_SUBNET_NAME, vnet_name, vnet_rg,
228+
)
229+
return
230+
231+
new_cidr = _find_free_subnet_cidr(vnet, new_prefix=ACI_SUBNET_PREFIX)
232+
if new_cidr is None:
233+
raise InvalidArgumentValueError(
234+
f"Could not find a free /{ACI_SUBNET_PREFIX} in VNET '{vnet_name}' to create subnet '{ACI_SUBNET_NAME}'."
235+
)
236+
237+
subnet_params = {
238+
"address_prefix": str(new_cidr),
239+
"delegations": [{
240+
"name": "aci-delegation",
241+
"service_name": ACI_DELEGATION_SERVICE_NAME,
242+
}],
243+
}
244+
245+
logger.info(
246+
"Creating subnet '%s' (%s) in VNET '%s' (resource group '%s') delegated to ACI...",
247+
ACI_SUBNET_NAME, new_cidr, vnet_name, vnet_rg,
248+
)
249+
network_client.subnets.begin_create_or_update(
250+
vnet_rg, vnet_name, ACI_SUBNET_NAME, subnet_params
251+
).result()
252+
253+
254+
def _find_free_subnet_cidr(vnet, new_prefix):
255+
used = []
256+
for s in (vnet.subnets or []):
257+
for p in ([s.address_prefix] if s.address_prefix else []) + list(s.address_prefixes or []):
258+
try:
259+
used.append(ipaddress.ip_network(p, strict=False))
260+
except ValueError:
261+
continue
262+
263+
for space in (vnet.address_space.address_prefixes or []):
264+
try:
265+
vnet_net = ipaddress.ip_network(space, strict=False)
266+
except ValueError:
267+
continue
268+
if vnet_net.prefixlen > new_prefix:
269+
continue
270+
for candidate in vnet_net.subnets(new_prefix=new_prefix):
271+
if not any(candidate.overlaps(u) for u in used):
272+
return candidate
273+
return None

‎testing/test/extensions/public/VirtualNodes.Tests.ps1‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
Describe 'Azure VirtualNodes Testing' {
2+
BeforeAll {
3+
$extensionType = "microsoft.virtualnodes"
4+
$extensionName = "virtualnodes"
5+
6+
. $PSScriptRoot/../../helper/Constants.ps1
7+
. $PSScriptRoot/../../helper/Helper.ps1
8+
}
9+
10+
It 'Creates the extension and checks that it onboards correctly' {
11+
$output = az $Env:K8sExtensionName create -c $($ENVCONFIG.aksClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type managedClusters --extension-type $extensionType -n $extensionName --no-wait
12+
$? | Should -BeTrue
13+
14+
$output = az $Env:K8sExtensionName show -c $($ENVCONFIG.aksClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type managedClusters -n $extensionName
15+
$? | Should -BeTrue
16+
17+
$isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion
18+
$isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue
19+
20+
$autoUpgradeMode = ($output | ConvertFrom-Json).autoUpgradeMode
21+
$autoUpgradeMode -eq "compatible" | Should -BeTrue
22+
23+
# Loop and retry until the extension installs
24+
$n = 0
25+
do
26+
{
27+
if (Has-ExtensionData $extensionName) {
28+
break
29+
}
30+
Start-Sleep -Seconds 10
31+
$n += 1
32+
} while ($n -le $MAX_RETRY_ATTEMPTS)
33+
$n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS
34+
}
35+
36+
It "Performs a show on the extension" {
37+
$output = az $Env:K8sExtensionName show -c $($ENVCONFIG.aksClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type managedClusters -n $extensionName
38+
$? | Should -BeTrue
39+
$output | Should -Not -BeNullOrEmpty
40+
}
41+
42+
It "Lists the extensions on the cluster" {
43+
$output = az $Env:K8sExtensionName list -c $($ENVCONFIG.aksClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type managedClusters
44+
$? | Should -BeTrue
45+
46+
$output | Should -Not -BeNullOrEmpty
47+
$extensionExists = $output | ConvertFrom-Json | Where-Object { $_.extensionType -eq $extensionType }
48+
$extensionExists | Should -Not -BeNullOrEmpty
49+
}
50+
51+
It "Deletes the extension from the cluster" {
52+
$output = az $Env:K8sExtensionName delete -c $($ENVCONFIG.aksClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type managedClusters -n $extensionName --force
53+
$? | Should -BeTrue
54+
55+
# Extension should not be found on the cluster
56+
$output = az $Env:K8sExtensionName show -c $($ENVCONFIG.aksClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type managedClusters -n $extensionName
57+
$? | Should -BeFalse
58+
$output | Should -BeNullOrEmpty
59+
}
60+
61+
It "Performs another list after the delete" {
62+
$output = az $Env:K8sExtensionName list -c $($ENVCONFIG.aksClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type managedClusters
63+
$? | Should -BeTrue
64+
$output | Should -Not -BeNullOrEmpty
65+
66+
$extensionExists = $output | ConvertFrom-Json | Where-Object { $_.name -eq $extensionName }
67+
$extensionExists | Should -BeNullOrEmpty
68+
}
69+
}

0 commit comments

Comments
 (0)