Skip to content

Commit e1b6c73

Browse files
authored
feat: consume NI kubernetes packages on MAR (#7134)
Signed-off-by: Billy Zha <jinzha1@microsoft.com>
1 parent 274c20a commit e1b6c73

109 files changed

Lines changed: 916 additions & 499 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

e2e/scenario_test.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2032,6 +2032,59 @@ func Test_Ubuntu2204_PMC_Install(t *testing.T) {
20322032
})
20332033
}
20342034

2035+
func Test_Ubuntu2204Gen2_ContainerdAirgappedNonAnonymousK8sNotCached_InstallPackage(t *testing.T) {
2036+
location := config.Config.DefaultLocation
2037+
ctx := newTestCtx(t)
2038+
identity, err := config.Azure.UserAssignedIdentities.Get(ctx, config.ResourceGroupName(location), config.VMIdentityName, nil)
2039+
require.NoError(t, err)
2040+
RunScenario(t, &Scenario{
2041+
Description: "Tests that a node using the Ubuntu 2204 VHD without k8s binary and is airgap can be properly bootstrapped",
2042+
Tags: Tags{
2043+
Airgap: true,
2044+
NonAnonymousACR: true,
2045+
},
2046+
Config: Config{
2047+
Cluster: ClusterKubenetAirgapNonAnon,
2048+
VHD: config.VHDUbuntu2204Gen2ContainerdAirgappedK8sNotCached,
2049+
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
2050+
nbc.TenantID = *identity.Properties.TenantID
2051+
nbc.UserAssignedIdentityClientID = *identity.Properties.ClientID
2052+
nbc.OutboundType = datamodel.OutboundTypeBlock
2053+
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
2054+
PrivateEgress: &datamodel.PrivateEgress{
2055+
Enabled: true,
2056+
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io", config.PrivateACRNameNotAnon(location)),
2057+
},
2058+
}
2059+
nbc.AgentPoolProfile.LocalDNSProfile = nil
2060+
nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = true
2061+
nbc.AgentPoolProfile.KubernetesConfig.UseManagedIdentity = true
2062+
// intentionally using private acr url to get kube binaries
2063+
nbc.AgentPoolProfile.KubernetesConfig.CustomKubeBinaryURL = fmt.Sprintf(
2064+
"%s.azurecr.io/oss/binaries/kubernetes/kubernetes-node:v%s-linux-amd64",
2065+
config.PrivateACRNameNotAnon(location),
2066+
nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
2067+
nbc.K8sComponents.LinuxCredentialProviderURL = fmt.Sprintf(
2068+
"https://packages.aks.azure.com/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-linux-amd64-v%s.tar.gz",
2069+
nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion,
2070+
nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
2071+
nbc.KubeletConfig["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml"
2072+
nbc.KubeletConfig["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider"
2073+
nbc.KubeletConfig["--pod-infra-container-image"] = "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6"
2074+
},
2075+
VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
2076+
if vmss.Tags == nil {
2077+
vmss.Tags = map[string]*string{}
2078+
}
2079+
vmss.Tags["ShouldEnforceKubePMCInstall"] = to.Ptr("true")
2080+
},
2081+
Validator: func(ctx context.Context, s *Scenario) {
2082+
ValidateDirectoryContent(ctx, s, "/run", []string{"outbound-check-skipped"})
2083+
},
2084+
},
2085+
})
2086+
}
2087+
20352088
func Test_AzureLinux3OSGuard_PMC_Install(t *testing.T) {
20362089
RunScenario(t, &Scenario{
20372090
Description: "Tests that a node using an Azure Linux V3 OS Guard VHD and install kube pkgs from PMC can be properly bootstrapped",

parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,4 +77,9 @@ cleanUpGPUDrivers() {
7777
stub
7878
}
7979

80-
#EOF
80+
installToolFromLocalRepo() {
81+
stub
82+
return 1
83+
}
84+
85+
#EOF

parts/linux/cloud-init/artifacts/cse_config.sh

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -525,14 +525,22 @@ EOF
525525
}
526526

527527
configureKubeletAndKubectl() {
528-
# Install kubelet and kubectl binaries from URL for Network Isolated, Custom Kube binary, and Private Kube binary
529-
if [ -n "${CUSTOM_KUBE_BINARY_DOWNLOAD_URL}" ] || [ -n "${PRIVATE_KUBE_BINARY_DOWNLOAD_URL}" ] || [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then
528+
# Install kubelet and kubectl binaries from URL for Custom Kube binary and Private Kube binary
529+
if [ -n "${CUSTOM_KUBE_BINARY_DOWNLOAD_URL}" ] || [ -n "${PRIVATE_KUBE_BINARY_DOWNLOAD_URL}" ]; then
530530
logs_to_events "AKS.CSE.configureKubeletAndKubectl.installKubeletKubectlFromURL" installKubeletKubectlFromURL
531531
# only install kube pkgs from pmc if k8s version >= 1.34.0 or skip_bypass_k8s_version_check is true
532532
elif [ "${SHOULD_ENFORCE_KUBE_PMC_INSTALL}" != "true" ] && ! semverCompare ${KUBERNETES_VERSION:-"0.0.0"} "1.34.0"; then
533533
logs_to_events "AKS.CSE.configureKubeletAndKubectl.installKubeletKubectlFromURL" installKubeletKubectlFromURL
534534
else
535-
if isMarinerOrAzureLinux "$OS"; then
535+
if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ] ; then
536+
# For network isolated clusters, try distro packages first and fallback to binary installation
537+
if ! logs_to_events "AKS.CSE.configureKubeletAndKubectl.installK8sToolsFromBootstrapProfileRegistry" "installK8sToolsFromBootstrapProfileRegistry ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ${KUBERNETES_VERSION}" && [ "${SHOULD_ENFORCE_KUBE_PMC_INSTALL}" != "true" ]; then
538+
# SHOULD_ENFORCE_KUBE_PMC_INSTALL will only be set for e2e tests, which should not fallback to reflect result of package installation behavior
539+
# TODO: remove SHOULD_ENFORCE_KUBE_PMC_INSTALL check when the test cluster supports > 1.34.0 case
540+
# if install from bootstrap profile registry fails, fallback to install from URL
541+
logs_to_events "AKS.CSE.configureKubeletAndKubectl.installKubeletKubectlFromURL-Fallback" installKubeletKubectlFromURL
542+
fi
543+
elif isMarinerOrAzureLinux "$OS"; then
536544
if [ "$OS_VERSION" = "2.0" ]; then
537545
# we do not publish packages to PMC for azurelinux V2
538546
logs_to_events "AKS.CSE.configureKubeletAndKubectl.installKubeletKubectlFromURL" installKubeletKubectlFromURL

parts/linux/cloud-init/artifacts/cse_helpers.sh

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -325,26 +325,37 @@ retrycmd_curl_file() {
325325
_retry_file_curl_internal "$curl_retries" "$wait_sleep" "$timeout" "$filepath" "$url" "$check_file_exists"
326326
}
327327

328-
retrycmd_get_tarball_from_registry_with_oras() {
329-
tar_retries=$1; wait_sleep=$2; tarball=$3; url=$4
330-
tar_folder=$(dirname "$tarball")
331-
echo "${tar_retries} retries"
332-
for i in $(seq 1 $tar_retries); do
333-
[ -f "$tarball" ] && tar -tzf "$tarball" && break || \
334-
if [ "$i" -eq "$tar_retries" ]; then
328+
retrycmd_pull_from_registry_with_oras() {
329+
pull_retries=$1; wait_sleep=$2; target_folder=$3; url=$4
330+
shift 4 # Remove first 4 parameters, remaining parameters are extra oras flags
331+
echo "${pull_retries} retries"
332+
for i in $(seq 1 $pull_retries); do
333+
if [ "$i" -eq "$pull_retries" ]; then
335334
return 1
335+
fi
336+
if [ "$i" -gt 1 ]; then
337+
sleep $wait_sleep
338+
fi
339+
timeout 60 oras pull "$url" -o "$target_folder" --registry-config "${ORAS_REGISTRY_CONFIG_FILE}" "$@" > $ORAS_OUTPUT 2>&1
340+
if [ "$?" -eq 0 ]; then
341+
return 0
336342
else
337-
if [ "$i" -gt 1 ]; then
338-
sleep $wait_sleep
339-
fi
340-
timeout 60 oras pull $url -o $tar_folder --registry-config ${ORAS_REGISTRY_CONFIG_FILE} > $ORAS_OUTPUT 2>&1
341-
if [ "$?" -ne 0 ]; then
342-
cat $ORAS_OUTPUT
343-
fi
343+
cat $ORAS_OUTPUT
344344
fi
345345
done
346346
}
347347

348+
retrycmd_get_tarball_from_registry_with_oras() {
349+
tar_retries=$1; wait_sleep=$2; tarball=$3; url=$4
350+
if [ -f "$tarball" ] && tar -tzf "$tarball" > /dev/null 2>&1; then
351+
# skip if tarball exists and is valid
352+
return 0
353+
fi
354+
355+
tar_folder=$(dirname "$tarball")
356+
retrycmd_pull_from_registry_with_oras "$tar_retries" "$wait_sleep" "$tar_folder" "$url"
357+
}
358+
348359
retrycmd_cp_oci_layout_with_oras() {
349360
retries=$1; wait_sleep=$2; path=$3; tag=$4; url=$5
350361
mkdir -p "$path"
@@ -1197,6 +1208,12 @@ extract_tarball() {
11971208
esac
11981209
}
11991210

1211+
# Returns a list of Kubernetes tool names that need to be installed
1212+
# Usage: for tool in $(get_kubernetes_tools); do ... done
1213+
get_kubernetes_tools() {
1214+
echo "kubelet kubectl"
1215+
}
1216+
12001217
function get_sandbox_image(){
12011218
sandbox_image=$(get_sandbox_image_from_containerd_config "/etc/containerd/config.toml")
12021219
if [ -z "$sandbox_image" ]; then

parts/linux/cloud-init/artifacts/cse_install.sh

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -526,6 +526,46 @@ extractKubeBinaries() {
526526
extractKubeBinariesToUsrLocalBin "${k8s_tgz_tmp}" "${k8s_version}" "${is_private_url}"
527527
}
528528

529+
installK8sToolsFromBootstrapProfileRegistry() {
530+
local registry_server=$1
531+
local kubernetes_version=$2
532+
533+
# Try to pull distro-specific packages (e.g., .deb for Ubuntu) from registry
534+
local download_root="/tmp/kubernetes/downloads" # /opt folder will return permission error
535+
536+
for tool_name in $(get_kubernetes_tools); do
537+
tool_package_url="${registry_server}/aks/packages/kubernetes/${tool_name}:v${kubernetes_version}"
538+
tool_download_dir="${download_root}/${tool_name}"
539+
mkdir -p "${tool_download_dir}"
540+
541+
# Construct platform string for ORAS pull
542+
os_version=$(getOsVersion) || return 1 # Platform-specific OS version detection
543+
platform_flag="--platform=linux/${CPU_ARCH}:${OS,,} ${os_version}"
544+
545+
echo "Attempting to pull ${tool_name} package from ${tool_package_url} with platform ${platform_flag}"
546+
# retrycmd_pull_from_registry_with_oras will pull all artifacts to the directory with platform selection
547+
if ! retrycmd_pull_from_registry_with_oras 10 5 "${tool_download_dir}" "${tool_package_url}" "${platform_flag}"; then
548+
echo "Failed to pull ${tool_name} package from registry"
549+
rm -rf "${tool_download_dir}"
550+
return 1
551+
fi
552+
553+
echo "Successfully pulled ${tool_name} package"
554+
555+
# Try to install using distro-specific package installer from local repo
556+
installation_root="/usr/local/bin"
557+
if ! installToolFromLocalRepo "${tool_name}" "${tool_download_dir}" "${installation_root}"; then
558+
echo "Failed to install ${tool_name} from local repo ${tool_download_dir}"
559+
rm -rf "${tool_download_dir}"
560+
return 1
561+
fi
562+
done
563+
564+
# All tools installed successfully
565+
rm -rf "${download_root}"
566+
return 0
567+
}
568+
529569
installKubeletKubectlFromURL() {
530570
# when both, custom and private urls for kubernetes packages are set, custom url will be used and private url will be ignored
531571
CUSTOM_KUBE_BINARY_DOWNLOAD_URL="${CUSTOM_KUBE_BINARY_URL:=}"

parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,14 @@ cleanUpGPUDrivers() {
3232
rm -Rf $GPU_DEST /opt/gpu
3333
}
3434

35+
installToolFromLocalRepo() {
36+
stub
37+
return 1
38+
}
39+
40+
getOsVersion() {
41+
stub
42+
return 1
43+
}
44+
3545
#EOF

parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,16 @@ installKubeletKubectlPkgFromPMC() {
175175
installRPMPackageFromFile "kubectl" $desiredVersion || exit $ERR_KUBECTL_INSTALL_FAIL
176176
}
177177

178+
installToolFromLocalRepo() {
179+
echo "installToolFromLocalRepo is not yet implemented for Mariner"
180+
return 1
181+
}
182+
183+
getOsVersion() {
184+
echo "getOsVersion is not yet implemented for Mariner"
185+
return 1
186+
}
187+
178188
updateDnfWithNvidiaPkg() {
179189
if [ "$OS_VERSION" != "3.0" ]; then
180190
echo "NVIDIA repo setup is only supported on Azure Linux 3.0"

parts/linux/cloud-init/artifacts/ubuntu/cse_helpers_ubuntu.sh

Lines changed: 89 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,18 @@ wait_for_apt_locks() {
2121
sleep 3
2222
done
2323
}
24-
apt_get_update() {
25-
retries=10
26-
apt_update_output=/tmp/apt-get-update.out
24+
# Core update function used by apt_get_update and apt_get_install_from_local_repo
25+
_apt_get_update() {
26+
local retries=$1
27+
local apt_opts=$2
28+
local apt_update_output=/tmp/apt-get-update.out
29+
2730
for i in $(seq 1 $retries); do
2831
wait_for_apt_locks
2932
export DEBIAN_FRONTEND=noninteractive
3033
dpkg --configure -a --force-confdef
31-
apt-get -f -y install
32-
! (apt-get update 2>&1 | tee $apt_update_output | grep -E "^([WE]:.*)|^([Ee][Rr][Rr][Oo][Rr].*)$") && \
34+
apt-get ${apt_opts} -f -y install
35+
! (apt-get ${apt_opts} update 2>&1 | tee $apt_update_output | grep -E "^([WE]:.*)|^([Ee][Rr][Rr][Oo][Rr].*)$") && \
3336
cat $apt_update_output && break || \
3437
cat $apt_update_output
3538
if [ $i -eq $retries ]; then
@@ -40,22 +43,37 @@ apt_get_update() {
4043
echo Executed apt-get update $i times
4144
wait_for_apt_locks
4245
}
43-
apt_get_install() {
44-
retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
46+
apt_get_update() {
47+
_apt_get_update 10 ""
48+
}
49+
_apt_get_install() {
50+
local retries=$1
51+
local wait_sleep=$2
52+
local apt_opts=$3
53+
shift && shift && shift
54+
4555
for i in $(seq 1 $retries); do
4656
wait_for_apt_locks
4757
export DEBIAN_FRONTEND=noninteractive
4858
dpkg --configure -a --force-confdef
49-
apt-get install -o Dpkg::Options::="--force-confold" --no-install-recommends -y ${@} && break || \
59+
60+
if apt-get install ${apt_opts} -o Dpkg::Options::="--force-confold" --no-install-recommends "${@}"; then
61+
echo "Executed apt-get install \"${packages[@]}\" $i times"
62+
wait_for_apt_locks
63+
return 0
64+
fi
65+
5066
if [ $i -eq $retries ]; then
5167
return 1
5268
else
5369
sleep $wait_sleep
5470
apt_get_update
5571
fi
5672
done
57-
echo Executed apt-get install --no-install-recommends -y \"$@\" $i times;
58-
wait_for_apt_locks
73+
}
74+
apt_get_install() {
75+
retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
76+
_apt_get_install $retries $wait_sleep "-y" "$@"
5977
}
6078
apt_get_purge() {
6179
retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
@@ -101,4 +119,65 @@ installDebPackageFromFile() {
101119
return 1
102120
fi
103121
}
122+
123+
apt_get_install_from_local_repo() {
124+
local local_repo_dir=$1
125+
local package_name=$2
126+
local installation_root=$3
127+
128+
# Convert to absolute path
129+
local_repo_dir=$(realpath "${local_repo_dir}")
130+
131+
if [ ! -d "${local_repo_dir}" ]; then
132+
echo "Local repo directory ${local_repo_dir} does not exist"
133+
return 1
134+
fi
135+
136+
# Check if Packages.gz exists in the repo
137+
if [ ! -f "${local_repo_dir}/Packages.gz" ]; then
138+
echo "Packages.gz not found in ${local_repo_dir}"
139+
return 1
140+
fi
141+
142+
wait_for_apt_locks
143+
144+
local tmp_list=$(mktemp)
145+
local tmp_dir=$(mktemp -d)
146+
147+
# Create temporary sources.list pointing to local repo
148+
printf 'deb [trusted=yes] file:%s ./\n' "${local_repo_dir}" > "${tmp_list}"
149+
150+
local opts="-o Dir::Etc::sourcelist=${tmp_list} -o Dir::Etc::sourceparts=${tmp_dir}"
151+
152+
# Update apt cache with local repo using core update function
153+
if ! _apt_get_update 10 "${opts}"; then
154+
echo "Failed to update apt cache from local repo ${local_repo_dir}"
155+
rm -f "${tmp_list}"
156+
rmdir "${tmp_dir}"
157+
return 1
158+
fi
159+
160+
# Install package from local repo using core installation function
161+
local retries=10
162+
local wait_sleep=5
163+
if ! _apt_get_install $retries $wait_sleep "${opts}" "${package_name}"; then
164+
echo "Failed to install ${package_name} from local repo"
165+
rm -f "${tmp_list}"
166+
rmdir "${tmp_dir}"
167+
return 1
168+
fi
169+
170+
# Cleanup
171+
rm -f "${tmp_list}"
172+
rmdir "${tmp_dir}"
173+
174+
# If installation_root is specified, copy binaries to that location
175+
if [ -n "${installation_root}" ]; then
176+
echo "Copying binaries ${package_name} to ${installation_root}"
177+
mkdir -p "${installation_root}"
178+
mv "/usr/bin/${package_name}" "/usr/local/bin/${package_name}"
179+
fi
180+
181+
return 0
182+
}
104183
#EOF

0 commit comments

Comments
 (0)