Simulate a failed agent upgrade scenario to test the error message display.

SherryPuffs · SherryPuffs · commit 9df2acec83e9 · 2025-06-24T17:38:17.000+08:00
diff --git a/src/connectedk8s/HISTORY.rst b/src/connectedk8s/HISTORY.rst
@@ -5,8 +5,8 @@ Release History
 
 1.10.8
 ++++++
-* Add checks for the Graytown bundle feature flag in the 'connectedk8s connect' and 'connectedk8s update' commands.
-* Display a more detailed error message from the agent-update-validator in the 'connectedk8s upgrade' command.
+* Add checks for the bundling feature flag in the 'connectedk8s connect' and 'connectedk8s update' commands.
+* Display a more detailed error message when the 'connectedk8s upgrade' command fails with the bundling feature enabled.
 
 1.10.7
 ++++++
diff --git a/src/connectedk8s/azext_connectedk8s/tests/latest/agent_update_validator_test_config/ArcAgentryValues.json b/src/connectedk8s/azext_connectedk8s/tests/latest/agent_update_validator_test_config/ArcAgentryValues.json
@@ -0,0 +1,7 @@
+{
+    "systemDefaultValues": {
+        "agent-update-validator": {
+            "config_dp_endpoint_override": "http://fake-dp-server-service.graytowntest.svc.cluster.local:80"
+        }
+    }
+}
diff --git a/src/connectedk8s/azext_connectedk8s/tests/latest/agent_update_validator_test_config/fake_ext_config.yml b/src/connectedk8s/azext_connectedk8s/tests/latest/agent_update_validator_test_config/fake_ext_config.yml
@@ -0,0 +1,19 @@
+apiVersion: clusterconfig.azure.com/v1beta1
+kind: ExtensionConfig
+metadata:
+  name: fake-ext-config
+  namespace: azure-arc
+spec:
+  dependentBundles:
+  - appdevaddons
+  extensionType: microsoft.graytown.testextension
+  isDependentOnBundle: true
+  parameter:
+    releaseTrain: Preview
+    repoUrl: https://graytownacr.azurecr.io/helm/graytowntest
+    scope: cluster
+    version: 1.302.0
+status:
+  syncStatus:
+    isSyncedWithAzure: true
+    lastSyncTime: '2025-06-23T21:14:39.000Z'
diff --git a/src/connectedk8s/azext_connectedk8s/tests/latest/test_connectedk8s_scenario.py b/src/connectedk8s/azext_connectedk8s/tests/latest/test_connectedk8s_scenario.py
@@ -12,11 +12,13 @@
 import stat
 import subprocess
 import time
+from datetime import datetime
 from subprocess import PIPE, Popen
 
 import oras.client  # type: ignore[import-untyped]
 import psutil
 import requests
+import yaml
 from azure.cli.core import get_default_cli
 from azure.cli.core.azclierror import (
     ArgumentUsageError,
@@ -343,7 +345,7 @@ def test_connect_withbundlefeatureflag(self, resource_group):
         with self.assertLogs(level="WARNING") as cm:
             self.cmd(
                 "connectedk8s connect -g {rg} -n {name} -l {location} --disable-auto-upgrade --kube-config {kubeconfig} \
-            --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='preview' --yes",
+                --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='preview' --yes",
                 checks=[
                     self.check("resourceGroup", "{rg}"),
                     self.check("name", "{name}"),
@@ -495,16 +497,16 @@ def test_update_withbundlefeatureflag(self, resource_group):
 
         self.cmd(
             "k8s-extension create --cluster-name {name} --resource-group {rg} --cluster-type connectedClusters \
-                 --extension-type microsoft.iotoperations.platform --name azure-iot-operations-platform \
-                 --release-train preview --auto-upgrade-minor-version False --config installTrustManager=true \
-                 --config installCertManager=true --version 0.7.6 --release-namespace cert-manager --scope cluster"
+            --extension-type microsoft.iotoperations.platform --name azure-iot-operations-platform \
+            --release-train preview --auto-upgrade-minor-version False --config installTrustManager=true \
+            --config installCertManager=true --version 0.7.6 --release-namespace cert-manager --scope cluster"
         )
 
         self.cmd(
             "k8s-extension create --cluster-name {name} --resource-group {rg} --cluster-type connectedClusters \
-                 --extension-type microsoft.azure.secretstore --name azure-secret-store --auto-upgrade-minor-version False \
-                 --config rotationPollIntervalInSeconds=120 --config validatingAdmissionPolicies.applyPolicies=false \
-                 --scope cluster"
+            --extension-type microsoft.azure.secretstore --name azure-secret-store --auto-upgrade-minor-version False \
+            --config rotationPollIntervalInSeconds=120 --config validatingAdmissionPolicies.applyPolicies=false \
+            --scope cluster"
         )
 
         # With bundle extensions installed on the cluster, the bundle feature flag cannot be set to 'disabled'
@@ -521,6 +523,144 @@ def test_update_withbundlefeatureflag(self, resource_group):
         )
         self.assertEqual(configmap_bundle_feature_flag, "enabled")
 
+    @live_only()
+    @ResourceGroupPreparer(
+        name_prefix="conk8stest", location=CONFIG["location"], random_name_length=16
+    )
+    def test_upgrade_with_agentupdatevalidator(self, resource_group):
+        managed_cluster_name = self.create_random_name(prefix="test-upgrade", length=24)
+        kubeconfig = _get_test_data_file(managed_cluster_name + "-config.yaml")
+        self.kwargs.update(
+            {
+                "rg": resource_group,
+                "name": self.create_random_name(prefix="cc-", length=12),
+                "kubeconfig": kubeconfig,
+                "managed_cluster_name": managed_cluster_name,
+                "location": CONFIG["location"],
+            }
+        )
+        self.cmd("aks create -g {rg} -n {managed_cluster_name} --generate-ssh-keys")
+        self.cmd(
+            "aks get-credentials -g {rg} -n {managed_cluster_name} -f {kubeconfig} --admin"
+        )
+
+        self.cmd(
+            "connectedk8s connect -g {rg} -n {name} -l {location} --disable-auto-upgrade --kube-config {kubeconfig} \
+            --kube-context {managed_cluster_name}-admin --config extensionSets.versionManagedExtensions='enabled'",
+            checks=[
+                self.check("resourceGroup", "{rg}"),
+                self.check("name", "{name}"),
+                self.check(
+                    "arcAgentryConfigurations[0].settings.versionManagedExtensions",
+                    "enabled",
+                ),
+            ],
+        )
+
+        ns = "azure-arc"
+        config_dir = os.path.join(
+            os.path.dirname(__file__), "agent_update_validator_test_config"
+        )
+        arc_agent_values_path = os.path.join(config_dir, "ArcAgentryValues.json")
+        fake_ext_config_path = os.path.join(config_dir, "fake_ext_config.yml")
+
+        with open(fake_ext_config_path) as file:
+            data = yaml.safe_load(file)
+
+        # Update the lastSyncTime to the current time
+        current_time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.000Z")
+        data["status"]["syncStatus"]["lastSyncTime"] = current_time
+
+        with open(fake_ext_config_path, "w") as file:
+            yaml.dump(data, file, default_flow_style=False)
+
+        kubectl_client_location = install_kubectl_client()
+
+        # Create the fake extension config to simulate an extension that depends on the bundle
+        subprocess.run(
+            [
+                kubectl_client_location,
+                "apply",
+                "-f",
+                fake_ext_config_path,
+                "--namespace",
+                ns,
+                "--kubeconfig",
+                kubeconfig,
+                "--context",
+                f"{managed_cluster_name}-admin",
+            ]
+        )
+
+        os.environ["HELMVALUESPATH"] = arc_agent_values_path
+
+        with self.assertRaisesRegex(
+            CLIError, "Error: Failed to validate agent update.*?no such host"
+        ):
+            self.cmd(
+                "connectedk8s upgrade -g {rg} -n {name} --agent-version 1.26.0 --kube-config {kubeconfig} \
+                --kube-context {managed_cluster_name}-admin",
+            )
+
+        os.environ["HELMVALUESPATH"] = ""
+
+        # Remove the finalizers from the fake extension config to allow deletion
+        subprocess.run(
+            [
+                kubectl_client_location,
+                "patch",
+                "extensionconfig",
+                "fake-ext-config",
+                "--namespace",
+                ns,
+                "--type=json",
+                "-p",
+                '[{"op": "remove", "path": "/metadata/finalizers"}]',
+                "--kubeconfig",
+                kubeconfig,
+                "--context",
+                f"{managed_cluster_name}-admin",
+            ]
+        )
+
+        subprocess.Popen(
+            [
+                kubectl_client_location,
+                "delete",
+                "extensionconfig",
+                "fake-ext-config",
+                "--namespace",
+                ns,
+                "--kubeconfig",
+                kubeconfig,
+                "--context",
+                f"{managed_cluster_name}-admin",
+            ]
+        )
+
+        time.sleep(60)
+
+        cmd_output = subprocess.Popen(
+            [
+                kubectl_client_location,
+                "get",
+                "extensionconfig",
+                "fake-ext-config",
+                "--namespace",
+                ns,
+                "--kubeconfig",
+                kubeconfig,
+                "--context",
+                f"{managed_cluster_name}-admin",
+            ],
+            stdout=PIPE,
+            stderr=PIPE,
+        )
+        _, error_get_ext_config = cmd_output.communicate()
+
+        # Should fail to get the extension config as it has been deleted
+        assert cmd_output.returncode != 0
+
     @live_only()
     @ResourceGroupPreparer(
         name_prefix="conk8stest", location=CONFIG["location"], random_name_length=16
diff --git a/testing/test/configurations/BundleFeatureFlag.Tests.ps1 b/testing/test/configurations/BundleFeatureFlag.Tests.ps1
@@ -149,6 +149,74 @@ Describe 'Setting Bundle Feature Flag Scenario' {
         $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS
     }
 
+    It "Verify the error message when failing to upgrade the agent with bundle feature flag enabled" {
+        az connectedk8s update -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --auto-upgrade false `
+            --config extensionSets.versionManagedExtensions='enabled'
+        $? | Should -BeTrue
+        Start-Sleep -Seconds 10
+
+        # Loop and retry until the configuration installs
+        $n = 0
+        do 
+        {
+            $output = az connectedk8s show -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup
+            $jsonOutput = [System.Text.Json.JsonDocument]::Parse($output)
+            $provisioningState = ($output | ConvertFrom-Json).provisioningState
+            $bundleFeatureFlag = $jsonOutput.RootElement.GetProperty("arcAgentryConfigurations")[0].GetProperty("settings").GetProperty("versionManagedExtensions").GetString()
+            Write-Host "Provisioning State: $provisioningState"
+            Write-Host "Bundle Feature Flag: $bundleFeatureFlag"
+            if ($provisioningState -eq $SUCCEEDED -and $bundleFeatureFlag -eq "enabled")
+            {
+                break
+            }
+            Start-Sleep -Seconds 10
+            $n += 1
+        } while ($n -le $MAX_RETRY_ATTEMPTS)
+        $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS
+
+        $ns = "azure-arc"
+        $rootDir = Resolve-Path "$PSScriptRoot/../../../"
+
+        $configDir = Join-Path $rootDir "src/connectedk8s/azext_connectedk8s/tests/latest/agent_update_validator_test_config"
+        $arcAgentValuesPath = Join-Path $configDir "ArcAgentryValues.json"
+        $fakeExtConfigPath = Join-Path $configDir "fake_ext_config.yml"
+
+        $tmpDir = Join-Path $rootDir "testing/tmp"
+
+        if (-not (Test-Path $tmpDir)) {
+            New-Item -ItemType Directory -Path $tmpDir | Out-Null
+        }
+
+        # Update lastSyncTime in fake_ext_config.yml
+        $updatedFakeExtConfigPath = Join-Path $tmpDir "fake_ext_config_updated.yml"
+        $data = Get-Content $fakeExtConfigPath | ConvertFrom-Yaml
+        $data.status.syncStatus.lastSyncTime = (Get-Date).ToString("yyyy-MM-ddTHH:mm:ss.000Z")
+        $data | ConvertTo-Yaml | Set-Content $updatedFakeExtConfigPath
+
+        # Create the fake extension config to simulate an extension that depends on the bundle
+        kubectl apply -f $updatedFakeExtConfigPath --namespace $ns
+        $? | Should -BeTrue
+
+        $ENV:HELMVALUESPATH = $arcAgentValuesPath
+
+        $output = & {
+            az connectedk8s upgrade -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --agent-version "1.26.0" 2>&1 | Out-String
+        }
+        $output | Should -Match "Error: Failed to validate agent update.*no such host"
+
+        $env:HELMVALUESPATH = ""
+
+        # Remove the finalizers from the fake extension config to allow deletion
+        kubectl patch extensionconfig fake-ext-config --namespace $ns --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]'
+
+        Start-Process kubectl -ArgumentList "delete extensionconfig fake-ext-config --namespace $ns" -NoNewWindow
+
+        Start-Sleep -Seconds 60
+
+        kubectl get extensionconfig fake-ext-config --namespace $ns 2>&1
+        $? | Should -BeFalse
+    }
+
     It "Delete the connected instance" {
         az connectedk8s delete -n $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --force -y
         $? | Should -BeTrue
