cks: fix k8s version upgrade (#6513)

Fixes #6514

On latest systemvm template used for CKS /usr/sbin is not present in the $PATH for normal user used during upgrade. This leads to failure for blkid command. Due to this during k8s version upgrade ISO is not being able to mount on the k8s cluster VMs and upgrade process is not carried out.
This PR fixes mounting of k8s version ISO and also returns failure for script when ISO mounting is failed.
Same failure is not seen during deployment of the cluster because setup-kube-system workflow is executed as ROOT user and it has a different value for $PATH.
From /etc/login.defs:

ENV_SUPATH	PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV_PATH	PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

Author: shwstppr

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java

@@ -22,6 +22,7 @@
 import java.util.List;
 
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Level;
 
 import com.cloud.hypervisor.Hypervisor;
@@ -36,7 +37,6 @@
 import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.ssh.SshHelper;
-import org.apache.commons.lang3.StringUtils;
 
 public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorker {
 
@@ -122,6 +122,9 @@ private void upgradeKubernetesClusterNodes() {
                     logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
                 }
             }
+            if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), i==0, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName)) {
+                logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node on VM : %s upgraded to version %s", kubernetesCluster.getName(), vm.getDisplayName(), upgradeVersion.getSemanticVersion()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
+            }
             if (LOGGER.isInfoEnabled()) {
                 LOGGER.info(String.format("Successfully upgraded node on VM %s in Kubernetes cluster %s with Kubernetes version(%s) ID: %s",
                         vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid()));

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java

@@ -32,19 +32,21 @@
 import javax.net.ssl.TrustManager;
 
 import org.apache.cloudstack.utils.security.SSLUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 
 import com.cloud.kubernetes.cluster.KubernetesCluster;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.Pair;
 import com.cloud.utils.nio.TrustAllManager;
 import com.cloud.utils.ssh.SshHelper;
-import org.apache.commons.lang3.StringUtils;
 
 public class KubernetesClusterUtil {
 
     protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterUtil.class);
 
+    public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl version --short";
+
     public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port,
                                                        String user, File sshKeyFile, String nodeName) throws Exception {
         Pair<Boolean, String> result = SshHelper.sshExecute(ipAddress, port,
@@ -324,4 +326,38 @@ public static String generateClusterHACertificateKey(final KubernetesCluster kub
         }
         return token.toString().substring(0, 64);
     }
+
+    public static boolean clusterNodeVersionMatches(final String version, boolean isControlNode,
+                                                    final String ipAddress, final int port,
+                                                    final String user, final File sshKeyFile,
+                                                    final String hostName) {
+        Pair<Boolean, String> result = null;
+        try {
+            result = SshHelper.sshExecute(
+                    ipAddress, port,
+                    user, sshKeyFile, null,
+                    CLUSTER_NODE_VERSION_COMMAND,
+                    10000, 10000, 20000);
+        } catch (Exception e) {
+            if (LOGGER.isDebugEnabled()) {
+                LOGGER.debug(String.format("Failed to retrieve Kubernetes version from cluster node : %s due to exception", hostName), e);
+            }
+            return false;
+        }
+        if (Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) {
+            return false;
+        }
+        String response = result.second();
+        boolean clientVersionPresent = false;
+        boolean serverVersionPresent = false;
+        for (String line : response.split("\n")) {
+            if (line.contains("Client Version") && line.contains(String.format("v%s", version))) {
+                clientVersionPresent = true;
+            }
+            if (isControlNode && line.contains("Server Version") && line.contains(String.format("v%s", version))) {
+                serverVersionPresent = true;
+            }
+        }
+        return clientVersionPresent && (!isControlNode || serverVersionPresent);
+    }
 }

plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh

@@ -37,6 +37,9 @@ if [ $# -gt 3 ]; then
 fi
 
 export PATH=$PATH:/opt/bin
+if [[ "$PATH" != *:/usr/sbin && "$PATH" != *:/usr/sbin:* ]]; then
+  export PATH=$PATH:/usr/sbin
+fi
 
 ISO_MOUNT_DIR=/mnt/k8sdisk
 BINARIES_DIR=${ISO_MOUNT_DIR}/
@@ -149,4 +152,7 @@ if [ -d "$BINARIES_DIR" ]; then
   if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
     eject "${iso_drive_path}"
   fi
+else
+  echo "ERROR: Unable to access Binaries directory for upgrade version ${UPGRADE_VERSION}"
+  exit 1
 fi

plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java

@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.kubernetes.cluster.utils;
+
+import java.io.File;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import com.cloud.utils.Pair;
+import com.cloud.utils.ssh.SshHelper;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(SshHelper.class)
+public class KubernetesClusterUtilTest {
+    String ipAddress = "10.1.1.1";
+    int port = 2222;
+    String user = "user";
+    File sshKeyFile = Mockito.mock(File.class);
+    String hostName = "host";
+
+    private void mockSshHelperExecuteThrowAndTestVersionMatch() {
+        try {
+            Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenThrow(Exception.class);
+        } catch (Exception e) {
+            Assert.fail(String.format("Exception: %s", e.getMessage()));
+        }
+        Assert.assertFalse(KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", false, ipAddress, port, user, sshKeyFile, hostName));
+    }
+
+    private void mockSshHelperExecuteAndTestVersionMatch(boolean status, String response, boolean isControlNode, boolean expectedResult) {
+        try {
+            Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenReturn(new Pair<>(status, response));
+        } catch (Exception e) {
+            Assert.fail(String.format("Exception: %s", e.getMessage()));
+        }
+        boolean result = KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", isControlNode, ipAddress, port, user, sshKeyFile, hostName);
+        Assert.assertEquals(expectedResult, result);
+    }
+
+    @Test
+    public void testClusterNodeVersionMatches() {
+        PowerMockito.mockStatic(SshHelper.class);
+        String v1233WorkerNodeOutput = "Client Version: v1.23.3\n" +
+                "The connection to the server localhost:8080 was refused - did you specify the right host or port?";
+        String v1240WorkerNodeOutput = "Client Version: v1.24.0\n" +
+                "Kustomize Version: v4.5.4\n" +
+                "The connection to the server localhost:8080 was refused - did you specify the right host or port?";
+        String v1240ControlNodeOutput = "Client Version: v1.24.0\n" +
+                "Kustomize Version: v4.5.4\n" +
+                "Server Version: v1.24.0";
+        mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, false, true);
+
+        mockSshHelperExecuteAndTestVersionMatch(true, v1240ControlNodeOutput, true, true);
+
+        mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, true, false);
+
+        mockSshHelperExecuteAndTestVersionMatch(false, v1240WorkerNodeOutput, false, false);
+
+        mockSshHelperExecuteAndTestVersionMatch(true, v1233WorkerNodeOutput, false, false);
+
+        mockSshHelperExecuteAndTestVersionMatch(true, "Client Version: v1.24.0\n" +
+                "Kustomize Version: v4.5.4\n" +
+                "Server Version: v1.23.0", true, false);
+
+        mockSshHelperExecuteAndTestVersionMatch(true, null, false, false);
+
+        mockSshHelperExecuteAndTestVersionMatch(false, "-\n-", false, false);
+
+        mockSshHelperExecuteAndTestVersionMatch(false, "1.24.0", false, false);
+
+        mockSshHelperExecuteThrowAndTestVersionMatch();
+    }
+}