Merge remote-tracking branch 'upstream/main' into ExternalDeploymentIntegration

Author: abh1sar

engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java

@@ -204,9 +204,9 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
                 "(CASE WHEN ISNULL(service_offering.speed) THEN custom_speed.value ELSE service_offering.speed end) AS speed, " +
                 "(CASE WHEN ISNULL(service_offering.ram_size) THEN custom_ram_size.value ELSE service_offering.ram_size end) AS ram_size " +
                 "FROM vm_instance vi LEFT JOIN service_offering ON(((vi.service_offering_id = service_offering.id))) " +
-                "LEFT JOIN user_vm_details custom_cpu ON(((custom_cpu.vm_id = vi.id) AND (custom_cpu.name = 'CpuNumber'))) " +
-                "LEFT JOIN user_vm_details custom_speed ON(((custom_speed.vm_id = vi.id) AND (custom_speed.name = 'CpuSpeed'))) " +
-                "LEFT JOIN user_vm_details custom_ram_size ON(((custom_ram_size.vm_id = vi.id) AND (custom_ram_size.name = 'memory'))) ";
+                "LEFT JOIN vm_instance_details custom_cpu ON(((custom_cpu.vm_id = vi.id) AND (custom_cpu.name = 'CpuNumber'))) " +
+                "LEFT JOIN vm_instance_details custom_speed ON(((custom_speed.vm_id = vi.id) AND (custom_speed.name = 'CpuSpeed'))) " +
+                "LEFT JOIN vm_instance_details custom_ram_size ON(((custom_ram_size.vm_id = vi.id) AND (custom_ram_size.name = 'memory'))) ";
 
     private static final String WHERE_STATE_IS_NOT_DESTRUCTIVE =
             "WHERE ISNULL(vi.removed) AND vi.state NOT IN ('Destroyed', 'Error', 'Expunging')";

engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java

@@ -336,7 +336,7 @@ public long removeEntriesByOwner(long ownerId, ResourceOwnerType ownerType) {
             + "    END)) as total "
             + " from vm_instance vm "
             + " join service_offering so on so.id = vm.service_offering_id "
-            + " left join user_vm_details vmd on vmd.vm_id = vm.id and vmd.name = '%s' "
+            + " left join vm_instance_details vmd on vmd.vm_id = vm.id and vmd.name = '%s' "
             + " where vm.type = 'User' and state not in ('Destroyed', 'Error', 'Expunging') and display_vm = true and account_id = ? ";
 
     @Override

engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java

@@ -124,8 +124,8 @@ public class UserVmDaoImpl extends GenericDaoBase<UserVmVO, Long> implements Use
             + "left join security_group on security_group_vm_map.security_group_id=security_group.id " + "left join nics on vm_instance.id=nics.instance_id "
             + "left join networks on nics.network_id=networks.id " + "left join user_ip_address on user_ip_address.vm_id=vm_instance.id " + "where vm_instance.id in (";
 
-    private static final String VMS_DETAIL_BY_NAME = "select vm_instance.instance_name, vm_instance.vm_type, vm_instance.id , user_vm_details.value, user_vm_details.name from vm_instance "
-            + "left join user_vm_details on vm_instance.id = user_vm_details.vm_id where (user_vm_details.name is null or user_vm_details.name = ? ) and vm_instance.instance_name in (";
+    private static final String VMS_DETAIL_BY_NAME = "select vm_instance.instance_name, vm_instance.vm_type, vm_instance.id , vm_instance_details.value, vm_instance_details.name from vm_instance "
+            + "left join vm_instance_details on vm_instance.id = vm_instance_details.vm_id where (vm_instance_details.name is null or vm_instance_details.name = ? ) and vm_instance.instance_name in (";
 
     private static final int VM_DETAILS_BATCH_SIZE = 100;
 
@@ -755,7 +755,7 @@ public List<Pair<Pair<String, VirtualMachine.Type>, Pair<Long, String>>> getVmsD
                 while (rs.next()) {
                     vmsDetailByNames.add(new Pair<Pair<String, VirtualMachine.Type>, Pair<Long, String>>(new Pair<String, VirtualMachine.Type>(
                             rs.getString("vm_instance.instance_name"), VirtualMachine.Type.valueOf(rs.getString("vm_type"))),
-                            new Pair<Long, String>(rs.getLong("vm_instance.id"), rs.getString("user_vm_details.value"))));
+                            new Pair<Long, String>(rs.getLong("vm_instance.id"), rs.getString("vm_instance_details.value"))));
                 }
             }
         } catch (SQLException e) {

framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java

@@ -632,7 +632,7 @@ private void migrateScaleIOStoragePoolDetails(Connection conn) {
 
     private void migrateUserVmDetails(Connection conn) {
         System.out.println("Begin migrate user vm details");
-        migrateDetails(conn, "user_vm_details", PASSWORD);
+        migrateDetails(conn, "vm_instance_details", PASSWORD);
         System.out.println("End migrate user vm details");
     }
 

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -3203,7 +3203,7 @@ protected void setNewVRamSizeVmVideoCard(VirtualMachineMO vmMo, long svgaVmramSi
     }
 
     /**
-     * Modifies vm vram size if it was set to a different size to the one provided in svga.vramSize (user_vm_details or template_vm_details) on {@code vmConfigSpec}
+     * Modifies vm vram size if it was set to a different size to the one provided in svga.vramSize (vm_instance_details or template_vm_details) on {@code vmConfigSpec}
      *
      * @param videoCard     vm's video card device
      * @param vmMo          virtual machine mo

scripts/util/create-kubernetes-binaries-iso.sh

@@ -19,41 +19,56 @@
 set -e
 
 if [ $# -lt 6 ]; then
-    echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG BUILD_NAME [ETCD_VERSION]"
-    echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml setup-v1.11.4 3.5.1"
+    echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG BUILD_NAME [ARCH] [ETCD_VERSION]"
+    echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml setup-v1.11.4 amd64"
     exit 1
 fi
 
+ARCH="amd64"
+ARCH_SUFFIX="x86_64"
+if [ -n "${8}" ]; then
+  if [ "${8}" = "x86_64" ] || [ "${8}" = "amd64" ]; then
+    ARCH="amd64"
+    ARCH_SUFFIX="x86_64"
+  elif [ "${8}" = "aarch64" ] || [ "${8}" = "arm64" ]; then
+    ARCH="arm64"
+    ARCH_SUFFIX="aarch64"
+  else
+    echo "ERROR: ARCH must be 'x86_64' or 'aarch64'. If the optional parameter ARCH is not set then 'x86_64' is used."
+    exit 1
+  fi
+fi
+
 RELEASE="v${2}"
 VAL="1.18.0"
 output_dir="${1}"
 start_dir="$PWD"
 iso_dir="/tmp/iso"
 working_dir="${iso_dir}/"
 mkdir -p "${working_dir}"
-build_name="${7}.iso"
-[ -z "${build_name}" ] && build_name="setup-${RELEASE}.iso"
+build_name="${7}-${ARCH_SUFFIX}.iso"
+[ -z "${build_name}" ] && build_name="setup-${RELEASE}-${ARCH_SUFFIX}.iso"
 
 CNI_VERSION="v${3}"
 echo "Downloading CNI ${CNI_VERSION}..."
 cni_dir="${working_dir}/cni/"
 mkdir -p "${cni_dir}"
-cni_status_code=$(curl -L  --write-out "%{http_code}\n" "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64.tgz")
+cni_status_code=$(curl -L  --write-out "%{http_code}\n" "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-${ARCH}.tgz")
 if [[ ${cni_status_code} -eq 404 ]] ; then
-  curl -L  --write-out "%{http_code}\n" "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64.tgz"
+  curl -L  --write-out "%{http_code}\n" "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-${ARCH}-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-${ARCH}.tgz"
 fi
 
 CRICTL_VERSION="v${4}"
 echo "Downloading CRI tools ${CRICTL_VERSION}..."
 crictl_dir="${working_dir}/cri-tools/"
 mkdir -p "${crictl_dir}"
-curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" -o "${crictl_dir}/crictl-linux-amd64.tar.gz"
+curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" -o "${crictl_dir}/crictl-linux-${ARCH}.tar.gz"
 
 echo "Downloading Kubernetes tools ${RELEASE}..."
 k8s_dir="${working_dir}/k8s"
 mkdir -p "${k8s_dir}"
 cd "${k8s_dir}"
-curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
+curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/${ARCH}/{kubeadm,kubelet,kubectl}
 kubeadm_file_permissions=`stat --format '%a' kubeadm`
 chmod +x kubeadm
 
@@ -148,11 +163,12 @@ chmod ${kubeadm_file_permissions} "${working_dir}/k8s/kubeadm"
 echo "Updating imagePullPolicy to IfNotPresent in yaml files..."
 sed -i "s/imagePullPolicy:.*/imagePullPolicy: IfNotPresent/g" ${working_dir}/*.yaml
 
-if [ -n "${8}" ]; then
+# Optional parameter ETCD_VERSION
+if [ -n "${9}" ]; then
   # Install etcd dependencies
   etcd_dir="${working_dir}/etcd"
   mkdir -p "${etcd_dir}"
-  ETCD_VERSION=v${8}
+  ETCD_VERSION=v${9}
   wget -q --show-progress "https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" -O ${etcd_dir}/etcd-linux-amd64.tar.gz
 fi
 

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -6343,7 +6343,7 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE
             }
         }
 
-        // Add extraConfig to user_vm_details table
+        // Add extraConfig to vm_instance_details table
         String extraConfig = cmd.getExtraConfig();
         if (StringUtils.isNotBlank(extraConfig)) {
             if (EnableAdditionalVmConfig.valueIn(callerId)) {
@@ -6501,7 +6501,7 @@ private void checkIfVolumeTemplateIsTheSameAsTheProvided(VolumeInfo volume, Long
     }
 
     /**
-     * Persist extra configuration data in the user_vm_details table as key/value pair
+     * Persist extra configuration data in the vm_instance_details table as key/value pair
      * @param decodedUrl String consisting of the extra config data to appended onto the vmx file for VMware instances
      */
     protected void persistExtraConfigVmware(String decodedUrl, UserVm vm) {
@@ -6525,7 +6525,7 @@ protected void persistExtraConfigVmware(String decodedUrl, UserVm vm) {
     }
 
     /**
-     * Used to persist extra configuration settings in user_vm_details table for the XenServer hypervisor
+     * Used to persist extra configuration settings in vm_instance_details table for the XenServer hypervisor
      * persists config as key/value pair e.g key = extraconfig-1 , value="PV-bootloader=pygrub" and so on to extraconfig-N where
      * N denotes the number of extra configuration settings passed by user
      *
@@ -6606,9 +6606,9 @@ protected boolean isValidXenOrVmwareConfiguration(String cfg, String[] allowedKe
 
     /**
      * Persist extra configuration data on KVM
-     * persisted in the user_vm_details DB as extraconfig-1, and so on depending on the number of configurations
+     * persisted in the vm_instance_details DB as extraconfig-1, and so on depending on the number of configurations
      * For KVM, extra config is passed as XML
-     * @param decodedUrl string containing xml configuration to be persisted into user_vm_details table
+     * @param decodedUrl string containing xml configuration to be persisted into vm_instance_details table
      * @param vm
      */
     protected void persistExtraConfigKvm(String decodedUrl, UserVm vm) {

test/integration/component/test_rootvolume_resize.py

@@ -308,7 +308,7 @@ def getDiskController(self, vm, diskcontroller="ide"):
                 vmid = int(qresultvmuuid[0][0])
                 qresult = self.dbclient.execute(
                     "select rootDiskController from"
-                    " user_vm_details where id = '%s';" % vmid
+                    " vm_instance_details where id = '%s';" % vmid
                 )
                 self.debug("Query result: %s" % qresult)
                 diskcontroller = qresult[0][0]

test/integration/component/test_update_vm.py

@@ -186,7 +186,7 @@ def test_update_vm_details_admin(self):
         vm_db_id = dbresult[0][0]
         self.debug("VM has database id %d" % vm_db_id)
 
-        self.dbclient.execute("insert into user_vm_details (vm_id, name, value, display) values (%d,'%s','HOST', 0)" % (vm_db_id, hidden_detail_name))
+        self.dbclient.execute("insert into vm_instance_details (vm_id, name, value, display) values (%d,'%s','HOST', 0)" % (vm_db_id, hidden_detail_name))
 
         vm = self.listVmById(self.virtual_machine.id)
         self.debug("VirtualMachine fetched with details: %s of type %s" % (vm.details, type(vm.details)))
@@ -227,7 +227,7 @@ def test_update_vm_details_admin(self):
 
 
     def detailInDatabase(self, vm_id, detail_name):
-        dbresult = self.dbclient.execute("select id from user_vm_details where vm_id=%s and name='%s'" % (vm_id, detail_name))
+        dbresult = self.dbclient.execute("select id from vm_instance_details where vm_id=%s and name='%s'" % (vm_id, detail_name))
         self.debug("Detail %s for VM %s: %s" % (detail_name, vm_id, dbresult))
         if validateList(dbresult)[0] == PASS:
             return True

test/integration/smoke/test_deploy_vgpu_enabled_vm.py

@@ -319,7 +319,7 @@ def test_3d_gpu_support(self):
 
         qresultset = self.dbclient.execute("select id from vm_instance where uuid = '%s';" % self.virtual_machine.id)
         vm_id = qresultset[0]
-        qresultset = self.dbclient.execute("select name, value from user_vm_details where vm_id = '%d';" % vm_id)
+        qresultset = self.dbclient.execute("select name, value from vm_instance_details where vm_id = '%d';" % vm_id)
         detailKeys = [x[0] for x in qresultset]
 
         self.assertTrue('mks.enable3d' in detailKeys and 'mks.use3dRenderer' in detailKeys and 'svga.autodetect' in detailKeys and 'svga.vramSize' in detailKeys, "VM details do not contain 3D GPU details")