Merge remote-tracking branch 'origin/4.17'

Author: yadvr

.github/workflows/codecov.yml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Coverage Check
+
+on: [pull_request, push]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: codecov
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up JDK11
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+          cache: 'maven'
+
+      - name: Build CloudStack with Quality Checks
+        run: |
+          git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss
+          cd nonoss && bash -x install-non-oss.sh && cd ..
+          mvn -P quality -Dsimulator -Dnoredist clean install
+
+      - uses: codecov/codecov-action@v3
+        with:
+          files: ./client/target/site/jacoco-aggregate/jacoco.xml
+          fail_ci_if_error: true
+          verbose: true
+          name: codecov

engine/schema/src/main/resources/META-INF/db/schema-41700to41710.sql

@@ -20,4 +20,109 @@
 --;
 
 UPDATE `cloud`.`configuration` SET `value` = 'false' WHERE `name` = 'network.disable.rpfilter' AND `value` != 'true';
-UPDATE `cloud`.`configuration` SET `value` = 'false' WHERE `name` = 'consoleproxy.disable.rpfilter' AND `value` != 'true';
+UPDATE `cloud`.`configuration` SET `value` = 'false' WHERE `name` = 'consoleproxy.disable.rpfilter' AND `value` != 'true';
+
+-- Retrieve the hypervisor_type from vm_instance
+DROP VIEW IF EXISTS `cloud`.`domain_router_view`;
+CREATE VIEW `cloud`.`domain_router_view` AS
+    select
+        vm_instance.id id,
+        vm_instance.name name,
+        account.id account_id,
+        account.uuid account_uuid,
+        account.account_name account_name,
+        account.type account_type,
+        domain.id domain_id,
+        domain.uuid domain_uuid,
+        domain.name domain_name,
+        domain.path domain_path,
+        projects.id project_id,
+        projects.uuid project_uuid,
+        projects.name project_name,
+        vm_instance.uuid uuid,
+        vm_instance.created created,
+        vm_instance.state state,
+        vm_instance.removed removed,
+        vm_instance.pod_id pod_id,
+        vm_instance.instance_name instance_name,
+        vm_instance.hypervisor_type,
+        host_pod_ref.uuid pod_uuid,
+        data_center.id data_center_id,
+        data_center.uuid data_center_uuid,
+        data_center.name data_center_name,
+        data_center.networktype data_center_type,
+        data_center.dns1 dns1,
+        data_center.dns2 dns2,
+        data_center.ip6_dns1 ip6_dns1,
+        data_center.ip6_dns2 ip6_dns2,
+        host.id host_id,
+        host.uuid host_uuid,
+        host.name host_name,
+        host.cluster_id cluster_id,
+        vm_template.id template_id,
+        vm_template.uuid template_uuid,
+        service_offering.id service_offering_id,
+        service_offering.uuid service_offering_uuid,
+        service_offering.name service_offering_name,
+        nics.id nic_id,
+        nics.uuid nic_uuid,
+        nics.network_id network_id,
+        nics.ip4_address ip_address,
+        nics.ip6_address ip6_address,
+        nics.ip6_gateway ip6_gateway,
+        nics.ip6_cidr ip6_cidr,
+        nics.default_nic is_default_nic,
+        nics.gateway gateway,
+        nics.netmask netmask,
+        nics.mac_address mac_address,
+        nics.broadcast_uri broadcast_uri,
+        nics.isolation_uri isolation_uri,
+        vpc.id vpc_id,
+        vpc.uuid vpc_uuid,
+        vpc.name vpc_name,
+        networks.uuid network_uuid,
+        networks.name network_name,
+        networks.network_domain network_domain,
+        networks.traffic_type traffic_type,
+        networks.guest_type guest_type,
+        async_job.id job_id,
+        async_job.uuid job_uuid,
+        async_job.job_status job_status,
+        async_job.account_id job_account_id,
+        domain_router.template_version template_version,
+        domain_router.scripts_version scripts_version,
+        domain_router.is_redundant_router is_redundant_router,
+        domain_router.redundant_state redundant_state,
+        domain_router.stop_pending stop_pending,
+        domain_router.role role,
+        domain_router.software_version software_version
+    from
+        `cloud`.`domain_router`
+            inner join
+        `cloud`.`vm_instance` ON vm_instance.id = domain_router.id
+            inner join
+        `cloud`.`account` ON vm_instance.account_id = account.id
+            inner join
+        `cloud`.`domain` ON vm_instance.domain_id = domain.id
+            left join
+        `cloud`.`host_pod_ref` ON vm_instance.pod_id = host_pod_ref.id
+            left join
+        `cloud`.`projects` ON projects.project_account_id = account.id
+            left join
+        `cloud`.`data_center` ON vm_instance.data_center_id = data_center.id
+            left join
+        `cloud`.`host` ON vm_instance.host_id = host.id
+            left join
+        `cloud`.`vm_template` ON vm_instance.vm_template_id = vm_template.id
+            left join
+        `cloud`.`service_offering` ON vm_instance.service_offering_id = service_offering.id
+            left join
+        `cloud`.`nics` ON vm_instance.id = nics.instance_id and nics.removed is null
+            left join
+        `cloud`.`networks` ON nics.network_id = networks.id
+            left join
+        `cloud`.`vpc` ON domain_router.vpc_id = vpc.id and vpc.removed is null
+            left join
+        `cloud`.`async_job` ON async_job.instance_id = vm_instance.id
+            and async_job.instance_type = 'DomainRouter'
+            and async_job.job_status = 0;

server/src/main/java/com/cloud/api/ApiDBUtils.java

@@ -329,7 +329,6 @@
 import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
-import org.apache.commons.lang3.StringUtils;
 
 import javax.annotation.PostConstruct;
 import javax.inject.Inject;
@@ -1795,17 +1794,7 @@ public static String findJobInstanceUuid(AsyncJob job) {
     ///////////////////////////////////////////////////////////////////////
 
     public static DomainRouterResponse newDomainRouterResponse(DomainRouterJoinVO vr, Account caller) {
-        DomainRouterResponse response = s_domainRouterJoinDao.newDomainRouterResponse(vr, caller);
-        if (StringUtils.isBlank(response.getHypervisor())) {
-            VMInstanceVO vm = ApiDBUtils.findVMInstanceById(vr.getId());
-            if (vm.getLastHostId() != null) {
-                HostVO lastHost = ApiDBUtils.findHostById(vm.getLastHostId());
-                if (lastHost != null) {
-                    response.setHypervisor(lastHost.getHypervisorType().toString());
-                }
-            }
-        }
-        return  response;
+        return s_domainRouterJoinDao.newDomainRouterResponse(vr, caller);
     }
 
     public static DomainRouterResponse fillRouterDetails(DomainRouterResponse vrData, DomainRouterJoinVO vr) {

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -1538,18 +1538,13 @@ public SystemVmResponse createSystemVmResponse(VirtualMachine vm) {
                 vmResponse.setTemplateName(template.getName());
             }
             vmResponse.setCreated(vm.getCreated());
+            vmResponse.setHypervisor(vm.getHypervisorType().toString());
 
             if (vm.getHostId() != null) {
                 Host host = ApiDBUtils.findHostById(vm.getHostId());
                 if (host != null) {
                     vmResponse.setHostId(host.getUuid());
                     vmResponse.setHostName(host.getName());
-                    vmResponse.setHypervisor(host.getHypervisorType().toString());
-                }
-            } else if (vm.getLastHostId() != null) {
-                Host lastHost = ApiDBUtils.findHostById(vm.getLastHostId());
-                if (lastHost != null) {
-                    vmResponse.setHypervisor(lastHost.getHypervisorType().toString());
                 }
             }
 

server/src/main/java/com/cloud/api/query/dao/DomainRouterJoinDaoImpl.java

@@ -118,6 +118,7 @@ public DomainRouterResponse newDomainRouterResponse(DomainRouterJoinVO router, A
             routerResponse.setRequiresUpgrade(true);
         }
 
+        routerResponse.setHypervisor(router.getHypervisorType().toString());
         routerResponse.setHasAnnotation(annotationDao.hasAnnotations(router.getUuid(), AnnotationService.EntityType.VR.name(),
                 _accountMgr.isRootAdmin(CallContext.current().getCallingAccount().getId())));
 
@@ -126,7 +127,6 @@ public DomainRouterResponse newDomainRouterResponse(DomainRouterJoinVO router, A
             if (router.getHostId() != null) {
                 routerResponse.setHostId(router.getHostUuid());
                 routerResponse.setHostName(router.getHostName());
-                routerResponse.setHypervisor(router.getHypervisorType().toString());
             }
             routerResponse.setPodId(router.getPodUuid());
             HostPodVO pod = ApiDBUtils.findPodById(router.getPodId());

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -630,6 +630,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             HypervisorType.Simulator
     ));
 
+    private static final List<HypervisorType> HYPERVISORS_THAT_CAN_DO_STORAGE_MIGRATION_ON_NON_USER_VMS = Arrays.asList(HypervisorType.KVM, HypervisorType.VMware);
+
     @Override
     public UserVmVO getVirtualMachine(long vmId) {
         return _vmDao.findById(vmId);
@@ -6093,16 +6095,18 @@ private VMInstanceVO preVmStorageMigrationCheck(Long vmId) {
             throw ex;
         }
 
-        if (vm.getType() != VirtualMachine.Type.User && !HypervisorType.VMware.equals(vm.getHypervisorType())) {
-            throw new InvalidParameterValueException("cannot do storage migration on non-user vm for hypervisor: " + vm.getHypervisorType().toString() + ", only supported for VMware");
+        HypervisorType hypervisorType = vm.getHypervisorType();
+        if (vm.getType() != VirtualMachine.Type.User && !HYPERVISORS_THAT_CAN_DO_STORAGE_MIGRATION_ON_NON_USER_VMS.contains(hypervisorType)) {
+            throw new InvalidParameterValueException(String.format("Unable to migrate storage of non-user VMs for hypervisor [%s]. Operation only supported for the following"
+                    + " hypervisors: [%s].", hypervisorType, HYPERVISORS_THAT_CAN_DO_STORAGE_MIGRATION_ON_NON_USER_VMS));
         }
 
         List<VolumeVO> vols = _volsDao.findByInstance(vm.getId());
         if (vols.size() > 1) {
             // OffLineVmwareMigration: data disks are not permitted, here!
             if (vols.size() > 1 &&
                     // OffLineVmwareMigration: allow multiple disks for vmware
-                    !HypervisorType.VMware.equals(vm.getHypervisorType())) {
+                    !HypervisorType.VMware.equals(hypervisorType)) {
                 throw new InvalidParameterValueException("Data disks attached to the vm, can not migrate. Need to detach data disks first");
             }
         }

test/integration/smoke/test_attach_multiple_volumes.py

@@ -28,9 +28,9 @@
                              DiskOffering,
                              )
 from marvin.lib.common import (get_domain,
-                                get_zone,
-                                get_template,
-                                find_storage_pool_type)
+                               get_zone,
+                               get_suitable_test_template,
+                               find_storage_pool_type)
 from marvin.codes import (
     PASS,
     FAILED,
@@ -69,13 +69,13 @@ def setUpClass(cls):
                                     )
         cls._cleanup.append(cls.disk_offering)
 
-        template = get_template(
-                            cls.apiclient,
-                            cls.zone.id,
-                            cls.services["ostype"]
-                            )
+        template = get_suitable_test_template(
+            cls.apiclient,
+            cls.zone.id,
+            cls.services["ostype"],
+            cls.hypervisor)
         if template == FAILED:
-            assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
+            assert False, "get_suitable_test_template() failed to return template with description %s" % cls.services["ostype"]
 
         cls.services["domainid"] = cls.domain.id
         cls.services["zoneid"] = cls.zone.id

test/integration/smoke/test_network_permissions.py

@@ -44,7 +44,7 @@
 
 from marvin.lib.common import (get_domain,
                                get_zone,
-                               get_template)
+                               get_suitable_test_template)
 
 NETWORK_FILTER_ACCOUNT = 'account'
 NETWORK_FILTER_DOMAIN = 'domain'
@@ -66,7 +66,12 @@ def setUpClass(cls):
 
         zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests())
         cls.zone = Zone(zone.__dict__)
-        cls.template = get_template(cls.apiclient, cls.zone.id)
+        cls.hypervisor = cls.testClient.getHypervisorInfo()
+        cls.template = get_suitable_test_template(
+            cls.apiclient,
+            cls.zone.id,
+            cls.services["ostype"],
+            cls.hypervisor)
         cls._cleanup = []
 
         cls.logger = logging.getLogger("TestNetworkPermissions")