Merge branch '4.22' into create-vol-on-stroage

Author: abh1sar

api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java

@@ -37,7 +37,7 @@ public interface VolumeImportUnmanageService extends PluggableService, Configura
             Arrays.asList(Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.VMware);
 
     List<Storage.StoragePoolType> SUPPORTED_STORAGE_POOL_TYPES_FOR_KVM = Arrays.asList(Storage.StoragePoolType.NetworkFilesystem,
-            Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD);
+            Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD, Storage.StoragePoolType.SharedMountPoint);
 
     ConfigKey<Boolean> AllowImportVolumeWithBackingFile = new ConfigKey<>(Boolean.class,
             "allow.import.volume.with.backing.file",

core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java

@@ -34,6 +34,7 @@ public class RestoreBackupCommand extends Command  {
     private List<String> backupVolumesUUIDs;
     private List<PrimaryDataStoreTO> restoreVolumePools;
     private List<String> restoreVolumePaths;
+    private List<Long> restoreVolumeSizes;
     private List<String> backupFiles;
     private String diskType;
     private Boolean vmExists;
@@ -92,6 +93,14 @@ public void setRestoreVolumePaths(List<String> restoreVolumePaths) {
         this.restoreVolumePaths = restoreVolumePaths;
     }
 
+    public List<Long> getRestoreVolumeSizes() {
+        return restoreVolumeSizes;
+    }
+
+    public void setRestoreVolumeSizes(List<Long> restoreVolumeSizes) {
+        this.restoreVolumeSizes = restoreVolumeSizes;
+    }
+
     public List<String> getBackupFiles() {
         return backupFiles;
     }

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java

@@ -45,4 +45,9 @@ public interface HostTagsDao extends GenericDao<HostTagVO, Long> {
     HostTagResponse newHostTagResponse(HostTagVO hostTag);
 
     List<HostTagVO> searchByIds(Long... hostTagIds);
+
+    /**
+     * List all host tags defined on hosts within a cluster
+     */
+    List<String> listByClusterId(Long clusterId);
 }

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java

@@ -23,6 +23,7 @@
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
@@ -43,9 +44,12 @@ public class HostTagsDaoImpl extends GenericDaoBase<HostTagVO, Long> implements
     private final SearchBuilder<HostTagVO> stSearch;
     private final SearchBuilder<HostTagVO> tagIdsearch;
     private final SearchBuilder<HostTagVO> ImplicitTagsSearch;
+    private final GenericSearchBuilder<HostTagVO, String> tagSearch;
 
     @Inject
     private ConfigurationDao _configDao;
+    @Inject
+    private HostDao hostDao;
 
     public HostTagsDaoImpl() {
         HostSearch = createSearchBuilder();
@@ -72,6 +76,11 @@ public HostTagsDaoImpl() {
         ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ);
         ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ);
         ImplicitTagsSearch.done();
+
+        tagSearch = createSearchBuilder(String.class);
+        tagSearch.selectFields(tagSearch.entity().getTag());
+        tagSearch.and("hostIdIN", tagSearch.entity().getHostId(), SearchCriteria.Op.IN);
+        tagSearch.done();
     }
 
     @Override
@@ -235,4 +244,15 @@ public List<HostTagVO> searchByIds(Long... tagIds) {
 
         return tagList;
     }
+
+    @Override
+    public List<String> listByClusterId(Long clusterId) {
+        List<Long> hostIds = hostDao.listIdsByClusterId(clusterId);
+        if (CollectionUtils.isEmpty(hostIds)) {
+            return new ArrayList<>();
+        }
+        SearchCriteria<String> sc = tagSearch.create();
+        sc.setParameters("hostIdIN", hostIds.toArray());
+        return customSearch(sc, null);
+    }
 }

engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java

@@ -193,6 +193,7 @@ protected void init() {
         PersistentNetworkSearch.and("id", PersistentNetworkSearch.entity().getId(), Op.NEQ);
         PersistentNetworkSearch.and("guestType", PersistentNetworkSearch.entity().getGuestType(), Op.IN);
         PersistentNetworkSearch.and("broadcastUri", PersistentNetworkSearch.entity().getBroadcastUri(), Op.EQ);
+        PersistentNetworkSearch.and("dc", PersistentNetworkSearch.entity().getDataCenterId(), Op.EQ);
         PersistentNetworkSearch.and("removed", PersistentNetworkSearch.entity().getRemoved(), Op.NULL);
         final SearchBuilder<NetworkOfferingVO> persistentNtwkOffJoin = _ntwkOffDao.createSearchBuilder();
         persistentNtwkOffJoin.and("persistent", persistentNtwkOffJoin.entity().isPersistent(), Op.EQ);

engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql

@@ -18,5 +18,3 @@
 --;
 -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0
 --;
-
-DROP VIEW IF EXISTS `cloud`.`account_netstats_view`;

engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql

@@ -35,3 +35,6 @@ UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU';
 UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent';
 
 UPDATE `cloud`.`vm_template` SET guest_os_id = 99 WHERE name = 'kvm-default-vm-import-dummy-template';
+
+-- Update existing vm_template records with NULL type to "USER"
+UPDATE `cloud`.`vm_template` SET `type` = 'USER' WHERE `type` IS NULL;

engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql

@@ -0,0 +1,31 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- cloud.account_netstats_view source
+
+
+DROP VIEW IF EXISTS `cloud`.`account_netstats_view`;
+
+CREATE VIEW `cloud`.`account_netstats_view` AS
+select
+    `user_statistics`.`account_id` AS `account_id`,
+    (sum(`user_statistics`.`net_bytes_received`) + sum(`user_statistics`.`current_bytes_received`)) AS `bytesReceived`,
+    (sum(`user_statistics`.`net_bytes_sent`) + sum(`user_statistics`.`current_bytes_sent`)) AS `bytesSent`
+from
+    `user_statistics`
+group by
+    `user_statistics`.`account_id`;

engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql

@@ -39,8 +39,8 @@ select
     `data_center`.`id` AS `data_center_id`,
     `data_center`.`uuid` AS `data_center_uuid`,
     `data_center`.`name` AS `data_center_name`,
-    `account_netstats`.`bytesReceived` AS `bytesReceived`,
-    `account_netstats`.`bytesSent` AS `bytesSent`,
+    `account_netstats_view`.`bytesReceived` AS `bytesReceived`,
+    `account_netstats_view`.`bytesSent` AS `bytesSent`,
     `vmlimit`.`max` AS `vmLimit`,
     `vmcount`.`count` AS `vmTotal`,
     `runningvm`.`vmcount` AS `runningVms`,
@@ -89,15 +89,8 @@ from
     `cloud`.`domain` ON account.domain_id = domain.id
         left join
     `cloud`.`data_center` ON account.default_zone_id = data_center.id
-        left join lateral (
-        select
-            coalesce(sum(`user_statistics`.`net_bytes_received` + `user_statistics`.`current_bytes_received`), 0) AS `bytesReceived`,
-            coalesce(sum(`user_statistics`.`net_bytes_sent` + `user_statistics`.`current_bytes_sent`), 0) AS `bytesSent`
-        from
-            `cloud`.`user_statistics`
-        where
-            `user_statistics`.`account_id` = `account`.`id`
-    ) AS `account_netstats` ON TRUE
+        left join
+    `cloud`.`account_netstats_view` ON account.id = account_netstats_view.account_id
         left join
     `cloud`.`resource_limit` vmlimit ON account.id = vmlimit.account_id
         and vmlimit.type = 'user_vm' and vmlimit.tag IS NULL

plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java

@@ -351,7 +351,8 @@ private Pair<List<PrimaryDataStoreTO>, List<String>> getVolumePoolsAndPaths(List
             volumePools.add(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null);
 
             String volumePathPrefix = getVolumePathPrefix(storagePool);
-            volumePaths.add(String.format("%s/%s", volumePathPrefix, volume.getPath()));
+            String volumePathSuffix = getVolumePathSuffix(storagePool);
+            volumePaths.add(String.format("%s%s%s", volumePathPrefix, volume.getPath(), volumePathSuffix));
         }
         return new Pair<>(volumePools, volumePaths);
     }
@@ -361,14 +362,24 @@ private String getVolumePathPrefix(StoragePoolVO storagePool) {
         if (ScopeType.HOST.equals(storagePool.getScope()) ||
                 Storage.StoragePoolType.SharedMountPoint.equals(storagePool.getPoolType()) ||
                 Storage.StoragePoolType.RBD.equals(storagePool.getPoolType())) {
-            volumePathPrefix = storagePool.getPath();
+            volumePathPrefix = storagePool.getPath() + "/";
+        } else if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) {
+            volumePathPrefix = "/dev/drbd/by-res/cs-";
         } else {
             // Should be Storage.StoragePoolType.NetworkFilesystem
-            volumePathPrefix = String.format("/mnt/%s", storagePool.getUuid());
+            volumePathPrefix = String.format("/mnt/%s/", storagePool.getUuid());
         }
         return volumePathPrefix;
     }
 
+    private String getVolumePathSuffix(StoragePoolVO storagePool) {
+        if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) {
+            return "/0";
+        } else {
+            return "";
+        }
+    }
+
     @Override
     public Pair<Boolean, String> restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair<String, VirtualMachine.State> vmNameAndState) {
         final VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid());
@@ -413,7 +424,9 @@ public Pair<Boolean, String> restoreBackedUpVolume(Backup backup, Backup.VolumeI
         restoreCommand.setBackupRepoType(backupRepository.getType());
         restoreCommand.setBackupRepoAddress(backupRepository.getAddress());
         restoreCommand.setVmName(vmNameAndState.first());
-        restoreCommand.setRestoreVolumePaths(Collections.singletonList(String.format("%s/%s", getVolumePathPrefix(pool), volumeUUID)));
+        String restoreVolumePath = String.format("%s%s%s", getVolumePathPrefix(pool), volumeUUID, getVolumePathSuffix(pool));
+        restoreCommand.setRestoreVolumePaths(Collections.singletonList(restoreVolumePath));
+        restoreCommand.setRestoreVolumeSizes(Collections.singletonList(backedUpVolumeSize));
         DataStore dataStore = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary);
         restoreCommand.setRestoreVolumePools(Collections.singletonList(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null));
         restoreCommand.setDiskType(backupVolumeInfo.getType().name().toLowerCase(Locale.ROOT));