veeam: fix issues with PreSetup and DVS and Solidfire (#9256)

* Veeam: find storage pool by path for PreSetup and VMFS

* Veeam: support VMware distributed virtual switch

* Veeam: sync volumes on Solidfire after backup restoration

user faced the issue that backup is restored but the DATA disk is gone (ROOT disk is ok)
```
2024-05-03 12:00:32,868 ERROR [o.a.c.b.BackupManagerImpl] (API-Job-Executor-13:ctx-aa8a1d85 job-149661 ctx-73328567) (logid:6510cf06) Failed to import VM [vmInternalName: i-169-9679-VM] from backup restoration [{"backupType":"Full","externalId":"821ca400-a5da-4282-bf3f-7c7e38a6cdb4","id":257,"uuid":"69399101-5cbd-461c-8a48-f0c70eac0b24","vmId":9679}] with hypervisor [type: VMware] due to: [Couldn't find storage pool -iqn.2010-01.com.solidfire:3p53.data-9679.221-0].
```

On managed storage, the datastore name of DATA disk is determined by the iscsi_name of the volume.

* Veeam: set correct path for DATA disks on solidfire

Author: weizhouapache

engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java

@@ -156,4 +156,6 @@ public interface VolumeDao extends GenericDao<VolumeVO, Long>, StateDao<Volume.S
     VolumeVO findByPoolIdAndPath(long id, String path);
 
     List<VolumeVO> listByIds(List<Long> ids);
+
+    VolumeVO findOneByIScsiName(String iScsiName);
 }

engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java

@@ -388,6 +388,7 @@ public VolumeDaoImpl() {
         AllFieldsSearch.and("updatedCount", AllFieldsSearch.entity().getUpdatedCount(), Op.EQ);
         AllFieldsSearch.and("name", AllFieldsSearch.entity().getName(), Op.EQ);
         AllFieldsSearch.and("passphraseId", AllFieldsSearch.entity().getPassphraseId(), Op.EQ);
+        AllFieldsSearch.and("iScsiName", AllFieldsSearch.entity().get_iScsiName(), Op.EQ);
         AllFieldsSearch.done();
 
         RootDiskStateSearch = createSearchBuilder();
@@ -840,4 +841,10 @@ public List<VolumeVO> listByIds(List<Long> ids) {
         sc.setParameters("idIN", ids.toArray());
         return listBy(sc, null);
     }
+
+    public VolumeVO findOneByIScsiName(String iScsiName) {
+        SearchCriteria<VolumeVO> sc = AllFieldsSearch.create();
+        sc.setParameters("iScsiName", iScsiName);
+        return findOneIncludingRemovedBy(sc);
+    }
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java

@@ -140,6 +140,8 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
 
     List<StoragePoolVO> findPoolsByStorageType(String storageType);
 
+    StoragePoolVO findPoolByZoneAndPath(long zoneId, String datastorePath);
+
     List<StoragePoolVO> listStoragePoolsWithActiveVolumesByOfferingId(long offeringid);
 
     Pair<List<Long>, Integer> searchForIdsAndCount(Long storagePoolId, String storagePoolName, Long zoneId,

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java

@@ -659,6 +659,16 @@ public List<StoragePoolVO> findPoolsByStorageType(String storageType) {
         return listBy(sc);
     }
 
+    @Override
+    public StoragePoolVO findPoolByZoneAndPath(long zoneId, String datastorePath) {
+        SearchCriteria<StoragePoolVO> sc = AllFieldSearch.create();
+        sc.setParameters("datacenterId", zoneId);
+        if (datastorePath != null) {
+            sc.addAnd("path", Op.LIKE,  "%/" + datastorePath);
+        }
+        return findOneBy(sc);
+    }
+
     @Override
     public List<StoragePoolVO> listStoragePoolsWithActiveVolumesByOfferingId(long offeringId) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java

@@ -18,6 +18,7 @@
 
 import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
 
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
@@ -149,16 +150,22 @@
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.VMInstanceDao;
 import com.google.gson.Gson;
+import com.vmware.vim25.DistributedVirtualPort;
+import com.vmware.vim25.DistributedVirtualSwitchPortConnection;
+import com.vmware.vim25.DistributedVirtualSwitchPortCriteria;
 import com.vmware.vim25.ManagedObjectReference;
+import com.vmware.vim25.VMwareDVSPortSetting;
 import com.vmware.vim25.VirtualDevice;
 import com.vmware.vim25.VirtualDeviceBackingInfo;
 import com.vmware.vim25.VirtualDeviceConnectInfo;
 import com.vmware.vim25.VirtualDisk;
 import com.vmware.vim25.VirtualDiskFlatVer2BackingInfo;
 import com.vmware.vim25.VirtualEthernetCard;
+import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo;
 import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo;
 import com.vmware.vim25.VirtualMachineConfigSummary;
 import com.vmware.vim25.VirtualMachineRuntimeInfo;
+import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec;
 
 public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Configurable {
     private static final Logger s_logger = Logger.getLogger(VMwareGuru.class);
@@ -533,25 +540,43 @@ private void checkBackingInfo(VirtualDeviceBackingInfo backingInfo) {
     /**
      * Get pool ID from datastore UUID
      */
-    private Long getPoolIdFromDatastoreUuid(String datastoreUuid) {
-        String poolUuid = UuidUtils.normalize(datastoreUuid);
-        StoragePoolVO pool = _storagePoolDao.findByUuid(poolUuid);
+    private Long getPoolIdFromDatastoreUuid(long zoneId, String datastoreUuid) {
+        StoragePoolVO pool = null;
+        try {
+            String poolUuid = UuidUtils.normalize(datastoreUuid);
+            s_logger.info("Trying to find pool by UUID: " + poolUuid);
+            pool = _storagePoolDao.findByUuid(poolUuid);
+        } catch (CloudRuntimeException ex) {
+            s_logger.warn("Unable to get pool by datastore UUID: " + ex.getMessage());
+        }
         if (pool == null) {
-            throw new CloudRuntimeException("Couldn't find storage pool " + poolUuid);
+            s_logger.info("Trying to find pool by path: " + datastoreUuid);
+            pool = _storagePoolDao.findPoolByZoneAndPath(zoneId, datastoreUuid);
+        }
+        if (pool == null && datastoreUuid.startsWith("-iqn") && datastoreUuid.endsWith("-0")) {
+            String iScsiName = "/iqn" + datastoreUuid.substring(4, datastoreUuid.length() - 2) + "/0";
+            s_logger.info("Trying to find volume by iScsi name: " + iScsiName);
+            VolumeVO volumeVO = _volumeDao.findOneByIScsiName(iScsiName);
+            if (volumeVO != null) {
+                pool = _storagePoolDao.findById(volumeVO.getPoolId());
+            }
+        }
+        if (pool == null) {
+            throw new CloudRuntimeException("Couldn't find storage pool " + datastoreUuid);
         }
         return pool.getId();
     }
 
     /**
      * Get pool ID for disk
      */
-    private Long getPoolId(VirtualDisk disk) {
+    private Long getPoolId(long zoneId, VirtualDisk disk) {
         VirtualDeviceBackingInfo backing = disk.getBacking();
         checkBackingInfo(backing);
         VirtualDiskFlatVer2BackingInfo info = (VirtualDiskFlatVer2BackingInfo)backing;
         String[] fileNameParts = info.getFileName().split(" ");
         String datastoreUuid = StringUtils.substringBetween(fileNameParts[0], "[", "]");
-        return getPoolIdFromDatastoreUuid(datastoreUuid);
+        return getPoolIdFromDatastoreUuid(zoneId, datastoreUuid);
     }
 
     /**
@@ -588,12 +613,12 @@ private VirtualMachineMO getTemplate(DatacenterMO dcMo, String templatePath) thr
     /**
      * Get template pool ID
      */
-    private Long getTemplatePoolId(VirtualMachineMO template) throws Exception {
+    private Long getTemplatePoolId(long zoneId, VirtualMachineMO template) throws Exception {
         VirtualMachineConfigSummary configSummary = template.getConfigSummary();
         String vmPathName = configSummary.getVmPathName();
         String[] pathParts = vmPathName.split(" ");
         String dataStoreUuid = pathParts[0].replace("[", "").replace("]", "");
-        return getPoolIdFromDatastoreUuid(dataStoreUuid);
+        return getPoolIdFromDatastoreUuid(zoneId, dataStoreUuid);
     }
 
     /**
@@ -643,14 +668,14 @@ private void updateTemplateRef(long templateId, Long poolId, String templatePath
     /**
      * Get template ID for VM being imported. If it is not found, it is created
      */
-    private Long getImportingVMTemplate(List<VirtualDisk> virtualDisks, DatacenterMO dcMo, String vmInternalName, Long guestOsId, long accountId, Map<VirtualDisk, VolumeVO> disksMapping, Backup backup) throws Exception {
+    private Long getImportingVMTemplate(List<VirtualDisk> virtualDisks, long zoneId, DatacenterMO dcMo, String vmInternalName, Long guestOsId, long accountId, Map<VirtualDisk, VolumeVO> disksMapping, Backup backup) throws Exception {
         for (VirtualDisk disk : virtualDisks) {
             if (isRootDisk(disk, disksMapping, backup)) {
                 VolumeVO volumeVO = disksMapping.get(disk);
                 if (volumeVO == null) {
                     String templatePath = getRootDiskTemplatePath(disk);
                     VirtualMachineMO template = getTemplate(dcMo, templatePath);
-                    Long poolId = getTemplatePoolId(template);
+                    Long poolId = getTemplatePoolId(zoneId, template);
                     Long templateSize = getTemplateSize(template, vmInternalName, disksMapping, backup);
                     long templateId = getTemplateId(templatePath, vmInternalName, guestOsId, accountId);
                     updateTemplateRef(templateId, poolId, templatePath, templateSize);
@@ -744,7 +769,11 @@ private long getDiskOfferingId(long size, Storage.ProvisioningType provisioningT
     protected VolumeVO updateVolume(VirtualDisk disk, Map<VirtualDisk, VolumeVO> disksMapping, VirtualMachineMO vmToImport, Long poolId, VirtualMachine vm) throws Exception {
         VolumeVO volume = disksMapping.get(disk);
         String volumeName = getVolumeName(disk, vmToImport);
-        volume.setPath(volumeName);
+        if (volume.get_iScsiName() != null) {
+            volume.setPath(String.format("[%s] %s.vmdk", volumeName, volumeName));
+        } else {
+            volume.setPath(volumeName);
+        }
         volume.setPoolId(poolId);
         VirtualMachineDiskInfo diskInfo = getDiskInfo(vmToImport, poolId, volumeName);
         volume.setChainInfo(GSON.toJson(diskInfo));
@@ -779,7 +808,7 @@ private void syncVMVolumes(VMInstanceVO vmInstanceVO, List<VirtualDisk> virtualD
 
         String operation = "";
         for (VirtualDisk disk : virtualDisks) {
-            Long poolId = getPoolId(disk);
+            Long poolId = getPoolId(zoneId, disk);
             Volume volume = null;
             if (disksMapping.containsKey(disk) && disksMapping.get(disk) != null) {
                 volume = updateVolume(disk, disksMapping, vmToImport, poolId, vmInstanceVO);
@@ -903,8 +932,13 @@ private Map<String, NetworkVO> getNetworksMapping(String[] vmNetworkNames, long
         Map<String, NetworkVO> mapping = new HashMap<>();
         for (String networkName : vmNetworkNames) {
             NetworkVO networkVO = getGuestNetworkFromNetworkMorName(networkName, accountId, zoneId, domainId);
-            s_logger.debug(String.format("Mapping network name [%s] to networkVO [id: %s].", networkName, networkVO.getUuid()));
-            mapping.put(networkName, networkVO);
+            URI broadcastUri = networkVO.getBroadcastUri();
+            if (broadcastUri == null) {
+                continue;
+            }
+            String vlan = broadcastUri.getHost();
+            s_logger.debug(String.format("Mapping network vlan [%s] to networkVO [id: %s].", vlan, networkVO.getUuid()));
+            mapping.put(vlan, networkVO);
         }
         return mapping;
     }
@@ -914,30 +948,88 @@ private Map<String, NetworkVO> getNetworksMapping(String[] vmNetworkNames, long
      */
     private NetworkMO getNetworkMO(VirtualEthernetCard nic, VmwareContext context) {
         VirtualDeviceConnectInfo connectable = nic.getConnectable();
-        VirtualEthernetCardNetworkBackingInfo info = (VirtualEthernetCardNetworkBackingInfo)nic.getBacking();
+        VirtualEthernetCardNetworkBackingInfo info = (VirtualEthernetCardNetworkBackingInfo) nic.getBacking();
         ManagedObjectReference networkMor = info.getNetwork();
         if (networkMor == null) {
             throw new CloudRuntimeException("Could not find network for NIC on: " + nic.getMacAddress());
         }
         return new NetworkMO(context, networkMor);
     }
 
-    private Pair<String, String> getNicMacAddressAndNetworkName(VirtualDevice nicDevice, VmwareContext context) throws Exception {
+    private Pair<String, String> getNicMacAddressAndVlan(VirtualDevice nicDevice, VmwareContext context) throws Exception {
         VirtualEthernetCard nic = (VirtualEthernetCard)nicDevice;
         String macAddress = nic.getMacAddress();
-        NetworkMO networkMO = getNetworkMO(nic, context);
-        String networkName = networkMO.getName();
-        return new Pair<>(macAddress, networkName);
+        VirtualDeviceBackingInfo backing = nic.getBacking();
+        if (backing instanceof VirtualEthernetCardNetworkBackingInfo) {
+            VirtualEthernetCardNetworkBackingInfo backingInfo = (VirtualEthernetCardNetworkBackingInfo) backing;
+            String deviceName = backingInfo.getDeviceName();
+            String vlan = getVlanFromDeviceName(deviceName);
+            return new Pair<>(macAddress, vlan);
+        } else if (backing instanceof VirtualEthernetCardDistributedVirtualPortBackingInfo) {
+            VirtualEthernetCardDistributedVirtualPortBackingInfo portInfo = (VirtualEthernetCardDistributedVirtualPortBackingInfo) backing;
+            DistributedVirtualSwitchPortConnection port = portInfo.getPort();
+            String portKey = port.getPortKey();
+            String portGroupKey = port.getPortgroupKey();
+            String dvSwitchUuid = port.getSwitchUuid();
+            String vlan = getVlanFromDvsPort(context, dvSwitchUuid, portGroupKey, portKey);
+            return new Pair<>(macAddress, vlan);
+        }
+        return new Pair<>(macAddress, null);
+    }
+
+    private String getVlanFromDeviceName(String networkName) {
+        String prefix = "cloud.guest.";
+        if (!networkName.startsWith(prefix)) {
+            return null;
+        }
+        String nameWithoutPrefix = networkName.replace(prefix, "");
+        String[] parts = nameWithoutPrefix.split("\\.");
+        String vlan = parts[0];
+        return vlan;
+    }
+
+    private String getVlanFromDvsPort(VmwareContext context, String dvSwitchUuid, String portGroupKey, String portKey) {
+        try {
+            ManagedObjectReference dvSwitchManager = context.getVimClient().getServiceContent().getDvSwitchManager();
+            ManagedObjectReference dvSwitch = context.getVimClient().getService().queryDvsByUuid(dvSwitchManager, dvSwitchUuid);
+
+            // Get all ports
+            DistributedVirtualSwitchPortCriteria criteria = new DistributedVirtualSwitchPortCriteria();
+            criteria.setInside(true);
+            criteria.getPortgroupKey().add(portGroupKey);
+            List<DistributedVirtualPort> dvPorts = context.getVimClient().getService().fetchDVPorts(dvSwitch, criteria);
+
+            for (DistributedVirtualPort dvPort : dvPorts) {
+                if (!portKey.equals(dvPort.getKey())) {
+                    continue;
+                }
+                VMwareDVSPortSetting settings = (VMwareDVSPortSetting) dvPort.getConfig().getSetting();
+                VmwareDistributedVirtualSwitchVlanIdSpec vlanId = (VmwareDistributedVirtualSwitchVlanIdSpec) settings.getVlan();
+                s_logger.debug("Found port " + dvPort.getKey() + " with vlan " + vlanId.getVlanId());
+                return String.valueOf(vlanId.getVlanId());
+            }
+        } catch (Exception ex) {
+            s_logger.error("Got exception while get vlan from DVS port: " + ex.getMessage());
+        }
+        return null;
     }
 
     private void syncVMNics(VirtualDevice[] nicDevices, DatacenterMO dcMo, Map<String, NetworkVO> networksMapping, VMInstanceVO vm) throws Exception {
         VmwareContext context = dcMo.getContext();
         List<NicVO> allNics = nicDao.listByVmId(vm.getId());
         for (VirtualDevice nicDevice : nicDevices) {
-            Pair<String, String> pair = getNicMacAddressAndNetworkName(nicDevice, context);
+            Pair<String, String> pair = getNicMacAddressAndVlan(nicDevice, context);
             String macAddress = pair.first();
-            String networkName = pair.second();
-            NetworkVO networkVO = networksMapping.get(networkName);
+            String vlanId = pair.second();
+            if (vlanId == null) {
+                s_logger.warn(String.format("vlanId for MAC address [%s] is null", macAddress));
+                continue;
+            }
+            NetworkVO networkVO = networksMapping.get(vlanId);
+            if (networkVO == null) {
+                s_logger.warn(String.format("Cannot find network for MAC address [%s] and vlanId [%s]", macAddress, vlanId));
+                continue;
+            }
             NicVO nicVO = nicDao.findByNetworkIdAndMacAddressIncludingRemoved(networkVO.getId(), macAddress);
             if (nicVO != null) {
                 s_logger.warn(String.format("Find NIC in DB with networkId [%s] and MAC Address [%s], so this NIC will be removed from list of unmapped NICs of VM [id: %s, name: %s].",
@@ -1068,7 +1160,7 @@ public VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId,
 
         long guestOsId = getImportingVMGuestOs(configSummary);
         long serviceOfferingId = getImportingVMServiceOffering(configSummary, runtimeInfo);
-        long templateId = getImportingVMTemplate(virtualDisks, dcMo, vmInternalName, guestOsId, accountId, disksMapping, backup);
+        long templateId = getImportingVMTemplate(virtualDisks, zoneId, dcMo, vmInternalName, guestOsId, accountId, disksMapping, backup);
 
         VMInstanceVO vm = getVM(vmInternalName, templateId, guestOsId, serviceOfferingId, zoneId, accountId, userId, domainId);
         syncVMVolumes(vm, virtualDisks, disksMapping, vmToImport, backup);