Skip to content

Commit c0cf895

Browse files
Pearl1594Pearl1594
committed
support live migration from clvm_ng to nfs and vice-versa
1 parent 22f9d0e commit c0cf895

File tree

4 files changed

+174
-4
lines changed

4 files changed

+174
-4
lines changed

‎core/src/main/java/com/cloud/agent/api/MigrateCommand.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
import com.cloud.agent.api.to.DpdkTO;
2828
import com.cloud.agent.api.to.VirtualMachineTO;
29+
import com.cloud.storage.Storage;
2930

3031
public class MigrateCommand extends Command {
3132
private String vmName;
@@ -184,6 +185,7 @@ public String toString() {
184185
private final String sourceText;
185186
private final String backingStoreText;
186187
private boolean isSourceDiskOnStorageFileSystem;
188+
private Storage.StoragePoolType destPoolType;
187189

188190
public MigrateDiskInfo(final String serialNumber, final DiskType diskType, final DriverType driverType, final Source source, final String sourceText) {
189191
this.serialNumber = serialNumber;
@@ -232,6 +234,14 @@ public boolean isSourceDiskOnStorageFileSystem() {
232234
public void setSourceDiskOnStorageFileSystem(boolean isDiskOnFileSystemStorage) {
233235
this.isSourceDiskOnStorageFileSystem = isDiskOnFileSystemStorage;
234236
}
237+
238+
public Storage.StoragePoolType getDestPoolType() {
239+
return destPoolType;
240+
}
241+
242+
public void setDestPoolType(Storage.StoragePoolType destPoolType) {
243+
this.destPoolType = destPoolType;
244+
}
235245
}
236246

237247
@Override

‎engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,12 +144,16 @@ protected boolean isDestinationNfsPrimaryStorageClusterWide(Map<VolumeInfo, Data
144144
}
145145

146146
/**
147-
* Configures a {@link MigrateDiskInfo} object configured for migrating a File System volume and calls rootImageProvisioning.
147+
* Configures a {@link MigrateDiskInfo} object configured for migrating a File System volume.
148148
*/
149149
@Override
150150
protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo srcVolumeInfo, String destPath, String backingPath) {
151-
return new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(), MigrateCommand.MigrateDiskInfo.DiskType.FILE, MigrateCommand.MigrateDiskInfo.DriverType.QCOW2,
152-
MigrateCommand.MigrateDiskInfo.Source.FILE, destPath, backingPath);
151+
return new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(),
152+
MigrateCommand.MigrateDiskInfo.DiskType.FILE,
153+
MigrateCommand.MigrateDiskInfo.DriverType.QCOW2,
154+
MigrateCommand.MigrateDiskInfo.Source.FILE,
155+
destPath,
156+
backingPath);
153157
}
154158

155159
/**
@@ -158,6 +162,17 @@ protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo src
158162
*/
159163
@Override
160164
protected String generateDestPath(Host destHost, StoragePoolVO destStoragePool, VolumeInfo destVolumeInfo) {
165+
if (destStoragePool.getPoolType() == StoragePoolType.CLVM || destStoragePool.getPoolType() == StoragePoolType.CLVM_NG) {
166+
String vgName = destStoragePool.getPath();
167+
if (StringUtils.isBlank(vgName)) {
168+
throw new CloudRuntimeException(String.format("CLVM/CLVM_NG destination pool [%s] has empty VG path", destStoragePool.getUuid()));
169+
}
170+
if (vgName.startsWith("/")) {
171+
vgName = vgName.substring(1);
172+
}
173+
return String.format("/dev/%s/%s", vgName, destVolumeInfo.getUuid());
174+
}
175+
161176
return new File(destStoragePool.getPath(), destVolumeInfo.getUuid()).getAbsolutePath();
162177
}
163178

‎engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.cloud.agent.api.CheckVirtualMachineCommand;
3737
import com.cloud.agent.api.PrepareForMigrationAnswer;
3838
import com.cloud.resource.ResourceManager;
39+
import com.cloud.storage.ClvmLockManager;
3940
import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
4041
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
4142
import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
@@ -2107,9 +2108,11 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeDataStoreMap, VirtualMach
21072108
} else {
21082109
String backingPath = generateBackingPath(destStoragePool, destVolumeInfo);
21092110
migrateDiskInfo = configureMigrateDiskInfo(srcVolumeInfo, destPath, backingPath);
2111+
migrateDiskInfo = updateMigrateDiskInfoForBlockDevice(migrateDiskInfo, destStoragePool);
21102112
migrateDiskInfo.setSourceDiskOnStorageFileSystem(isStoragePoolTypeOfFile(sourceStoragePool));
21112113
migrateDiskInfoList.add(migrateDiskInfo);
21122114
}
2115+
migrateDiskInfo.setDestPoolType(destVolumeInfo.getStoragePoolType());
21132116
prepareDiskWithSecretConsumerDetail(vmTO, srcVolumeInfo, destVolumeInfo.getPath());
21142117

21152118
migrateStorage.put(srcVolumeInfo.getPath(), migrateDiskInfo);
@@ -2328,6 +2331,39 @@ protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo src
23282331
MigrateCommand.MigrateDiskInfo.Source.DEV, destPath, backingPath);
23292332
}
23302333

2334+
/**
2335+
* UpdatesMigrateDiskInfo for CLVM/CLVM_NG block devices by returning a new instance with corrected disk type, driver type, and source.
2336+
* For CLVM/CLVM_NG destinations, returns a new MigrateDiskInfo with BLOCK disk type, DEV source, and appropriate driver type (QCOW2 for CLVM_NG, RAW for CLVM).
2337+
* For other storage types, returns the original MigrateDiskInfo unchanged.
2338+
*
2339+
* @param migrateDiskInfo The original MigrateDiskInfo object
2340+
* @param destStoragePool The destination storage pool
2341+
* @return A new MigrateDiskInfo with updated values for CLVM/CLVM_NG, or the original for other storage types
2342+
*/
2343+
protected MigrateCommand.MigrateDiskInfo updateMigrateDiskInfoForBlockDevice(MigrateCommand.MigrateDiskInfo migrateDiskInfo,
2344+
StoragePoolVO destStoragePool) {
2345+
if (ClvmLockManager.isClvmPoolType(destStoragePool.getPoolType())) {
2346+
2347+
MigrateCommand.MigrateDiskInfo.DriverType driverType =
2348+
(destStoragePool.getPoolType() == StoragePoolType.CLVM_NG) ?
2349+
MigrateCommand.MigrateDiskInfo.DriverType.QCOW2 :
2350+
MigrateCommand.MigrateDiskInfo.DriverType.RAW;
2351+
2352+
logger.debug("Updating MigrateDiskInfo for {} destination: setting BLOCK disk type, DEV source, and {} driver type",
2353+
destStoragePool.getPoolType(), driverType);
2354+
2355+
return new MigrateCommand.MigrateDiskInfo(
2356+
migrateDiskInfo.getSerialNumber(),
2357+
MigrateCommand.MigrateDiskInfo.DiskType.BLOCK,
2358+
driverType,
2359+
MigrateCommand.MigrateDiskInfo.Source.DEV,
2360+
migrateDiskInfo.getSourceText(),
2361+
migrateDiskInfo.getBackingStoreText());
2362+
}
2363+
2364+
return migrateDiskInfo;
2365+
}
2366+
23312367
/**
23322368
* Sets the volume path as the iScsi name in case of a configured iScsi.
23332369
*/

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java‎

Lines changed: 110 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import com.cloud.hypervisor.kvm.resource.LibvirtXMLParser;
4747
import com.cloud.resource.CommandWrapper;
4848
import com.cloud.resource.ResourceWrapper;
49+
import com.cloud.storage.Storage;
4950
import com.cloud.utils.Ternary;
5051
import com.cloud.utils.exception.CloudRuntimeException;
5152
import com.cloud.vm.VirtualMachine;
@@ -243,14 +244,33 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0.
243244
final ExecutorService executor = Executors.newFixedThreadPool(1);
244245
boolean migrateNonSharedInc = command.isMigrateNonSharedInc() && !migrateStorageManaged;
245246

247+
// If any of the destination disks target CLVM/CLVM_NG pools, exclude them from
248+
// the migration disk list. These disks are already created/activated on the
249+
// destination storage before VM migration, so libvirt should not try to migrate them.
250+
// Only non-CLVM disks will be migrated by libvirt.
251+
boolean effectiveMigrateStorage = migrateStorage;
252+
if (migrateStorage && migrateDiskLabels != null && hasClvmDestinationDisks(mapMigrateStorage)) {
253+
logger.info("Filtering out CLVM/CLVM_NG disks from migration for VM {} as they are pre-created on destination", vmName);
254+
migrateDiskLabels = filterOutClvmDisks(migrateDiskLabels, disks, mapMigrateStorage);
255+
logger.debug("Remaining disks to migrate via libvirt: {}", migrateDiskLabels);
256+
257+
// If all disks were filtered out (only CLVM/CLVM_NG), disable storage migration entirely
258+
// to prevent libvirt from trying to handle the block devices
259+
if (migrateDiskLabels != null && migrateDiskLabels.isEmpty()) {
260+
logger.info("All disks are CLVM/CLVM_NG and pre-created on destination. Disabling storage migration for VM {}", vmName);
261+
effectiveMigrateStorage = false;
262+
migrateDiskLabels = null;
263+
}
264+
}
265+
246266
// add cancel hook before we start. If migration fails to start and hook is called, it's non-fatal
247267
cancelHook = new MigrationCancelHook(dm);
248268
libvirtComputingResource.addDisconnectHook(cancelHook);
249269

250270
libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.PROCESSING);
251271

252272
final Callable<Domain> worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc,
253-
migrateStorage, migrateNonSharedInc,
273+
effectiveMigrateStorage, migrateNonSharedInc,
254274
command.isAutoConvergence(), vmName, command.getDestinationIp(), migrateDiskLabels);
255275
final Future<Domain> migrateThread = executor.submit(worker);
256276
executor.shutdown();
@@ -1073,4 +1093,93 @@ public static String maskSensitiveInfoInXML(String xmlDesc) {
10731093
return xmlDesc.replaceAll("(graphics\\s+[^>]*type=['\"]vnc['\"][^>]*passwd=['\"])([^'\"]*)(['\"])",
10741094
"$1*****$3");
10751095
}
1096+
1097+
/**
1098+
* Checks if any of the destination disks in the migration target a CLVM or CLVM_NG storage pool.
1099+
* This is used to determine if incremental migration should be disabled to avoid libvirt
1100+
* precreate errors with QCOW2-on-LVM setups.
1101+
*
1102+
* @param mapMigrateStorage the map containing migration disk information with destination pool types
1103+
* @return true if any destination disk targets CLVM or CLVM_NG, false otherwise
1104+
*/
1105+
protected boolean hasClvmDestinationDisks(Map<String, MigrateCommand.MigrateDiskInfo> mapMigrateStorage) {
1106+
if (MapUtils.isEmpty(mapMigrateStorage)) {
1107+
return false;
1108+
}
1109+
1110+
try {
1111+
for (Map.Entry<String, MigrateCommand.MigrateDiskInfo> entry : mapMigrateStorage.entrySet()) {
1112+
MigrateCommand.MigrateDiskInfo diskInfo = entry.getValue();
1113+
if (isClvmBlockDevice(diskInfo)) {
1114+
logger.debug("Found disk targeting CLVM/CLVM_NG destination pool");
1115+
return true;
1116+
}
1117+
}
1118+
} catch (final Exception e) {
1119+
logger.debug("Failed to check for CLVM destination disks: {}. Assuming no CLVM disks.", e.getMessage());
1120+
}
1121+
1122+
return false;
1123+
}
1124+
1125+
/**
1126+
* Filters out disk labels that target CLVM or CLVM_NG destination pools from the migration disk labels set.
1127+
* CLVM/CLVM_NG disks are pre-created/activated on the destination before VM migration,
1128+
* so they should not be migrated by libvirt.
1129+
*
1130+
* @param migrateDiskLabels the original set of disk labels to migrate
1131+
* @param diskDefinitions the list of disk definitions to map labels to paths
1132+
* @param mapMigrateStorage the map containing migration disk information with destination pool types
1133+
* @return a new set with CLVM/CLVM_NG disk labels removed
1134+
*/
1135+
protected Set<String> filterOutClvmDisks(Set<String> migrateDiskLabels,
1136+
List<DiskDef> diskDefinitions,
1137+
Map<String, MigrateCommand.MigrateDiskInfo> mapMigrateStorage) {
1138+
if (migrateDiskLabels == null || migrateDiskLabels.isEmpty()) {
1139+
return migrateDiskLabels;
1140+
}
1141+
1142+
Set<String> filteredLabels = new HashSet<>(migrateDiskLabels);
1143+
1144+
try {
1145+
Set<String> clvmDiskPaths = new HashSet<>();
1146+
for (Map.Entry<String, MigrateCommand.MigrateDiskInfo> entry : mapMigrateStorage.entrySet()) {
1147+
MigrateCommand.MigrateDiskInfo diskInfo = entry.getValue();
1148+
if (isClvmBlockDevice(diskInfo)) {
1149+
clvmDiskPaths.add(entry.getKey());
1150+
logger.debug("Identified CLVM/CLVM_NG destination disk: {}", entry.getKey());
1151+
}
1152+
}
1153+
1154+
// Map disk paths to labels and remove CLVM disk labels from the migration set
1155+
if (!clvmDiskPaths.isEmpty()) {
1156+
for (String clvmDiskPath : clvmDiskPaths) {
1157+
for (DiskDef diskDef : diskDefinitions) {
1158+
String diskPath = diskDef.getDiskPath();
1159+
if (diskPath != null && diskPath.contains(clvmDiskPath)) {
1160+
String label = diskDef.getDiskLabel();
1161+
if (filteredLabels.remove(label)) {
1162+
logger.info("Excluded disk label {} (path: {}) from libvirt migration - CLVM/CLVM_NG destination",
1163+
label, clvmDiskPath);
1164+
}
1165+
break;
1166+
}
1167+
}
1168+
}
1169+
}
1170+
1171+
} catch (final Exception e) {
1172+
logger.warn("Failed to filter CLVM disks: {}. Proceeding with original disk list.", e.getMessage());
1173+
return migrateDiskLabels;
1174+
}
1175+
1176+
return filteredLabels;
1177+
}
1178+
1179+
private boolean isClvmBlockDevice(MigrateCommand.MigrateDiskInfo diskInfo) {
1180+
if (diskInfo == null ||diskInfo.getDestPoolType() == null) {
1181+
return false;
1182+
}
1183+
return (Storage.StoragePoolType.CLVM.equals(diskInfo.getDestPoolType()) || Storage.StoragePoolType.CLVM_NG.equals(diskInfo.getDestPoolType()));
1184+
}
10761185
}

0 commit comments

Comments
 (0)