Add backup validation

Author: João Jandre

agent/conf/agent.properties

@@ -461,3 +461,12 @@ iscsi.session.cleanup.enabled=false
 # Timeout (in seconds) for QCOW2 delta merge operations, mainly used for classic volume snapshots, disk-only VM snapshots on file-based storage, and the KNIB plugin.
 # If a value of 0 or less is informed, the default will be used.
 # qcow2.delta.merge.timeout=259200
+
+
+# Maximum number of backup validation jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many validations as possible will be done at
+# the same time.
+# backup.validation.max.concurrent.operations.per.host=
+
+# Maximum number of backup compression jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many compressions as possible will be
+# done at the same time.
+# backup.compression.max.concurrent.operations.per.host=

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -887,6 +887,18 @@ public Property<Integer> getWorkers() {
     public static final Property<Boolean> CREATE_FULL_CLONE = new Property<>("create.full.clone", false);
 
 
+    /**
+     * Maximum number of backup validation jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many validations as possible will be done at
+     * the same time.
+     */
+    public static final Property<Integer> BACKUP_VALIDATION_MAX_CONCURRENT_OPERATIONS_PER_HOST = new Property<>("backup.validation.max.concurrent.operations.per.host", null, Integer.class);
+
+    /**
+     * Maximum number of backup compression jobs that can be executed at the same time. Values lower than 0 remove the limit, meaning that as many compressions as possible will be
+     * done at the same time.
+     */
+    public static final Property<Integer> BACKUP_COMPRESSION_MAX_CONCURRENT_OPERATIONS_PER_HOST = new Property<>("backup.compression.max.concurrent.operations.per.host", null, Integer.class);
+
     public static class Property <T>{
         private String name;
         private T defaultValue;

api/src/main/java/com/cloud/event/EventTypes.java

@@ -643,6 +643,7 @@ public class EventTypes {
     public static final String EVENT_VM_BACKUP_USAGE_METRIC = "BACKUP.USAGE.METRIC";
     public static final String EVENT_VM_BACKUP_EDIT = "BACKUP.OFFERING.EDIT";
     public static final String EVENT_VM_CREATE_FROM_BACKUP = "VM.CREATE.FROM.BACKUP";
+    public static final String EVENT_SCREENSHOT_DOWNLOAD = "BACKUP.VALIDATION.SCREENSHOT.DOWNLOAD";
 
     // external network device events
     public static final String EVENT_EXTERNAL_NVP_CONTROLLER_ADD = "PHYSICAL.NVPCONTROLLER.ADD";

api/src/main/java/com/cloud/storage/Storage.java

@@ -35,7 +35,8 @@ public static enum ImageFormat {
         VDI(true, true, false, "vdi"),
         TAR(false, false, false, "tar"),
         ZIP(false, false, false, "zip"),
-        DIR(false, false, false, "dir");
+        DIR(false, false, false, "dir"),
+        PNG(false, false, false, "png");
 
         private final boolean supportThinProvisioning;
         private final boolean supportSparse;

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -182,7 +182,7 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
 
     boolean validateConditionsToReplaceDiskOfferingOfVolume(Volume volume, DiskOffering newDiskOffering, StoragePool destPool);
 
-    Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge);
+    Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge, Boolean countDisplayFalseInResourceCount);
 
     void destroyVolume(long volumeId);
 

api/src/main/java/com/cloud/user/ResourceLimitService.java

@@ -259,7 +259,7 @@ void checkVolumeResourceLimitForDiskOfferingChange(Account owner, Boolean displa
     void checkPrimaryStorageResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException;
 
     void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
-    void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
+    void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering, Boolean countDisplayFalseInResourceCount);
 
     void updateVmResourceCountForTemplateChange(long accountId, Boolean display, ServiceOffering offering, VirtualMachineTemplate currentTemplate, VirtualMachineTemplate newTemplate);
 
@@ -274,8 +274,8 @@ void updateVolumeResourceCountForDiskOfferingChange(long accountId, Boolean disp
     void incrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
     void decrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering);
     void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) throws ResourceAllocationException;
-    void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template);
-    void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template);
+    void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Boolean countDisplayFalseInResourceLimit);
+    void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Boolean countDisplayFalseInResourceCount);
 
     void checkVmResourceLimitsForServiceOfferingChange(Account owner, Boolean display, Long currentCpu, Long newCpu,
             Long currentMemory, Long newMemory, ServiceOffering currentOffering, ServiceOffering newOffering, VirtualMachineTemplate template) throws ResourceAllocationException;

api/src/main/java/com/cloud/vm/UserVmService.java

@@ -73,11 +73,13 @@ public interface UserVmService {
     /**
      * Destroys one virtual machine
      *
-     * @param cmd the API Command Object containg the parameters to use for this service action
+     * @param cmd
+     *         the API Command Object containg the parameters to use for this service action
+     * @param checkExpunge
      * @throws ConcurrentOperationException
      * @throws ResourceUnavailableException
      */
-    UserVm destroyVm(DestroyVMCmd cmd) throws ResourceUnavailableException, ConcurrentOperationException;
+    UserVm destroyVm(DestroyVMCmd cmd, boolean checkExpunge) throws ResourceUnavailableException, ConcurrentOperationException;
 
     /**
      * Destroys one virtual machine

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -132,4 +132,10 @@ public interface VmDetailConstants {
 
     // KNIB specific
     String LINKED_VOLUMES_SECONDARY_STORAGE_UUIDS = "linkedVolumesSecondaryStorageUuids";
+    String VALIDATION_COMMAND = "backupValidationCommand";
+    String VALIDATION_COMMAND_ARGUMENTS = "backupValidationCommandArguments";
+    String VALIDATION_COMMAND_EXPECTED_RESULT = "backupValidationCommandExpectedResult";
+    String VALIDATION_COMMAND_TIMEOUT = "backupValidationCommandTimeout";
+    String VALIDATION_SCREENSHOT_WAIT = "backupValidationScreenshotWait";
+    String VALIDATION_BOOT_TIMEOUT = "backupValidationBootTimeout";
 }

api/src/main/java/org/apache/cloudstack/alert/AlertService.java

@@ -83,6 +83,9 @@ private AlertType(short type, String name, boolean isDefault) {
         public static final AlertType ALERT_TYPE_VPN_GATEWAY_OBSOLETE_PARAMETERS = new AlertType((short)34, "ALERT.S2S.VPN.GATEWAY.OBSOLETE.PARAMETERS", true, true);
         public static final AlertType ALERT_TYPE_BACKUP_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_BACKUP_STORAGE, "ALERT.STORAGE.BACKUP", true);
         public static final AlertType ALERT_TYPE_OBJECT_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, "ALERT.STORAGE.OBJECT", true);
+        public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_FAILED = new AlertType((short)35, "ALERT.BACKUP.VALIDATION.FAILED", true, true);
+        public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_UNABLE_TO_VALIDATE = new AlertType((short)36, "ALERT.BACKUP.VALIDATION.UNABLE.TO.VALIDATE", true, true);
+        public static final AlertType ALERT_TYPE_BACKUP_VALIDATION_CLEANUP_FAILED = new AlertType((short)37, "ALERT.BACKUP.VALIDATION.CLEANUP_FAILED", true, true);
 
         public short getType() {
             return type;

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -572,6 +572,7 @@ public class ApiConstants {
     public static final String STATS = "stats";
     public static final String STATUS = "status";
     public static final String COMPRESSION_STATUS = "compressionstatus";
+    public static final String VALIDATION_STATUS = "validationstatus";
     public static final String STORAGE_TYPE = "storagetype";
     public static final String STORAGE_POLICY = "storagepolicy";
     public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
@@ -1389,6 +1390,8 @@ public class ApiConstants {
 
     public static final String VALIDATE = "validate";
 
+    public static final String VALIDATION_STEPS = "validationsteps";
+
     public static final String ALLOW_QUICK_RESTORE = "allowquickrestore";
 
     public static final String ALLOW_EXTRACT_FILE = "allowextractfile";