[WIP] backup: veeam kvm integration

agent/conf/agent.properties

@@ -78,6 +78,14 @@ zone=default
 # Generated with "uuidgen".
 local.storage.uuid=
 
+# Enable TLS for image server transfers. The keys are read from:
+# cert file = /etc/cloudstack/agent/cloud.crt
+# key file = /etc/cloudstack/agent/cloud.key
+image.server.tls.enabled=true
+
+# The Address for the network interface that the image server listens on. If not specified, it will listen on the Management network.
+#image.server.listen.address=
+
 # Location for KVM virtual router scripts.
 # The path defined in this property is relative to the directory "/usr/share/cloudstack-common/".
 domr.scripts.dir=scripts/network/domr/kvm

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -123,6 +123,20 @@ public class AgentProperties{
      */
     public static final Property<String> LOCAL_STORAGE_PATH = new Property<>("local.storage.path", "/var/lib/libvirt/images/");
 
+    /**
+     * Enables TLS on the KVM image server transfer endpoint.<br>
+     * Data type: Boolean.<br>
+     * Default value: <code>true</code>
+     */
+    public static final Property<Boolean> IMAGE_SERVER_TLS_ENABLED = new Property<>("image.server.tls.enabled", true);
+
+    /**
+     * The IP address that the KVM image server listens on.<br>
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> IMAGE_SERVER_LISTEN_ADDRESS = new Property<>("image.server.listen.address", null, String.class);
+
     /**
      * Directory where Qemu sockets are placed.<br>
      * These sockets are for the Qemu Guest Agent and SSVM provisioning.<br>

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.dc.DataCenter;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.offering.DiskOffering;
 import com.cloud.user.Account;
@@ -70,6 +71,10 @@ public interface VolumeApiService {
      */
     Volume allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationException;
 
+    Volume allocVolume(long ownerId, Long zoneId, Long diskOfferingId, Long vmId, Long snapshotId, String name,
+           Long cmdSize, Boolean displayVolume, Long cmdMinIops, Long cmdMaxIops, String customId)
+            throws ResourceAllocationException;
+
     /**
      * Creates the volume based on the given criteria
      *
@@ -80,6 +85,8 @@ public interface VolumeApiService {
      */
     Volume createVolume(CreateVolumeCmd cmd);
 
+    Volume createVolume(long volumeId, Long vmId, Long snapshotId, Long storageId, Boolean display);
+
     /**
      * Resizes the volume based on the given criteria
      *
@@ -203,4 +210,6 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
     Pair<String, String> checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException;
 
     Long getVolumePhysicalSize(Storage.ImageFormat format, String path, String chainInfo);
+
+    Long getCustomDiskOfferingIdForVolumeUpload(Account owner, DataCenter zone);
 }

api/src/main/java/com/cloud/user/AccountService.java

@@ -88,10 +88,14 @@ User createUser(String userName, String password, String firstName, String lastN
 
     Account getActiveAccountById(long accountId);
 
+    Account getActiveAccountByUuid(String accountUuid);
+
     Account getAccount(long accountId);
 
     User getActiveUser(long userId);
 
+    User getOneActiveUserForAccount(Account account);
+
     User getUserIncludingRemoved(long userId);
 
     boolean isRootAdmin(Long accountId);

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -130,4 +130,10 @@ public interface VmDetailConstants {
     String EXTERNAL_DETAIL_PREFIX = "External:";
     String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details";
     String CLOUDSTACK_VLAN = "cloudstack.vlan";
+
+    // KVM Checkpoints related
+    String ACTIVE_CHECKPOINT_ID = "active.checkpoint.id";
+    String ACTIVE_CHECKPOINT_CREATE_TIME = "active.checkpoint.create.time";
+    String LAST_CHECKPOINT_ID = "last.checkpoint.id";
+    String LAST_CHECKPOINT_CREATE_TIME = "last.checkpoint.create.time";
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -77,6 +77,7 @@ public class ApiConstants {
     public static final String BOOTABLE = "bootable";
     public static final String BIND_DN = "binddn";
     public static final String BIND_PASSWORD = "bindpass";
+    public static final String BLANK_INSTANCE = "blankinstance";
     public static final String BUS_ADDRESS = "busaddress";
     public static final String BYTES_READ_RATE = "bytesreadrate";
     public static final String BYTES_READ_RATE_MAX = "bytesreadratemax";
@@ -216,6 +217,7 @@ public class ApiConstants {
     public static final String DOMAIN_PATH = "domainpath";
     public static final String DOMAIN_ID = "domainid";
     public static final String DOMAIN__ID = "domainId";
+    public static final String DUMMY = "dummy";
     public static final String DURATION = "duration";
     public static final String ELIGIBLE = "eligible";
     public static final String EMAIL = "email";
@@ -331,6 +333,7 @@ public class ApiConstants {
     public static final String IS_2FA_VERIFIED = "is2faverified";
 
     public static final String IS_2FA_MANDATED = "is2famandated";
+    public static final String IS_ACTIVE = "isactive";
     public static final String IS_ASYNC = "isasync";
     public static final String IP_AVAILABLE = "ipavailable";
     public static final String IP_LIMIT = "iplimit";

api/src/main/java/org/apache/cloudstack/api/ApiServerService.java

@@ -21,8 +21,11 @@
 
 import javax.servlet.http.HttpSession;
 
+import org.apache.cloudstack.context.CallContext;
+
 import com.cloud.domain.Domain;
 import com.cloud.exception.CloudAuthenticationException;
+import com.cloud.user.Account;
 import com.cloud.user.UserAccount;
 
 public interface ApiServerService {
@@ -52,4 +55,20 @@ public ResponseObject loginUser(HttpSession session, String username, String pas
     String getDomainId(Map<String, Object[]> params);
 
     boolean isPostRequestsAndTimestampsEnforced();
+
+    AsyncCmdResult processAsyncCmd(BaseAsyncCmd cmdObj, Map<String, String> params, CallContext ctx, Long callerUserId, Account caller) throws Exception;
+
+    class AsyncCmdResult {
+        public final Long objectId;
+        public final String objectUuid;
+        public final BaseAsyncCmd asyncCmd;
+        public final long jobId;
+
+        public AsyncCmdResult(Long objectId, String objectUuid, BaseAsyncCmd asyncCmd, long jobId) {
+            this.objectId = objectId;
+            this.objectUuid = objectUuid;
+            this.asyncCmd = asyncCmd;
+            this.jobId = jobId;
+        }
+    }
 }

api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java

@@ -0,0 +1,98 @@
+//Licensed to the Apache Software Foundation (ASF) under one
+//or more contributor license agreements.  See the NOTICE file
+//distributed with this work for additional information
+//regarding copyright ownership.  The ASF licenses this file
+//to you under the Apache License, Version 2.0 (the
+//"License"); you may not use this file except in compliance
+//the License.  You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+//Unless required by applicable law or agreed to in writing,
+//software distributed under the License is distributed on an
+//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+//KIND, either express or implied.  See the License for the
+//specific language governing permissions and limitations
+//under the License.
+
+package org.apache.cloudstack.api.command.admin.backup;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
+import org.apache.cloudstack.api.response.BackupResponse;
+import org.apache.cloudstack.api.response.ImageTransferResponse;
+import org.apache.cloudstack.api.response.VolumeResponse;
+import org.apache.cloudstack.backup.ImageTransfer;
+import org.apache.cloudstack.backup.KVMBackupExportService;
+import org.apache.cloudstack.context.CallContext;
+
+import com.cloud.utils.EnumUtils;
+
+@APICommand(name = "createImageTransfer",
+        description = "Create image transfer for a disk in backup. This API is intended for testing only and is disabled by default.",
+        responseObject = ImageTransferResponse.class,
+        since = "4.23.0",
+        authorized = {RoleType.Admin})
+public class CreateImageTransferCmd extends BaseCmd implements AdminCmd {
+
+    @Inject
+    private KVMBackupExportService kvmBackupExportService;
+
+    @Parameter(name = ApiConstants.BACKUP_ID,
+            type = CommandType.UUID,
+            entityType = BackupResponse.class,
+            description = "ID of the backup")
+    private Long backupId;
+
+    @Parameter(name = ApiConstants.VOLUME_ID,
+            type = CommandType.UUID,
+            entityType = VolumeResponse.class,
+            required = true,
+            description = "ID of the disk/volume")
+    private Long volumeId;
+
+    @Parameter(name = ApiConstants.DIRECTION,
+            type = CommandType.STRING,
+            required = true,
+            description = "Direction of the transfer: upload, download")
+    private String direction;
+
+    @Parameter(name = ApiConstants.FORMAT,
+            type = CommandType.STRING,
+            description = "Format of the image: cow/raw. Currently only raw is supported for download. Defaults to raw if not provided")
+    private String format;
+
+    public Long getBackupId() {
+        return backupId;
+    }
+
+    public Long getVolumeId() {
+        return volumeId;
+    }
+
+    public ImageTransfer.Direction getDirection() {
+        return ImageTransfer.Direction.valueOf(direction);
+    }
+
+    public ImageTransfer.Format getFormat() {
+        return EnumUtils.getEnum(ImageTransfer.Format.class, format);
+    }
+
+   @Override
+    public void execute() {
+        ImageTransferResponse response = kvmBackupExportService.createImageTransfer(this);
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccount().getId();
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java

@@ -0,0 +1,85 @@
+//Licensed to the Apache Software Foundation (ASF) under one
+//or more contributor license agreements.  See the NOTICE file
+//distributed with this work for additional information
+//regarding copyright ownership.  The ASF licenses this file
+//to you under the Apache License, Version 2.0 (the
+//"License"); you may not use this file except in compliance
+//the License.  You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+//Unless required by applicable law or agreed to in writing,
+//software distributed under the License is distributed on an
+//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+//KIND, either express or implied.  See the License for the
+//specific language governing permissions and limitations
+//under the License.
+
+package org.apache.cloudstack.api.command.admin.backup;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.api.response.UserVmResponse;
+import org.apache.cloudstack.backup.KVMBackupExportService;
+import org.apache.cloudstack.context.CallContext;
+
+@APICommand(name = "deleteVirtualMachineCheckpoint",
+        description = "Delete a VM checkpoint. This API is intended for testing only and is disabled by default.",
+        responseObject = SuccessResponse.class,
+        since = "4.23.0",
+        authorized = {RoleType.Admin})
+public class DeleteVmCheckpointCmd extends BaseCmd implements AdminCmd {
+
+    @Inject
+    private KVMBackupExportService kvmBackupExportService;
+
+    @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID,
+            type = CommandType.UUID,
+            entityType = UserVmResponse.class,
+            required = true,
+            description = "ID of the VM")
+    private Long vmId;
+
+    @Parameter(name = "checkpointid",
+            type = CommandType.STRING,
+            required = true,
+            description = "Checkpoint ID")
+    private String checkpointId;
+
+    public Long getVmId() {
+        return vmId;
+    }
+
+    public String getCheckpointId() {
+        return checkpointId;
+    }
+
+    public void setVmId(Long vmId) {
+        this.vmId = vmId;
+    }
+
+    public void setCheckpointId(String checkpointId) {
+        this.checkpointId = checkpointId;
+    }
+
+    @Override
+    public void execute() {
+        boolean result = kvmBackupExportService.deleteVmCheckpoint(this);
+        SuccessResponse response = new SuccessResponse(getCommandName());
+        response.setSuccess(result);
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccount().getId();
+    }
+}