backup: veeam kvm integration (#12991)

This PR introduces the initial implementation of Veeam integration support for KVM in CloudStack by adding a UHAPI-compatible server and image server components.

Veeam Backup & Replication interacts with virtualization platforms using its Universal Hypervisor API (UHAPI). To enable backup and restore workflows for CloudStack-managed KVM environments, this change introduces a UHAPI server that exposes CloudStack resources through a UHAPI-compatible interface.

In addition to the control plane APIs, an image server component is introduced to handle the data transfer operations required during backup and restore workflows.


The integration consists of two main components:

1. UHAPI Server (Control Plane) named CloudStack Veeam Control Service

A lightweight UHAPI server runs inside the CloudStack management server and exposes endpoints under:

/ovirt-engine
    - /api - For APIs
    - /sso - For authentication
    - /services/pki-resource - For certificates

This server provides inventory discovery APIs required by Veeam and translates CloudStack resources into the structures expected by UHAPI.

The server:

- exposes infrastructure inventory
- handles authentication and session tokens
- maps CloudStack resources to UHAPI-compatible representations


2. Image Server (Data Plane) named CloudStack Image Service

A separate image server component is introduced to handle backup and restore data transfer operations.

This component:

- serves disk image data during backup
- receives image data during restore operations
- exposes endpoints used by Veeam worker components
- integrates with CloudStack storage to read and write VM disk data

The separation between both these components server ensures that:

- metadata APIs and control operations remain lightweight
- bulk image transfer operations are handled independently

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
Co-authored-by: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com>
Co-authored-by: abh1sar <abhisar.sinha@gmail.com>
Co-authored-by: Wei Zhou <weizhou@apache.org>

Author: 3 people

agent/conf/agent.properties

@@ -78,6 +78,14 @@ zone=default
 # Generated with "uuidgen".
 local.storage.uuid=
 
+# Enable TLS for image server transfers. The keys are read from:
+# cert file = /etc/cloudstack/agent/cloud.crt
+# key file = /etc/cloudstack/agent/cloud.key
+image.server.tls.enabled=true
+
+# The Address for the network interface that the image server listens on. If not specified, it will listen on the Management network.
+#image.server.listen.address=
+
 # Location for KVM virtual router scripts.
 # The path defined in this property is relative to the directory "/usr/share/cloudstack-common/".
 domr.scripts.dir=scripts/network/domr/kvm

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -123,6 +123,20 @@ public class AgentProperties{
      */
     public static final Property<String> LOCAL_STORAGE_PATH = new Property<>("local.storage.path", "/var/lib/libvirt/images/");
 
+    /**
+     * Enables TLS on the KVM image server transfer endpoint.<br>
+     * Data type: Boolean.<br>
+     * Default value: <code>true</code>
+     */
+    public static final Property<Boolean> IMAGE_SERVER_TLS_ENABLED = new Property<>("image.server.tls.enabled", true);
+
+    /**
+     * The IP address that the KVM image server listens on.<br>
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> IMAGE_SERVER_LISTEN_ADDRESS = new Property<>("image.server.listen.address", null, String.class);
+
     /**
      * Directory where Qemu sockets are placed.<br>
      * These sockets are for the Qemu Guest Agent and SSVM provisioning.<br>

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.dc.DataCenter;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.offering.DiskOffering;
 import com.cloud.user.Account;
@@ -70,6 +71,10 @@ public interface VolumeApiService {
      */
     Volume allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationException;
 
+    Volume allocVolume(long ownerId, Long zoneId, Long diskOfferingId, Long vmId, Long snapshotId, String name,
+           Long cmdSize, Boolean displayVolume, Long cmdMinIops, Long cmdMaxIops, String customId)
+            throws ResourceAllocationException;
+
     /**
      * Creates the volume based on the given criteria
      *
@@ -80,6 +85,8 @@ public interface VolumeApiService {
      */
     Volume createVolume(CreateVolumeCmd cmd);
 
+    Volume createVolume(long volumeId, Long vmId, Long snapshotId, Long storageId, Boolean display);
+
     /**
      * Resizes the volume based on the given criteria
      *
@@ -203,4 +210,6 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
     Pair<String, String> checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException;
 
     Long getVolumePhysicalSize(Storage.ImageFormat format, String path, String chainInfo);
+
+    Long getCustomDiskOfferingIdForVolumeUpload(Account owner, DataCenter zone, boolean encryptEnabledOnly);
 }

api/src/main/java/com/cloud/user/AccountService.java

@@ -88,10 +88,16 @@ User createUser(String userName, String password, String firstName, String lastN
 
     Account getActiveAccountById(long accountId);
 
+    Account getActiveAccountByUuid(String accountUuid);
+
     Account getAccount(long accountId);
 
+    Account getAccountByUuid(String accountUuid);
+
     User getActiveUser(long userId);
 
+    User getOneActiveUserForAccount(Account account);
+
     User getUserIncludingRemoved(long userId);
 
     boolean isRootAdmin(Long accountId);

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -130,4 +130,10 @@ public interface VmDetailConstants {
     String EXTERNAL_DETAIL_PREFIX = "External:";
     String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details";
     String CLOUDSTACK_VLAN = "cloudstack.vlan";
+
+    // KVM Checkpoints related
+    String ACTIVE_CHECKPOINT_ID = "active.checkpoint.id";
+    String ACTIVE_CHECKPOINT_CREATE_TIME = "active.checkpoint.create.time";
+    String LAST_CHECKPOINT_ID = "last.checkpoint.id";
+    String LAST_CHECKPOINT_CREATE_TIME = "last.checkpoint.create.time";
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -79,6 +79,7 @@ public class ApiConstants {
     public static final String BOOTABLE = "bootable";
     public static final String BIND_DN = "binddn";
     public static final String BIND_PASSWORD = "bindpass";
+    public static final String BLANK_INSTANCE = "blankinstance";
     public static final String BUS_ADDRESS = "busaddress";
     public static final String BYTES_READ_RATE = "bytesreadrate";
     public static final String BYTES_READ_RATE_MAX = "bytesreadratemax";
@@ -220,6 +221,7 @@ public class ApiConstants {
     public static final String DOMAIN_PATH = "domainpath";
     public static final String DOMAIN_ID = "domainid";
     public static final String DOMAIN__ID = "domainId";
+    public static final String DUMMY = "dummy";
     public static final String DURATION = "duration";
     public static final String ELIGIBLE = "eligible";
     public static final String EMAIL = "email";
@@ -263,6 +265,7 @@ public class ApiConstants {
     public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork";
     public static final String FOR_SYSTEM_VMS = "forsystemvms";
     public static final String FOR_PROVIDER = "forprovider";
+    public static final String FROM_CHECKPOINT_ID = "fromcheckpointid";
     public static final String FULL_PATH = "fullpath";
     public static final String GATEWAY = "gateway";
     public static final String IP6_GATEWAY = "ip6gateway";
@@ -335,6 +338,7 @@ public class ApiConstants {
     public static final String IS_2FA_VERIFIED = "is2faverified";
 
     public static final String IS_2FA_MANDATED = "is2famandated";
+    public static final String IS_ACTIVE = "isactive";
     public static final String IS_ASYNC = "isasync";
     public static final String IP_AVAILABLE = "ipavailable";
     public static final String IP_LIMIT = "iplimit";
@@ -611,6 +615,7 @@ public class ApiConstants {
     public static final String TENANT_NAME = "tenantname";
     public static final String TOTAL = "total";
     public static final String TOTAL_SUBNETS = "totalsubnets";
+    public static final String TO_CHECKPOINT_ID = "tocheckpointid";
     public static final String TOTAL_QUOTA = "totalquota";
     public static final String TYPE = "type";
     public static final String TRUST_STORE = "truststore";

api/src/main/java/org/apache/cloudstack/api/ApiServerService.java

@@ -21,8 +21,11 @@
 
 import javax.servlet.http.HttpSession;
 
+import org.apache.cloudstack.context.CallContext;
+
 import com.cloud.domain.Domain;
 import com.cloud.exception.CloudAuthenticationException;
+import com.cloud.user.Account;
 import com.cloud.user.UserAccount;
 
 public interface ApiServerService {
@@ -52,4 +55,20 @@ public ResponseObject loginUser(HttpSession session, String username, String pas
     String getDomainId(Map<String, Object[]> params);
 
     boolean isPostRequestsAndTimestampsEnforced();
+
+    AsyncCmdResult processAsyncCmd(BaseAsyncCmd cmdObj, Map<String, String> params, CallContext ctx, Long callerUserId, Account caller) throws Exception;
+
+    class AsyncCmdResult {
+        public final Long objectId;
+        public final String objectUuid;
+        public final BaseAsyncCmd asyncCmd;
+        public final long jobId;
+
+        public AsyncCmdResult(Long objectId, String objectUuid, BaseAsyncCmd asyncCmd, long jobId) {
+            this.objectId = objectId;
+            this.objectUuid = objectUuid;
+            this.asyncCmd = asyncCmd;
+            this.jobId = jobId;
+        }
+    }
 }

api/src/main/java/org/apache/cloudstack/api/BaseAsyncCreateCustomIdCmd.java

@@ -20,7 +20,7 @@ public abstract class BaseAsyncCreateCustomIdCmd extends BaseAsyncCreateCmd {
     @Parameter(name = ApiConstants.CUSTOM_ID,
                type = CommandType.STRING,
                description = "An optional field, in case you want to set a custom id to the resource. Allowed to Root Admins only")
-    private String customId;
+    protected String customId;
 
     public String getCustomId() {
         return customId;

api/src/main/java/org/apache/cloudstack/api/command/admin/backup/CreateImageTransferCmd.java

@@ -0,0 +1,100 @@
+//Licensed to the Apache Software Foundation (ASF) under one
+//or more contributor license agreements.  See the NOTICE file
+//distributed with this work for additional information
+//regarding copyright ownership.  The ASF licenses this file
+//to you under the Apache License, Version 2.0 (the
+//"License"); you may not use this file except in compliance
+//the License.  You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+//Unless required by applicable law or agreed to in writing,
+//software distributed under the License is distributed on an
+//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+//KIND, either express or implied.  See the License for the
+//specific language governing permissions and limitations
+//under the License.
+
+package org.apache.cloudstack.api.command.admin.backup;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
+import org.apache.cloudstack.api.response.BackupResponse;
+import org.apache.cloudstack.api.response.ImageTransferResponse;
+import org.apache.cloudstack.api.response.VolumeResponse;
+import org.apache.cloudstack.backup.ImageTransfer;
+import org.apache.cloudstack.backup.KVMBackupExportService;
+import org.apache.cloudstack.context.CallContext;
+
+import com.cloud.utils.EnumUtils;
+
+@APICommand(name = "createImageTransfer",
+        description = "Create image transfer for a disk in backup. This API is intended for testing only and is disabled by default.",
+        responseObject = ImageTransferResponse.class,
+        since = "4.23.0",
+        authorized = {RoleType.Admin})
+public class CreateImageTransferCmd extends BaseCmd implements AdminCmd {
+
+    @Inject
+    private KVMBackupExportService kvmBackupExportService;
+
+    @Parameter(name = ApiConstants.BACKUP_ID,
+            type = CommandType.UUID,
+            entityType = BackupResponse.class,
+            description = "ID of the backup")
+    private Long backupId;
+
+    @Parameter(name = ApiConstants.VOLUME_ID,
+            type = CommandType.UUID,
+            entityType = VolumeResponse.class,
+            required = true,
+            description = "ID of the disk/volume")
+    private Long volumeId;
+
+    @Parameter(name = ApiConstants.DIRECTION,
+            type = CommandType.STRING,
+            required = true,
+            description = "Direction of the transfer: upload, download")
+    private String direction;
+
+    @Parameter(name = ApiConstants.FORMAT,
+            type = CommandType.STRING,
+            description = "Format for the image transfer: raw/cow. 'raw' will create an NBD backend. 'cow' will use the File backend." +
+                    "For download, only the 'raw' format is supported. Default: raw")
+    private String format;
+
+    public Long getBackupId() {
+        return backupId;
+    }
+
+    public Long getVolumeId() {
+        return volumeId;
+    }
+
+    public ImageTransfer.Direction getDirection() {
+        return ImageTransfer.Direction.valueOf(direction);
+    }
+
+    public ImageTransfer.Format getFormat() {
+        return EnumUtils.getEnum(ImageTransfer.Format.class, format);
+    }
+
+   @Override
+    public void execute() {
+        ImageTransferResponse response = kvmBackupExportService.createImageTransfer(this);
+        response.setObjectName(ImageTransfer.class.getSimpleName().toLowerCase());
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccount().getId();
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/backup/DeleteVmCheckpointCmd.java

@@ -0,0 +1,85 @@
+//Licensed to the Apache Software Foundation (ASF) under one
+//or more contributor license agreements.  See the NOTICE file
+//distributed with this work for additional information
+//regarding copyright ownership.  The ASF licenses this file
+//to you under the Apache License, Version 2.0 (the
+//"License"); you may not use this file except in compliance
+//the License.  You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+//Unless required by applicable law or agreed to in writing,
+//software distributed under the License is distributed on an
+//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+//KIND, either express or implied.  See the License for the
+//specific language governing permissions and limitations
+//under the License.
+
+package org.apache.cloudstack.api.command.admin.backup;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.api.response.UserVmResponse;
+import org.apache.cloudstack.backup.KVMBackupExportService;
+import org.apache.cloudstack.context.CallContext;
+
+@APICommand(name = "deleteVirtualMachineCheckpoint",
+        description = "Delete a VM checkpoint. This API is intended for testing only and is disabled by default.",
+        responseObject = SuccessResponse.class,
+        since = "4.23.0",
+        authorized = {RoleType.Admin})
+public class DeleteVmCheckpointCmd extends BaseCmd implements AdminCmd {
+
+    @Inject
+    private KVMBackupExportService kvmBackupExportService;
+
+    @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID,
+            type = CommandType.UUID,
+            entityType = UserVmResponse.class,
+            required = true,
+            description = "ID of the VM")
+    private Long vmId;
+
+    @Parameter(name = "checkpointid",
+            type = CommandType.STRING,
+            required = true,
+            description = "Checkpoint ID")
+    private String checkpointId;
+
+    public Long getVmId() {
+        return vmId;
+    }
+
+    public String getCheckpointId() {
+        return checkpointId;
+    }
+
+    public void setVmId(Long vmId) {
+        this.vmId = vmId;
+    }
+
+    public void setCheckpointId(String checkpointId) {
+        this.checkpointId = checkpointId;
+    }
+
+    @Override
+    public void execute() {
+        boolean result = kvmBackupExportService.deleteVmCheckpoint(this);
+        SuccessResponse response = new SuccessResponse(getCommandName());
+        response.setSuccess(result);
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccount().getId();
+    }
+}