AutoScaling: support Managed User Data

api/src/main/java/org/apache/cloudstack/api/BaseCmd.java

@@ -21,8 +21,10 @@
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
@@ -42,6 +44,7 @@
 import org.apache.cloudstack.query.QueryService;
 import org.apache.cloudstack.storage.ImageStoreService;
 import org.apache.cloudstack.usage.UsageService;
+import org.apache.commons.collections.MapUtils;
 import org.apache.log4j.Logger;
 
 import com.cloud.configuration.ConfigurationService;
@@ -456,4 +459,18 @@ public ApiCommandResourceType getApiResourceType() {
         return ApiCommandResourceType.None;
     }
 
+    public Map<String, String> convertDetailsToMap(Map details) {
+        Map<String, String> detailsMap = new HashMap<String, String>();
+        if (MapUtils.isNotEmpty(details)) {
+            Collection parameterCollection = details.values();
+            Iterator iter = parameterCollection.iterator();
+            while (iter.hasNext()) {
+                HashMap<String, String> value = (HashMap<String, String>)iter.next();
+                for (Map.Entry<String,String> entry: value.entrySet()) {
+                    detailsMap.put(entry.getKey(),entry.getValue());
+                }
+            }
+        }
+        return detailsMap;
+    }
 }

api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java

@@ -35,6 +35,7 @@
 import org.apache.cloudstack.api.response.ProjectResponse;
 import org.apache.cloudstack.api.response.ServiceOfferingResponse;
 import org.apache.cloudstack.api.response.TemplateResponse;
+import org.apache.cloudstack.api.response.UserDataResponse;
 import org.apache.cloudstack.api.response.UserResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.context.CallContext;
@@ -107,6 +108,12 @@ public class CreateAutoScaleVmProfileCmd extends BaseAsyncCreateCmd {
             since = "4.18.0")
     private String userData;
 
+    @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.18.1")
+    private Long userDataId;
+
+    @Parameter(name = ApiConstants.USER_DATA_DETAILS, type = CommandType.MAP, description = "used to specify the parameters values for the variables in userdata.", since = "4.18.1")
+    private Map userDataDetails;
+
     @Parameter(name = ApiConstants.AUTOSCALE_USER_ID,
                type = CommandType.UUID,
                entityType = UserResponse.class,
@@ -163,6 +170,14 @@ public String getUserData() {
         return userData;
     }
 
+    public Long getUserDataId() {
+        return userDataId;
+    }
+
+    public Map<String, String> getUserDataDetails() {
+        return convertDetailsToMap(userDataDetails);
+    }
+
     public Long getAutoscaleUserId() {
         return autoscaleUserId;
     }

api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/UpdateAutoScaleVmProfileCmd.java

@@ -35,6 +35,7 @@
 import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse;
 import org.apache.cloudstack.api.response.ServiceOfferingResponse;
 import org.apache.cloudstack.api.response.TemplateResponse;
+import org.apache.cloudstack.api.response.UserDataResponse;
 import org.apache.cloudstack.api.response.UserResponse;
 import org.apache.cloudstack.context.CallContext;
 
@@ -102,6 +103,14 @@ public class UpdateAutoScaleVmProfileCmd extends BaseAsyncCustomIdCmd {
             since = "4.18.0")
     private String userData;
 
+    @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the userdata",
+            since = "4.18.1")
+    private Long userdataId;
+
+    @Parameter(name = ApiConstants.USER_DATA_DETAILS, type = CommandType.MAP, description = "used to specify the parameters values for the variables in userdata.",
+            since = "4.18.1")
+    private Map userdataDetails;
+
     @Parameter(name = ApiConstants.AUTOSCALE_USER_ID,
                type = CommandType.UUID,
                entityType = UserResponse.class,
@@ -156,6 +165,14 @@ public String getUserData() {
         return userData;
     }
 
+    public Long getUserdataId() {
+        return userdataId;
+    }
+
+    public Map<String, String> getUserdataDetails() {
+        return convertDetailsToMap(userdataDetails);
+    }
+
     public Long getAutoscaleUserId() {
         return autoscaleUserId;
     }

api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java

@@ -309,17 +309,8 @@ public ApiConstants.BootType getBootType() {
     }
 
     public Map<String, String> getDetails() {
-        Map<String, String> customparameterMap = new HashMap<String, String>();
-        if (details != null && details.size() != 0) {
-            Collection parameterCollection = details.values();
-            Iterator iter = parameterCollection.iterator();
-            while (iter.hasNext()) {
-                HashMap<String, String> value = (HashMap<String, String>)iter.next();
-                for (Map.Entry<String,String> entry: value.entrySet()) {
-                    customparameterMap.put(entry.getKey(),entry.getValue());
-                }
-            }
-        }
+        Map<String, String> customparameterMap = convertDetailsToMap(details);
+
         if (getBootType() != null) {
             customparameterMap.put(getBootType().toString(), getBootMode().toString());
         }
@@ -450,18 +441,7 @@ public Long getUserdataId() {
     }
 
     public Map<String, String> getUserdataDetails() {
-        Map<String, String> userdataDetailsMap = new HashMap<String, String>();
-        if (userdataDetails != null && userdataDetails.size() != 0) {
-            Collection parameterCollection = userdataDetails.values();
-            Iterator iter = parameterCollection.iterator();
-            while (iter.hasNext()) {
-                HashMap<String, String> value = (HashMap<String, String>)iter.next();
-                for (Map.Entry<String,String> entry: value.entrySet()) {
-                    userdataDetailsMap.put(entry.getKey(),entry.getValue());
-                }
-            }
-        }
-        return userdataDetailsMap;
+        return convertDetailsToMap(userdataDetails);
     }
 
     public Long getZoneId() {

api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMUserDataCmd.java

@@ -39,9 +39,6 @@
 import org.apache.cloudstack.context.CallContext;
 import org.apache.log4j.Logger;
 
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
 import java.util.Map;
 
 @APICommand(name = "resetUserDataForVirtualMachine", responseObject = UserVmResponse.class, description = "Resets the UserData for virtual machine. " +
@@ -117,18 +114,7 @@ public Long getUserdataId() {
     }
 
     public Map<String, String> getUserdataDetails() {
-        Map<String, String> userdataDetailsMap = new HashMap<String, String>();
-        if (userdataDetails != null && userdataDetails.size() != 0) {
-            Collection parameterCollection = userdataDetails.values();
-            Iterator iter = parameterCollection.iterator();
-            while (iter.hasNext()) {
-                HashMap<String, String> value = (HashMap<String, String>)iter.next();
-                for (Map.Entry<String,String> entry: value.entrySet()) {
-                    userdataDetailsMap.put(entry.getKey(),entry.getValue());
-                }
-            }
-        }
-        return userdataDetailsMap;
+        return convertDetailsToMap(userdataDetails);
     }
 
     @Override

api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java

@@ -18,7 +18,6 @@
 
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -176,18 +175,7 @@ public Long getUserdataId() {
     }
 
     public Map<String, String> getUserdataDetails() {
-        Map<String, String> userdataDetailsMap = new HashMap<String, String>();
-        if (userdataDetails != null && userdataDetails.size() != 0) {
-            Collection parameterCollection = userdataDetails.values();
-            Iterator iter = parameterCollection.iterator();
-            while (iter.hasNext()) {
-                HashMap<String, String> value = (HashMap<String, String>)iter.next();
-                for (Map.Entry<String,String> entry: value.entrySet()) {
-                    userdataDetailsMap.put(entry.getKey(),entry.getValue());
-                }
-            }
-        }
-        return userdataDetailsMap;
+        return convertDetailsToMap(userdataDetails);
     }
 
     public Boolean getDisplayVm() {

engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmProfileVO.java

@@ -88,6 +88,12 @@ public class AutoScaleVmProfileVO implements AutoScaleVmProfile, Identity, Inter
     @Basic(fetch = FetchType.LAZY)
     private String userData;
 
+    @Column(name = "user_data_id", nullable = true)
+    private Long userDataId = null;
+
+    @Column(name = "user_data_details", updatable = true, length = 4096)
+    private String userDataDetails;
+
     @Column(name = GenericDao.REMOVED_COLUMN)
     protected Date removed;
 
@@ -228,6 +234,22 @@ public String getUserData() {
         return userData;
     }
 
+    public Long getUserDataId() {
+        return userDataId;
+    }
+
+    public void setUserDataId(Long userDataId) {
+        this.userDataId = userDataId;
+    }
+
+    public String getUserDataDetails() {
+        return userDataDetails;
+    }
+
+    public void setUserDataDetails(String userDataDetails) {
+        this.userDataDetails = userDataDetails;
+    }
+
     @Override
     public String getUuid() {
         return uuid;

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java

@@ -58,6 +58,7 @@ public InputStream[] getPrepareScripts() {
     @Override
     public void performDataMigration(Connection conn) {
         fixForeignKeyNames(conn);
+        addForeignKeyToAutoscaleVmprofiles(conn);
     }
 
     @Override
@@ -111,4 +112,8 @@ private void fixForeignKeyNames(Connection conn) {
         DbUpgradeUtils.dropKeysIfExist(conn, "cloud.volumes", keys, false);
         DbUpgradeUtils.addForeignKey(conn, "volumes", "passphrase_id","passphrase", "id");
     }
+
+    private void addForeignKeyToAutoscaleVmprofiles(Connection conn) {
+        DbUpgradeUtils.addForeignKey(conn, "autoscale_vmprofiles", "user_data_id","user_data", "id");
+    }
 }

engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql

@@ -31,3 +31,7 @@ CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VM
 CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VMware', '8.0', 'windows2019srvNext_64Guest');
 CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VMware', '8.0.0.1', 'windows2019srvNext_64Guest');
 CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'Xenserver', '8.2.0', 'Windows Server 2022 (64-bit)');
+
+-- Support userdata ids and details in VM AutoScaling
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.autoscale_vmprofiles', 'user_data_id', 'bigint unsigned DEFAULT NULL COMMENT "id of the user data" AFTER `user_data`');
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.autoscale_vmprofiles', 'user_data_details', 'mediumtext DEFAULT NULL COMMENT "value of the comma-separated list of parameters" AFTER `user_data_id`');

server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java

@@ -532,7 +532,6 @@ public AutoScaleVmProfile createAutoScaleVmProfile(CreateAutoScaleVmProfileCmd c
         long zoneId = cmd.getZoneId();
         long serviceOfferingId = cmd.getServiceOfferingId();
         Long autoscaleUserId = cmd.getAutoscaleUserId();
-        String userData = cmd.getUserData();
 
         DataCenter zone = entityMgr.findById(DataCenter.class, zoneId);
 
@@ -545,6 +544,11 @@ public AutoScaleVmProfile createAutoScaleVmProfile(CreateAutoScaleVmProfileCmd c
             throw new InvalidParameterValueException("Unable to find service offering by id");
         }
 
+        VirtualMachineTemplate template = entityMgr.findById(VirtualMachineTemplate.class, cmd.getTemplateId());
+        if (template == null) {
+            throw new InvalidParameterValueException("Unable to find template by id " + cmd.getTemplateId());
+        }
+
         // validations
         HashMap<String, String> deployParams = cmd.getDeployParamMap();
         /*
@@ -562,9 +566,23 @@ public AutoScaleVmProfile createAutoScaleVmProfile(CreateAutoScaleVmProfileCmd c
             profileVO.setDisplay(cmd.getDisplay());
         }
 
+        String userData = cmd.getUserData();
+        Long userDataId = cmd.getUserDataId();
+        String userDataDetails = null;
+        if (MapUtils.isNotEmpty(cmd.getUserDataDetails())) {
+            userDataDetails = cmd.getUserDataDetails().toString();
+        }
+        userData = userVmMgr.finalizeUserData(userData, userDataId, template);
+        userData = userVmMgr.validateUserData(userData, cmd.getHttpMethod());
         if (userData != null) {
             profileVO.setUserData(userData);
         }
+        if (userDataId != null) {
+            profileVO.setUserDataId(userDataId);
+        }
+        if (userDataDetails != null) {
+            profileVO.setUserDataDetails(userDataDetails);
+        }
 
         profileVO = checkValidityAndPersist(profileVO, true);
         s_logger.info("Successfully create AutoScale Vm Profile with Id: " + profileVO.getId());
@@ -582,12 +600,19 @@ public AutoScaleVmProfile updateAutoScaleVmProfile(UpdateAutoScaleVmProfileCmd c
         Map<String, HashMap<String, String>> otherDeployParams = cmd.getOtherDeployParams();
         Map counterParamList = cmd.getCounterParamList();
         String userData = cmd.getUserData();
+        Long userDataId = cmd.getUserdataId();
+        String userDataDetails = null;
+        if (MapUtils.isNotEmpty(cmd.getUserdataDetails())) {
+            userDataDetails = cmd.getUserdataDetails().toString();
+        }
+        boolean userdataUpdate = userData != null || userDataId != null || MapUtils.isNotEmpty(cmd.getUserdataDetails());
 
         Integer expungeVmGracePeriod = cmd.getExpungeVmGracePeriod();
 
         AutoScaleVmProfileVO vmProfile = getEntityInDatabase(CallContext.current().getCallingAccount(), "Auto Scale Vm Profile", profileId, autoScaleVmProfileDao);
 
-        boolean physicalParameterUpdate = (templateId != null || autoscaleUserId != null || counterParamList != null || otherDeployParams != null || expungeVmGracePeriod != null || userData != null);
+        boolean physicalParameterUpdate = (templateId != null || autoscaleUserId != null || counterParamList != null
+                || otherDeployParams != null || expungeVmGracePeriod != null || userdataUpdate);
 
         if (serviceOfferingId != null) {
             vmProfile.setServiceOfferingId(serviceOfferingId);
@@ -609,10 +634,6 @@ public AutoScaleVmProfile updateAutoScaleVmProfile(UpdateAutoScaleVmProfileCmd c
             vmProfile.setCounterParamsForUpdate(counterParamList);
         }
 
-        if (userData != null) {
-            vmProfile.setUserData(userData);
-        }
-
         if (expungeVmGracePeriod != null) {
             vmProfile.setExpungeVmGracePeriod(expungeVmGracePeriod);
         }
@@ -625,6 +646,18 @@ public AutoScaleVmProfile updateAutoScaleVmProfile(UpdateAutoScaleVmProfileCmd c
             vmProfile.setDisplay(cmd.getDisplay());
         }
 
+        if (userdataUpdate) {
+            if (templateId == null) {
+                templateId = vmProfile.getTemplateId();
+            }
+            VirtualMachineTemplate template = entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, templateId);
+            userData = userVmMgr.finalizeUserData(userData, userDataId, template);
+            userData = userVmMgr.validateUserData(userData, cmd.getHttpMethod());
+            vmProfile.setUserDataId(userDataId);
+            vmProfile.setUserData(userData);
+            vmProfile.setUserDataDetails(userDataDetails);
+        }
+
         List<AutoScaleVmGroupVO> vmGroupList = autoScaleVmGroupDao.listByAll(null, profileId);
         for (AutoScaleVmGroupVO vmGroupVO : vmGroupList) {
             if (physicalParameterUpdate && !vmGroupVO.getState().equals(AutoScaleVmGroup.State.DISABLED)) {
@@ -1740,6 +1773,8 @@ protected long createNewVM(AutoScaleVmGroupVO asGroup) {
             }
 
             String userData = profileVo.getUserData();
+            Long userDataId = profileVo.getUserDataId();
+            String userDataDetails = profileVo.getUserDataDetails();
 
             UserVm vm = null;
             IpAddresses addrs = new IpAddresses(null, null);
@@ -1763,20 +1798,20 @@ protected long createNewVM(AutoScaleVmGroupVO asGroup) {
             if (zone.getNetworkType() == NetworkType.Basic) {
                 vm = userVmService.createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, null, owner, vmHostName,
                         vmHostName, diskOfferingId, dataDiskSize, null,
-                        hypervisorType, HTTPMethod.GET, userData, null, null, sshKeyPairs,
+                        hypervisorType, HTTPMethod.GET, userData, userDataId, userDataDetails, sshKeyPairs,
                         null, null, true, null, affinityGroupIdList, customParameters, null, null, null,
                         null, true, overrideDiskOfferingId);
             } else {
                 if (zone.isSecurityGroupEnabled()) {
                     vm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, networkIds, null,
                             owner, vmHostName,vmHostName, diskOfferingId, dataDiskSize, null,
-                            hypervisorType, HTTPMethod.GET, userData, null, null, sshKeyPairs,
+                            hypervisorType, HTTPMethod.GET, userData, userDataId, userDataDetails, sshKeyPairs,
                             null, null, true, null, affinityGroupIdList, customParameters, null, null, null,
                             null, true, overrideDiskOfferingId, null);
                 } else {
                     vm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, vmHostName, vmHostName,
                             diskOfferingId, dataDiskSize, null,
-                            hypervisorType, HTTPMethod.GET, userData, null, null, sshKeyPairs,
+                            hypervisorType, HTTPMethod.GET, userData, userDataId, userDataDetails, sshKeyPairs,
                             null, addrs, true, null, affinityGroupIdList, customParameters, null, null, null,
                             null, true, null, overrideDiskOfferingId);
                 }