From 790b3762e0f0909b31425543354f0657b17cb9ae Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 3 May 2022 19:12:24 +0200 Subject: [PATCH 001/360] AutoScaling: update description of existing counter/condition apis --- .../api/command/admin/autoscale/CreateCounterCmd.java | 2 +- .../api/command/admin/autoscale/DeleteCounterCmd.java | 2 +- .../api/command/user/autoscale/CreateConditionCmd.java | 2 +- .../api/command/user/autoscale/DeleteConditionCmd.java | 2 +- .../api/command/user/autoscale/ListConditionsCmd.java | 2 +- .../cloudstack/api/command/user/autoscale/ListCountersCmd.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/CreateCounterCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/CreateCounterCmd.java index 6c49a23aa586..66b8f401cce4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/CreateCounterCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/CreateCounterCmd.java @@ -32,7 +32,7 @@ import com.cloud.network.as.Counter; import com.cloud.user.Account; -@APICommand(name = "createCounter", description = "Adds metric counter", responseObject = CounterResponse.class, +@APICommand(name = "createCounter", description = "Adds metric counter for VM auto scaling", responseObject = CounterResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class CreateCounterCmd extends BaseAsyncCreateCmd { public static final Logger s_logger = Logger.getLogger(CreateCounterCmd.class.getName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/DeleteCounterCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/DeleteCounterCmd.java index 6575bf940baf..16406b730e14 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/DeleteCounterCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/DeleteCounterCmd.java @@ -33,7 +33,7 @@ import com.cloud.exception.ResourceInUseException; import com.cloud.user.Account; -@APICommand(name = "deleteCounter", description = "Deletes a counter", responseObject = SuccessResponse.class, +@APICommand(name = "deleteCounter", description = "Deletes a counter for VM auto scaling", responseObject = SuccessResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class DeleteCounterCmd extends BaseAsyncCmd { public static final Logger s_logger = Logger.getLogger(DeleteCounterCmd.class.getName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java index 655fcfceb362..373ecbb1c8ce 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java @@ -35,7 +35,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.network.as.Condition; -@APICommand(name = "createCondition", description = "Creates a condition", responseObject = ConditionResponse.class, entityType = {Condition.class}, +@APICommand(name = "createCondition", description = "Creates a condition for VM auto scaling", responseObject = ConditionResponse.class, entityType = {Condition.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class CreateConditionCmd extends BaseAsyncCreateCmd { public static final Logger s_logger = Logger.getLogger(CreateConditionCmd.class.getName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/DeleteConditionCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/DeleteConditionCmd.java index 8c3e56d43ba7..3d21c3c04125 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/DeleteConditionCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/DeleteConditionCmd.java @@ -36,7 +36,7 @@ import com.cloud.network.as.Condition; import com.cloud.user.Account; -@APICommand(name = "deleteCondition", description = "Removes a condition", responseObject = SuccessResponse.class, entityType = {Condition.class}, +@APICommand(name = "deleteCondition", description = "Removes a condition for VM auto scaling", responseObject = SuccessResponse.class, entityType = {Condition.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class DeleteConditionCmd extends BaseAsyncCmd { public static final Logger s_logger = Logger.getLogger(DeleteConditionCmd.class.getName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListConditionsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListConditionsCmd.java index d75d64942f90..b21d60fef94e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListConditionsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListConditionsCmd.java @@ -33,7 +33,7 @@ import com.cloud.network.as.Condition; -@APICommand(name = "listConditions", description = "List Conditions for the specific user", responseObject = ConditionResponse.class, +@APICommand(name = "listConditions", description = "List Conditions for VM auto scaling", responseObject = ConditionResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListConditionsCmd extends BaseListAccountResourcesCmd { public static final Logger s_logger = Logger.getLogger(ListConditionsCmd.class.getName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListCountersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListCountersCmd.java index 6a9ac8dfdcbc..add6f007b32a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListCountersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListCountersCmd.java @@ -32,7 +32,7 @@ import com.cloud.network.as.Counter; import com.cloud.user.Account; -@APICommand(name = "listCounters", description = "List the counters", responseObject = CounterResponse.class, +@APICommand(name = "listCounters", description = "List the counters for VM auto scaling", responseObject = CounterResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListCountersCmd extends BaseListCmd { public static final Logger s_logger = Logger.getLogger(ListCountersCmd.class.getName()); From e323035763e47147068ef0cbeb6a8cba7da94c6d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 5 May 2022 14:32:38 +0200 Subject: [PATCH 002/360] AS: reformat scripts/vm/hypervisor/xenserver/perfmon.py --- scripts/vm/hypervisor/xenserver/perfmon.py | 38 +++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/vm/hypervisor/xenserver/perfmon.py b/scripts/vm/hypervisor/xenserver/perfmon.py index e4b0101a5201..dcd8204b982c 100755 --- a/scripts/vm/hypervisor/xenserver/perfmon.py +++ b/scripts/vm/hypervisor/xenserver/perfmon.py @@ -75,7 +75,7 @@ def get_vm_param_list(self, uuid): return report.keys() def get_total_cpu_core(self, uuid): - report = self.vm_reports[uuid] + report = self.vm_reports[uuid] if not report: return 0 else: @@ -87,8 +87,8 @@ def get_total_cpu_core(self, uuid): return result def get_vm_data(self, uuid, param, row): - #pp = pprint.PrettyPrinter(indent=4) - #pp.pprint(self.vm_reports) + #pp = pprint.PrettyPrinter(indent=4) + #pp.pprint(self.vm_reports) report = self.vm_reports[uuid] col = report[param] return self.__lookup_data(col, row) @@ -135,15 +135,15 @@ def refresh(self, login, starttime, session, override_params): paramstr = "&".join(["%s=%s" % (k, params[k]) for k in params]) # this is better than urllib.urlopen() as it raises an Exception on http 401 'Unauthorised' error # rather than drop into interactive mode - for host in login.host.get_all(): - #print "http://" + str(login.host.get_address(host)) + "/rrd_updates?%s" % paramstr - sock = urllib.URLopener().open("http://" + str(login.host.get_address(host)) + "/rrd_updates?%s" % paramstr) - xmlsource = sock.read() - sock.close() - xmldoc = minidom.parseString(xmlsource) - self.__parse_xmldoc(xmldoc) - # Update the time used on the next run - self.params['start'] = self.end_time + 1 # avoid retrieving same data twice + for host in login.host.get_all(): + #print "http://" + str(login.host.get_address(host)) + "/rrd_updates?%s" % paramstr + sock = urllib.URLopener().open("http://" + str(login.host.get_address(host)) + "/rrd_updates?%s" % paramstr) + xmlsource = sock.read() + sock.close() + xmldoc = minidom.parseString(xmlsource) + self.__parse_xmldoc(xmldoc) + # Update the time used on the next run + self.params['start'] = self.end_time + 1 # avoid retrieving same data twice def __parse_xmldoc(self, xmldoc): # The 1st node contains meta data (description of the data) @@ -165,7 +165,7 @@ def lookup_metadata_bytag(name): self.legend = self.meta_node.getElementsByTagName('legend')[0] # vm_reports matches uuid to per VM report if not hasattr(self,'vm_reports'): - self.vm_reports = {} + self.vm_reports = {} # There is just one host_report and its uuid should not change! self.host_report = None # Handle each column. (I.e. each variable) @@ -198,7 +198,7 @@ def __handle_col(self, col): def getuuid(vm_name): status, output = commands.getstatusoutput("xe vm-list | grep "+vm_name+" -B 1 | head -n 1 | awk -F':' '{print $2}' | tr -d ' '") if (status != 0): - raise PerfMonException("Invalid vm name: %s" % vm_name) + raise PerfMonException("Invalid vm name: %s" % vm_name) return output def get_vm_group_perfmon(args={}): @@ -228,19 +228,19 @@ def get_vm_group_perfmon(args={}): #for uuid in rrd_updates.get_vm_list(): for vm_count in xrange(1, total_vm + 1): - vm_name = args['vmname' + str(vm_count)] + vm_name = args['vmname' + str(vm_count)] vm_uuid = getuuid(vm_name) #print "Got values for VM: " + str(vm_count) + " " + vm_uuid for counter_count in xrange(1, total_counter + 1): - #refresh average - average_cpu = 0 - average_memory = 0 + #refresh average + average_cpu = 0 + average_memory = 0 counter = args['counter' + str(counter_count)] total_row = rrd_updates.get_nrows() duration = int(args['duration' + str(counter_count)]) / 60 duration_diff = total_row - duration if counter == "cpu": - total_cpu = rrd_updates.get_total_cpu_core(vm_uuid) + total_cpu = rrd_updates.get_total_cpu_core(vm_uuid) for row in xrange(duration_diff, total_row): for cpu in xrange(0, total_cpu): average_cpu += rrd_updates.get_vm_data(vm_uuid, "cpu" + str(cpu), row) From d6df48c4ce1e05db94911ab9a671bb7681ab1fec Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 18 May 2022 11:52:33 +0200 Subject: [PATCH 003/360] AS: add provider to counter --- .../java/com/cloud/network/as/Counter.java | 5 ++++- .../admin/autoscale/CreateCounterCmd.java | 7 ++++++ .../user/autoscale/ListCountersCmd.java | 7 ++++++ .../api/response/CounterResponse.java | 8 +++++++ .../java/com/cloud/network/as/CounterVO.java | 12 +++++++++- .../com/cloud/network/as/dao/CounterDao.java | 2 +- .../cloud/network/as/dao/CounterDaoImpl.java | 5 ++++- .../java/com/cloud/api/ApiResponseHelper.java | 1 + .../network/as/AutoScaleManagerImpl.java | 22 ++++++++++++++++--- 9 files changed, 62 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/com/cloud/network/as/Counter.java b/api/src/main/java/com/cloud/network/as/Counter.java index 286bf517c79d..bdcbdd074c1b 100644 --- a/api/src/main/java/com/cloud/network/as/Counter.java +++ b/api/src/main/java/com/cloud/network/as/Counter.java @@ -26,7 +26,8 @@ public static enum Source { netscaler, snmp, cpu, - memory + memory, + cloudstack } String getName(); @@ -35,4 +36,6 @@ public static enum Source { Source getSource(); + String getProvider(); + } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/CreateCounterCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/CreateCounterCmd.java index 66b8f401cce4..018a07ae8bb3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/CreateCounterCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/autoscale/CreateCounterCmd.java @@ -51,6 +51,9 @@ public class CreateCounterCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.VALUE, type = CommandType.STRING, required = true, description = "Value of the counter e.g. oid in case of snmp.") private String value; + @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, required = true, description = "Provider of the counter.") + private String provider; + // ///////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// // /////////////////////////////////////////////////// @@ -67,6 +70,10 @@ public String getValue() { return value; } + public String getProvider() { + return provider; + } + // /////////////////////////////////////////////////// // ///////////// API Implementation/////////////////// // /////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListCountersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListCountersCmd.java index add6f007b32a..72382248a404 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListCountersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/ListCountersCmd.java @@ -51,6 +51,9 @@ public class ListCountersCmd extends BaseListCmd { @Parameter(name = ApiConstants.SOURCE, type = CommandType.STRING, description = "Source of the counter.") private String source; + @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, description = "Provider of the counter.") + private String provider; + // /////////////////////////////////////////////////// // ///////////// API Implementation/////////////////// // /////////////////////////////////////////////////// @@ -92,6 +95,10 @@ public String getSource() { return source; } + public String getProvider() { + return provider; + } + @Override public long getEntityOwnerId() { return Account.ACCOUNT_ID_SYSTEM; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CounterResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CounterResponse.java index a9bac00c8ac0..f013690b64cc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/CounterResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/CounterResponse.java @@ -49,6 +49,10 @@ public class CounterResponse extends BaseResponse { @Param(description = "zone id of counter") private String zoneId; + @SerializedName(value = ApiConstants.PROVIDER) + @Param(description = "Provider of the counter.") + private String provider; + @Override public String getObjectId() { return this.id; @@ -69,4 +73,8 @@ public void setSource(String source) { public void setValue(String value) { this.value = value; } + + public void setProvider(String provider) { + this.provider = provider; + } } diff --git a/engine/schema/src/main/java/com/cloud/network/as/CounterVO.java b/engine/schema/src/main/java/com/cloud/network/as/CounterVO.java index 2ba71f0ced35..e5ab9886dda7 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/CounterVO.java +++ b/engine/schema/src/main/java/com/cloud/network/as/CounterVO.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; +import com.cloud.network.Network; import com.cloud.utils.db.GenericDao; @Entity @@ -62,14 +63,18 @@ public class CounterVO implements Counter, Identity, InternalIdentity { @Column(name = GenericDao.CREATED_COLUMN) Date created; + @Column(name = "provider") + private String provider; + public CounterVO() { } - public CounterVO(Source source, String name, String value) { + public CounterVO(Source source, String name, String value, Network.Provider provider) { this.source = source; this.name = name; this.value = value; this.uuid = UUID.randomUUID().toString(); + this.provider = provider.getName(); } @Override @@ -109,4 +114,9 @@ public Date getRemoved() { public Date getCreated() { return created; } + + @Override + public String getProvider() { + return provider; + } } diff --git a/engine/schema/src/main/java/com/cloud/network/as/dao/CounterDao.java b/engine/schema/src/main/java/com/cloud/network/as/dao/CounterDao.java index 47b0d4876faa..4c9df8e749fe 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/dao/CounterDao.java +++ b/engine/schema/src/main/java/com/cloud/network/as/dao/CounterDao.java @@ -24,6 +24,6 @@ import com.cloud.utils.db.GenericDao; public interface CounterDao extends GenericDao { - public List listCounters(Long id, String name, String source, String keyword, Filter filter); + public List listCounters(Long id, String name, String source, String provider, String keyword, Filter filter); } diff --git a/engine/schema/src/main/java/com/cloud/network/as/dao/CounterDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/as/dao/CounterDaoImpl.java index dde7d0307843..9aefa17e0663 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/dao/CounterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/as/dao/CounterDaoImpl.java @@ -42,7 +42,7 @@ protected CounterDaoImpl() { } @Override - public List listCounters(Long id, String name, String source, String keyword, Filter filter) { + public List listCounters(Long id, String name, String source, String provider, String keyword, Filter filter) { SearchCriteria sc = AllFieldsSearch.create(); if (keyword != null) { @@ -62,6 +62,9 @@ public List listCounters(Long id, String name, String source, String if (source != null) { sc.addAnd("source", SearchCriteria.Op.EQ, source); } + if (provider != null) { + sc.addAnd("provider", SearchCriteria.Op.EQ, provider); + } return listBy(sc, filter); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index fe0b9a5c0ebc..b13f12783cdf 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -3291,6 +3291,7 @@ public CounterResponse createCounterResponse(Counter counter) { response.setSource(counter.getSource().toString()); response.setName(counter.getName()); response.setValue(counter.getValue()); + response.setProvider(counter.getProvider()); response.setObjectName("counter"); return response; } diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 2757525eca40..f6b0769d1136 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -76,6 +76,7 @@ import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.network.Network; import com.cloud.network.Network.Capability; import com.cloud.network.Network.IpAddresses; import com.cloud.network.as.AutoScaleCounter.AutoScaleCounterParam; @@ -1112,10 +1113,16 @@ public Counter createCounter(CreateCounterCmd cmd) { throw new InvalidParameterValueException("The Source " + source + " does not exist; Unable to create Counter"); } + // Validate Provider + Network.Provider provider = Network.Provider.getProvider(cmd.getProvider());; + if (provider == null) { + throw new InvalidParameterValueException("The Provider " + cmd.getProvider() + " does not exist; Unable to create Counter"); + } + CounterVO counter = null; s_logger.debug("Adding Counter " + name); - counter = _counterDao.persist(new CounterVO(src, name, cmd.getValue())); + counter = _counterDao.persist(new CounterVO(src, name, cmd.getValue(), provider)); CallContext.current().setEventDetails(" Id: " + counter.getId() + " Name: " + name); return counter; @@ -1156,12 +1163,21 @@ public List listCounters(ListCountersCmd cmd) { String name = cmd.getName(); Long id = cmd.getId(); String source = cmd.getSource(); - if (source != null) + if (source != null) { source = source.toLowerCase(); + } + String providerStr = cmd.getProvider(); + if (providerStr != null) { + Network.Provider provider = Network.Provider.getProvider(providerStr); + if (provider == null) { + throw new InvalidParameterValueException("The Provider " + providerStr + " does not exist; Unable to list Counter"); + } + providerStr = provider.getName(); + } Filter searchFilter = new Filter(CounterVO.class, "created", false, cmd.getStartIndex(), cmd.getPageSizeVal()); - List counters = _counterDao.listCounters(id, name, source, cmd.getKeyword(), searchFilter); + List counters = _counterDao.listCounters(id, name, source, providerStr, cmd.getKeyword(), searchFilter); return counters; } From af72e9d3c3a2a5cb40b2c66cdb4cc3aced95f1e7 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 18 May 2022 15:48:38 +0200 Subject: [PATCH 004/360] AS: add AutoScaleVmGroup.State --- .../cloud/agent/api/to/LoadBalancerTO.java | 10 +++--- .../cloud/network/as/AutoScaleVmGroup.java | 28 ++++++++++++--- .../cloud/network/lb/LoadBalancingRule.java | 6 ++-- .../network/lb/LoadBalancingRulesManager.java | 3 +- .../cloud/network/as/AutoScaleVmGroupVO.java | 8 ++--- .../network/resource/NetscalerResource.java | 9 ++--- .../java/com/cloud/api/ApiResponseHelper.java | 2 +- .../network/as/AutoScaleManagerImpl.java | 34 +++++++++---------- .../lb/LoadBalancingRulesManagerImpl.java | 14 ++++---- .../java/com/cloud/server/StatsCollector.java | 3 +- 10 files changed, 69 insertions(+), 48 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/to/LoadBalancerTO.java b/api/src/main/java/com/cloud/agent/api/to/LoadBalancerTO.java index 3a3dfa475021..1c98a744e4c0 100644 --- a/api/src/main/java/com/cloud/agent/api/to/LoadBalancerTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/LoadBalancerTO.java @@ -548,11 +548,11 @@ public static class AutoScaleVmGroupTO implements Serializable { private final int interval; private final List policies; private final AutoScaleVmProfileTO profile; - private final String state; - private final String currentState; + private final AutoScaleVmGroup.State state; + private final AutoScaleVmGroup.State currentState; AutoScaleVmGroupTO(String uuid, int minMembers, int maxMembers, int memberPort, int interval, List policies, AutoScaleVmProfileTO profile, - String state, String currentState) { + AutoScaleVmGroup.State state, AutoScaleVmGroup.State currentState) { this.uuid = uuid; this.minMembers = minMembers; this.maxMembers = maxMembers; @@ -592,11 +592,11 @@ public AutoScaleVmProfileTO getProfile() { return profile; } - public String getState() { + public AutoScaleVmGroup.State getState() { return state; } - public String getCurrentState() { + public AutoScaleVmGroup.State getCurrentState() { return currentState; } } diff --git a/api/src/main/java/com/cloud/network/as/AutoScaleVmGroup.java b/api/src/main/java/com/cloud/network/as/AutoScaleVmGroup.java index cf2c15c42166..e6c434e5780f 100644 --- a/api/src/main/java/com/cloud/network/as/AutoScaleVmGroup.java +++ b/api/src/main/java/com/cloud/network/as/AutoScaleVmGroup.java @@ -19,16 +19,34 @@ import java.util.Date; +import com.cloud.exception.InvalidParameterValueException; + import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.Displayable; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.StringUtils; public interface AutoScaleVmGroup extends ControlledEntity, InternalIdentity, Displayable { - String State_New = "new"; - String State_Revoke = "revoke"; - String State_Enabled = "enabled"; - String State_Disabled = "disabled"; + enum State { + New, Revoke, Enabled, Disabled; + + public static State fromValue(String state) { + if (StringUtils.isBlank(state)) { + return null; + } else if (state.equalsIgnoreCase("new")) { + return New; + } else if (state.equalsIgnoreCase("revoke")) { + return Revoke; + } else if (state.equalsIgnoreCase("enabled")) { + return Enabled; + } else if (state.equalsIgnoreCase("disabled")) { + return Disabled; + } else { + throw new InvalidParameterValueException("Unexpected AutoScale VM group state : " + state); + } + } + } @Override long getId(); @@ -50,7 +68,7 @@ public interface AutoScaleVmGroup extends ControlledEntity, InternalIdentity, Di Date getLastInterval(); - String getState(); + State getState(); String getUuid(); diff --git a/api/src/main/java/com/cloud/network/lb/LoadBalancingRule.java b/api/src/main/java/com/cloud/network/lb/LoadBalancingRule.java index be69332ea780..0edf72044fe0 100644 --- a/api/src/main/java/com/cloud/network/lb/LoadBalancingRule.java +++ b/api/src/main/java/com/cloud/network/lb/LoadBalancingRule.java @@ -406,9 +406,9 @@ public static class LbAutoScaleVmGroup { AutoScaleVmGroup vmGroup; private final List policies; private final LbAutoScaleVmProfile profile; - private final String currentState; + private final AutoScaleVmGroup.State currentState; - public LbAutoScaleVmGroup(AutoScaleVmGroup vmGroup, List policies, LbAutoScaleVmProfile profile, String currentState) { + public LbAutoScaleVmGroup(AutoScaleVmGroup vmGroup, List policies, LbAutoScaleVmProfile profile, AutoScaleVmGroup.State currentState) { this.vmGroup = vmGroup; this.policies = policies; this.profile = profile; @@ -427,7 +427,7 @@ public LbAutoScaleVmProfile getProfile() { return profile; } - public String getCurrentState() { + public AutoScaleVmGroup.State getCurrentState() { return currentState; } } diff --git a/engine/components-api/src/main/java/com/cloud/network/lb/LoadBalancingRulesManager.java b/engine/components-api/src/main/java/com/cloud/network/lb/LoadBalancingRulesManager.java index 945ee9783b53..585f1d50f1eb 100644 --- a/engine/components-api/src/main/java/com/cloud/network/lb/LoadBalancingRulesManager.java +++ b/engine/components-api/src/main/java/com/cloud/network/lb/LoadBalancingRulesManager.java @@ -22,6 +22,7 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.as.AutoScaleVmGroup; import com.cloud.network.lb.LoadBalancingRule.LbDestination; import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy; import com.cloud.network.lb.LoadBalancingRule.LbSslCert; @@ -61,7 +62,7 @@ LoadBalancer createPublicLoadBalancer(String xId, String name, String descriptio String getLBCapability(long networkid, String capabilityName); - boolean configureLbAutoScaleVmGroup(long vmGroupid, String currentState) throws ResourceUnavailableException; + boolean configureLbAutoScaleVmGroup(long vmGroupid, AutoScaleVmGroup.State currentState) throws ResourceUnavailableException; boolean revokeLoadBalancersForNetwork(long networkId, Scheme scheme) throws ResourceUnavailableException; diff --git a/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupVO.java b/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupVO.java index ea8eaf68806d..218e394ab1e9 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupVO.java +++ b/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupVO.java @@ -85,7 +85,7 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity { protected Date created; @Column(name = "state") - private String state; + private State state; @Column(name = "display", updatable = true, nullable = false) protected boolean display = true; @@ -95,7 +95,7 @@ public AutoScaleVmGroupVO() { public AutoScaleVmGroupVO(long lbRuleId, long zoneId, long domainId, long accountId, int minMembers, int maxMembers, int memberPort, - int interval, Date lastInterval, long profileId, String state) { + int interval, Date lastInterval, long profileId, State state) { uuid = UUID.randomUUID().toString(); loadBalancerId = lbRuleId; @@ -179,11 +179,11 @@ public Date getCreated() { } @Override - public String getState() { + public State getState() { return state; } - public void setState(String state) { + public void setState(State state) { this.state = state; } diff --git a/plugins/network-elements/netscaler/src/main/java/com/cloud/network/resource/NetscalerResource.java b/plugins/network-elements/netscaler/src/main/java/com/cloud/network/resource/NetscalerResource.java index ac9b0c6b99d9..03c72d0d11c7 100644 --- a/plugins/network-elements/netscaler/src/main/java/com/cloud/network/resource/NetscalerResource.java +++ b/plugins/network-elements/netscaler/src/main/java/com/cloud/network/resource/NetscalerResource.java @@ -115,6 +115,7 @@ import com.cloud.agent.api.to.StaticNatRuleTO; import com.cloud.host.Host; import com.cloud.host.Host.Type; +import com.cloud.network.as.AutoScaleVmGroup; import com.cloud.network.lb.LoadBalancingRule.LbSslCert; import com.cloud.network.rules.LbStickinessMethod.StickinessMethodType; import com.cloud.resource.ServerResource; @@ -2838,7 +2839,7 @@ public synchronized void applyAutoScaleConfig(final LoadBalancerTO loadBalancer) if (!isAutoScaleSupportedInNetScaler()) { throw new ExecutionException("AutoScale not supported in this version of NetScaler"); } - if (loadBalancer.isRevoked() || vmGroupTO.getState().equals("revoke")) { + if (loadBalancer.isRevoked() || vmGroupTO.getState().equals(AutoScaleVmGroup.State.Revoke)) { removeAutoScaleConfig(loadBalancer); } else { createAutoScaleConfig(loadBalancer); @@ -2895,10 +2896,10 @@ private synchronized boolean createAutoScaleConfig(final LoadBalancerTO loadBala } // Create the autoscale config - if (!loadBalancerTO.getAutoScaleVmGroupTO().getState().equals("disabled")) { + if (!loadBalancerTO.getAutoScaleVmGroupTO().getState().equals(AutoScaleVmGroup.State.Disabled)) { // on restart of network, there might be vmgrps in disabled state, no need to create autoscale config for them enableAutoScaleConfig(loadBalancerTO, false); - } else if (loadBalancerTO.getAutoScaleVmGroupTO().getState().equals("disabled")) { + } else if (loadBalancerTO.getAutoScaleVmGroupTO().getState().equals(AutoScaleVmGroup.State.Disabled)) { disableAutoScaleConfig(loadBalancerTO, false); } @@ -2913,7 +2914,7 @@ private synchronized boolean removeAutoScaleConfig(final LoadBalancerTO loadBala final String nsVirtualServerName = generateNSVirtualServerName(srcIp, srcPort); final String serviceGroupName = generateAutoScaleServiceGroupName(loadBalancerTO); - if (loadBalancerTO.getAutoScaleVmGroupTO().getCurrentState().equals("enabled")) { + if (loadBalancerTO.getAutoScaleVmGroupTO().getCurrentState().equals(AutoScaleVmGroup.State.Enabled)) { disableAutoScaleConfig(loadBalancerTO, false); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index b13f12783cdf..eae42deb5053 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -3373,7 +3373,7 @@ public AutoScaleVmGroupResponse createAutoScaleVmGroupResponse(AutoScaleVmGroup response.setId(vmGroup.getUuid()); response.setMinMembers(vmGroup.getMinMembers()); response.setMaxMembers(vmGroup.getMaxMembers()); - response.setState(vmGroup.getState()); + response.setState(vmGroup.getState().toString()); response.setInterval(vmGroup.getInterval()); response.setForDisplay(vmGroup.isDisplay()); AutoScaleVmProfileVO profile = ApiDBUtils.findAutoScaleVmProfileById(vmGroup.getProfileId()); diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index f6b0769d1136..6cb45e1c0280 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -427,7 +427,7 @@ public AutoScaleVmProfile updateAutoScaleVmProfile(UpdateAutoScaleVmProfileCmd c List vmGroupList = _autoScaleVmGroupDao.listByAll(null, profileId); for (AutoScaleVmGroupVO vmGroupVO : vmGroupList) { - if (physicalParameterUpdate && !vmGroupVO.getState().equals(AutoScaleVmGroup.State_Disabled)) { + if (physicalParameterUpdate && !vmGroupVO.getState().equals(AutoScaleVmGroup.State.Disabled)) { throw new InvalidParameterValueException("The AutoScale Vm Profile can be updated only if the Vm Group it is associated with is disabled in state"); } } @@ -743,7 +743,7 @@ public AutoScalePolicy updateAutoScalePolicy(UpdateAutoScalePolicyCmd cmd) { continue; } - if (!vmGroupVO.getState().equals(AutoScaleVmGroup.State_Disabled)) { + if (!vmGroupVO.getState().equals(AutoScaleVmGroup.State.Disabled)) { throw new InvalidParameterValueException("The AutoScale Policy can be updated only if the Vm Group it is associated with is disabled in state"); } if (policy.getDuration() < vmGroupVO.getInterval()) { @@ -782,7 +782,7 @@ public AutoScaleVmGroup createAutoScaleVmGroup(CreateAutoScaleVmGroupCmd cmd) { } AutoScaleVmGroupVO vmGroupVO = new AutoScaleVmGroupVO(cmd.getLbRuleId(), zoneId, loadBalancer.getDomainId(), loadBalancer.getAccountId(), minMembers, maxMembers, - loadBalancer.getDefaultPortStart(), interval, null, cmd.getProfileId(), AutoScaleVmGroup.State_New); + loadBalancer.getDefaultPortStart(), interval, null, cmd.getProfileId(), AutoScaleVmGroup.State.New); if (forDisplay != null) { vmGroupVO.setDisplay(forDisplay); @@ -797,14 +797,14 @@ public AutoScaleVmGroup createAutoScaleVmGroup(CreateAutoScaleVmGroupCmd cmd) { @Override @ActionEvent(eventType = EventTypes.EVENT_AUTOSCALEVMGROUP_CREATE, eventDescription = "creating autoscale vm group", async = true) public boolean configureAutoScaleVmGroup(CreateAutoScaleVmGroupCmd cmd) throws ResourceUnavailableException { - return configureAutoScaleVmGroup(cmd.getEntityId(), AutoScaleVmGroup.State_New); + return configureAutoScaleVmGroup(cmd.getEntityId(), AutoScaleVmGroup.State.New); } public boolean isLoadBalancerBasedAutoScaleVmGroup(AutoScaleVmGroup vmGroup) { return vmGroup.getLoadBalancerId() != null; } - private boolean configureAutoScaleVmGroup(long vmGroupid, String currentState) throws ResourceUnavailableException { + private boolean configureAutoScaleVmGroup(long vmGroupid, AutoScaleVmGroup.State currentState) throws ResourceUnavailableException { AutoScaleVmGroup vmGroup = _autoScaleVmGroupDao.findById(vmGroupid); if (isLoadBalancerBasedAutoScaleVmGroup(vmGroup)) { @@ -828,12 +828,12 @@ private boolean configureAutoScaleVmGroup(long vmGroupid, String currentState) t public boolean deleteAutoScaleVmGroup(final long id) { AutoScaleVmGroupVO autoScaleVmGroupVO = getEntityInDatabase(CallContext.current().getCallingAccount(), "AutoScale Vm Group", id, _autoScaleVmGroupDao); - if (autoScaleVmGroupVO.getState().equals(AutoScaleVmGroup.State_New)) { + if (autoScaleVmGroupVO.getState().equals(AutoScaleVmGroup.State.New)) { /* This condition is for handling failures during creation command */ return _autoScaleVmGroupDao.remove(id); } - String bakupState = autoScaleVmGroupVO.getState(); - autoScaleVmGroupVO.setState(AutoScaleVmGroup.State_Revoke); + AutoScaleVmGroup.State bakupState = autoScaleVmGroupVO.getState(); + autoScaleVmGroupVO.setState(AutoScaleVmGroup.State.Revoke); _autoScaleVmGroupDao.persist(autoScaleVmGroupVO); boolean success = false; @@ -1013,7 +1013,7 @@ public AutoScaleVmGroup updateAutoScaleVmGroup(UpdateAutoScaleVmGroupCmd cmd) { boolean physicalParametersUpdate = (minMembers != null || maxMembers != null || interval != null); - if (physicalParametersUpdate && !vmGroupVO.getState().equals(AutoScaleVmGroup.State_Disabled)) { + if (physicalParametersUpdate && !vmGroupVO.getState().equals(AutoScaleVmGroup.State.Disabled)) { throw new InvalidParameterValueException("An AutoScale Vm Group can be updated with minMembers/maxMembers/Interval only when it is in disabled state"); } @@ -1051,16 +1051,16 @@ public AutoScaleVmGroup updateAutoScaleVmGroup(UpdateAutoScaleVmGroupCmd cmd) { public AutoScaleVmGroup enableAutoScaleVmGroup(Long id) { AutoScaleVmGroupVO vmGroup = getEntityInDatabase(CallContext.current().getCallingAccount(), "AutoScale Vm Group", id, _autoScaleVmGroupDao); boolean success = false; - if (!vmGroup.getState().equals(AutoScaleVmGroup.State_Disabled)) { + if (!vmGroup.getState().equals(AutoScaleVmGroup.State.Disabled)) { throw new InvalidParameterValueException("Only a AutoScale Vm Group which is in Disabled state can be enabled."); } try { - vmGroup.setState(AutoScaleVmGroup.State_Enabled); + vmGroup.setState(AutoScaleVmGroup.State.Enabled); vmGroup = _autoScaleVmGroupDao.persist(vmGroup); - success = configureAutoScaleVmGroup(id, AutoScaleVmGroup.State_Disabled); + success = configureAutoScaleVmGroup(id, AutoScaleVmGroup.State.Disabled); } catch (ResourceUnavailableException e) { - vmGroup.setState(AutoScaleVmGroup.State_Disabled); + vmGroup.setState(AutoScaleVmGroup.State.Disabled); _autoScaleVmGroupDao.persist(vmGroup); } finally { if (!success) { @@ -1078,16 +1078,16 @@ public AutoScaleVmGroup enableAutoScaleVmGroup(Long id) { public AutoScaleVmGroup disableAutoScaleVmGroup(Long id) { AutoScaleVmGroupVO vmGroup = getEntityInDatabase(CallContext.current().getCallingAccount(), "AutoScale Vm Group", id, _autoScaleVmGroupDao); boolean success = false; - if (!vmGroup.getState().equals(AutoScaleVmGroup.State_Enabled)) { + if (!vmGroup.getState().equals(AutoScaleVmGroup.State.Enabled)) { throw new InvalidParameterValueException("Only a AutoScale Vm Group which is in Enabled state can be disabled."); } try { - vmGroup.setState(AutoScaleVmGroup.State_Disabled); + vmGroup.setState(AutoScaleVmGroup.State.Disabled); vmGroup = _autoScaleVmGroupDao.persist(vmGroup); - success = configureAutoScaleVmGroup(id, AutoScaleVmGroup.State_Enabled); + success = configureAutoScaleVmGroup(id, AutoScaleVmGroup.State.Enabled); } catch (ResourceUnavailableException e) { - vmGroup.setState(AutoScaleVmGroup.State_Enabled); + vmGroup.setState(AutoScaleVmGroup.State.Enabled); _autoScaleVmGroupDao.persist(vmGroup); } finally { if (!success) { diff --git a/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java b/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java index 3227516e04d7..749b9f75394f 100644 --- a/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java +++ b/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java @@ -291,7 +291,7 @@ public String getLBCapability(long networkid, String capabilityName) { return null; } - private LbAutoScaleVmGroup getLbAutoScaleVmGroup(AutoScaleVmGroupVO vmGroup, String currentState, LoadBalancerVO lb) { + private LbAutoScaleVmGroup getLbAutoScaleVmGroup(AutoScaleVmGroupVO vmGroup, AutoScaleVmGroup.State currentState, LoadBalancerVO lb) { long lbNetworkId = lb.getNetworkId(); String lbName = lb.getName(); List vmGroupPolicyMapList = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(vmGroup.getId()); @@ -349,7 +349,7 @@ private LbAutoScaleVmGroup getLbAutoScaleVmGroup(AutoScaleVmGroupVO vmGroup, Str return new LbAutoScaleVmGroup(vmGroup, autoScalePolicies, lbAutoScaleVmProfile, currentState); } - private boolean applyAutoScaleConfig(LoadBalancerVO lb, AutoScaleVmGroupVO vmGroup, String currentState) throws ResourceUnavailableException { + private boolean applyAutoScaleConfig(LoadBalancerVO lb, AutoScaleVmGroupVO vmGroup, AutoScaleVmGroup.State currentState) throws ResourceUnavailableException { LbAutoScaleVmGroup lbAutoScaleVmGroup = getLbAutoScaleVmGroup(vmGroup, currentState, lb); /* * Regular config like destinations need not be packed for applying @@ -389,7 +389,7 @@ private Ip getSourceIp(LoadBalancer lb) { @Override @DB - public boolean configureLbAutoScaleVmGroup(final long vmGroupid, String currentState) throws ResourceUnavailableException { + public boolean configureLbAutoScaleVmGroup(final long vmGroupid, AutoScaleVmGroup.State currentState) throws ResourceUnavailableException { final AutoScaleVmGroupVO vmGroup = _autoScaleVmGroupDao.findById(vmGroupid); boolean success = false; @@ -397,10 +397,10 @@ public boolean configureLbAutoScaleVmGroup(final long vmGroupid, String currentS FirewallRule.State backupState = loadBalancer.getState(); - if (vmGroup.getState().equals(AutoScaleVmGroup.State_New)) { + if (vmGroup.getState().equals(AutoScaleVmGroup.State.New)) { loadBalancer.setState(FirewallRule.State.Add); _lbDao.persist(loadBalancer); - } else if (loadBalancer.getState() == FirewallRule.State.Active && vmGroup.getState().equals(AutoScaleVmGroup.State_Revoke)) { + } else if (loadBalancer.getState() == FirewallRule.State.Active && vmGroup.getState().equals(AutoScaleVmGroup.State.Revoke)) { loadBalancer.setState(FirewallRule.State.Add); _lbDao.persist(loadBalancer); } @@ -422,14 +422,14 @@ public boolean configureLbAutoScaleVmGroup(final long vmGroupid, String currentS } if (success) { - if (vmGroup.getState().equals(AutoScaleVmGroup.State_New)) { + if (vmGroup.getState().equals(AutoScaleVmGroup.State.New)) { Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { loadBalancer.setState(FirewallRule.State.Active); s_logger.debug("LB rule " + loadBalancer.getId() + " state is set to Active"); _lbDao.persist(loadBalancer); - vmGroup.setState(AutoScaleVmGroup.State_Enabled); + vmGroup.setState(AutoScaleVmGroup.State.Enabled); _autoScaleVmGroupDao.persist(vmGroup); s_logger.debug("LB Auto Scale Vm Group with Id: " + vmGroupid + " is set to Enabled state."); } diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 4a353a1aa1c8..4ec1c3c79d86 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -110,6 +110,7 @@ import com.cloud.network.as.AutoScaleManager; import com.cloud.network.as.AutoScalePolicyConditionMapVO; import com.cloud.network.as.AutoScalePolicyVO; +import com.cloud.network.as.AutoScaleVmGroup; import com.cloud.network.as.AutoScaleVmGroupPolicyMapVO; import com.cloud.network.as.AutoScaleVmGroupVO; import com.cloud.network.as.AutoScaleVmGroupVmMapVO; @@ -1772,7 +1773,7 @@ protected void runInContext() { List asGroups = _asGroupDao.listAll(); for (AutoScaleVmGroupVO asGroup : asGroups) { // check group state - if ((asGroup.getState().equals("enabled")) && (is_native(asGroup.getId()))) { + if ((asGroup.getState().equals(AutoScaleVmGroup.State.Enabled)) && (is_native(asGroup.getId()))) { // check minimum vm of group Integer currentVM = _asGroupVmDao.countByGroup(asGroup.getId()); if (currentVM < asGroup.getMinMembers()) { From 2fcff7b1ac1c0274891ba59042831ad64f73eea1 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 20 May 2022 09:00:27 +0200 Subject: [PATCH 005/360] AS: add AutoScalePolicy.Action --- .../cloud/agent/api/to/LoadBalancerTO.java | 6 ++--- .../com/cloud/network/as/AutoScalePolicy.java | 21 +++++++++++++++++- .../cloud/network/as/AutoScalePolicyVO.java | 6 ++--- .../network/resource/NetscalerResource.java | 5 +++-- .../main/java/com/cloud/api/ApiDBUtils.java | 4 ++-- .../java/com/cloud/api/ApiResponseHelper.java | 2 +- .../network/as/AutoScaleManagerImpl.java | 22 +++++++++++++------ .../java/com/cloud/server/StatsCollector.java | 7 +++--- 8 files changed, 51 insertions(+), 22 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/to/LoadBalancerTO.java b/api/src/main/java/com/cloud/agent/api/to/LoadBalancerTO.java index 1c98a744e4c0..9b63e86f7167 100644 --- a/api/src/main/java/com/cloud/agent/api/to/LoadBalancerTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/LoadBalancerTO.java @@ -420,11 +420,11 @@ public static class AutoScalePolicyTO implements Serializable { private final long id; private final int duration; private final int quietTime; - private String action; + private AutoScalePolicy.Action action; boolean revoked; private final List conditions; - public AutoScalePolicyTO(long id, int duration, int quietTime, String action, List conditions, boolean revoked) { + public AutoScalePolicyTO(long id, int duration, int quietTime, AutoScalePolicy.Action action, List conditions, boolean revoked) { this.id = id; this.duration = duration; this.quietTime = quietTime; @@ -445,7 +445,7 @@ public int getQuietTime() { return quietTime; } - public String getAction() { + public AutoScalePolicy.Action getAction() { return action; } diff --git a/api/src/main/java/com/cloud/network/as/AutoScalePolicy.java b/api/src/main/java/com/cloud/network/as/AutoScalePolicy.java index 56ea3f5f7f72..5a8cae1dfe70 100644 --- a/api/src/main/java/com/cloud/network/as/AutoScalePolicy.java +++ b/api/src/main/java/com/cloud/network/as/AutoScalePolicy.java @@ -19,11 +19,30 @@ import java.util.Date; +import com.cloud.exception.InvalidParameterValueException; + import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.StringUtils; public interface AutoScalePolicy extends ControlledEntity, InternalIdentity { + enum Action { + ScaleUp, ScaleDown; + + public static Action fromValue(String action) { + if (StringUtils.isBlank(action)) { + return null; + } else if (action.equalsIgnoreCase("ScaleUp")) { + return ScaleUp; + } else if (action.equalsIgnoreCase("ScaleDown")) { + return ScaleDown; + } else { + throw new InvalidParameterValueException("Unexpected AutoScale action : " + action); + } + } + } + @Override long getId(); @@ -35,6 +54,6 @@ public interface AutoScalePolicy extends ControlledEntity, InternalIdentity { public Date getLastQuiteTime(); - public String getAction(); + public Action getAction(); } diff --git a/engine/schema/src/main/java/com/cloud/network/as/AutoScalePolicyVO.java b/engine/schema/src/main/java/com/cloud/network/as/AutoScalePolicyVO.java index 6379f65c30d2..c774436b22dd 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/AutoScalePolicyVO.java +++ b/engine/schema/src/main/java/com/cloud/network/as/AutoScalePolicyVO.java @@ -64,7 +64,7 @@ public class AutoScalePolicyVO implements AutoScalePolicy, InternalIdentity { private Date lastQuiteTime; @Column(name = "action", updatable = false, nullable = false) - private String action; + private Action action; @Column(name = GenericDao.REMOVED_COLUMN) protected Date removed; @@ -76,7 +76,7 @@ public AutoScalePolicyVO() { } public AutoScalePolicyVO(long domainId, long accountId, int duration, - int quietTime, Date lastQuiteTime, String action) { + int quietTime, Date lastQuiteTime, Action action) { uuid = UUID.randomUUID().toString(); this.domainId = domainId; this.accountId = accountId; @@ -127,7 +127,7 @@ public Date getLastQuiteTime() { } @Override - public String getAction() { + public Action getAction() { return action; } diff --git a/plugins/network-elements/netscaler/src/main/java/com/cloud/network/resource/NetscalerResource.java b/plugins/network-elements/netscaler/src/main/java/com/cloud/network/resource/NetscalerResource.java index 03c72d0d11c7..586da09ca23a 100644 --- a/plugins/network-elements/netscaler/src/main/java/com/cloud/network/resource/NetscalerResource.java +++ b/plugins/network-elements/netscaler/src/main/java/com/cloud/network/resource/NetscalerResource.java @@ -115,6 +115,7 @@ import com.cloud.agent.api.to.StaticNatRuleTO; import com.cloud.host.Host; import com.cloud.host.Host.Type; +import com.cloud.network.as.AutoScalePolicy; import com.cloud.network.as.AutoScaleVmGroup; import com.cloud.network.lb.LoadBalancingRule.LbSslCert; import com.cloud.network.rules.LbStickinessMethod.StickinessMethodType; @@ -3512,11 +3513,11 @@ private boolean isAutoScaleSupportedInNetScaler() throws ExecutionException { } private boolean isScaleUpPolicy(final AutoScalePolicyTO autoScalePolicyTO) { - return autoScalePolicyTO.getAction().equals("scaleup"); + return autoScalePolicyTO.getAction().equals(AutoScalePolicy.Action.ScaleUp); } private boolean isScaleDownPolicy(final AutoScalePolicyTO autoScalePolicyTO) { - return autoScalePolicyTO.getAction().equals("scaledown"); + return autoScalePolicyTO.getAction().equals(AutoScalePolicy.Action.ScaleDown); } private void saveConfiguration() throws ExecutionException { diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index c874aa19b572..d5fd1ac4a7c6 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -1553,7 +1553,7 @@ public static void getAutoScaleVmGroupPolicyIds(long vmGroupId, List scale List vos = s_asVmGroupPolicyMapDao.listByVmGroupId(vmGroupId); for (AutoScaleVmGroupPolicyMapVO vo : vos) { AutoScalePolicy autoScalePolicy = s_asPolicyDao.findById(vo.getPolicyId()); - if (autoScalePolicy.getAction().equals("scaleup")) { + if (autoScalePolicy.getAction().equals(AutoScalePolicy.Action.ScaleUp)) { scaleUpPolicyIds.add(autoScalePolicy.getId()); } else { scaleDownPolicyIds.add(autoScalePolicy.getId()); @@ -1578,7 +1578,7 @@ public static void getAutoScaleVmGroupPolicies(long vmGroupId, List vos = s_asVmGroupPolicyMapDao.listByVmGroupId(vmGroupId); for (AutoScaleVmGroupPolicyMapVO vo : vos) { AutoScalePolicy autoScalePolicy = s_asPolicyDao.findById(vo.getPolicyId()); - if (autoScalePolicy.getAction().equals("scaleup")) { + if (autoScalePolicy.getAction().equals(AutoScalePolicy.Action.ScaleUp)) { scaleUpPolicies.add(autoScalePolicy); } else { scaleDownPolicies.add(autoScalePolicy); diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index eae42deb5053..229db588a106 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -3352,7 +3352,7 @@ public AutoScalePolicyResponse createAutoScalePolicyResponse(AutoScalePolicy pol response.setId(policy.getUuid()); response.setDuration(policy.getDuration()); response.setQuietTime(policy.getQuietTime()); - response.setAction(policy.getAction()); + response.setAction(policy.getAction().toString()); List vos = ApiDBUtils.getAutoScalePolicyConditions(policy.getId()); ArrayList conditions = new ArrayList(vos.size()); for (ConditionVO vo : vos) { diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 6cb45e1c0280..1b2be4d43acf 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -19,6 +19,7 @@ import java.security.InvalidParameterException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -247,7 +248,7 @@ private VO getEntityInDatabase(Account caller, Str } private boolean isAutoScaleScaleUpPolicy(AutoScalePolicy policyVO) { - return policyVO.getAction().equals("scaleup"); + return policyVO.getAction().equals(AutoScalePolicy.Action.ScaleUp); } private List getAutoScalePolicies(String paramName, List policyIds, List counters, int interval, boolean scaleUpPolicies) { @@ -569,12 +570,12 @@ public AutoScalePolicy createAutoScalePolicy(CreateAutoScalePolicyCmd cmd) { quietTime = NetUtils.DEFAULT_AUTOSCALE_POLICY_QUIET_TIME; } - action = action.toLowerCase(); - if (!NetUtils.isValidAutoScaleAction(action)) { - throw new InvalidParameterValueException("action is invalid, only 'scaleup' and 'scaledown' is supported"); + AutoScalePolicy.Action scaleAction = AutoScalePolicy.Action.fromValue(action); + if (scaleAction == null) { + throw new InvalidParameterValueException("action is invalid. Supported actions are: " + Arrays.toString(AutoScalePolicy.Action.values())); } - AutoScalePolicyVO policyVO = new AutoScalePolicyVO(cmd.getDomainId(), cmd.getAccountId(), duration, quietTime, null, action); + AutoScalePolicyVO policyVO = new AutoScalePolicyVO(cmd.getDomainId(), cmd.getAccountId(), duration, quietTime, null, scaleAction); policyVO = checkValidityAndPersist(policyVO, cmd.getConditionIds()); s_logger.info("Successfully created AutoScale Policy with Id: " + policyVO.getId()); @@ -681,6 +682,13 @@ public List listAutoScalePolicies(ListAutoScalePolici String action = cmd.getAction(); Long vmGroupId = cmd.getVmGroupId(); + if (action != null) { + AutoScalePolicy.Action scaleAction = AutoScalePolicy.Action.fromValue(action); + if (scaleAction == null) { + throw new InvalidParameterValueException("action is invalid. Supported actions are: " + Arrays.toString(AutoScalePolicy.Action.values())); + } + } + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); sb.and("action", sb.entity().getAction(), SearchCriteria.Op.EQ); @@ -1483,7 +1491,7 @@ public void doScaleUp(long groupId, Integer numVm) { for (AutoScaleVmGroupPolicyMapVO GroupPolicyVO : GroupPolicyVOs) { AutoScalePolicyVO vo = _autoScalePolicyDao .findById(GroupPolicyVO.getPolicyId()); - if (vo.getAction().equals("scaleup")) { + if (vo.getAction().equals(AutoScalePolicy.Action.ScaleUp)) { vo.setLastQuiteTime(new Date()); _autoScalePolicyDao.persist(vo); break; @@ -1521,7 +1529,7 @@ public void doScaleDown(final long groupId) { List GroupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); for (AutoScaleVmGroupPolicyMapVO GroupPolicyVO : GroupPolicyVOs) { AutoScalePolicyVO vo = _autoScalePolicyDao.findById(GroupPolicyVO.getPolicyId()); - if (vo.getAction().equals("scaledown")) { + if (vo.getAction().equals(AutoScalePolicy.Action.ScaleUp)) { vo.setLastQuiteTime(new Date()); _autoScalePolicyDao.persist(vo); break; diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 4ec1c3c79d86..ab75276a62d1 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -108,6 +108,7 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.as.AutoScaleManager; +import com.cloud.network.as.AutoScalePolicy; import com.cloud.network.as.AutoScalePolicyConditionMapVO; import com.cloud.network.as.AutoScalePolicyVO; import com.cloud.network.as.AutoScaleVmGroup; @@ -1881,10 +1882,10 @@ protected void runInContext() { } } - String scaleAction = getAutoscaleAction(avgCounter, asGroup.getId(), currentVM, params); + AutoScalePolicy.Action scaleAction = getAutoscaleAction(avgCounter, asGroup.getId(), currentVM, params); if (scaleAction != null) { LOGGER.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); - if (scaleAction.equals("scaleup")) { + if (AutoScalePolicy.Action.ScaleUp.equals(scaleAction)) { _asManager.doScaleUp(asGroup.getId(), 1); } else { _asManager.doScaleDown(asGroup.getId()); @@ -1920,7 +1921,7 @@ private boolean is_native(long groupId) { return false; } - private String getAutoscaleAction(HashMap avgCounter, long groupId, long currentVM, Map params) { + private AutoScalePolicy.Action getAutoscaleAction(HashMap avgCounter, long groupId, long currentVM, Map params) { List listMap = _asGroupPolicyDao.listByVmGroupId(groupId); if ((listMap == null) || (listMap.size() == 0)) From b896177baa34b14c5448f3f068991b2e00422a5e Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 19 May 2022 10:10:43 +0200 Subject: [PATCH 006/360] AS: add global configuration AutoScaleStatsInterval --- .../com/cloud/network/as/AutoScaleManager.java | 8 ++++++++ .../cloud/network/as/AutoScaleManagerImpl.java | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManager.java b/server/src/main/java/com/cloud/network/as/AutoScaleManager.java index 410fb8f923ef..dd42183aaed2 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManager.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManager.java @@ -16,8 +16,16 @@ // under the License. package com.cloud.network.as; +import org.apache.cloudstack.framework.config.ConfigKey; + public interface AutoScaleManager extends AutoScaleService { + ConfigKey AutoScaleStatsInterval = new ConfigKey<>("Advanced", Integer.class, + "autoscale.stats.interval", + "0", + "The interval (in milliseconds) when VM auto scaling stats retrieved from agents and/or other resources. Less than 1 means disabled.", + false); + void cleanUpAutoScaleResources(Long accountId); void doScaleUp(long groupId, Integer numVm); diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 1b2be4d43acf..1fb3d6cbdf38 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -58,6 +58,8 @@ import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import com.cloud.api.ApiDBUtils; @@ -127,7 +129,7 @@ import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmService; -public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManager, AutoScaleService { +public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManager, AutoScaleService, Configurable { private static final Logger s_logger = Logger.getLogger(AutoScaleManagerImpl.class); private ScheduledExecutorService _executor = Executors.newScheduledThreadPool(1); @@ -1560,4 +1562,15 @@ public void run() { } } + @Override + public String getConfigComponentName() { + return AutoScaleManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { + AutoScaleStatsInterval + }; + } } From 0a03bed23670f5018e49f041cb9370621e374378 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 20 May 2022 13:20:42 +0200 Subject: [PATCH 007/360] AS: move autoscaling methods from StatsCollector to AutoScaleManagerImpl --- .../cloud/network/as/AutoScaleManager.java | 2 + .../network/as/AutoScaleManagerImpl.java | 276 ++++++++++++++++ .../java/com/cloud/server/StatsCollector.java | 297 +----------------- 3 files changed, 279 insertions(+), 296 deletions(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManager.java b/server/src/main/java/com/cloud/network/as/AutoScaleManager.java index dd42183aaed2..193df1e0dc61 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManager.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManager.java @@ -31,4 +31,6 @@ public interface AutoScaleManager extends AutoScaleService { void doScaleUp(long groupId, Integer numVm); void doScaleDown(long groupId); + + void checkAutoScaleVmGroup(AutoScaleVmGroupVO asGroup); } diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 1fb3d6cbdf38..769af52f9e22 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -62,6 +62,9 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.PerformanceMonitorCommand; import com.cloud.api.ApiDBUtils; import com.cloud.api.dispatch.DispatchChainFactory; import com.cloud.api.dispatch.DispatchTask; @@ -101,6 +104,8 @@ import com.cloud.network.lb.LoadBalancingRulesService; import com.cloud.offering.ServiceOffering; import com.cloud.projects.Project.ListProjectResourcesCriteria; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.template.TemplateManager; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; @@ -128,6 +133,8 @@ import com.cloud.utils.net.NetUtils; import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmService; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManager, AutoScaleService, Configurable { private static final Logger s_logger = Logger.getLogger(AutoScaleManagerImpl.class); @@ -187,6 +194,12 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale LoadBalancerVMMapDao _lbVmMapDao; @Inject LoadBalancingRulesService _loadBalancingRulesService; + @Inject + private VMInstanceDao _vmInstanceDao; + @Inject + private AgentManager _agentMgr; + @Inject + private ServiceOfferingDao _serviceOfferingDao; public List getSupportedAutoScaleCounters(long networkid) { String capability = _lbRulesMgr.getLBCapability(networkid, Capability.AutoScaleCounters.getName()); @@ -1573,4 +1586,267 @@ public ConfigKey[] getConfigKeys() { AutoScaleStatsInterval }; } + + @Override + public void checkAutoScaleVmGroup(AutoScaleVmGroupVO asGroup) { + // check group state + if ((asGroup.getState().equals(AutoScaleVmGroup.State.Enabled)) && (is_native(asGroup.getId()))) { + // check minimum vm of group + Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId()); + if (currentVM < asGroup.getMinMembers()) { + doScaleUp(asGroup.getId(), asGroup.getMinMembers() - currentVM); + return; + } + + //check interval + long now = (new Date()).getTime(); + if (asGroup.getLastInterval() != null && (now - asGroup.getLastInterval().getTime()) < asGroup.getInterval()) { + return; + } + + // update last_interval + asGroup.setLastInterval(new Date()); + _autoScaleVmGroupDao.persist(asGroup); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("[Netscaler AutoScale] Collecting RRDs data..."); + } + Map params = new HashMap(); + List asGroupVmVOs = _autoScaleVmGroupVmMapDao.listByGroup(asGroup.getId()); + params.put("total_vm", String.valueOf(asGroupVmVOs.size())); + for (int i = 0; i < asGroupVmVOs.size(); i++) { + long vmId = asGroupVmVOs.get(i).getInstanceId(); + VMInstanceVO vmVO = _vmInstanceDao.findById(vmId); + //xe vm-list | grep vmname -B 1 | head -n 1 | awk -F':' '{print $2}' + params.put("vmname" + String.valueOf(i + 1), vmVO.getInstanceName()); + params.put("vmid" + String.valueOf(i + 1), String.valueOf(vmVO.getId())); + + } + // get random hostid because all vms are in a cluster + long vmId = asGroupVmVOs.get(0).getInstanceId(); + VMInstanceVO vmVO = _vmInstanceDao.findById(vmId); + Long receiveHost = vmVO.getHostId(); + + // setup parameters phase: duration and counter + // list pair [counter, duration] + List> lstPair = getPairofCounternameAndDuration(asGroup.getId()); + int total_counter = 0; + String[] lstCounter = new String[lstPair.size()]; + for (int i = 0; i < lstPair.size(); i++) { + Pair pair = lstPair.get(i); + String strCounterNames = pair.first(); + Integer duration = pair.second(); + + lstCounter[i] = strCounterNames.split(",")[0]; + total_counter++; + params.put("duration" + String.valueOf(total_counter), duration.toString()); + params.put("counter" + String.valueOf(total_counter), lstCounter[i]); + params.put("con" + String.valueOf(total_counter), strCounterNames.split(",")[1]); + } + params.put("total_counter", String.valueOf(total_counter)); + + PerformanceMonitorCommand perfMon = new PerformanceMonitorCommand(params, 20); + + try { + Answer answer = _agentMgr.send(receiveHost, perfMon); + if (answer == null || !answer.getResult()) { + s_logger.debug("Failed to send data to node !"); + } else { + String result = answer.getDetails(); + s_logger.debug("[AutoScale] RRDs collection answer: " + result); + HashMap avgCounter = new HashMap(); + + // extract data + String[] counterElements = result.split(","); + if ((counterElements != null) && (counterElements.length > 0)) { + for (String string : counterElements) { + try { + String[] counterVals = string.split(":"); + String[] counter_vm = counterVals[0].split("\\."); + + Long counterId = Long.parseLong(counter_vm[1]); + Long conditionId = Long.parseLong(params.get("con" + counter_vm[1])); + Double coVal = Double.parseDouble(counterVals[1]); + + // Summary of all counter by counterId key + if (avgCounter.get(counterId) == null) { + /* initialize if data is not set */ + avgCounter.put(counterId, new Double(0)); + } + + String counterName = getCounternamebyCondition(conditionId.longValue()); + if (Counter.Source.memory.toString().equals(counterName)) { + // calculate memory in percent + Long profileId = asGroup.getProfileId(); + AutoScaleVmProfileVO profileVo = _autoScaleVmProfileDao.findById(profileId); + ServiceOfferingVO serviceOff = _serviceOfferingDao.findById(profileVo.getServiceOfferingId()); + int maxRAM = serviceOff.getRamSize(); + + // get current RAM percent + coVal = coVal / maxRAM; + } else { + // cpu + coVal = coVal * 100; + } + + // update data entry + avgCounter.put(counterId, avgCounter.get(counterId) + coVal); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + AutoScalePolicy.Action scaleAction = getAutoscaleAction(avgCounter, asGroup.getId(), currentVM, params); + if (scaleAction != null) { + s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); + if (AutoScalePolicy.Action.ScaleUp.equals(scaleAction)) { + doScaleUp(asGroup.getId(), 1); + } else { + doScaleDown(asGroup.getId()); + } + } + } + } + + } catch (Exception e) { + e.printStackTrace(); + } + + } + } + + private boolean is_native(long groupId) { + List vos = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); + for (AutoScaleVmGroupPolicyMapVO vo : vos) { + List ConditionPolicies = _autoScalePolicyConditionMapDao.findByPolicyId(vo.getPolicyId()); + for (AutoScalePolicyConditionMapVO ConditionPolicy : ConditionPolicies) { + ConditionVO condition = _conditionDao.findById(ConditionPolicy.getConditionId()); + CounterVO counter = _counterDao.findById(condition.getCounterid()); + if (counter.getSource() == Counter.Source.cpu || counter.getSource() == Counter.Source.memory) + return true; + } + } + return false; + } + + private AutoScalePolicy.Action getAutoscaleAction(HashMap avgCounter, long groupId, long currentVM, Map params) { + + List listMap = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); + if ((listMap == null) || (listMap.size() == 0)) + return null; + for (AutoScaleVmGroupPolicyMapVO asVmgPmap : listMap) { + AutoScalePolicyVO policyVO = _autoScalePolicyDao.findById(asVmgPmap.getPolicyId()); + if (policyVO != null) { + int quitetime = policyVO.getQuietTime(); + Date quitetimeDate = policyVO.getLastQuiteTime(); + long last_quitetime = 0L; + if (quitetimeDate != null) { + last_quitetime = policyVO.getLastQuiteTime().getTime(); + } + long current_time = (new Date()).getTime(); + + // check quite time for this policy + if ((current_time - last_quitetime) >= (long)quitetime) { + + // list all condition of this policy + boolean bValid = true; + List lstConditions = getConditionsbyPolicyId(policyVO.getId()); + if ((lstConditions != null) && (lstConditions.size() > 0)) { + // check whole conditions of this policy + for (ConditionVO conditionVO : lstConditions) { + long thresholdValue = conditionVO.getThreshold(); + Double thresholdPercent = (double)thresholdValue / 100; + CounterVO counterVO = _counterDao.findById(conditionVO.getCounterid()); + long counter_count = 1; + do { + String counter_param = params.get("counter" + String.valueOf(counter_count)); + Counter.Source counter_source = counterVO.getSource(); + if (counter_param.equals(counter_source.toString())) + break; + counter_count++; + } while (true); + + Double sum = avgCounter.get(counter_count); + Double avg = sum / currentVM; + Condition.Operator op = conditionVO.getRelationalOperator(); + boolean bConditionCheck = ((op == com.cloud.network.as.Condition.Operator.EQ) && (thresholdPercent.equals(avg))) + || ((op == com.cloud.network.as.Condition.Operator.GE) && (avg.doubleValue() >= thresholdPercent.doubleValue())) + || ((op == com.cloud.network.as.Condition.Operator.GT) && (avg.doubleValue() > thresholdPercent.doubleValue())) + || ((op == com.cloud.network.as.Condition.Operator.LE) && (avg.doubleValue() <= thresholdPercent.doubleValue())) + || ((op == com.cloud.network.as.Condition.Operator.LT) && (avg.doubleValue() < thresholdPercent.doubleValue())); + + if (!bConditionCheck) { + bValid = false; + break; + } + } + if (bValid) { + return policyVO.getAction(); + } + } + } + } + } + return null; + } + + private List getConditionsbyPolicyId(long policyId) { + List conditionMap = _autoScalePolicyConditionMapDao.findByPolicyId(policyId); + if ((conditionMap == null) || (conditionMap.size() == 0)) + return null; + + List lstResult = new ArrayList(); + for (AutoScalePolicyConditionMapVO asPCmap : conditionMap) { + lstResult.add(_conditionDao.findById(asPCmap.getConditionId())); + } + + return lstResult; + } + + public List> getPairofCounternameAndDuration(long groupId) { + AutoScaleVmGroupVO groupVo = _autoScaleVmGroupDao.findById(groupId); + if (groupVo == null) + return null; + List> result = new ArrayList>(); + //list policy map + List groupPolicymap = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupVo.getId()); + if (groupPolicymap == null) + return null; + for (AutoScaleVmGroupPolicyMapVO gpMap : groupPolicymap) { + //get duration + AutoScalePolicyVO policyVo = _autoScalePolicyDao.findById(gpMap.getPolicyId()); + Integer duration = policyVo.getDuration(); + //get collection of counter name + + StringBuffer buff = new StringBuffer(); + List lstPCmap = _autoScalePolicyConditionMapDao.findByPolicyId(policyVo.getId()); + for (AutoScalePolicyConditionMapVO pcMap : lstPCmap) { + String counterName = getCounternamebyCondition(pcMap.getConditionId()); + buff.append(counterName); + buff.append(","); + buff.append(pcMap.getConditionId()); + } + // add to result + Pair pair = new Pair(buff.toString(), duration); + result.add(pair); + } + + return result; + } + + public String getCounternamebyCondition(long conditionId) { + + ConditionVO condition = _conditionDao.findById(conditionId); + if (condition == null) + return ""; + + long counterId = condition.getCounterid(); + CounterVO counter = _counterDao.findById(counterId); + if (counter == null) + return ""; + + return counter.getSource().toString(); + } + } diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index ab75276a62d1..d9eb163b5026 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -79,7 +79,6 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.GetStorageStatsCommand; import com.cloud.agent.api.HostStatsEntry; -import com.cloud.agent.api.PerformanceMonitorCommand; import com.cloud.agent.api.VgpuTypesInfo; import com.cloud.agent.api.VmDiskStatsEntry; import com.cloud.agent.api.VmNetworkStatsEntry; @@ -108,32 +107,12 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.as.AutoScaleManager; -import com.cloud.network.as.AutoScalePolicy; -import com.cloud.network.as.AutoScalePolicyConditionMapVO; -import com.cloud.network.as.AutoScalePolicyVO; -import com.cloud.network.as.AutoScaleVmGroup; -import com.cloud.network.as.AutoScaleVmGroupPolicyMapVO; import com.cloud.network.as.AutoScaleVmGroupVO; -import com.cloud.network.as.AutoScaleVmGroupVmMapVO; -import com.cloud.network.as.AutoScaleVmProfileVO; -import com.cloud.network.as.Condition.Operator; -import com.cloud.network.as.ConditionVO; -import com.cloud.network.as.Counter; -import com.cloud.network.as.CounterVO; -import com.cloud.network.as.dao.AutoScalePolicyConditionMapDao; -import com.cloud.network.as.dao.AutoScalePolicyDao; import com.cloud.network.as.dao.AutoScaleVmGroupDao; -import com.cloud.network.as.dao.AutoScaleVmGroupPolicyMapDao; -import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; -import com.cloud.network.as.dao.AutoScaleVmProfileDao; -import com.cloud.network.as.dao.ConditionDao; -import com.cloud.network.as.dao.CounterDao; import com.cloud.org.Cluster; import com.cloud.resource.ResourceManager; import com.cloud.resource.ResourceState; import com.cloud.serializer.GsonHelper; -import com.cloud.service.ServiceOfferingVO; -import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.ImageStoreDetailsUtil; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; @@ -149,7 +128,6 @@ import com.cloud.user.dao.VmDiskStatisticsDao; import com.cloud.utils.LogUtils; import com.cloud.utils.NumbersUtil; -import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentMethodInterceptable; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; @@ -342,26 +320,10 @@ public String toString() { @Inject private AutoScaleVmGroupDao _asGroupDao; @Inject - private AutoScaleVmGroupVmMapDao _asGroupVmDao; - @Inject private AutoScaleManager _asManager; @Inject private VMInstanceDao _vmInstance; @Inject - private AutoScaleVmGroupPolicyMapDao _asGroupPolicyDao; - @Inject - private AutoScalePolicyDao _asPolicyDao; - @Inject - private AutoScalePolicyConditionMapDao _asConditionMapDao; - @Inject - private ConditionDao _asConditionDao; - @Inject - private CounterDao _asCounterDao; - @Inject - private AutoScaleVmProfileDao _asProfileDao; - @Inject - private ServiceOfferingDao _serviceOfferingDao; - @Inject private HostGpuGroupsDao _hostGpuGroupsDao; @Inject private ImageStoreDetailsUtil imageStoreDetailsUtil; @@ -1773,132 +1735,7 @@ protected void runInContext() { // list all AS VMGroups List asGroups = _asGroupDao.listAll(); for (AutoScaleVmGroupVO asGroup : asGroups) { - // check group state - if ((asGroup.getState().equals(AutoScaleVmGroup.State.Enabled)) && (is_native(asGroup.getId()))) { - // check minimum vm of group - Integer currentVM = _asGroupVmDao.countByGroup(asGroup.getId()); - if (currentVM < asGroup.getMinMembers()) { - _asManager.doScaleUp(asGroup.getId(), asGroup.getMinMembers() - currentVM); - continue; - } - - //check interval - long now = (new Date()).getTime(); - if (asGroup.getLastInterval() != null) - if ((now - asGroup.getLastInterval().getTime()) < asGroup.getInterval()) { - continue; - } - - // update last_interval - asGroup.setLastInterval(new Date()); - _asGroupDao.persist(asGroup); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("[AutoScale] Collecting RRDs data..."); - } - Map params = new HashMap(); - List asGroupVmVOs = _asGroupVmDao.listByGroup(asGroup.getId()); - params.put("total_vm", String.valueOf(asGroupVmVOs.size())); - for (int i = 0; i < asGroupVmVOs.size(); i++) { - long vmId = asGroupVmVOs.get(i).getInstanceId(); - VMInstanceVO vmVO = _vmInstance.findById(vmId); - //xe vm-list | grep vmname -B 1 | head -n 1 | awk -F':' '{print $2}' - params.put("vmname" + String.valueOf(i + 1), vmVO.getInstanceName()); - params.put("vmid" + String.valueOf(i + 1), String.valueOf(vmVO.getId())); - - } - // get random hostid because all vms are in a cluster - long vmId = asGroupVmVOs.get(0).getInstanceId(); - VMInstanceVO vmVO = _vmInstance.findById(vmId); - Long receiveHost = vmVO.getHostId(); - - // setup parameters phase: duration and counter - // list pair [counter, duration] - List> lstPair = getPairofCounternameAndDuration(asGroup.getId()); - int total_counter = 0; - String[] lstCounter = new String[lstPair.size()]; - for (int i = 0; i < lstPair.size(); i++) { - Pair pair = lstPair.get(i); - String strCounterNames = pair.first(); - Integer duration = pair.second(); - - lstCounter[i] = strCounterNames.split(",")[0]; - total_counter++; - params.put("duration" + String.valueOf(total_counter), duration.toString()); - params.put("counter" + String.valueOf(total_counter), lstCounter[i]); - params.put("con" + String.valueOf(total_counter), strCounterNames.split(",")[1]); - } - params.put("total_counter", String.valueOf(total_counter)); - - PerformanceMonitorCommand perfMon = new PerformanceMonitorCommand(params, 20); - - try { - Answer answer = _agentMgr.send(receiveHost, perfMon); - if (answer == null || !answer.getResult()) { - LOGGER.debug("Failed to send data to node !"); - } else { - String result = answer.getDetails(); - LOGGER.debug("[AutoScale] RRDs collection answer: " + result); - HashMap avgCounter = new HashMap(); - - // extract data - String[] counterElements = result.split(","); - if ((counterElements != null) && (counterElements.length > 0)) { - for (String string : counterElements) { - try { - String[] counterVals = string.split(":"); - String[] counter_vm = counterVals[0].split("\\."); - - Long counterId = Long.parseLong(counter_vm[1]); - Long conditionId = Long.parseLong(params.get("con" + counter_vm[1])); - Double coVal = Double.parseDouble(counterVals[1]); - - // Summary of all counter by counterId key - if (avgCounter.get(counterId) == null) { - /* initialize if data is not set */ - avgCounter.put(counterId, new Double(0)); - } - - String counterName = getCounternamebyCondition(conditionId.longValue()); - if (Counter.Source.memory.toString().equals(counterName)) { - // calculate memory in percent - Long profileId = asGroup.getProfileId(); - AutoScaleVmProfileVO profileVo = _asProfileDao.findById(profileId); - ServiceOfferingVO serviceOff = _serviceOfferingDao.findById(profileVo.getServiceOfferingId()); - int maxRAM = serviceOff.getRamSize(); - - // get current RAM percent - coVal = coVal / maxRAM; - } else { - // cpu - coVal = coVal * 100; - } - - // update data entry - avgCounter.put(counterId, avgCounter.get(counterId) + coVal); - - } catch (Exception e) { - e.printStackTrace(); - } - } - - AutoScalePolicy.Action scaleAction = getAutoscaleAction(avgCounter, asGroup.getId(), currentVM, params); - if (scaleAction != null) { - LOGGER.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); - if (AutoScalePolicy.Action.ScaleUp.equals(scaleAction)) { - _asManager.doScaleUp(asGroup.getId(), 1); - } else { - _asManager.doScaleDown(asGroup.getId()); - } - } - } - } - - } catch (Exception e) { - e.printStackTrace(); - } - - } + _asManager.checkAutoScaleVmGroup(asGroup); } } catch (Throwable t) { @@ -1907,138 +1744,6 @@ protected void runInContext() { } - private boolean is_native(long groupId) { - List vos = _asGroupPolicyDao.listByVmGroupId(groupId); - for (AutoScaleVmGroupPolicyMapVO vo : vos) { - List ConditionPolicies = _asConditionMapDao.findByPolicyId(vo.getPolicyId()); - for (AutoScalePolicyConditionMapVO ConditionPolicy : ConditionPolicies) { - ConditionVO condition = _asConditionDao.findById(ConditionPolicy.getConditionId()); - CounterVO counter = _asCounterDao.findById(condition.getCounterid()); - if (counter.getSource() == Counter.Source.cpu || counter.getSource() == Counter.Source.memory) - return true; - } - } - return false; - } - - private AutoScalePolicy.Action getAutoscaleAction(HashMap avgCounter, long groupId, long currentVM, Map params) { - - List listMap = _asGroupPolicyDao.listByVmGroupId(groupId); - if ((listMap == null) || (listMap.size() == 0)) - return null; - for (AutoScaleVmGroupPolicyMapVO asVmgPmap : listMap) { - AutoScalePolicyVO policyVO = _asPolicyDao.findById(asVmgPmap.getPolicyId()); - if (policyVO != null) { - int quitetime = policyVO.getQuietTime(); - Date quitetimeDate = policyVO.getLastQuiteTime(); - long last_quitetime = 0L; - if (quitetimeDate != null) { - last_quitetime = policyVO.getLastQuiteTime().getTime(); - } - long current_time = (new Date()).getTime(); - - // check quite time for this policy - if ((current_time - last_quitetime) >= (long)quitetime) { - - // list all condition of this policy - boolean bValid = true; - List lstConditions = getConditionsbyPolicyId(policyVO.getId()); - if ((lstConditions != null) && (lstConditions.size() > 0)) { - // check whole conditions of this policy - for (ConditionVO conditionVO : lstConditions) { - long thresholdValue = conditionVO.getThreshold(); - Double thresholdPercent = (double)thresholdValue / 100; - CounterVO counterVO = _asCounterDao.findById(conditionVO.getCounterid()); - long counter_count = 1; - do { - String counter_param = params.get("counter" + String.valueOf(counter_count)); - Counter.Source counter_source = counterVO.getSource(); - if (counter_param.equals(counter_source.toString())) - break; - counter_count++; - } while (true); - - Double sum = avgCounter.get(counter_count); - Double avg = sum / currentVM; - Operator op = conditionVO.getRelationalOperator(); - boolean bConditionCheck = ((op == com.cloud.network.as.Condition.Operator.EQ) && (thresholdPercent.equals(avg))) - || ((op == com.cloud.network.as.Condition.Operator.GE) && (avg.doubleValue() >= thresholdPercent.doubleValue())) - || ((op == com.cloud.network.as.Condition.Operator.GT) && (avg.doubleValue() > thresholdPercent.doubleValue())) - || ((op == com.cloud.network.as.Condition.Operator.LE) && (avg.doubleValue() <= thresholdPercent.doubleValue())) - || ((op == com.cloud.network.as.Condition.Operator.LT) && (avg.doubleValue() < thresholdPercent.doubleValue())); - - if (!bConditionCheck) { - bValid = false; - break; - } - } - if (bValid) { - return policyVO.getAction(); - } - } - } - } - } - return null; - } - - private List getConditionsbyPolicyId(long policyId) { - List conditionMap = _asConditionMapDao.findByPolicyId(policyId); - if ((conditionMap == null) || (conditionMap.size() == 0)) - return null; - - List lstResult = new ArrayList(); - for (AutoScalePolicyConditionMapVO asPCmap : conditionMap) { - lstResult.add(_asConditionDao.findById(asPCmap.getConditionId())); - } - - return lstResult; - } - - public List> getPairofCounternameAndDuration(long groupId) { - AutoScaleVmGroupVO groupVo = _asGroupDao.findById(groupId); - if (groupVo == null) - return null; - List> result = new ArrayList>(); - //list policy map - List groupPolicymap = _asGroupPolicyDao.listByVmGroupId(groupVo.getId()); - if (groupPolicymap == null) - return null; - for (AutoScaleVmGroupPolicyMapVO gpMap : groupPolicymap) { - //get duration - AutoScalePolicyVO policyVo = _asPolicyDao.findById(gpMap.getPolicyId()); - Integer duration = policyVo.getDuration(); - //get collection of counter name - - StringBuffer buff = new StringBuffer(); - List lstPCmap = _asConditionMapDao.findByPolicyId(policyVo.getId()); - for (AutoScalePolicyConditionMapVO pcMap : lstPCmap) { - String counterName = getCounternamebyCondition(pcMap.getConditionId()); - buff.append(counterName); - buff.append(","); - buff.append(pcMap.getConditionId()); - } - // add to result - Pair pair = new Pair(buff.toString(), duration); - result.add(pair); - } - - return result; - } - - public String getCounternamebyCondition(long conditionId) { - - ConditionVO condition = _asConditionDao.findById(conditionId); - if (condition == null) - return ""; - - long counterId = condition.getCounterid(); - CounterVO counter = _asCounterDao.findById(counterId); - if (counter == null) - return ""; - - return counter.getSource().toString(); - } } /** From 3a2ee8f5d91bb9751e69b7c55d0ab97963fc0831 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 20 May 2022 16:48:54 +0200 Subject: [PATCH 008/360] AS: monitor AS group only if provider is netscaler --- .../network/as/AutoScaleManagerImpl.java | 273 ++++++++++-------- 1 file changed, 152 insertions(+), 121 deletions(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 769af52f9e22..7fdc53babd00 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -58,6 +58,7 @@ import org.apache.cloudstack.api.command.user.vm.DeployVMCmd; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -130,6 +131,7 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmService; @@ -200,6 +202,8 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale private AgentManager _agentMgr; @Inject private ServiceOfferingDao _serviceOfferingDao; + @Inject + NetworkOrchestrationService _networkMgr; public List getSupportedAutoScaleCounters(long networkid) { String capability = _lbRulesMgr.getLBCapability(networkid, Capability.AutoScaleCounters.getName()); @@ -1591,128 +1595,12 @@ public ConfigKey[] getConfigKeys() { public void checkAutoScaleVmGroup(AutoScaleVmGroupVO asGroup) { // check group state if ((asGroup.getState().equals(AutoScaleVmGroup.State.Enabled)) && (is_native(asGroup.getId()))) { - // check minimum vm of group - Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId()); - if (currentVM < asGroup.getMinMembers()) { - doScaleUp(asGroup.getId(), asGroup.getMinMembers() - currentVM); - return; - } - - //check interval - long now = (new Date()).getTime(); - if (asGroup.getLastInterval() != null && (now - asGroup.getLastInterval().getTime()) < asGroup.getInterval()) { - return; - } - - // update last_interval - asGroup.setLastInterval(new Date()); - _autoScaleVmGroupDao.persist(asGroup); - - if (s_logger.isDebugEnabled()) { - s_logger.debug("[Netscaler AutoScale] Collecting RRDs data..."); - } - Map params = new HashMap(); - List asGroupVmVOs = _autoScaleVmGroupVmMapDao.listByGroup(asGroup.getId()); - params.put("total_vm", String.valueOf(asGroupVmVOs.size())); - for (int i = 0; i < asGroupVmVOs.size(); i++) { - long vmId = asGroupVmVOs.get(i).getInstanceId(); - VMInstanceVO vmVO = _vmInstanceDao.findById(vmId); - //xe vm-list | grep vmname -B 1 | head -n 1 | awk -F':' '{print $2}' - params.put("vmname" + String.valueOf(i + 1), vmVO.getInstanceName()); - params.put("vmid" + String.valueOf(i + 1), String.valueOf(vmVO.getId())); - - } - // get random hostid because all vms are in a cluster - long vmId = asGroupVmVOs.get(0).getInstanceId(); - VMInstanceVO vmVO = _vmInstanceDao.findById(vmId); - Long receiveHost = vmVO.getHostId(); - - // setup parameters phase: duration and counter - // list pair [counter, duration] - List> lstPair = getPairofCounternameAndDuration(asGroup.getId()); - int total_counter = 0; - String[] lstCounter = new String[lstPair.size()]; - for (int i = 0; i < lstPair.size(); i++) { - Pair pair = lstPair.get(i); - String strCounterNames = pair.first(); - Integer duration = pair.second(); - - lstCounter[i] = strCounterNames.split(",")[0]; - total_counter++; - params.put("duration" + String.valueOf(total_counter), duration.toString()); - params.put("counter" + String.valueOf(total_counter), lstCounter[i]); - params.put("con" + String.valueOf(total_counter), strCounterNames.split(",")[1]); + Network.Provider provider = getLoadBalancerServiceProvider(asGroup); + if (Network.Provider.Netscaler.equals(provider)) { + monitorNetScalerAsGroup(asGroup); + } else if (Network.Provider.VirtualRouter.equals(provider) || Network.Provider.VPCVirtualRouter.equals(provider)) { + monitorCloudStackAsGroup(asGroup); } - params.put("total_counter", String.valueOf(total_counter)); - - PerformanceMonitorCommand perfMon = new PerformanceMonitorCommand(params, 20); - - try { - Answer answer = _agentMgr.send(receiveHost, perfMon); - if (answer == null || !answer.getResult()) { - s_logger.debug("Failed to send data to node !"); - } else { - String result = answer.getDetails(); - s_logger.debug("[AutoScale] RRDs collection answer: " + result); - HashMap avgCounter = new HashMap(); - - // extract data - String[] counterElements = result.split(","); - if ((counterElements != null) && (counterElements.length > 0)) { - for (String string : counterElements) { - try { - String[] counterVals = string.split(":"); - String[] counter_vm = counterVals[0].split("\\."); - - Long counterId = Long.parseLong(counter_vm[1]); - Long conditionId = Long.parseLong(params.get("con" + counter_vm[1])); - Double coVal = Double.parseDouble(counterVals[1]); - - // Summary of all counter by counterId key - if (avgCounter.get(counterId) == null) { - /* initialize if data is not set */ - avgCounter.put(counterId, new Double(0)); - } - - String counterName = getCounternamebyCondition(conditionId.longValue()); - if (Counter.Source.memory.toString().equals(counterName)) { - // calculate memory in percent - Long profileId = asGroup.getProfileId(); - AutoScaleVmProfileVO profileVo = _autoScaleVmProfileDao.findById(profileId); - ServiceOfferingVO serviceOff = _serviceOfferingDao.findById(profileVo.getServiceOfferingId()); - int maxRAM = serviceOff.getRamSize(); - - // get current RAM percent - coVal = coVal / maxRAM; - } else { - // cpu - coVal = coVal * 100; - } - - // update data entry - avgCounter.put(counterId, avgCounter.get(counterId) + coVal); - - } catch (Exception e) { - e.printStackTrace(); - } - } - - AutoScalePolicy.Action scaleAction = getAutoscaleAction(avgCounter, asGroup.getId(), currentVM, params); - if (scaleAction != null) { - s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); - if (AutoScalePolicy.Action.ScaleUp.equals(scaleAction)) { - doScaleUp(asGroup.getId(), 1); - } else { - doScaleDown(asGroup.getId()); - } - } - } - } - - } catch (Exception e) { - e.printStackTrace(); - } - } } @@ -1849,4 +1737,147 @@ public String getCounternamebyCondition(long conditionId) { return counter.getSource().toString(); } + private Network.Provider getLoadBalancerServiceProvider(AutoScaleVmGroup asGroup) { + final LoadBalancerVO loadBalancer = _lbDao.findById(asGroup.getLoadBalancerId()); + if (loadBalancer == null) { + throw new CloudRuntimeException(String.format("Unable to find load balancer with id: % ", asGroup.getLoadBalancerId())); + } + Network network = _networkDao.findById(loadBalancer.getNetworkId()); + if (network == null) { + throw new CloudRuntimeException(String.format("Unable to find network with id: % ", loadBalancer.getNetworkId())); + } + List providers = _networkMgr.getProvidersForServiceInNetwork(network, Network.Service.Lb); + if (providers == null || providers.size() == 0) { + throw new CloudRuntimeException(String.format("Unable to find LB provider for network with id: % ", loadBalancer.getNetworkId())); + } + return providers.get(0); + } + + private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { + // check minimum vm of group + Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId()); + if (currentVM < asGroup.getMinMembers()) { + doScaleUp(asGroup.getId(), asGroup.getMinMembers() - currentVM); + return; + } + + //check interval + long now = (new Date()).getTime(); + if (asGroup.getLastInterval() != null && (now - asGroup.getLastInterval().getTime()) < asGroup.getInterval()) { + return; + } + + // update last_interval + asGroup.setLastInterval(new Date()); + _autoScaleVmGroupDao.persist(asGroup); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("[Netscaler AutoScale] Collecting RRDs data..."); + } + Map params = new HashMap(); + List asGroupVmVOs = _autoScaleVmGroupVmMapDao.listByGroup(asGroup.getId()); + params.put("total_vm", String.valueOf(asGroupVmVOs.size())); + for (int i = 0; i < asGroupVmVOs.size(); i++) { + long vmId = asGroupVmVOs.get(i).getInstanceId(); + VMInstanceVO vmVO = _vmInstanceDao.findById(vmId); + //xe vm-list | grep vmname -B 1 | head -n 1 | awk -F':' '{print $2}' + params.put("vmname" + String.valueOf(i + 1), vmVO.getInstanceName()); + params.put("vmid" + String.valueOf(i + 1), String.valueOf(vmVO.getId())); + + } + // get random hostid because all vms are in a cluster + long vmId = asGroupVmVOs.get(0).getInstanceId(); + VMInstanceVO vmVO = _vmInstanceDao.findById(vmId); + Long receiveHost = vmVO.getHostId(); + + // setup parameters phase: duration and counter + // list pair [counter, duration] + List> lstPair = getPairofCounternameAndDuration(asGroup.getId()); + int total_counter = 0; + String[] lstCounter = new String[lstPair.size()]; + for (int i = 0; i < lstPair.size(); i++) { + Pair pair = lstPair.get(i); + String strCounterNames = pair.first(); + Integer duration = pair.second(); + + lstCounter[i] = strCounterNames.split(",")[0]; + total_counter++; + params.put("duration" + String.valueOf(total_counter), duration.toString()); + params.put("counter" + String.valueOf(total_counter), lstCounter[i]); + params.put("con" + String.valueOf(total_counter), strCounterNames.split(",")[1]); + } + params.put("total_counter", String.valueOf(total_counter)); + + PerformanceMonitorCommand perfMon = new PerformanceMonitorCommand(params, 20); + + try { + Answer answer = _agentMgr.send(receiveHost, perfMon); + if (answer == null || !answer.getResult()) { + s_logger.debug("Failed to send data to node !"); + } else { + String result = answer.getDetails(); + s_logger.debug("[AutoScale] RRDs collection answer: " + result); + HashMap avgCounter = new HashMap(); + + // extract data + String[] counterElements = result.split(","); + if ((counterElements != null) && (counterElements.length > 0)) { + for (String string : counterElements) { + try { + String[] counterVals = string.split(":"); + String[] counter_vm = counterVals[0].split("\\."); + + Long counterId = Long.parseLong(counter_vm[1]); + Long conditionId = Long.parseLong(params.get("con" + counter_vm[1])); + Double coVal = Double.parseDouble(counterVals[1]); + + // Summary of all counter by counterId key + if (avgCounter.get(counterId) == null) { + /* initialize if data is not set */ + avgCounter.put(counterId, new Double(0)); + } + + String counterName = getCounternamebyCondition(conditionId.longValue()); + if (Counter.Source.memory.toString().equals(counterName)) { + // calculate memory in percent + Long profileId = asGroup.getProfileId(); + AutoScaleVmProfileVO profileVo = _autoScaleVmProfileDao.findById(profileId); + ServiceOfferingVO serviceOff = _serviceOfferingDao.findById(profileVo.getServiceOfferingId()); + int maxRAM = serviceOff.getRamSize(); + + // get current RAM percent + coVal = coVal / maxRAM; + } else { + // cpu + coVal = coVal * 100; + } + + // update data entry + avgCounter.put(counterId, avgCounter.get(counterId) + coVal); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + AutoScalePolicy.Action scaleAction = getAutoscaleAction(avgCounter, asGroup.getId(), currentVM, params); + if (scaleAction != null) { + s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); + if (AutoScalePolicy.Action.ScaleUp.equals(scaleAction)) { + doScaleUp(asGroup.getId(), 1); + } else { + doScaleDown(asGroup.getId()); + } + } + } + } + + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void monitorCloudStackAsGroup(AutoScaleVmGroupVO asGroup) { + //TODO + } } From fb81f15374a0a92830690e3ee58ccc4f9aa993e1 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 20 May 2022 22:06:38 +0200 Subject: [PATCH 009/360] AS: update relationalOperator description --- .../api/command/user/autoscale/CreateConditionCmd.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java index 373ecbb1c8ce..fd0149ccb244 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java @@ -48,7 +48,7 @@ public class CreateConditionCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.COUNTER_ID, type = CommandType.UUID, entityType = CounterResponse.class, required = true, description = "ID of the Counter.") private long counterId; - @Parameter(name = ApiConstants.RELATIONAL_OPERATOR, type = CommandType.STRING, required = true, description = "Relational Operator to be used with threshold.") + @Parameter(name = ApiConstants.RELATIONAL_OPERATOR, type = CommandType.STRING, required = true, description = "Relational Operator to be used with threshold. Valid values are EQ, GT, LT, GE, LE.") private String relationalOperator; @Parameter(name = ApiConstants.THRESHOLD, type = CommandType.LONG, required = true, description = "Threshold value.") From 5abb3846b995667647bf91cc049761da967d11c2 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 23 May 2022 10:54:39 +0200 Subject: [PATCH 010/360] AS: add autoscaler counter with source="virtualrouter" --- .../java/com/cloud/network/as/Counter.java | 2 +- .../network/as/AutoScaleManagerImpl.java | 11 +++++----- .../network/element/VirtualRouterElement.java | 20 +++++++++++++++++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/com/cloud/network/as/Counter.java b/api/src/main/java/com/cloud/network/as/Counter.java index bdcbdd074c1b..f67d0b4840b1 100644 --- a/api/src/main/java/com/cloud/network/as/Counter.java +++ b/api/src/main/java/com/cloud/network/as/Counter.java @@ -27,7 +27,7 @@ public static enum Source { snmp, cpu, memory, - cloudstack + virtualrouter } String getName(); diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 7fdc53babd00..85bd1c37a604 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1370,20 +1370,21 @@ private long createNewVM(AutoScaleVmGroupVO asGroup) { UserVm vm = null; IpAddresses addrs = new IpAddresses(null, null); + HypervisorType hypervisorType = template.getHypervisorType(); if (zone.getNetworkType() == NetworkType.Basic) { vm = _userVmService.createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), - "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null, null, + "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, hypervisorType, HTTPMethod.GET, null, null, null, null, true, null, null, null, null, null, null, null, true, null); } else { if (zone.isSecurityGroupEnabled()) { vm = _userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, null, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), - "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null,null, null, true, null, null, null, null, null, null, null, true, null, null); + "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, hypervisorType, HTTPMethod.GET, null, null,null, null, true, null, null, null, null, null, null, null, true, null, null); } else { vm = _userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), - null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null, null, addrs, true, null, null, null, null, null, null, null, true, null, null); + null, null, null, hypervisorType, HTTPMethod.GET, null, null, null, addrs, true, null, null, null, null, null, null, null, true, null, null); } } @@ -1599,7 +1600,7 @@ public void checkAutoScaleVmGroup(AutoScaleVmGroupVO asGroup) { if (Network.Provider.Netscaler.equals(provider)) { monitorNetScalerAsGroup(asGroup); } else if (Network.Provider.VirtualRouter.equals(provider) || Network.Provider.VPCVirtualRouter.equals(provider)) { - monitorCloudStackAsGroup(asGroup); + monitorVirtualRouterAsGroup(asGroup); } } } @@ -1877,7 +1878,7 @@ private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { } } - private void monitorCloudStackAsGroup(AutoScaleVmGroupVO asGroup) { + private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { //TODO } } diff --git a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java index 0900509aad5c..1da2431f2a8c 100644 --- a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java @@ -124,6 +124,8 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl private static final Logger s_logger = Logger.getLogger(VirtualRouterElement.class); public static final AutoScaleCounterType AutoScaleCounterCpu = new AutoScaleCounterType("cpu"); public static final AutoScaleCounterType AutoScaleCounterMemory = new AutoScaleCounterType("memory"); + public static final AutoScaleCounterType AutoScaleCounterVirtualRouter = new AutoScaleCounterType("virtualrouter"); + protected static final Map> capabilities = setCapabilities(); @Inject @@ -545,6 +547,8 @@ private static Map> setCapabilities() { counterList.add(counter); counter = new AutoScaleCounter(AutoScaleCounterMemory); counterList.add(counter); + counter = new AutoScaleCounter(AutoScaleCounterVirtualRouter); + counterList.add(counter); final Gson gson = new Gson(); final String autoScaleCounterList = gson.toJson(counterList); lbCapabilities.put(Capability.AutoScaleCounters, autoScaleCounterList); @@ -1401,4 +1405,20 @@ private void updateToFailedState(Network network){ _routerDao.persist(router); } } + + enum AutoScaleCounterVirtualRouterValue { + NetworkReceive ("virtual.network.receive"), + NetworkTransmit ("virtual.network.transmit"), + LbAverageConnections ("virtual.network.lb.average.connections"); + + String _value; + AutoScaleCounterVirtualRouterValue(String value) { + _value = value; + } + @Override + public String toString() { + return _value; + } + } + } From 407969dec6e389a81171242a42f6fbe1c9422e22 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 23 May 2022 11:32:07 +0200 Subject: [PATCH 011/360] AS: fix java.lang.UnsupportedOperationException in _rules.clear() --- .../com/cloud/network/lb/LoadBalancingRulesManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java b/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java index 749b9f75394f..bd83101bfa14 100644 --- a/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java +++ b/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java @@ -368,7 +368,7 @@ private boolean applyAutoScaleConfig(LoadBalancerVO lb, AutoScaleVmGroupVO vmGro List rules = Arrays.asList(rule); - if (!applyLbRules(rules, false)) { + if (!applyLbRules(new ArrayList(rules), false)) { s_logger.debug("LB rules' autoscale config are not completely applied"); return false; } From 38094e5c21e9aa714eaaccec7686ba1d3cf039f7 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 23 May 2022 12:09:51 +0200 Subject: [PATCH 012/360] AS: check global config 'autoscale.stats.interval' when create autoscale vm group --- .../main/java/com/cloud/network/as/AutoScaleManagerImpl.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 85bd1c37a604..37f15db5461e 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -352,6 +352,11 @@ protected AutoScaleVmProfileVO checkValidityAndPersist(AutoScaleVmProfileVO vmPr throw new InvalidParameterValueException(String.format("Global setting %s has to be set to the Management Server's API end point", ApiServiceConfiguration.ApiServletPath.key())); } + Integer autoScaleStatsInterval = AutoScaleStatsInterval.value(); + if (autoScaleStatsInterval <= 0) { + throw new InvalidParameterValueException(String.format("Global setting %s has to be set to larger than 0", AutoScaleStatsInterval.key())); + } + vmProfile = _autoScaleVmProfileDao.persist(vmProfile); return vmProfile; From 69402e761115a08a40528db3df12d1f877f30f64 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 23 May 2022 17:56:58 +0200 Subject: [PATCH 013/360] AS: create first vm in autoscale vmgroup --- .../java/com/cloud/network/as/Counter.java | 7 ++- .../network/as/AutoScaleManagerImpl.java | 53 +++++++++++++++---- .../network/element/VirtualRouterElement.java | 7 +-- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/com/cloud/network/as/Counter.java b/api/src/main/java/com/cloud/network/as/Counter.java index f67d0b4840b1..70d1f861e032 100644 --- a/api/src/main/java/com/cloud/network/as/Counter.java +++ b/api/src/main/java/com/cloud/network/as/Counter.java @@ -20,9 +20,12 @@ import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; +import java.util.Arrays; +import java.util.List; + public interface Counter extends InternalIdentity, Identity { - public static enum Source { + static enum Source { netscaler, snmp, cpu, @@ -30,6 +33,8 @@ public static enum Source { virtualrouter } + static final List NativeSources = Arrays.asList(Source.cpu, Source.memory, Source.virtualrouter); + String getName(); String getValue(); diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 37f15db5461e..bb25689960c3 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1376,20 +1376,33 @@ private long createNewVM(AutoScaleVmGroupVO asGroup) { UserVm vm = null; IpAddresses addrs = new IpAddresses(null, null); HypervisorType hypervisorType = template.getHypervisorType(); + final Network network = getNetwork(asGroup); + final List networkIds = new ArrayList<>(); + networkIds.add(network.getId()); + Map customParameters = new HashMap(); + List sshKeyPairs = new ArrayList<>(); if (zone.getNetworkType() == NetworkType.Basic) { vm = _userVmService.createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), - "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, hypervisorType, HTTPMethod.GET, null, null, null, - null, true, null, null, null, null, null, null, null, true, null); + "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, + hypervisorType, HTTPMethod.GET, null, sshKeyPairs, null, + null, true, null, null, customParameters, null, null, null, + null, true, null); } else { if (zone.isSecurityGroupEnabled()) { - vm = _userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, null, null, + vm = _userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, networkIds, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), - "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, hypervisorType, HTTPMethod.GET, null, null,null, null, true, null, null, null, null, null, null, null, true, null, null); + "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, + hypervisorType, HTTPMethod.GET, null, sshKeyPairs,null, + null, true, null, null, customParameters, null, null, null, + null, true, null, null); } else { - vm = _userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + + vm = _userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), - null, null, null, hypervisorType, HTTPMethod.GET, null, null, null, addrs, true, null, null, null, null, null, null, null, true, null, null); + null, null, null, + hypervisorType, HTTPMethod.GET, null, sshKeyPairs,null, + addrs, true, null, null, customParameters, null, null, null, + null, true, null, null); } } @@ -1617,8 +1630,9 @@ private boolean is_native(long groupId) { for (AutoScalePolicyConditionMapVO ConditionPolicy : ConditionPolicies) { ConditionVO condition = _conditionDao.findById(ConditionPolicy.getConditionId()); CounterVO counter = _counterDao.findById(condition.getCounterid()); - if (counter.getSource() == Counter.Source.cpu || counter.getSource() == Counter.Source.memory) + if (Counter.NativeSources.contains(counter.getSource())) { return true; + } } } return false; @@ -1743,7 +1757,7 @@ public String getCounternamebyCondition(long conditionId) { return counter.getSource().toString(); } - private Network.Provider getLoadBalancerServiceProvider(AutoScaleVmGroup asGroup) { + private Network getNetwork(AutoScaleVmGroup asGroup) { final LoadBalancerVO loadBalancer = _lbDao.findById(asGroup.getLoadBalancerId()); if (loadBalancer == null) { throw new CloudRuntimeException(String.format("Unable to find load balancer with id: % ", asGroup.getLoadBalancerId())); @@ -1752,9 +1766,14 @@ private Network.Provider getLoadBalancerServiceProvider(AutoScaleVmGroup asGroup if (network == null) { throw new CloudRuntimeException(String.format("Unable to find network with id: % ", loadBalancer.getNetworkId())); } + return network; + } + + private Network.Provider getLoadBalancerServiceProvider(AutoScaleVmGroup asGroup) { + Network network = getNetwork(asGroup); List providers = _networkMgr.getProvidersForServiceInNetwork(network, Network.Service.Lb); if (providers == null || providers.size() == 0) { - throw new CloudRuntimeException(String.format("Unable to find LB provider for network with id: % ", loadBalancer.getNetworkId())); + throw new CloudRuntimeException(String.format("Unable to find LB provider for network with id: % ", network.getId())); } return providers.get(0); } @@ -1884,6 +1903,22 @@ private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { } private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { + // check minimum vm of group + Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId()); + if (currentVM < asGroup.getMinMembers()) { + doScaleUp(asGroup.getId(), asGroup.getMinMembers() - currentVM); + return; + } + + //check interval + long now = (new Date()).getTime(); + if (asGroup.getLastInterval() != null && (now - asGroup.getLastInterval().getTime()) < asGroup.getInterval()) { + return; + } + + // update last_interval + asGroup.setLastInterval(new Date()); + _autoScaleVmGroupDao.persist(asGroup); //TODO } } diff --git a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java index 1da2431f2a8c..1689513a751a 100644 --- a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java @@ -24,6 +24,7 @@ import javax.inject.Inject; +import com.cloud.network.as.Counter; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; @@ -122,9 +123,9 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl StaticNatServiceProvider, FirewallServiceProvider, LoadBalancingServiceProvider, PortForwardingServiceProvider, RemoteAccessVPNServiceProvider, IpDeployer, NetworkMigrationResponder, AggregatedCommandExecutor, RedundantResource, DnsServiceProvider{ private static final Logger s_logger = Logger.getLogger(VirtualRouterElement.class); - public static final AutoScaleCounterType AutoScaleCounterCpu = new AutoScaleCounterType("cpu"); - public static final AutoScaleCounterType AutoScaleCounterMemory = new AutoScaleCounterType("memory"); - public static final AutoScaleCounterType AutoScaleCounterVirtualRouter = new AutoScaleCounterType("virtualrouter"); + public static final AutoScaleCounterType AutoScaleCounterCpu = new AutoScaleCounterType(Counter.Source.cpu.name()); + public static final AutoScaleCounterType AutoScaleCounterMemory = new AutoScaleCounterType(Counter.Source.memory.name()); + public static final AutoScaleCounterType AutoScaleCounterVirtualRouter = new AutoScaleCounterType(Counter.Source.virtualrouter.name()); protected static final Map> capabilities = setCapabilities(); From 245ddde55fbc263222fe263151935a39983535e0 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 24 May 2022 21:46:08 +0200 Subject: [PATCH 014/360] AS: get vm cpu/memory stats in as vm group --- .../java/com/cloud/network/as/Counter.java | 2 +- .../network/as/AutoScaleManagerImpl.java | 228 ++++++++++++++---- 2 files changed, 183 insertions(+), 47 deletions(-) diff --git a/api/src/main/java/com/cloud/network/as/Counter.java b/api/src/main/java/com/cloud/network/as/Counter.java index 70d1f861e032..e1bd21628ded 100644 --- a/api/src/main/java/com/cloud/network/as/Counter.java +++ b/api/src/main/java/com/cloud/network/as/Counter.java @@ -33,7 +33,7 @@ static enum Source { virtualrouter } - static final List NativeSources = Arrays.asList(Source.cpu, Source.memory, Source.virtualrouter); + static final List NativeSources = Arrays.asList(Source.cpu, Source.memory); String getName(); diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index bb25689960c3..3197dd4dc5d5 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -64,8 +65,9 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import com.cloud.agent.AgentManager; -import com.cloud.agent.api.Answer; +import com.cloud.agent.api.PerformanceMonitorAnswer; import com.cloud.agent.api.PerformanceMonitorCommand; +import com.cloud.agent.api.VmStatsEntry; import com.cloud.api.ApiDBUtils; import com.cloud.api.dispatch.DispatchChainFactory; import com.cloud.api.dispatch.DispatchTask; @@ -82,6 +84,8 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.Network; import com.cloud.network.Network.Capability; @@ -135,7 +139,9 @@ import com.cloud.utils.net.NetUtils; import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmService; +import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManager, AutoScaleService, Configurable { @@ -204,6 +210,12 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale private ServiceOfferingDao _serviceOfferingDao; @Inject NetworkOrchestrationService _networkMgr; + @Inject + private UserVmManager _userVmMgr; + @Inject + private UserVmDao _userVmDao; + @Inject + private HostDao _hostDao; public List getSupportedAutoScaleCounters(long networkid) { String capability = _lbRulesMgr.getLBCapability(networkid, Capability.AutoScaleCounters.getName()); @@ -1613,7 +1625,7 @@ public ConfigKey[] getConfigKeys() { @Override public void checkAutoScaleVmGroup(AutoScaleVmGroupVO asGroup) { // check group state - if ((asGroup.getState().equals(AutoScaleVmGroup.State.Enabled)) && (is_native(asGroup.getId()))) { + if (asGroup.getState().equals(AutoScaleVmGroup.State.Enabled)) { Network.Provider provider = getLoadBalancerServiceProvider(asGroup); if (Network.Provider.Netscaler.equals(provider)) { monitorNetScalerAsGroup(asGroup); @@ -1638,6 +1650,21 @@ private boolean is_native(long groupId) { return false; } + private boolean has_source_virtual_router(long groupId) { + List vos = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); + for (AutoScaleVmGroupPolicyMapVO vo : vos) { + List ConditionPolicies = _autoScalePolicyConditionMapDao.findByPolicyId(vo.getPolicyId()); + for (AutoScalePolicyConditionMapVO ConditionPolicy : ConditionPolicies) { + ConditionVO condition = _conditionDao.findById(ConditionPolicy.getConditionId()); + CounterVO counter = _counterDao.findById(condition.getCounterid()); + if (Counter.Source.virtualrouter.equals(counter.getSource())) { + return true; + } + } + } + return false; + } + private AutoScalePolicy.Action getAutoscaleAction(HashMap avgCounter, long groupId, long currentVM, Map params) { List listMap = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); @@ -1664,14 +1691,18 @@ private AutoScalePolicy.Action getAutoscaleAction(HashMap avgCount // check whole conditions of this policy for (ConditionVO conditionVO : lstConditions) { long thresholdValue = conditionVO.getThreshold(); - Double thresholdPercent = (double)thresholdValue / 100; + Double thresholdPercent = (double)thresholdValue; CounterVO counterVO = _counterDao.findById(conditionVO.getCounterid()); long counter_count = 1; do { String counter_param = params.get("counter" + String.valueOf(counter_count)); Counter.Source counter_source = counterVO.getSource(); - if (counter_param.equals(counter_source.toString())) + String counter_param_value = params.get("value" + String.valueOf(counter_count)); + String counter_value = counterVO.getValue(); + if (counter_param.equals(counter_source.toString()) && + (counter_param_value == null || counter_param_value.equals(counter_value))) { break; + } counter_count++; } while (true); @@ -1734,6 +1765,9 @@ public List> getPairofCounternameAndDuration(long groupId) buff.append(counterName); buff.append(","); buff.append(pcMap.getConditionId()); + String counterValue = getCountervaluebyCondition(pcMap.getConditionId()); + buff.append(","); + buff.append(counterValue); } // add to result Pair pair = new Pair(buff.toString(), duration); @@ -1757,6 +1791,20 @@ public String getCounternamebyCondition(long conditionId) { return counter.getSource().toString(); } + public String getCountervaluebyCondition(long conditionId) { + + ConditionVO condition = _conditionDao.findById(conditionId); + if (condition == null) + return ""; + + long counterId = condition.getCounterid(); + CounterVO counter = _counterDao.findById(counterId); + if (counter == null) + return ""; + + return counter.getValue(); + } + private Network getNetwork(AutoScaleVmGroup asGroup) { final LoadBalancerVO loadBalancer = _lbDao.findById(asGroup.getLoadBalancerId()); if (loadBalancer == null) { @@ -1779,6 +1827,9 @@ private Network.Provider getLoadBalancerServiceProvider(AutoScaleVmGroup asGroup } private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { + if (!is_native(asGroup.getId())) { + return; + } // check minimum vm of group Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId()); if (currentVM < asGroup.getMinMembers()) { @@ -1815,28 +1866,12 @@ private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { VMInstanceVO vmVO = _vmInstanceDao.findById(vmId); Long receiveHost = vmVO.getHostId(); - // setup parameters phase: duration and counter - // list pair [counter, duration] - List> lstPair = getPairofCounternameAndDuration(asGroup.getId()); - int total_counter = 0; - String[] lstCounter = new String[lstPair.size()]; - for (int i = 0; i < lstPair.size(); i++) { - Pair pair = lstPair.get(i); - String strCounterNames = pair.first(); - Integer duration = pair.second(); - - lstCounter[i] = strCounterNames.split(",")[0]; - total_counter++; - params.put("duration" + String.valueOf(total_counter), duration.toString()); - params.put("counter" + String.valueOf(total_counter), lstCounter[i]); - params.put("con" + String.valueOf(total_counter), strCounterNames.split(",")[1]); - } - params.put("total_counter", String.valueOf(total_counter)); + setPerformanceMonitorCommandParams(asGroup, params); PerformanceMonitorCommand perfMon = new PerformanceMonitorCommand(params, 20); try { - Answer answer = _agentMgr.send(receiveHost, perfMon); + PerformanceMonitorAnswer answer = (PerformanceMonitorAnswer) _agentMgr.send(receiveHost, perfMon); if (answer == null || !answer.getResult()) { s_logger.debug("Failed to send data to node !"); } else { @@ -1856,29 +1891,7 @@ private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { Long conditionId = Long.parseLong(params.get("con" + counter_vm[1])); Double coVal = Double.parseDouble(counterVals[1]); - // Summary of all counter by counterId key - if (avgCounter.get(counterId) == null) { - /* initialize if data is not set */ - avgCounter.put(counterId, new Double(0)); - } - - String counterName = getCounternamebyCondition(conditionId.longValue()); - if (Counter.Source.memory.toString().equals(counterName)) { - // calculate memory in percent - Long profileId = asGroup.getProfileId(); - AutoScaleVmProfileVO profileVo = _autoScaleVmProfileDao.findById(profileId); - ServiceOfferingVO serviceOff = _serviceOfferingDao.findById(profileVo.getServiceOfferingId()); - int maxRAM = serviceOff.getRamSize(); - - // get current RAM percent - coVal = coVal / maxRAM; - } else { - // cpu - coVal = coVal * 100; - } - - // update data entry - avgCounter.put(counterId, avgCounter.get(counterId) + coVal); + updateCountersMap(avgCounter, asGroup, counterId, conditionId, coVal); } catch (Exception e) { e.printStackTrace(); @@ -1902,6 +1915,55 @@ private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { } } + private void setPerformanceMonitorCommandParams(AutoScaleVmGroupVO asGroup, Map params) { + // setup parameters phase: duration and counter + // list pair [counter, duration] + List> lstPair = getPairofCounternameAndDuration(asGroup.getId()); + int total_counter = 0; + String[] lstCounter = new String[lstPair.size()]; + for (int i = 0; i < lstPair.size(); i++) { + Pair pair = lstPair.get(i); + String strCounterNames = pair.first(); + Integer duration = pair.second(); + + lstCounter[i] = strCounterNames.split(",")[0]; + total_counter++; + params.put("duration" + String.valueOf(total_counter), duration.toString()); + params.put("counter" + String.valueOf(total_counter), lstCounter[i]); + params.put("con" + String.valueOf(total_counter), strCounterNames.split(",")[1]); + if (strCounterNames.split(",").length >= 3) { + params.put("value" + String.valueOf(total_counter), strCounterNames.split(",")[2]); + } + } + params.put("total_counter", String.valueOf(total_counter)); + } + + private void updateCountersMap(Map avgCounter, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { + // Summary of all counter by counterId key + if (avgCounter.get(counterId) == null) { + /* initialize if data is not set */ + avgCounter.put(counterId, new Double(0)); + } + + String counterName = getCounternamebyCondition(conditionId.longValue()); + if (Counter.Source.memory.toString().equals(counterName)) { + // calculate memory in percent + Long profileId = asGroup.getProfileId(); + AutoScaleVmProfileVO profileVo = _autoScaleVmProfileDao.findById(profileId); + ServiceOfferingVO serviceOff = _serviceOfferingDao.findById(profileVo.getServiceOfferingId()); + int maxRAM = serviceOff.getRamSize(); + + // get current RAM percent + coVal = coVal / maxRAM * 100; + } else if (Counter.Source.cpu.toString().equals(counterName)) { + // cpu + coVal = coVal * 100; + } + + // update data entry + avgCounter.put(counterId, avgCounter.get(counterId) + coVal); + } + private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { // check minimum vm of group Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId()); @@ -1919,6 +1981,80 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { // update last_interval asGroup.setLastInterval(new Date()); _autoScaleVmGroupDao.persist(asGroup); - //TODO + + s_logger.debug("[AutoScale] Collecting performance data for hosts and virtual routers ..."); + + HashMap counters = new HashMap(); + if (is_native(asGroup.getId())) { + // group vms by host id + Map> hostAndVmIdsMap = new HashMap>(); + + List asGroupVmVOs = _autoScaleVmGroupVmMapDao.listByGroup(asGroup.getId()); + for (AutoScaleVmGroupVmMapVO asGroupVmVO : asGroupVmVOs) { + Long vmId = asGroupVmVO.getInstanceId(); + UserVmVO vm = _userVmDao.findById(vmId); + if (vm.getHostId() != null) { + List vmIds = hostAndVmIdsMap.get(vm.getHostId()); + if (vmIds == null) { + vmIds = new ArrayList(); + } + vmIds.add(vmId); + hostAndVmIdsMap.put(vm.getHostId(), vmIds); + } + } + + // get vm stats from each host and update database + for (Long hostId : hostAndVmIdsMap.keySet()) { + Date timestamp = new Date(); + HostVO host = _hostDao.findById(hostId); + List vmIds = hostAndVmIdsMap.get(hostId); + + try { + Map vmStatsById = _userVmMgr.getVirtualMachineStatistics(host.getId(), host.getName(), vmIds); + + if (vmStatsById != null) { + Set vmIdSet = vmStatsById.keySet(); + for (Long vmId : vmIdSet) { + VmStatsEntry vmStats = vmStatsById.get(vmId); + } + } + } catch (Exception e) { + s_logger.debug("Failed to get VM stats for host with ID: " + host.getId()); + continue; + } + } + + // update counter with average percentage + } + + if (has_source_virtual_router(asGroup.getId())) { + // create PerformanceMonitorCommand for the host where is virtual router is running on + + // process PerformanceMonitorAnswer from the host + // 1. update database (aggregation or average or instant) + // 2. get data in this period + // average: do nothing + // aggregation: end - start + // instant: calculate the average + } + + // get scale action + Map params = new HashMap(); + setPerformanceMonitorCommandParams(asGroup, params); + if (String.valueOf(counters.size()) != params.get("total_counter")) { + s_logger.info(String.format("[AutoScale] Skipping AutoScaling action as the size of counter values (%d) is not equals to the total counter number (%s)", counters.size(), params.get("total_counter"))); + return; + } + AutoScalePolicy.Action scaleAction = getAutoscaleAction(counters, asGroup.getId(), currentVM, params); + if (scaleAction != null) { + s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); + if (AutoScalePolicy.Action.ScaleUp.equals(scaleAction)) { + doScaleUp(asGroup.getId(), 1); + } else { + doScaleDown(asGroup.getId()); + } + } + + // remove old records from database } } From 794f470c78d6add5f629567493b78cc63290760c Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 25 May 2022 08:57:30 +0200 Subject: [PATCH 015/360] AS: get VM stats for AS vm group --- .../network/as/AutoScaleManagerImpl.java | 76 ++++++++++++++----- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 3197dd4dc5d5..a1ba00ce1f80 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1665,7 +1665,21 @@ private boolean has_source_virtual_router(long groupId) { return false; } - private AutoScalePolicy.Action getAutoscaleAction(HashMap avgCounter, long groupId, long currentVM, Map params) { + private Map getConditionsMap(long groupId) { + Map conditionsMap = new HashMap<>(); + List vos = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); + for (AutoScaleVmGroupPolicyMapVO vo : vos) { + List ConditionPolicies = _autoScalePolicyConditionMapDao.findByPolicyId(vo.getPolicyId()); + for (AutoScalePolicyConditionMapVO ConditionPolicy : ConditionPolicies) { + ConditionVO condition = _conditionDao.findById(ConditionPolicy.getConditionId()); + CounterVO counter = _counterDao.findById(condition.getCounterid()); + conditionsMap.put(condition.getId(), counter); + } + } + return conditionsMap; + } + + private AutoScalePolicy.Action getAutoscaleAction(Map countersMap, Map countersNumberMap, long groupId, Map params) { List listMap = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); if ((listMap == null) || (listMap.size() == 0)) @@ -1706,8 +1720,13 @@ private AutoScalePolicy.Action getAutoscaleAction(HashMap avgCount counter_count++; } while (true); - Double sum = avgCounter.get(counter_count); - Double avg = sum / currentVM; + Double sum = countersMap.get(counter_count); + Integer number = countersNumberMap.get(counter_count); + if (number == null || number == 0) { + bValid = false; + break; + } + Double avg = sum / number; Condition.Operator op = conditionVO.getRelationalOperator(); boolean bConditionCheck = ((op == com.cloud.network.as.Condition.Operator.EQ) && (thresholdPercent.equals(avg))) || ((op == com.cloud.network.as.Condition.Operator.GE) && (avg.doubleValue() >= thresholdPercent.doubleValue())) @@ -1877,7 +1896,8 @@ private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { } else { String result = answer.getDetails(); s_logger.debug("[AutoScale] RRDs collection answer: " + result); - HashMap avgCounter = new HashMap(); + HashMap countersMap = new HashMap(); + HashMap countersNumberMap = new HashMap(); // extract data String[] counterElements = result.split(","); @@ -1891,14 +1911,14 @@ private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { Long conditionId = Long.parseLong(params.get("con" + counter_vm[1])); Double coVal = Double.parseDouble(counterVals[1]); - updateCountersMap(avgCounter, asGroup, counterId, conditionId, coVal); + updateCountersMap(countersMap, countersNumberMap, asGroup, counterId, conditionId, coVal); } catch (Exception e) { e.printStackTrace(); } } - AutoScalePolicy.Action scaleAction = getAutoscaleAction(avgCounter, asGroup.getId(), currentVM, params); + AutoScalePolicy.Action scaleAction = getAutoscaleAction(countersMap, countersNumberMap, asGroup.getId(), params); if (scaleAction != null) { s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); if (AutoScalePolicy.Action.ScaleUp.equals(scaleAction)) { @@ -1938,11 +1958,15 @@ private void setPerformanceMonitorCommandParams(AutoScaleVmGroupVO asGroup, Map< params.put("total_counter", String.valueOf(total_counter)); } - private void updateCountersMap(Map avgCounter, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { + private void updateCountersMap(Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { // Summary of all counter by counterId key - if (avgCounter.get(counterId) == null) { + if (countersMap.get(counterId) == null) { + /* initialize if data is not set */ + countersMap.put(counterId, Double.valueOf(0)); + } + if (countersNumberMap.get(counterId) == null) { /* initialize if data is not set */ - avgCounter.put(counterId, new Double(0)); + countersNumberMap.put(counterId, 0); } String counterName = getCounternamebyCondition(conditionId.longValue()); @@ -1961,7 +1985,9 @@ private void updateCountersMap(Map avgCounter, AutoScaleVmGroupVO } // update data entry - avgCounter.put(counterId, avgCounter.get(counterId) + coVal); + s_logger.debug(String.format("Updating countersMap for counterId = %d from %f to %f", counterId, countersMap.get(counterId), countersMap.get(counterId) + coVal)); + countersMap.put(counterId, countersMap.get(counterId) + coVal); + countersNumberMap.put(counterId, countersNumberMap.get(counterId) + 1); } private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { @@ -1982,10 +2008,16 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { asGroup.setLastInterval(new Date()); _autoScaleVmGroupDao.persist(asGroup); - s_logger.debug("[AutoScale] Collecting performance data for hosts and virtual routers ..."); + s_logger.debug("[AutoScale] Collecting performance data ..."); + + Map conditionsMap = getConditionsMap(asGroup.getId()); + + Map countersMap = new HashMap(); + Map countersNumberMap = new HashMap(); - HashMap counters = new HashMap(); if (is_native(asGroup.getId())) { + s_logger.debug("[AutoScale] Collecting performance data from hosts ..."); + // group vms by host id Map> hostAndVmIdsMap = new HashMap>(); @@ -2014,12 +2046,20 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { if (vmStatsById != null) { Set vmIdSet = vmStatsById.keySet(); + for (Long vmId : vmIdSet) { VmStatsEntry vmStats = vmStatsById.get(vmId); + for (Long conditionId : conditionsMap.keySet()) { + CounterVO counter = conditionsMap.get(conditionId); + if (Counter.NativeSources.contains(counter.getSource())) { + Double counterValue = Counter.Source.cpu.equals(counter.getSource()) ? vmStats.getCPUUtilization() : vmStats.getMemoryKBs() / 1024; + updateCountersMap(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, counterValue); + } + } } } } catch (Exception e) { - s_logger.debug("Failed to get VM stats for host with ID: " + host.getId()); + s_logger.debug("Failed to get VM stats from host : " + host.getName()); continue; } } @@ -2028,6 +2068,8 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { } if (has_source_virtual_router(asGroup.getId())) { + s_logger.debug("[AutoScale] Collecting performance data from virtual router ..."); + // create PerformanceMonitorCommand for the host where is virtual router is running on // process PerformanceMonitorAnswer from the host @@ -2041,11 +2083,11 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { // get scale action Map params = new HashMap(); setPerformanceMonitorCommandParams(asGroup, params); - if (String.valueOf(counters.size()) != params.get("total_counter")) { - s_logger.info(String.format("[AutoScale] Skipping AutoScaling action as the size of counter values (%d) is not equals to the total counter number (%s)", counters.size(), params.get("total_counter"))); - return; + if (String.valueOf(countersMap.size()) != params.get("total_counter") || String.valueOf(countersNumberMap.size()) != params.get("total_counter")) { + s_logger.warn(String.format("[AutoScale] AutoScaling action might be wrong as the size of counter values (%d) or counter number map (%d) is not equals to the total counter number (%s)", + countersMap.size(), countersNumberMap.size(), params.get("total_counter"))); } - AutoScalePolicy.Action scaleAction = getAutoscaleAction(counters, asGroup.getId(), currentVM, params); + AutoScalePolicy.Action scaleAction = getAutoscaleAction(countersMap, countersNumberMap, asGroup.getId(), params); if (scaleAction != null) { s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); if (AutoScalePolicy.Action.ScaleUp.equals(scaleAction)) { From 247406978766bdc7908ee43686bc3a71c2d4b104 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 25 May 2022 14:04:43 +0200 Subject: [PATCH 016/360] AS: fix issue if policy has multiple conditions --- .../network/as/AutoScaleManagerImpl.java | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index a1ba00ce1f80..cf2e0bab5d1f 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1680,6 +1680,7 @@ private Map getConditionsMap(long groupId) { } private AutoScalePolicy.Action getAutoscaleAction(Map countersMap, Map countersNumberMap, long groupId, Map params) { + s_logger.debug("[AutoScale] Getting autoscale action for group : " + groupId); List listMap = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); if ((listMap == null) || (listMap.size() == 0)) @@ -1707,12 +1708,12 @@ private AutoScalePolicy.Action getAutoscaleAction(Map countersMap, long thresholdValue = conditionVO.getThreshold(); Double thresholdPercent = (double)thresholdValue; CounterVO counterVO = _counterDao.findById(conditionVO.getCounterid()); + Counter.Source counter_source = counterVO.getSource(); + String counter_value = counterVO.getValue(); long counter_count = 1; do { String counter_param = params.get("counter" + String.valueOf(counter_count)); - Counter.Source counter_source = counterVO.getSource(); String counter_param_value = params.get("value" + String.valueOf(counter_count)); - String counter_value = counterVO.getValue(); if (counter_param.equals(counter_source.toString()) && (counter_param_value == null || counter_param_value.equals(counter_value))) { break; @@ -1720,8 +1721,9 @@ private AutoScalePolicy.Action getAutoscaleAction(Map countersMap, counter_count++; } while (true); - Double sum = countersMap.get(counter_count); - Integer number = countersNumberMap.get(counter_count); + Double sum = countersMap.get(counterVO.getId()); + Integer number = countersNumberMap.get(counterVO.getId()); + s_logger.debug(String.format("policyId = %d, conditionId = %d, counter = %s, sum = %f, number = %s", policyVO.getId(), conditionVO.getId(), counterVO.getSource(), sum, number)); if (number == null || number == 0) { bValid = false; break; @@ -1777,9 +1779,9 @@ public List> getPairofCounternameAndDuration(long groupId) Integer duration = policyVo.getDuration(); //get collection of counter name - StringBuffer buff = new StringBuffer(); List lstPCmap = _autoScalePolicyConditionMapDao.findByPolicyId(policyVo.getId()); for (AutoScalePolicyConditionMapVO pcMap : lstPCmap) { + StringBuffer buff = new StringBuffer(); String counterName = getCounternamebyCondition(pcMap.getConditionId()); buff.append(counterName); buff.append(","); @@ -1787,10 +1789,10 @@ public List> getPairofCounternameAndDuration(long groupId) String counterValue = getCountervaluebyCondition(pcMap.getConditionId()); buff.append(","); buff.append(counterValue); + // add to result + Pair pair = new Pair(buff.toString(), duration); + result.add(pair); } - // add to result - Pair pair = new Pair(buff.toString(), duration); - result.add(pair); } return result; @@ -2052,7 +2054,7 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { for (Long conditionId : conditionsMap.keySet()) { CounterVO counter = conditionsMap.get(conditionId); if (Counter.NativeSources.contains(counter.getSource())) { - Double counterValue = Counter.Source.cpu.equals(counter.getSource()) ? vmStats.getCPUUtilization() : vmStats.getMemoryKBs() / 1024; + Double counterValue = Counter.Source.cpu.equals(counter.getSource()) ? vmStats.getCPUUtilization() / 100: vmStats.getMemoryKBs() / 1024; updateCountersMap(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, counterValue); } } @@ -2083,10 +2085,7 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { // get scale action Map params = new HashMap(); setPerformanceMonitorCommandParams(asGroup, params); - if (String.valueOf(countersMap.size()) != params.get("total_counter") || String.valueOf(countersNumberMap.size()) != params.get("total_counter")) { - s_logger.warn(String.format("[AutoScale] AutoScaling action might be wrong as the size of counter values (%d) or counter number map (%d) is not equals to the total counter number (%s)", - countersMap.size(), countersNumberMap.size(), params.get("total_counter"))); - } + AutoScalePolicy.Action scaleAction = getAutoscaleAction(countersMap, countersNumberMap, asGroup.getId(), params); if (scaleAction != null) { s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); From c560b2b0729f27154161eaef69810258729cdc15 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 25 May 2022 14:16:17 +0200 Subject: [PATCH 017/360] AS: fix typo in UpdateAutoScaleVmGroupCmd --- .../main/java/com/cloud/network/as/AutoScaleManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index cf2e0bab5d1f..ec7cb45307bf 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1069,7 +1069,7 @@ public AutoScaleVmGroup updateAutoScaleVmGroup(UpdateAutoScaleVmGroupCmd cmd) { vmGroupVO.setMaxMembers(maxMembers); } - if (maxMembers != null) { + if (interval != null) { vmGroupVO.setInterval(interval); } From a7d2e1d3e272a5c19b07cd5843d4bc7032dedeca Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 27 May 2022 13:19:08 +0200 Subject: [PATCH 018/360] AS: re-organize AutoScaleCounterType values --- .../java/com/cloud/network/as/AutoScaleCounter.java | 9 +++++---- .../com/cloud/network/element/NetscalerElement.java | 6 ++---- .../cloud/network/element/VirtualRouterElement.java | 11 +++-------- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/api/src/main/java/com/cloud/network/as/AutoScaleCounter.java b/api/src/main/java/com/cloud/network/as/AutoScaleCounter.java index f25d6699a006..8011de2213df 100644 --- a/api/src/main/java/com/cloud/network/as/AutoScaleCounter.java +++ b/api/src/main/java/com/cloud/network/as/AutoScaleCounter.java @@ -25,10 +25,11 @@ public class AutoScaleCounter { public static class AutoScaleCounterType { private String _name; - public static final AutoScaleCounterType Snmp = new AutoScaleCounterType("snmp"); - public static final AutoScaleCounterType Netscaler = new AutoScaleCounterType("netscaler"); - public static final AutoScaleCounterType Cpu = new AutoScaleCounterType("cpu"); - public static final AutoScaleCounterType Memory = new AutoScaleCounterType("memory"); + public static final AutoScaleCounterType Snmp = new AutoScaleCounterType(Counter.Source.snmp.name()); + public static final AutoScaleCounterType Cpu = new AutoScaleCounterType(Counter.Source.cpu.name()); + public static final AutoScaleCounterType Memory = new AutoScaleCounterType(Counter.Source.memory.name()); + public static final AutoScaleCounterType Netscaler = new AutoScaleCounterType(Counter.Source.netscaler.name()); + public static final AutoScaleCounterType VirtualRouter = new AutoScaleCounterType(Counter.Source.virtualrouter.name()); public AutoScaleCounterType(String name) { _name = name; diff --git a/plugins/network-elements/netscaler/src/main/java/com/cloud/network/element/NetscalerElement.java b/plugins/network-elements/netscaler/src/main/java/com/cloud/network/element/NetscalerElement.java index 1df16404d399..a05ec7ad85c7 100644 --- a/plugins/network-elements/netscaler/src/main/java/com/cloud/network/element/NetscalerElement.java +++ b/plugins/network-elements/netscaler/src/main/java/com/cloud/network/element/NetscalerElement.java @@ -159,8 +159,6 @@ public class NetscalerElement extends ExternalLoadBalancerDeviceManagerImpl IpDeployer, StaticNatServiceProvider, GslbServiceProvider { private static final Logger s_logger = Logger.getLogger(NetscalerElement.class); - public static final AutoScaleCounterType AutoScaleCounterSnmp = new AutoScaleCounterType("snmp"); - public static final AutoScaleCounterType AutoScaleCounterNetscaler = new AutoScaleCounterType("netscaler"); @Inject NetworkModel _networkManager; @@ -514,13 +512,13 @@ public Map> getCapabilities() { // list of counters it supports AutoScaleCounter counter; List counterList = new ArrayList(); - counter = new AutoScaleCounter(AutoScaleCounterSnmp); + counter = new AutoScaleCounter(AutoScaleCounterType.Snmp); counterList.add(counter); counter.addParam("snmpcommunity", true, "the community string that has to be used to do a SNMP GET on the AutoScaled Vm", false); counter.addParam("snmpport", false, "the port at which SNMP agent is running on the AutoScaled Vm", false); - counter = new AutoScaleCounter(AutoScaleCounterNetscaler); + counter = new AutoScaleCounter(AutoScaleCounterType.Netscaler); counterList.add(counter); Gson gson = new Gson(); diff --git a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java index 1689513a751a..e9741d451278 100644 --- a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java @@ -24,7 +24,6 @@ import javax.inject.Inject; -import com.cloud.network.as.Counter; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; @@ -123,10 +122,6 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl StaticNatServiceProvider, FirewallServiceProvider, LoadBalancingServiceProvider, PortForwardingServiceProvider, RemoteAccessVPNServiceProvider, IpDeployer, NetworkMigrationResponder, AggregatedCommandExecutor, RedundantResource, DnsServiceProvider{ private static final Logger s_logger = Logger.getLogger(VirtualRouterElement.class); - public static final AutoScaleCounterType AutoScaleCounterCpu = new AutoScaleCounterType(Counter.Source.cpu.name()); - public static final AutoScaleCounterType AutoScaleCounterMemory = new AutoScaleCounterType(Counter.Source.memory.name()); - public static final AutoScaleCounterType AutoScaleCounterVirtualRouter = new AutoScaleCounterType(Counter.Source.virtualrouter.name()); - protected static final Map> capabilities = setCapabilities(); @Inject @@ -544,11 +539,11 @@ private static Map> setCapabilities() { // counters it supports AutoScaleCounter counter; final List counterList = new ArrayList(); - counter = new AutoScaleCounter(AutoScaleCounterCpu); + counter = new AutoScaleCounter(AutoScaleCounterType.Cpu); counterList.add(counter); - counter = new AutoScaleCounter(AutoScaleCounterMemory); + counter = new AutoScaleCounter(AutoScaleCounterType.Memory); counterList.add(counter); - counter = new AutoScaleCounter(AutoScaleCounterVirtualRouter); + counter = new AutoScaleCounter(AutoScaleCounterType.VirtualRouter); counterList.add(counter); final Gson gson = new Gson(); final String autoScaleCounterList = gson.toJson(counterList); From ceda2b7d1886294cf1fbf98a7f78fdb10a9420a7 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 27 May 2022 15:52:52 +0200 Subject: [PATCH 019/360] AS: add VO/DAO for autoscale_vmgroup_statistics --- .../as/AutoScaleVmGroupStatisticsVO.java | 103 ++++++++++++++++++ .../as/dao/AutoScaleVmGroupStatisticsDao.java | 36 ++++++ .../AutoScaleVmGroupStatisticsDaoImpl.java | 88 +++++++++++++++ ...spring-engine-schema-core-daos-context.xml | 1 + 4 files changed, 228 insertions(+) create mode 100644 engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupStatisticsVO.java create mode 100644 engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupStatisticsDao.java create mode 100644 engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupStatisticsDaoImpl.java diff --git a/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupStatisticsVO.java b/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupStatisticsVO.java new file mode 100644 index 000000000000..6400cafc05f4 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupStatisticsVO.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.as; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import com.cloud.server.ResourceTag; +import org.apache.cloudstack.api.InternalIdentity; + +@Entity +@Table(name = "autoscale_vmgroup_statistics") +public class AutoScaleVmGroupStatisticsVO implements InternalIdentity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "vmgroup_id") + private long vmGroupId; + + @Column(name = "counter_id") + private long counterId; + + @Column(name = "resource_id") + private long resourceId; + + @Column(name = "resource_type") + @Enumerated(value = EnumType.STRING) + private ResourceTag.ResourceObjectType resourceType; + + @Column(name = "raw_value") + private Double rawValue = null; + + @Column(name = "created") + @Temporal(value = TemporalType.TIMESTAMP) + private Date created = null; + + public AutoScaleVmGroupStatisticsVO() { + } + + public AutoScaleVmGroupStatisticsVO(long vmGroupId, long counterId, long resourceId, ResourceTag.ResourceObjectType resourceType, Double rawValue, Date created) { + this.vmGroupId = vmGroupId; + this.counterId = counterId; + this.resourceId = resourceId; + this.resourceType = resourceType; + this.rawValue = rawValue; + this.created = created; + } + + @Override + public long getId() { + return id; + } + + public long getVmGroupId() { + return vmGroupId; + } + + public long getCounterId() { + return counterId; + } + + public long getResourceId() { + return resourceId; + } + + public ResourceTag.ResourceObjectType getResourceType() { + return resourceType; + } + + public Double getRawValue() { + return rawValue; + } + + public Date getCreated() { + return created; + } +} diff --git a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupStatisticsDao.java b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupStatisticsDao.java new file mode 100644 index 000000000000..ec25419450fc --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupStatisticsDao.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.as.dao; + +import java.util.Date; +import java.util.List; + +import com.cloud.network.as.AutoScaleVmGroupStatisticsVO; +import com.cloud.utils.db.GenericDao; + +public interface AutoScaleVmGroupStatisticsDao extends GenericDao { + boolean removeByGroupId(long vmGroupId); + + boolean removeByGroupAndCounter(long vmGroupId, long counterId); + + boolean removeByGroupAndCounter(long vmGroupId, long counterId, Date beforeDate); + + List listByVmGroupId(long vmGroupId); + + List listByVmGroupIdAndCounterId(long vmGroupId, long counterId, Date afterDate); + +} diff --git a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupStatisticsDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupStatisticsDaoImpl.java new file mode 100644 index 000000000000..1d1c999fc21f --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupStatisticsDaoImpl.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.as.dao; + +import java.util.Date; +import java.util.List; + + +import org.springframework.stereotype.Component; + +import com.cloud.network.as.AutoScaleVmGroupStatisticsVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; + +import javax.annotation.PostConstruct; + +@Component +public class AutoScaleVmGroupStatisticsDaoImpl extends GenericDaoBase implements AutoScaleVmGroupStatisticsDao { + + SearchBuilder groupAndCounterSearch; + + @PostConstruct + protected void init() { + groupAndCounterSearch = createSearchBuilder(); + groupAndCounterSearch.and("vmGroupId", groupAndCounterSearch.entity().getVmGroupId(), Op.EQ); + groupAndCounterSearch.and("counterId", groupAndCounterSearch.entity().getCounterId(), Op.EQ); + groupAndCounterSearch.and("createdLT", groupAndCounterSearch.entity().getCreated(), Op.LT); + groupAndCounterSearch.and("createdGT", groupAndCounterSearch.entity().getCreated(), Op.GT); + groupAndCounterSearch.done(); + } + + @Override + public boolean removeByGroupId(long vmGroupId) { + SearchCriteria sc = groupAndCounterSearch.create(); + sc.addAnd("vmGroupId", SearchCriteria.Op.EQ, vmGroupId); + + return expunge(sc) > 0; + } + + @Override + public boolean removeByGroupAndCounter(long vmGroupId, long counterId) { + SearchCriteria sc = groupAndCounterSearch.create(); + sc.setParameters("vmGroupId", vmGroupId); + sc.setParameters("counterId", counterId); + return expunge(sc) > 0; + } + + @Override + public boolean removeByGroupAndCounter(long vmGroupId, long counterId, Date beforeDate) { + SearchCriteria sc = groupAndCounterSearch.create(); + sc.setParameters("vmGroupId", vmGroupId); + sc.setParameters("counterId", counterId); + sc.setParameters("createdLT", beforeDate); + return expunge(sc) > 0; + } + + @Override + public List listByVmGroupId(long vmGroupId) { + SearchCriteria sc = groupAndCounterSearch.create(); + sc.setParameters("vmGroupId", vmGroupId); + return listBy(sc); + } + + @Override + public List listByVmGroupIdAndCounterId(long vmGroupId, long counterId, Date afterDate) { + SearchCriteria sc = groupAndCounterSearch.create(); + sc.setParameters("vmGroupId", vmGroupId); + sc.setParameters("counterId", counterId); + sc.setParameters("createdGT", afterDate); + return listBy(sc); + } +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index f13ae6e68031..d87ed05bcc73 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -54,6 +54,7 @@ + From 3f1461535f5c987bb39162fb2d5fe90d6c4e01c9 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 27 May 2022 16:37:33 +0200 Subject: [PATCH 020/360] AS: persist cpu/memory usage of user vms --- .../network/as/AutoScaleManagerImpl.java | 124 +++++++++++------- 1 file changed, 77 insertions(+), 47 deletions(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index ec7cb45307bf..e68afd981a72 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -32,6 +32,7 @@ import javax.inject.Inject; import com.cloud.offering.DiskOffering; +import com.cloud.server.ResourceTag; import org.apache.log4j.Logger; import com.google.gson.Gson; @@ -95,6 +96,7 @@ import com.cloud.network.as.dao.AutoScalePolicyDao; import com.cloud.network.as.dao.AutoScaleVmGroupDao; import com.cloud.network.as.dao.AutoScaleVmGroupPolicyMapDao; +import com.cloud.network.as.dao.AutoScaleVmGroupStatisticsDao; import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; import com.cloud.network.as.dao.AutoScaleVmProfileDao; import com.cloud.network.as.dao.ConditionDao; @@ -216,6 +218,8 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale private UserVmDao _userVmDao; @Inject private HostDao _hostDao; + @Inject + private AutoScaleVmGroupStatisticsDao _asGroupStatisticsDao; public List getSupportedAutoScaleCounters(long networkid) { String capability = _lbRulesMgr.getLBCapability(networkid, Capability.AutoScaleCounters.getName()); @@ -1679,6 +1683,26 @@ private Map getConditionsMap(long groupId) { return conditionsMap; } + private List getCounters(long groupId) { + List counterIds = new ArrayList<>(); + List vos = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); + for (AutoScaleVmGroupPolicyMapVO vo : vos) { + List ConditionPolicies = _autoScalePolicyConditionMapDao.findByPolicyId(vo.getPolicyId()); + for (AutoScalePolicyConditionMapVO ConditionPolicy : ConditionPolicies) { + ConditionVO condition = _conditionDao.findById(ConditionPolicy.getConditionId()); + CounterVO counter = _counterDao.findById(condition.getCounterid()); + if (!counterIds.contains(counter.getId())) { + counterIds.add(counter.getId()); + } + } + } + List counters = new ArrayList<>(); + for (Long counterId : counterIds) { + counters.add(_counterDao.findById(counterId)); + } + return counters; + } + private AutoScalePolicy.Action getAutoscaleAction(Map countersMap, Map countersNumberMap, long groupId, Map params) { s_logger.debug("[AutoScale] Getting autoscale action for group : " + groupId); @@ -2012,59 +2036,13 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { s_logger.debug("[AutoScale] Collecting performance data ..."); - Map conditionsMap = getConditionsMap(asGroup.getId()); - Map countersMap = new HashMap(); Map countersNumberMap = new HashMap(); if (is_native(asGroup.getId())) { s_logger.debug("[AutoScale] Collecting performance data from hosts ..."); - // group vms by host id - Map> hostAndVmIdsMap = new HashMap>(); - - List asGroupVmVOs = _autoScaleVmGroupVmMapDao.listByGroup(asGroup.getId()); - for (AutoScaleVmGroupVmMapVO asGroupVmVO : asGroupVmVOs) { - Long vmId = asGroupVmVO.getInstanceId(); - UserVmVO vm = _userVmDao.findById(vmId); - if (vm.getHostId() != null) { - List vmIds = hostAndVmIdsMap.get(vm.getHostId()); - if (vmIds == null) { - vmIds = new ArrayList(); - } - vmIds.add(vmId); - hostAndVmIdsMap.put(vm.getHostId(), vmIds); - } - } - - // get vm stats from each host and update database - for (Long hostId : hostAndVmIdsMap.keySet()) { - Date timestamp = new Date(); - HostVO host = _hostDao.findById(hostId); - List vmIds = hostAndVmIdsMap.get(hostId); - - try { - Map vmStatsById = _userVmMgr.getVirtualMachineStatistics(host.getId(), host.getName(), vmIds); - - if (vmStatsById != null) { - Set vmIdSet = vmStatsById.keySet(); - - for (Long vmId : vmIdSet) { - VmStatsEntry vmStats = vmStatsById.get(vmId); - for (Long conditionId : conditionsMap.keySet()) { - CounterVO counter = conditionsMap.get(conditionId); - if (Counter.NativeSources.contains(counter.getSource())) { - Double counterValue = Counter.Source.cpu.equals(counter.getSource()) ? vmStats.getCPUUtilization() / 100: vmStats.getMemoryKBs() / 1024; - updateCountersMap(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, counterValue); - } - } - } - } - } catch (Exception e) { - s_logger.debug("Failed to get VM stats from host : " + host.getName()); - continue; - } - } + getVmStatsFromHosts(asGroup); // update counter with average percentage } @@ -2098,4 +2076,56 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { // remove old records from database } + + private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { + // group vms by host id + Map> hostAndVmIdsMap = new HashMap>(); + + List asGroupVmVOs = _autoScaleVmGroupVmMapDao.listByGroup(asGroup.getId()); + for (AutoScaleVmGroupVmMapVO asGroupVmVO : asGroupVmVOs) { + Long vmId = asGroupVmVO.getInstanceId(); + UserVmVO vm = _userVmDao.findById(vmId); + if (vm.getHostId() != null) { + List vmIds = hostAndVmIdsMap.get(vm.getHostId()); + if (vmIds == null) { + vmIds = new ArrayList(); + } + vmIds.add(vmId); + hostAndVmIdsMap.put(vm.getHostId(), vmIds); + } + } + + List counters = getCounters(asGroup.getId()); + + // get vm stats from each host and update database + for (Long hostId : hostAndVmIdsMap.keySet()) { + Date timestamp = new Date(); + HostVO host = _hostDao.findById(hostId); + List vmIds = hostAndVmIdsMap.get(hostId); + + try { + Map vmStatsById = _userVmMgr.getVirtualMachineStatistics(host.getId(), host.getName(), vmIds); + + if (vmStatsById != null) { + Set vmIdSet = vmStatsById.keySet(); + + for (Long vmId : vmIdSet) { + VmStatsEntry vmStats = vmStatsById.get(vmId); + for (CounterVO counter : counters) { + if (Counter.Source.cpu.equals(counter.getSource())) { + Double counterValue = vmStats.getCPUUtilization() / 100; + _asGroupStatisticsDao.persist(new AutoScaleVmGroupStatisticsVO(asGroup.getId(), counter.getId(), vmId, ResourceTag.ResourceObjectType.UserVm, counterValue, timestamp)); + } else if (Counter.Source.memory.equals(counter.getSource())) { + Double counterValue = vmStats.getMemoryKBs() / 1024; + _asGroupStatisticsDao.persist(new AutoScaleVmGroupStatisticsVO(asGroup.getId(), counter.getId(), vmId, ResourceTag.ResourceObjectType.UserVm, counterValue, timestamp)); + } + } + } + } + } catch (Exception e) { + s_logger.debug("Failed to get VM stats from host : " + host.getName()); + continue; + } + } + } } From 8f447a38cc056863b8c28185435a8f0d78a466bc Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 27 May 2022 20:48:27 +0200 Subject: [PATCH 021/360] AS: updateCountersMap when get vm stats from hosts --- .../network/as/AutoScaleManagerImpl.java | 82 ++++++++++++------- 1 file changed, 54 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index e68afd981a72..564eb03d6bde 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -33,6 +33,7 @@ import com.cloud.offering.DiskOffering; import com.cloud.server.ResourceTag; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.google.gson.Gson; @@ -1669,16 +1670,13 @@ private boolean has_source_virtual_router(long groupId) { return false; } - private Map getConditionsMap(long groupId) { + private Map getConditionsMap(long policyId) { Map conditionsMap = new HashMap<>(); - List vos = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); - for (AutoScaleVmGroupPolicyMapVO vo : vos) { - List ConditionPolicies = _autoScalePolicyConditionMapDao.findByPolicyId(vo.getPolicyId()); - for (AutoScalePolicyConditionMapVO ConditionPolicy : ConditionPolicies) { - ConditionVO condition = _conditionDao.findById(ConditionPolicy.getConditionId()); - CounterVO counter = _counterDao.findById(condition.getCounterid()); - conditionsMap.put(condition.getId(), counter); - } + List ConditionPolicies = _autoScalePolicyConditionMapDao.findByPolicyId(policyId); + for (AutoScalePolicyConditionMapVO ConditionPolicy : ConditionPolicies) { + ConditionVO condition = _conditionDao.findById(ConditionPolicy.getConditionId()); + CounterVO counter = _counterDao.findById(condition.getCounterid()); + conditionsMap.put(condition.getId(), counter); } return conditionsMap; } @@ -1703,7 +1701,7 @@ private List getCounters(long groupId) { return counters; } - private AutoScalePolicy.Action getAutoscaleAction(Map countersMap, Map countersNumberMap, long groupId, Map params) { + private AutoScalePolicy.Action getAutoscaleAction(Map countersMap, Map countersNumberMap, long groupId, Map params) { s_logger.debug("[AutoScale] Getting autoscale action for group : " + groupId); List listMap = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); @@ -1745,8 +1743,9 @@ private AutoScalePolicy.Action getAutoscaleAction(Map countersMap, counter_count++; } while (true); - Double sum = countersMap.get(counterVO.getId()); - Integer number = countersNumberMap.get(counterVO.getId()); + String key = generateKey(conditionVO.getId(), counterVO.getId()); + Double sum = countersMap.get(key); + Integer number = countersNumberMap.get(key); s_logger.debug(String.format("policyId = %d, conditionId = %d, counter = %s, sum = %f, number = %s", policyVO.getId(), conditionVO.getId(), counterVO.getSource(), sum, number)); if (number == null || number == 0) { bValid = false; @@ -1922,8 +1921,8 @@ private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { } else { String result = answer.getDetails(); s_logger.debug("[AutoScale] RRDs collection answer: " + result); - HashMap countersMap = new HashMap(); - HashMap countersNumberMap = new HashMap(); + HashMap countersMap = new HashMap<>(); + HashMap countersNumberMap = new HashMap<>(); // extract data String[] counterElements = result.split(","); @@ -1984,19 +1983,23 @@ private void setPerformanceMonitorCommandParams(AutoScaleVmGroupVO asGroup, Map< params.put("total_counter", String.valueOf(total_counter)); } - private void updateCountersMap(Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { + private void updateCountersMap(Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { // Summary of all counter by counterId key - if (countersMap.get(counterId) == null) { + String key = generateKey(conditionId, counterId); + if (countersMap.get(key) == null) { /* initialize if data is not set */ - countersMap.put(counterId, Double.valueOf(0)); + countersMap.put(key, Double.valueOf(0)); } - if (countersNumberMap.get(counterId) == null) { + if (countersNumberMap.get(key) == null) { /* initialize if data is not set */ - countersNumberMap.put(counterId, 0); + countersNumberMap.put(key, 0); } - String counterName = getCounternamebyCondition(conditionId.longValue()); - if (Counter.Source.memory.toString().equals(counterName)) { + CounterVO counter = _counterDao.findById(counterId); + if (counter == null) { + return; + } + if (Counter.Source.memory.equals(counter.getSource())) { // calculate memory in percent Long profileId = asGroup.getProfileId(); AutoScaleVmProfileVO profileVo = _autoScaleVmProfileDao.findById(profileId); @@ -2005,15 +2008,15 @@ private void updateCountersMap(Map countersMap, Map // get current RAM percent coVal = coVal / maxRAM * 100; - } else if (Counter.Source.cpu.toString().equals(counterName)) { + } else if (Counter.Source.cpu.equals(counter.getSource())) { // cpu coVal = coVal * 100; } // update data entry - s_logger.debug(String.format("Updating countersMap for counterId = %d from %f to %f", counterId, countersMap.get(counterId), countersMap.get(counterId) + coVal)); - countersMap.put(counterId, countersMap.get(counterId) + coVal); - countersNumberMap.put(counterId, countersNumberMap.get(counterId) + 1); + s_logger.debug(String.format("Updating countersMap for conditionId = %s, counterId = %d from %f to %f", conditionId, counterId, countersMap.get(key), countersMap.get(key) + coVal)); + countersMap.put(key, countersMap.get(key) + coVal); + countersNumberMap.put(key, countersNumberMap.get(key) + 1); } private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { @@ -2036,15 +2039,15 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { s_logger.debug("[AutoScale] Collecting performance data ..."); - Map countersMap = new HashMap(); - Map countersNumberMap = new HashMap(); + Map countersMap = new HashMap<>(); + Map countersNumberMap = new HashMap<>(); if (is_native(asGroup.getId())) { s_logger.debug("[AutoScale] Collecting performance data from hosts ..."); getVmStatsFromHosts(asGroup); - // update counter with average percentage + updateCountersMap(asGroup, countersMap, countersNumberMap); } if (has_source_virtual_router(asGroup.getId())) { @@ -2128,4 +2131,27 @@ private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { } } } + + private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map countersMap, Map countersNumberMap) { + List groupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(asGroup.getId()); + for (AutoScaleVmGroupPolicyMapVO groupPolicyVO : groupPolicyVOs) { + AutoScalePolicyVO vo = _autoScalePolicyDao.findById(groupPolicyVO.getPolicyId()); + Map conditionsMap = getConditionsMap(groupPolicyVO.getPolicyId()); + for (Long conditionId : conditionsMap.keySet()) { + CounterVO counter = conditionsMap.get(conditionId); + List stats = _asGroupStatisticsDao.listByVmGroupIdAndCounterId(asGroup.getId(), counter.getId(), new Date(System.currentTimeMillis() - ((long)vo.getDuration() << 10))); + if (CollectionUtils.isEmpty(stats)) { + continue; + } + for (AutoScaleVmGroupStatisticsVO stat : stats) { + updateCountersMap(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, stat.getRawValue()); + } + } + + } + } + + private String generateKey(Long conditionId, Long counterId) { + return conditionId + "-" + counterId; + } } From 08549cd5753f920e870c25551db45af78ff64a58 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 30 May 2022 12:19:43 +0200 Subject: [PATCH 022/360] AS: cleanup old statistics --- .../network/as/AutoScaleManagerImpl.java | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 564eb03d6bde..6b243ba5c104 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -32,7 +32,6 @@ import javax.inject.Inject; import com.cloud.offering.DiskOffering; -import com.cloud.server.ResourceTag; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -112,6 +111,7 @@ import com.cloud.network.lb.LoadBalancingRulesService; import com.cloud.offering.ServiceOffering; import com.cloud.projects.Project.ListProjectResourcesCriteria; +import com.cloud.server.ResourceTag; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.template.TemplateManager; @@ -123,6 +123,7 @@ import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserDao; import com.cloud.uservm.UserVm; +import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentContext; @@ -222,6 +223,15 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale @Inject private AutoScaleVmGroupStatisticsDao _asGroupStatisticsDao; + private long autoScaleStatsInterval = -1L; + private static final Long ONE_MINUTE_IN_MILLISCONDS = 60000L; + + @Override + public boolean start() { + autoScaleStatsInterval = NumbersUtil.parseLong(_configDao.getValue("autoscale.stats.interval"), ONE_MINUTE_IN_MILLISCONDS); + return true; + } + public List getSupportedAutoScaleCounters(long networkid) { String capability = _lbRulesMgr.getLBCapability(networkid, Capability.AutoScaleCounters.getName()); if (capability == null) { @@ -1743,7 +1753,7 @@ private AutoScalePolicy.Action getAutoscaleAction(Map countersMa counter_count++; } while (true); - String key = generateKey(conditionVO.getId(), counterVO.getId()); + String key = generateKeyFromConditionAndCounter(conditionVO.getId(), counterVO.getId()); Double sum = countersMap.get(key); Integer number = countersNumberMap.get(key); s_logger.debug(String.format("policyId = %d, conditionId = %d, counter = %s, sum = %f, number = %s", policyVO.getId(), conditionVO.getId(), counterVO.getSource(), sum, number)); @@ -1985,7 +1995,7 @@ private void setPerformanceMonitorCommandParams(AutoScaleVmGroupVO asGroup, Map< private void updateCountersMap(Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { // Summary of all counter by counterId key - String key = generateKey(conditionId, counterId); + String key = generateKeyFromConditionAndCounter(conditionId, counterId); if (countersMap.get(key) == null) { /* initialize if data is not set */ countersMap.put(key, Double.valueOf(0)); @@ -2077,7 +2087,8 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { } } - // remove old records from database + // Remove old statistics from database + cleanupAsVmGroupStatistics(asGroup); } private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { @@ -2133,17 +2144,24 @@ private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { } private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map countersMap, Map countersNumberMap) { + s_logger.debug("Updating countersMap for as group: " + asGroup.getId()); List groupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(asGroup.getId()); for (AutoScaleVmGroupPolicyMapVO groupPolicyVO : groupPolicyVOs) { + s_logger.debug(String.format("Updating countersMap for policy %d in as group %d: ", groupPolicyVO.getPolicyId(), asGroup.getId())); AutoScalePolicyVO vo = _autoScalePolicyDao.findById(groupPolicyVO.getPolicyId()); Map conditionsMap = getConditionsMap(groupPolicyVO.getPolicyId()); for (Long conditionId : conditionsMap.keySet()) { + Date afterDate = new Date(System.currentTimeMillis() - ((long)vo.getDuration() << 10)); + s_logger.debug(String.format("Updating countersMap for condition %d in policy %d in as group %d: ", conditionId, groupPolicyVO.getPolicyId(), asGroup.getId())); + s_logger.debug(String.format("Updating countersMap with stats in %d seconds : between %s and %s", vo.getDuration(), new Date(), afterDate)); CounterVO counter = conditionsMap.get(conditionId); - List stats = _asGroupStatisticsDao.listByVmGroupIdAndCounterId(asGroup.getId(), counter.getId(), new Date(System.currentTimeMillis() - ((long)vo.getDuration() << 10))); + List stats = _asGroupStatisticsDao.listByVmGroupIdAndCounterId(asGroup.getId(), counter.getId(), afterDate); if (CollectionUtils.isEmpty(stats)) { continue; } + s_logger.debug(String.format("Updating countersMap with %d stats", stats.size())); for (AutoScaleVmGroupStatisticsVO stat : stats) { + s_logger.debug(String.format("Updating countersMap with %s (%s): %f, created on %s", counter.getSource(), counter.getValue(), stat.getRawValue(), stat.getCreated())); updateCountersMap(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, stat.getRawValue()); } } @@ -2151,7 +2169,21 @@ private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map } } - private String generateKey(Long conditionId, Long counterId) { + private String generateKeyFromConditionAndCounter(Long conditionId, Long counterId) { return conditionId + "-" + counterId; } + + private void cleanupAsVmGroupStatistics(AutoScaleVmGroup asGroup) { + List groupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(asGroup.getId()); + for (AutoScaleVmGroupPolicyMapVO groupPolicyVO : groupPolicyVOs) { + AutoScalePolicyVO vo = _autoScalePolicyDao.findById(groupPolicyVO.getPolicyId()); + Date beforeDate = new Date(System.currentTimeMillis() - ((long)vo.getDuration() << 10) - autoScaleStatsInterval); + s_logger.debug(String.format("Removing stats for policy %d in as group %d, before %s", groupPolicyVO.getPolicyId(), asGroup.getId(), beforeDate)); + Map conditionsMap = getConditionsMap(groupPolicyVO.getPolicyId()); + for (Long conditionId : conditionsMap.keySet()) { + CounterVO counter = conditionsMap.get(conditionId); + _asGroupStatisticsDao.removeByGroupAndCounter(asGroup.getId(), counter.getId(), beforeDate); + } + } + } } From 1516de16a61c9a2b6077d51bdf7bd34f5eeb3f52 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 9 Jun 2022 13:48:20 +0200 Subject: [PATCH 023/360] AS: update DB in schema-41710to41800.sql --- .../META-INF/db/schema-41710to41800.sql | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41710to41800.sql b/engine/schema/src/main/resources/META-INF/db/schema-41710to41800.sql index 2465dcc37520..77466ca85d71 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41710to41800.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41710to41800.sql @@ -22,4 +22,44 @@ -- Enable CPU cap for default system offerings; UPDATE `cloud`.`service_offering` so SET so.limit_cpu_use = 1 -WHERE so.default_use = 1 AND so.vm_type IN ('domainrouter', 'secondarystoragevm', 'consoleproxy', 'internalloadbalancervm', 'elasticloadbalancervm'); \ No newline at end of file +WHERE so.default_use = 1 AND so.vm_type IN ('domainrouter', 'secondarystoragevm', 'consoleproxy', 'internalloadbalancervm', 'elasticloadbalancervm'); + +-- VM autoscaling + +-- Add column 'provider' and update values + +ALTER TABLE `cloud`.`counter` ADD COLUMN `provider` varchar(255) NOT NULL COMMENT 'Network provider name' AFTER `uuid`; + +UPDATE `cloud`.`counter` SET provider = 'Netscaler'; + +-- Add new counters for VM autoscaling + +INSERT INTO `cloud`.`counter` (uuid, provider, source, name, value, created) VALUES (UUID(), 'VirtualRouter', 'cpu', 'VM CPU - average percentage', 'vm.cpu.average.percentage', NOW()); +INSERT INTO `cloud`.`counter` (uuid, provider, source, name, value, created) VALUES (UUID(), 'VirtualRouter', 'memory', 'VM Memory - average percentage', 'vm.memory.average.percentage', NOW()); +INSERT INTO `cloud`.`counter` (uuid, provider, source, name, value, created) VALUES (UUID(), 'VirtualRouter', 'virtualrouter', 'Virtual Network - Receive (in Bytes)', 'virtual.network.receive', NOW()); +INSERT INTO `cloud`.`counter` (uuid, provider, source, name, value, created) VALUES (UUID(), 'VirtualRouter', 'virtualrouter', 'Virtual Network - Transmit (in Bytes)', 'virtual.network.transmit', NOW()); +INSERT INTO `cloud`.`counter` (uuid, provider, source, name, value, created) VALUES (UUID(), 'VirtualRouter', 'virtualrouter', 'Load Balancer - average connections per vm', 'virtual.network.lb.average.connections', NOW()); + +-- Update autoscale_vmgroups to new state + +UPDATE `cloud`.`autoscale_vmgroups` SET state='New' WHERE state='new'; +UPDATE `cloud`.`autoscale_vmgroups` SET state='Enabled' WHERE state='enabled'; +UPDATE `cloud`.`autoscale_vmgroups` SET state='Disabled' WHERE state='disabled'; +UPDATE `cloud`.`autoscale_vmgroups` SET state='Revoke' WHERE state='revoke'; + +-- Create table for VM autoscaling historic data + +CREATE TABLE `cloud`.`autoscale_vmgroup_statistics` ( + `id` bigint unsigned NOT NULL auto_increment, + `vmgroup_id` bigint unsigned NOT NULL, + `counter_id` bigint unsigned NOT NULL, + `resource_id` bigint unsigned NOT NULL, + `resource_type` varchar(255) NOT NULL, + `raw_value` double NOT NULL, + `created` datetime NOT NULL COMMENT 'Date this data is created', + PRIMARY KEY (`id`), + CONSTRAINT `fk_autoscale_vmgroup_statistics__vmgroup_id` FOREIGN KEY `fk_autoscale_vmgroup_statistics__vmgroup_id` (`vmgroup_id`) REFERENCES `autoscale_vmgroups` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_autoscale_vmgroup_statistics__counter_id` FOREIGN KEY `fk_autoscale_vmgroup_statistics__counter_id` (`counter_id`) REFERENCES `counter` (`id`), + INDEX `i_autoscale_vmgroup_statistics__vmgroup_id`(`vmgroup_id`), + INDEX `i_autoscale_vmgroup_statistics__counter_id`(`counter_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; From e2733f4f8f65d984b69f8eb858647bafc16d668b Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 10 Jun 2022 11:55:43 +0200 Subject: [PATCH 024/360] AS: Add GetAutoScaleMetricsCommand and GetAutoScaleMetricsAnswer --- .../router/VirtualRouterAutoScale.java | 111 ++++++++++++++++++ .../routing/GetAutoScaleMetricsAnswer.java | 48 ++++++++ .../routing/GetAutoScaleMetricsCommand.java | 40 +++++++ .../network/as/AutoScaleManagerImpl.java | 28 +++-- .../network/element/VirtualRouterElement.java | 16 --- 5 files changed, 215 insertions(+), 28 deletions(-) create mode 100644 api/src/main/java/com/cloud/network/router/VirtualRouterAutoScale.java create mode 100644 core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsAnswer.java create mode 100644 core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java diff --git a/api/src/main/java/com/cloud/network/router/VirtualRouterAutoScale.java b/api/src/main/java/com/cloud/network/router/VirtualRouterAutoScale.java new file mode 100644 index 000000000000..28b8cc50b118 --- /dev/null +++ b/api/src/main/java/com/cloud/network/router/VirtualRouterAutoScale.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.router; + +/** + * bridge internal and external traffic. + */ +public interface VirtualRouterAutoScale { + + enum AutoScaleCounter { + NetworkReceive ("virtual.network.receive"), + NetworkTransmit ("virtual.network.transmit"), + LbAverageConnections ("virtual.network.lb.average.connections"); + + String _value; + AutoScaleCounter(String value) { + _value = value; + } + + String getValue() { + return _value; + } + public AutoScaleCounter fromValue(String value) { + AutoScaleCounter[] values = AutoScaleCounter.values(); + for(AutoScaleCounter v : values) { + if(v.getValue().equals(value)) { + return v; + } + } + return null; + } + + @Override + public String toString() { + return _value; + } + } + + enum AutoScaleValueType { + AVERAGE, INSTANT, AGGREGATED + } + + public class AutoScaleMetrics { + AutoScaleCounter counter; + Long conditionId; + Long counterId; + Integer duration; + + public AutoScaleMetrics(AutoScaleCounter counter, Long conditionId, Long counterId, Integer duration) { + this.counter = counter; + this.conditionId = conditionId; + this.counterId = counterId; + this.duration = duration; + } + + public AutoScaleCounter getCounter() { + return counter; + } + + public Long getConditionId() { + return conditionId; + } + + public Long getCounterId() { + return counterId; + } + + public Integer getDuration() { + return duration; + } + } + + public class AutoScaleMetricsValue { + AutoScaleMetrics metrics; + AutoScaleValueType type; + Double value; + + public AutoScaleMetricsValue(AutoScaleMetrics metrics, AutoScaleValueType type, Double value) { + this.metrics = metrics; + this.type = type; + this.value = value; + } + + public AutoScaleMetrics getMetrics() { + return metrics; + } + + public AutoScaleValueType getType() { + return type; + } + + public Double getValue() { + return value; + } + } + +} diff --git a/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsAnswer.java b/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsAnswer.java new file mode 100644 index 000000000000..b0583a1a94c7 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsAnswer.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api.routing; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.network.router.VirtualRouterAutoScale.AutoScaleMetricsValue; + +import java.util.ArrayList; +import java.util.List; + +public class GetAutoScaleMetricsAnswer extends Answer { + private List values; + + public GetAutoScaleMetricsAnswer(Command cmd, boolean success) { + super(cmd, success, null); + this.values = new ArrayList<>(); + } + + public GetAutoScaleMetricsAnswer(Command cmd, boolean success, List values) { + super(cmd, success, null); + this.values = values; + } + + + public void addValue(AutoScaleMetricsValue value) { + this.values.add(value); + } + + public List getValues() { + return values; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java b/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java new file mode 100644 index 000000000000..06b168513811 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api.routing; + +import com.cloud.network.router.VirtualRouterAutoScale.AutoScaleMetrics; + +import java.util.List; + +public class GetAutoScaleMetricsCommand extends NetworkElementCommand { + + private List metrics; + + @Override + public boolean isQuery() { + return true; + } + + protected GetAutoScaleMetricsCommand(List metrics) { + this.metrics = metrics; + } + + public List getMetrics() { + return metrics; + } +} \ No newline at end of file diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 6b243ba5c104..b2046c520076 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -2054,25 +2054,17 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { if (is_native(asGroup.getId())) { s_logger.debug("[AutoScale] Collecting performance data from hosts ..."); - getVmStatsFromHosts(asGroup); - - updateCountersMap(asGroup, countersMap, countersNumberMap); } if (has_source_virtual_router(asGroup.getId())) { s_logger.debug("[AutoScale] Collecting performance data from virtual router ..."); - - // create PerformanceMonitorCommand for the host where is virtual router is running on - - // process PerformanceMonitorAnswer from the host - // 1. update database (aggregation or average or instant) - // 2. get data in this period - // average: do nothing - // aggregation: end - start - // instant: calculate the average + getVmStatsFromVirtualRouter(asGroup); } + // update counter maps in memory + updateCountersMap(asGroup, countersMap, countersNumberMap); + // get scale action Map params = new HashMap(); setPerformanceMonitorCommandParams(asGroup, params); @@ -2143,6 +2135,18 @@ private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { } } + private void getVmStatsFromVirtualRouter(AutoScaleVmGroupVO asGroup) { + // create GetAutoScaleMetricsCommand for the host where is virtual router is running on + + // process GetAutoScaleMetricsAnswer from the host + + // 1. update database (aggregation or average or instant) + // 2. get data in this period + // average: do nothing + // aggregation: end - start + // instant: calculate the average + } + private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map countersMap, Map countersNumberMap) { s_logger.debug("Updating countersMap for as group: " + asGroup.getId()); List groupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(asGroup.getId()); diff --git a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java index e9741d451278..1ad96d6ff77b 100644 --- a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java @@ -1401,20 +1401,4 @@ private void updateToFailedState(Network network){ _routerDao.persist(router); } } - - enum AutoScaleCounterVirtualRouterValue { - NetworkReceive ("virtual.network.receive"), - NetworkTransmit ("virtual.network.transmit"), - LbAverageConnections ("virtual.network.lb.average.connections"); - - String _value; - AutoScaleCounterVirtualRouterValue(String value) { - _value = value; - } - @Override - public String toString() { - return _value; - } - } - } From 825416d59e02106a4d248171f3726378d764937e Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 10 Jun 2022 12:42:18 +0200 Subject: [PATCH 025/360] AS: add global config 'autoscale.stats.cleanup.delay' --- .../main/java/com/cloud/network/as/AutoScaleManager.java | 9 ++++++++- .../java/com/cloud/network/as/AutoScaleManagerImpl.java | 6 ++++-- .../src/main/java/com/cloud/server/StatsCollector.java | 6 ++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManager.java b/server/src/main/java/com/cloud/network/as/AutoScaleManager.java index 193df1e0dc61..7f71ef46a8e2 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManager.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManager.java @@ -23,7 +23,14 @@ public interface AutoScaleManager extends AutoScaleService { ConfigKey AutoScaleStatsInterval = new ConfigKey<>("Advanced", Integer.class, "autoscale.stats.interval", "0", - "The interval (in milliseconds) when VM auto scaling stats retrieved from agents and/or other resources. Less than 1 means disabled.", + "The interval (in milliseconds) when VM auto scaling statistics are retrieved from hypervisors and/or other resources. Less than 1 means disabled.", + false); + + ConfigKey AutoScaleStatsCleanupDelay = new ConfigKey<>(Integer.class, + "autoscale.stats.cleanup.delay", + "Advanced", + "86400", + "Determines how long (in seconds) to wait before actually removing auto scaling statistics from database.", false); void cleanUpAutoScaleResources(Long accountId); diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index b2046c520076..fb30c7fcc382 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1633,7 +1633,8 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] { - AutoScaleStatsInterval + AutoScaleStatsInterval, + AutoScaleStatsCleanupDelay }; } @@ -2137,6 +2138,7 @@ private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { private void getVmStatsFromVirtualRouter(AutoScaleVmGroupVO asGroup) { // create GetAutoScaleMetricsCommand for the host where is virtual router is running on + // which virtual router ? PRIMARY or both ? // process GetAutoScaleMetricsAnswer from the host @@ -2181,7 +2183,7 @@ private void cleanupAsVmGroupStatistics(AutoScaleVmGroup asGroup) { List groupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(asGroup.getId()); for (AutoScaleVmGroupPolicyMapVO groupPolicyVO : groupPolicyVOs) { AutoScalePolicyVO vo = _autoScalePolicyDao.findById(groupPolicyVO.getPolicyId()); - Date beforeDate = new Date(System.currentTimeMillis() - ((long)vo.getDuration() << 10) - autoScaleStatsInterval); + Date beforeDate = new Date(System.currentTimeMillis() - ((long)AutoScaleStatsCleanupDelay.value() << 10)); s_logger.debug(String.format("Removing stats for policy %d in as group %d, before %s", groupPolicyVO.getPolicyId(), asGroup.getId(), beforeDate)); Map conditionsMap = getConditionsMap(groupPolicyVO.getPolicyId()); for (Long conditionId : conditionsMap.keySet()) { diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index d9eb163b5026..66af56b9579b 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -1732,6 +1732,12 @@ protected void runInContext() { if (LOGGER.isDebugEnabled()) { LOGGER.debug("AutoScaling Monitor is running..."); } + //msHost in UP state with min id should run the job + ManagementServerHostVO msHost = managementServerHostDao.findOneInUpState(new Filter(ManagementServerHostVO.class, "id", true, 0L, 1L)); + if (msHost == null || (msHost.getMsid() != mgmtSrvrId)) { + LOGGER.debug("Skipping AutoScaling Monitor"); + return; + } // list all AS VMGroups List asGroups = _asGroupDao.listAll(); for (AutoScaleVmGroupVO asGroup : asGroups) { From 5603e361c03b537e9ca0792898234f15d2c59515 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 10 Jun 2022 12:59:18 +0200 Subject: [PATCH 026/360] AS: add value_type to autoscale_vmgroup_statistics --- .../network/as/AutoScaleVmGroupStatisticsVO.java | 13 ++++++++++++- .../resources/META-INF/db/schema-41710to41800.sql | 1 + .../com/cloud/network/as/AutoScaleManagerImpl.java | 7 +++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupStatisticsVO.java b/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupStatisticsVO.java index 6400cafc05f4..448d5ee613a0 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupStatisticsVO.java +++ b/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupStatisticsVO.java @@ -29,6 +29,7 @@ import javax.persistence.Temporal; import javax.persistence.TemporalType; +import com.cloud.network.router.VirtualRouterAutoScale; import com.cloud.server.ResourceTag; import org.apache.cloudstack.api.InternalIdentity; @@ -56,6 +57,10 @@ public class AutoScaleVmGroupStatisticsVO implements InternalIdentity { @Column(name = "raw_value") private Double rawValue = null; + @Column(name = "value_type") + @Enumerated(value = EnumType.STRING) + private VirtualRouterAutoScale.AutoScaleValueType valueType; + @Column(name = "created") @Temporal(value = TemporalType.TIMESTAMP) private Date created = null; @@ -63,12 +68,14 @@ public class AutoScaleVmGroupStatisticsVO implements InternalIdentity { public AutoScaleVmGroupStatisticsVO() { } - public AutoScaleVmGroupStatisticsVO(long vmGroupId, long counterId, long resourceId, ResourceTag.ResourceObjectType resourceType, Double rawValue, Date created) { + public AutoScaleVmGroupStatisticsVO(long vmGroupId, long counterId, long resourceId, ResourceTag.ResourceObjectType resourceType, + Double rawValue, VirtualRouterAutoScale.AutoScaleValueType valueType, Date created) { this.vmGroupId = vmGroupId; this.counterId = counterId; this.resourceId = resourceId; this.resourceType = resourceType; this.rawValue = rawValue; + this.valueType = valueType; this.created = created; } @@ -97,6 +104,10 @@ public Double getRawValue() { return rawValue; } + public VirtualRouterAutoScale.AutoScaleValueType getValueType() { + return valueType; + } + public Date getCreated() { return created; } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41710to41800.sql b/engine/schema/src/main/resources/META-INF/db/schema-41710to41800.sql index 77466ca85d71..47e123c70da0 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41710to41800.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41710to41800.sql @@ -56,6 +56,7 @@ CREATE TABLE `cloud`.`autoscale_vmgroup_statistics` ( `resource_id` bigint unsigned NOT NULL, `resource_type` varchar(255) NOT NULL, `raw_value` double NOT NULL, + `value_type` varchar(255) NOT NULL, `created` datetime NOT NULL COMMENT 'Date this data is created', PRIMARY KEY (`id`), CONSTRAINT `fk_autoscale_vmgroup_statistics__vmgroup_id` FOREIGN KEY `fk_autoscale_vmgroup_statistics__vmgroup_id` (`vmgroup_id`) REFERENCES `autoscale_vmgroups` (`id`) ON DELETE CASCADE, diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index fb30c7fcc382..52c2d19c62cc 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -31,6 +31,7 @@ import javax.inject.Inject; +import com.cloud.network.router.VirtualRouterAutoScale; import com.cloud.offering.DiskOffering; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -2121,10 +2122,12 @@ private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { for (CounterVO counter : counters) { if (Counter.Source.cpu.equals(counter.getSource())) { Double counterValue = vmStats.getCPUUtilization() / 100; - _asGroupStatisticsDao.persist(new AutoScaleVmGroupStatisticsVO(asGroup.getId(), counter.getId(), vmId, ResourceTag.ResourceObjectType.UserVm, counterValue, timestamp)); + _asGroupStatisticsDao.persist(new AutoScaleVmGroupStatisticsVO(asGroup.getId(), counter.getId(), vmId, ResourceTag.ResourceObjectType.UserVm, + counterValue, VirtualRouterAutoScale.AutoScaleValueType.INSTANT, timestamp)); } else if (Counter.Source.memory.equals(counter.getSource())) { Double counterValue = vmStats.getMemoryKBs() / 1024; - _asGroupStatisticsDao.persist(new AutoScaleVmGroupStatisticsVO(asGroup.getId(), counter.getId(), vmId, ResourceTag.ResourceObjectType.UserVm, counterValue, timestamp)); + _asGroupStatisticsDao.persist(new AutoScaleVmGroupStatisticsVO(asGroup.getId(), counter.getId(), vmId, ResourceTag.ResourceObjectType.UserVm, + counterValue, VirtualRouterAutoScale.AutoScaleValueType.INSTANT, timestamp)); } } } From 9844fa854f34edf6d1598990f2272d0a81b673a0 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 10 Jun 2022 15:46:23 +0200 Subject: [PATCH 027/360] AS: setup GetAutoScaleMetricsCommand --- .../router/VirtualRouterAutoScale.java | 16 ++--- .../routing/GetAutoScaleMetricsCommand.java | 2 +- .../VirtualRoutingResource.java | 8 +++ .../network/as/AutoScaleManagerImpl.java | 69 ++++++++++++++++--- 4 files changed, 78 insertions(+), 17 deletions(-) diff --git a/api/src/main/java/com/cloud/network/router/VirtualRouterAutoScale.java b/api/src/main/java/com/cloud/network/router/VirtualRouterAutoScale.java index 28b8cc50b118..aebd1518f018 100644 --- a/api/src/main/java/com/cloud/network/router/VirtualRouterAutoScale.java +++ b/api/src/main/java/com/cloud/network/router/VirtualRouterAutoScale.java @@ -21,22 +21,22 @@ */ public interface VirtualRouterAutoScale { - enum AutoScaleCounter { + enum VirtualRouterAutoScaleCounter { NetworkReceive ("virtual.network.receive"), NetworkTransmit ("virtual.network.transmit"), LbAverageConnections ("virtual.network.lb.average.connections"); String _value; - AutoScaleCounter(String value) { + VirtualRouterAutoScaleCounter(String value) { _value = value; } String getValue() { return _value; } - public AutoScaleCounter fromValue(String value) { - AutoScaleCounter[] values = AutoScaleCounter.values(); - for(AutoScaleCounter v : values) { + public static VirtualRouterAutoScaleCounter fromValue(String value) { + VirtualRouterAutoScaleCounter[] values = VirtualRouterAutoScaleCounter.values(); + for(VirtualRouterAutoScaleCounter v : values) { if(v.getValue().equals(value)) { return v; } @@ -55,19 +55,19 @@ enum AutoScaleValueType { } public class AutoScaleMetrics { - AutoScaleCounter counter; + VirtualRouterAutoScaleCounter counter; Long conditionId; Long counterId; Integer duration; - public AutoScaleMetrics(AutoScaleCounter counter, Long conditionId, Long counterId, Integer duration) { + public AutoScaleMetrics(VirtualRouterAutoScaleCounter counter, Long conditionId, Long counterId, Integer duration) { this.counter = counter; this.conditionId = conditionId; this.counterId = counterId; this.duration = duration; } - public AutoScaleCounter getCounter() { + public VirtualRouterAutoScaleCounter getCounter() { return counter; } diff --git a/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java b/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java index 06b168513811..3efac08f42b4 100644 --- a/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java +++ b/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java @@ -30,7 +30,7 @@ public boolean isQuery() { return true; } - protected GetAutoScaleMetricsCommand(List metrics) { + public GetAutoScaleMetricsCommand(List metrics) { this.metrics = metrics; } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 8a265a78123b..992acc221fd7 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -59,6 +59,8 @@ import com.cloud.agent.api.GetRouterAlertsAnswer; import com.cloud.agent.api.routing.AggregationControlCommand; import com.cloud.agent.api.routing.AggregationControlCommand.Action; +import com.cloud.agent.api.routing.GetAutoScaleMetricsAnswer; +import com.cloud.agent.api.routing.GetAutoScaleMetricsCommand; import com.cloud.agent.api.routing.GetRouterAlertsCommand; import com.cloud.agent.api.routing.GetRouterMonitorResultsAnswer; import com.cloud.agent.api.routing.GetRouterMonitorResultsCommand; @@ -210,6 +212,8 @@ private Answer executeQueryCommand(NetworkElementCommand cmd) { return execute((DeleteFileInVrCommand)cmd); } else if (cmd instanceof GetRouterMonitorResultsCommand) { return execute((GetRouterMonitorResultsCommand)cmd); + } else if (cmd instanceof GetAutoScaleMetricsCommand) { + return execute((GetAutoScaleMetricsCommand)cmd); } else { s_logger.error("Unknown query command in VirtualRoutingResource!"); return Answer.createUnsupportedCommandAnswer(cmd); @@ -287,6 +291,10 @@ private List getFailingChecks(String line) { return failingChecks; } + private GetAutoScaleMetricsAnswer execute(GetAutoScaleMetricsCommand cmd) { + return new GetAutoScaleMetricsAnswer(cmd, true); + } + private GetRouterMonitorResultsAnswer parseLinesForHealthChecks(GetRouterMonitorResultsCommand cmd, String executionResult) { List failingChecks = new ArrayList<>(); StringBuilder monitorResults = new StringBuilder(); diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 52c2d19c62cc..99696e365139 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -31,8 +31,6 @@ import javax.inject.Inject; -import com.cloud.network.router.VirtualRouterAutoScale; -import com.cloud.offering.DiskOffering; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -70,6 +68,9 @@ import com.cloud.agent.api.PerformanceMonitorAnswer; import com.cloud.agent.api.PerformanceMonitorCommand; import com.cloud.agent.api.VmStatsEntry; +import com.cloud.agent.api.routing.GetAutoScaleMetricsAnswer; +import com.cloud.agent.api.routing.GetAutoScaleMetricsCommand; +import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.api.ApiDBUtils; import com.cloud.api.dispatch.DispatchChainFactory; import com.cloud.api.dispatch.DispatchTask; @@ -110,6 +111,12 @@ import com.cloud.network.dao.NetworkDao; import com.cloud.network.lb.LoadBalancingRulesManager; import com.cloud.network.lb.LoadBalancingRulesService; +import com.cloud.network.router.RouterControlHelper; +import com.cloud.network.router.VirtualRouter; +import com.cloud.network.router.VirtualRouterAutoScale.AutoScaleMetrics; +import com.cloud.network.router.VirtualRouterAutoScale.AutoScaleValueType; +import com.cloud.network.router.VirtualRouterAutoScale.VirtualRouterAutoScaleCounter; +import com.cloud.offering.DiskOffering; import com.cloud.offering.ServiceOffering; import com.cloud.projects.Project.ListProjectResourcesCriteria; import com.cloud.server.ResourceTag; @@ -142,10 +149,13 @@ import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; +import com.cloud.vm.DomainRouterVO; import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmService; import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; @@ -223,6 +233,10 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale private HostDao _hostDao; @Inject private AutoScaleVmGroupStatisticsDao _asGroupStatisticsDao; + @Inject + private DomainRouterDao _routerDao; + @Inject + protected RouterControlHelper _routerControlHelper; private long autoScaleStatsInterval = -1L; private static final Long ONE_MINUTE_IN_MILLISCONDS = 60000L; @@ -2123,11 +2137,11 @@ private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { if (Counter.Source.cpu.equals(counter.getSource())) { Double counterValue = vmStats.getCPUUtilization() / 100; _asGroupStatisticsDao.persist(new AutoScaleVmGroupStatisticsVO(asGroup.getId(), counter.getId(), vmId, ResourceTag.ResourceObjectType.UserVm, - counterValue, VirtualRouterAutoScale.AutoScaleValueType.INSTANT, timestamp)); + counterValue, AutoScaleValueType.INSTANT, timestamp)); } else if (Counter.Source.memory.equals(counter.getSource())) { Double counterValue = vmStats.getMemoryKBs() / 1024; _asGroupStatisticsDao.persist(new AutoScaleVmGroupStatisticsVO(asGroup.getId(), counter.getId(), vmId, ResourceTag.ResourceObjectType.UserVm, - counterValue, VirtualRouterAutoScale.AutoScaleValueType.INSTANT, timestamp)); + counterValue, AutoScaleValueType.INSTANT, timestamp)); } } } @@ -2140,10 +2154,27 @@ private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { } private void getVmStatsFromVirtualRouter(AutoScaleVmGroupVO asGroup) { - // create GetAutoScaleMetricsCommand for the host where is virtual router is running on - // which virtual router ? PRIMARY or both ? - - // process GetAutoScaleMetricsAnswer from the host + Network network = getNetwork(asGroup); + List routers = _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER); + if (CollectionUtils.isEmpty(routers)) { + return; + } + List metrics = setGetAutoScaleMetricsCommandMetrics(asGroup); + final GetAutoScaleMetricsCommand command = new GetAutoScaleMetricsCommand(metrics); + for (DomainRouterVO router : routers) { + if (VirtualMachine.State.Running.equals(router.getState())) { + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId())); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + command.setWait(30); + GetAutoScaleMetricsAnswer answer = (GetAutoScaleMetricsAnswer) _agentMgr.easySend(router.getHostId(), command); + if (answer == null || !answer.getResult()) { + s_logger.error("Failed to get autoscale metrics from virtual router " + router.getName()); + continue; + } + // process GetAutoScaleMetricsAnswer from the host + // TODO + } + } // 1. update database (aggregation or average or instant) // 2. get data in this period @@ -2152,6 +2183,28 @@ private void getVmStatsFromVirtualRouter(AutoScaleVmGroupVO asGroup) { // instant: calculate the average } + private List setGetAutoScaleMetricsCommandMetrics(AutoScaleVmGroupVO asGroup) { + List metrics = new ArrayList<>(); + List groupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(asGroup.getId()); + for (AutoScaleVmGroupPolicyMapVO groupPolicyVO : groupPolicyVOs) { + AutoScalePolicyVO vo = _autoScalePolicyDao.findById(groupPolicyVO.getPolicyId()); + Map conditionsMap = getConditionsMap(groupPolicyVO.getPolicyId()); + for (Long conditionId : conditionsMap.keySet()) { + CounterVO counter = conditionsMap.get(conditionId); + String provider = counter.getProvider(); + if (! Network.Provider.VirtualRouter.getName().equals(provider) && ! Network.Provider.VPCVirtualRouter.getName().equals(provider)) { + continue; + } + VirtualRouterAutoScaleCounter vrCounter = VirtualRouterAutoScaleCounter.fromValue(counter.getValue()); + if (vrCounter == null) { + continue; + } + metrics.add(new AutoScaleMetrics(vrCounter, conditionId, counter.getId(), vo.getDuration())); + } + } + return metrics; + } + private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map countersMap, Map countersNumberMap) { s_logger.debug("Updating countersMap for as group: " + asGroup.getId()); List groupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(asGroup.getId()); From 4d2c2aacf4f2a5f10c4659a5144f3159914c19cc Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 10 Jun 2022 20:50:55 +0200 Subject: [PATCH 028/360] AS: add processGetAutoScaleMetricsAnswer --- .../java/com/cloud/server/ResourceTag.java | 1 + .../network/as/AutoScaleManagerImpl.java | 60 +++++++++++++++---- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/api/src/main/java/com/cloud/server/ResourceTag.java b/api/src/main/java/com/cloud/server/ResourceTag.java index ba5ab8ad0ff5..d502b6f9d81b 100644 --- a/api/src/main/java/com/cloud/server/ResourceTag.java +++ b/api/src/main/java/com/cloud/server/ResourceTag.java @@ -31,6 +31,7 @@ public enum ResourceObjectType { Snapshot(true, false), Backup(true, false), Network(true, true, true), + DomainRouter(false, false), Nic(false, true), LoadBalancer(true, true), PortForwardingRule(true, true), diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 99696e365139..93497b2a4b42 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -32,6 +32,7 @@ import javax.inject.Inject; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; import com.google.gson.Gson; @@ -114,6 +115,7 @@ import com.cloud.network.router.RouterControlHelper; import com.cloud.network.router.VirtualRouter; import com.cloud.network.router.VirtualRouterAutoScale.AutoScaleMetrics; +import com.cloud.network.router.VirtualRouterAutoScale.AutoScaleMetricsValue; import com.cloud.network.router.VirtualRouterAutoScale.AutoScaleValueType; import com.cloud.network.router.VirtualRouterAutoScale.VirtualRouterAutoScaleCounter; import com.cloud.offering.DiskOffering; @@ -2075,7 +2077,7 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { if (has_source_virtual_router(asGroup.getId())) { s_logger.debug("[AutoScale] Collecting performance data from virtual router ..."); - getVmStatsFromVirtualRouter(asGroup); + getNetworkStatsFromVirtualRouter(asGroup); } // update counter maps in memory @@ -2153,7 +2155,7 @@ private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { } } - private void getVmStatsFromVirtualRouter(AutoScaleVmGroupVO asGroup) { + private void getNetworkStatsFromVirtualRouter(AutoScaleVmGroupVO asGroup) { Network network = getNetwork(asGroup); List routers = _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER); if (CollectionUtils.isEmpty(routers)) { @@ -2171,16 +2173,9 @@ private void getVmStatsFromVirtualRouter(AutoScaleVmGroupVO asGroup) { s_logger.error("Failed to get autoscale metrics from virtual router " + router.getName()); continue; } - // process GetAutoScaleMetricsAnswer from the host - // TODO + processGetAutoScaleMetricsAnswer(asGroup, answer.getValues(), router.getId()); } } - - // 1. update database (aggregation or average or instant) - // 2. get data in this period - // average: do nothing - // aggregation: end - start - // instant: calculate the average } private List setGetAutoScaleMetricsCommandMetrics(AutoScaleVmGroupVO asGroup) { @@ -2205,6 +2200,15 @@ private List setGetAutoScaleMetricsCommandMetrics(AutoScaleVmG return metrics; } + private void processGetAutoScaleMetricsAnswer(AutoScaleVmGroupVO asGroup, List values, Long routerId) { + Date timestamp = new Date(); + for (AutoScaleMetricsValue value : values) { + AutoScaleMetrics metrics = value.getMetrics(); + _asGroupStatisticsDao.persist(new AutoScaleVmGroupStatisticsVO(asGroup.getId(), metrics.getCounterId(), routerId, ResourceTag.ResourceObjectType.DomainRouter, + value.getValue(), value.getType(), timestamp)); + } + } + private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map countersMap, Map countersNumberMap) { s_logger.debug("Updating countersMap for as group: " + asGroup.getId()); List groupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(asGroup.getId()); @@ -2222,9 +2226,41 @@ private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map continue; } s_logger.debug(String.format("Updating countersMap with %d stats", stats.size())); + Map> aggregatedRecords = new HashMap<>(); + List incorrectRecords = new ArrayList<>(); for (AutoScaleVmGroupStatisticsVO stat : stats) { - s_logger.debug(String.format("Updating countersMap with %s (%s): %f, created on %s", counter.getSource(), counter.getValue(), stat.getRawValue(), stat.getCreated())); - updateCountersMap(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, stat.getRawValue()); + if (AutoScaleValueType.INSTANT.equals(stat.getValueType())) { + s_logger.debug(String.format("Updating countersMap with %s (%s): %f, created on %s", counter.getSource(), counter.getValue(), stat.getRawValue(), stat.getCreated())); + updateCountersMap(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, stat.getRawValue()); + } else if (AutoScaleValueType.AGGREGATED.equals(stat.getValueType())) { + String key = stat.getCounterId() + "-" + stat.getResourceId(); + if (incorrectRecords.contains(key)) { + continue; + } + if (!aggregatedRecords.containsKey(key)) { + List aggregatedRecordList = new ArrayList<>(); + aggregatedRecordList.add(stat); + aggregatedRecords.put(key, aggregatedRecordList); + continue; + } + List aggregatedRecordList = aggregatedRecords.get(key); + AutoScaleVmGroupStatisticsVO lastRecord = aggregatedRecordList.get(aggregatedRecordList.size() - 1); + if (stat.getCreated().after(lastRecord.getCreated())) { + if (stat.getRawValue() >= lastRecord.getRawValue()) { + aggregatedRecordList.add(stat); + } else { + s_logger.info("The new raw value is less than the previous raw value, which means the data is incorrect. The key is " + key); + aggregatedRecords.remove(key); + incorrectRecords.add(key); + } + } + } + } + if (MapUtils.isNotEmpty(aggregatedRecords)) { + s_logger.debug("Processing aggregated data"); + for (List records : aggregatedRecords.values()) { + //TODO + } } } From 34ac6ecfb4b4864805698936436ff72a82123489 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 10 Jun 2022 22:01:06 +0200 Subject: [PATCH 029/360] AS: update GetAutoScaleMetricsCommand --- .../routing/GetAutoScaleMetricsCommand.java | 20 ++++++++++++++++++- .../network/as/AutoScaleManagerImpl.java | 16 ++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java b/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java index 3efac08f42b4..d8e67362c098 100644 --- a/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java +++ b/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java @@ -23,6 +23,9 @@ public class GetAutoScaleMetricsCommand extends NetworkElementCommand { + private String privateIP; + boolean forVpc = false; + private String publicIP; private List metrics; @Override @@ -30,10 +33,25 @@ public boolean isQuery() { return true; } - public GetAutoScaleMetricsCommand(List metrics) { + public GetAutoScaleMetricsCommand(String privateIP, boolean forVpc, String publicIP, List metrics) { + this.privateIP = privateIP; + this.forVpc = forVpc; + this.publicIP = publicIP; this.metrics = metrics; } + public String getPrivateIP() { + return privateIP; + } + + public boolean isForVpc() { + return forVpc; + } + + public String getPublicIP() { + return publicIP; + } + public List getMetrics() { return metrics; } diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 93497b2a4b42..18edacc48454 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -105,6 +105,7 @@ import com.cloud.network.as.dao.ConditionDao; import com.cloud.network.as.dao.CounterDao; import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVMMapDao; import com.cloud.network.dao.LoadBalancerVMMapVO; @@ -1889,6 +1890,18 @@ private Network getNetwork(AutoScaleVmGroup asGroup) { return network; } + private String getPublicIp(AutoScaleVmGroup asGroup) { + final LoadBalancerVO loadBalancer = _lbDao.findById(asGroup.getLoadBalancerId()); + if (loadBalancer == null) { + throw new CloudRuntimeException(String.format("Unable to find load balancer with id: % ", asGroup.getLoadBalancerId())); + } + IPAddressVO ipAddress = _ipAddressDao.findById(loadBalancer.getSourceIpAddressId()); + if (ipAddress == null) { + throw new CloudRuntimeException(String.format("Unable to find IP Address with id: % ", loadBalancer.getSourceIpAddressId())); + } + return ipAddress.getAddress().addr(); + } + private Network.Provider getLoadBalancerServiceProvider(AutoScaleVmGroup asGroup) { Network network = getNetwork(asGroup); List providers = _networkMgr.getProvidersForServiceInNetwork(network, Network.Service.Lb); @@ -2157,14 +2170,15 @@ private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { private void getNetworkStatsFromVirtualRouter(AutoScaleVmGroupVO asGroup) { Network network = getNetwork(asGroup); + String publicIpAddr = getPublicIp(asGroup); List routers = _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER); if (CollectionUtils.isEmpty(routers)) { return; } List metrics = setGetAutoScaleMetricsCommandMetrics(asGroup); - final GetAutoScaleMetricsCommand command = new GetAutoScaleMetricsCommand(metrics); for (DomainRouterVO router : routers) { if (VirtualMachine.State.Running.equals(router.getState())) { + final GetAutoScaleMetricsCommand command = new GetAutoScaleMetricsCommand(router.getPrivateIpAddress(), network.getVpcId() != null, publicIpAddr, metrics); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); command.setWait(30); From 18001123b03c072c1388ec06cdd1682e7cb85511 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 13 Jun 2022 09:58:24 +0200 Subject: [PATCH 030/360] AS: get haproxy stats in VR --- .../routing/GetAutoScaleMetricsCommand.java | 8 +- .../cloud/network/HAProxyConfigurator.java | 2 +- .../resource/LibvirtComputingResource.java | 29 +++++++ ...virtGetAutoScaleMetricsCommandWrapper.java | 75 +++++++++++++++++++ .../network/as/AutoScaleManagerImpl.java | 8 +- .../debian/opt/cloud/bin/get_haproxy_stats.sh | 29 +++++++ 6 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetAutoScaleMetricsCommandWrapper.java create mode 100755 systemvm/debian/opt/cloud/bin/get_haproxy_stats.sh diff --git a/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java b/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java index d8e67362c098..27c2a19eb825 100644 --- a/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java +++ b/core/src/main/java/com/cloud/agent/api/routing/GetAutoScaleMetricsCommand.java @@ -26,6 +26,7 @@ public class GetAutoScaleMetricsCommand extends NetworkElementCommand { private String privateIP; boolean forVpc = false; private String publicIP; + private Integer port; private List metrics; @Override @@ -33,10 +34,11 @@ public boolean isQuery() { return true; } - public GetAutoScaleMetricsCommand(String privateIP, boolean forVpc, String publicIP, List metrics) { + public GetAutoScaleMetricsCommand(String privateIP, boolean forVpc, String publicIP, Integer port, List metrics) { this.privateIP = privateIP; this.forVpc = forVpc; this.publicIP = publicIP; + this.port = port; this.metrics = metrics; } @@ -52,6 +54,10 @@ public String getPublicIP() { return publicIP; } + public Integer getPort() { + return port; + } + public List getMetrics() { return metrics; } diff --git a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java index 7bf3bb8ca57f..00db651c2410 100644 --- a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java +++ b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java @@ -43,7 +43,7 @@ public class HAProxyConfigurator implements LoadBalancerConfigurator { private static final Logger s_logger = Logger.getLogger(HAProxyConfigurator.class); private static final String blankLine = "\t "; private static String[] globalSection = {"global", "\tlog 127.0.0.1:3914 local0 warning", "\tmaxconn 4096", "\tmaxpipes 1024", "\tchroot /var/lib/haproxy", - "\tuser haproxy", "\tgroup haproxy", "\tdaemon"}; + "\tuser haproxy", "\tgroup haproxy", "\tstats socket /run/haproxy/admin.sock", "\tdaemon"}; private static String[] defaultsSection = {"defaults", "\tlog global", "\tmode tcp", "\toption dontlognull", "\tretries 3", "\toption redispatch", "\toption forwardfor", "\toption forceclose", "\ttimeout connect 5000", "\ttimeout client 50000", "\ttimeout server 50000"}; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 37db67257f98..f43bfe64fcd7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -2289,6 +2289,35 @@ public long[] getNetworkStats(final String privateIP) { return stats; } + public String getHaproxyStats(final String privateIP, final String publicIp, final Integer port) { + final Script getHaproxyStatsScript = new Script(_routerProxyPath, s_logger); + getHaproxyStatsScript.add("get_haproxy_stats.sh"); + getHaproxyStatsScript.add(privateIP); + getHaproxyStatsScript.add(publicIp); + getHaproxyStatsScript.add(String.valueOf(port)); + + final OutputInterpreter.OneLineParser statsParser = new OutputInterpreter.OneLineParser(); + final String result = getHaproxyStatsScript.execute(statsParser); + if (result != null) { + s_logger.debug("Failed to execute haproxy stats:" + result); + return null; + } + return statsParser.getLine(); + } + + public long[] getNetworkLbStats(final String privateIP, final String publicIp, final Integer port) { + final String result = getHaproxyStats(privateIP, publicIp, port); + final long[] stats = new long[1]; + if (result != null) { + final String[] splitResult = result.split(","); + int i = 0; + while (i < splitResult.length - 1) { + stats[0] += Long.parseLong(splitResult[i++]); + } + } + return stats; + } + public String configureVPCNetworkUsage(final String privateIpAddress, final String publicIp, final String option, final String vpcCIDR) { final Script getUsage = new Script(_routerProxyPath, s_logger); getUsage.add("vpc_netusage.sh"); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetAutoScaleMetricsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetAutoScaleMetricsCommandWrapper.java new file mode 100644 index 000000000000..da05a8ffaeed --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetAutoScaleMetricsCommandWrapper.java @@ -0,0 +1,75 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.routing.GetAutoScaleMetricsAnswer; +import com.cloud.agent.api.routing.GetAutoScaleMetricsCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.network.router.VirtualRouterAutoScale.AutoScaleMetrics; +import com.cloud.network.router.VirtualRouterAutoScale.AutoScaleMetricsValue; +import com.cloud.network.router.VirtualRouterAutoScale.AutoScaleValueType; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; + +import java.util.ArrayList; +import java.util.List; + +@ResourceWrapper(handles = GetAutoScaleMetricsCommand.class) +public final class LibvirtGetAutoScaleMetricsCommandWrapper extends CommandWrapper { + + final static String NETWORK_USAGE_GET_OPTION = "get"; + + @Override + public Answer execute(final GetAutoScaleMetricsCommand command, final LibvirtComputingResource libvirtComputingResource) { + + Long bytesSent; + Long bytesReceived; + if (command.isForVpc()) { + final long[] stats = libvirtComputingResource.getVPCNetworkStats(command.getPrivateIP(), command.getPublicIP(), NETWORK_USAGE_GET_OPTION); + bytesSent = stats[0]; + bytesReceived = stats[1]; + } else { + final long [] stats = libvirtComputingResource.getNetworkStats(command.getPrivateIP()); + bytesSent = stats[0]; + bytesReceived = stats[1]; + } + final long [] lbStats = libvirtComputingResource.getNetworkLbStats(command.getPrivateIP(), command.getPublicIP(), command.getPort()); + final long lbConnections = lbStats[0]; + + List values = new ArrayList<>(); + + for (AutoScaleMetrics metrics : command.getMetrics()) { + switch (metrics.getCounter()) { + case NetworkReceive: + values.add(new AutoScaleMetricsValue(metrics, AutoScaleValueType.AGGREGATED, Double.valueOf(bytesReceived))); + break; + case NetworkTransmit: + values.add(new AutoScaleMetricsValue(metrics, AutoScaleValueType.AGGREGATED, Double.valueOf(bytesSent))); + break; + case LbAverageConnections: + values.add(new AutoScaleMetricsValue(metrics, AutoScaleValueType.INSTANT, Double.valueOf(lbConnections))); + break; + } + } + + return new GetAutoScaleMetricsAnswer(command, true, values); + } +} diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 18edacc48454..bd427269b7ae 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1890,7 +1890,7 @@ private Network getNetwork(AutoScaleVmGroup asGroup) { return network; } - private String getPublicIp(AutoScaleVmGroup asGroup) { + private Pair getPublicIpAndPort(AutoScaleVmGroup asGroup) { final LoadBalancerVO loadBalancer = _lbDao.findById(asGroup.getLoadBalancerId()); if (loadBalancer == null) { throw new CloudRuntimeException(String.format("Unable to find load balancer with id: % ", asGroup.getLoadBalancerId())); @@ -1899,7 +1899,7 @@ private String getPublicIp(AutoScaleVmGroup asGroup) { if (ipAddress == null) { throw new CloudRuntimeException(String.format("Unable to find IP Address with id: % ", loadBalancer.getSourceIpAddressId())); } - return ipAddress.getAddress().addr(); + return new Pair<>(ipAddress.getAddress().addr(), loadBalancer.getSourcePortStart()); } private Network.Provider getLoadBalancerServiceProvider(AutoScaleVmGroup asGroup) { @@ -2170,7 +2170,7 @@ private void getVmStatsFromHosts(AutoScaleVmGroupVO asGroup) { private void getNetworkStatsFromVirtualRouter(AutoScaleVmGroupVO asGroup) { Network network = getNetwork(asGroup); - String publicIpAddr = getPublicIp(asGroup); + Pair publicIpAddr = getPublicIpAndPort(asGroup); List routers = _routerDao.listByNetworkAndRole(network.getId(), VirtualRouter.Role.VIRTUAL_ROUTER); if (CollectionUtils.isEmpty(routers)) { return; @@ -2178,7 +2178,7 @@ private void getNetworkStatsFromVirtualRouter(AutoScaleVmGroupVO asGroup) { List metrics = setGetAutoScaleMetricsCommandMetrics(asGroup); for (DomainRouterVO router : routers) { if (VirtualMachine.State.Running.equals(router.getState())) { - final GetAutoScaleMetricsCommand command = new GetAutoScaleMetricsCommand(router.getPrivateIpAddress(), network.getVpcId() != null, publicIpAddr, metrics); + final GetAutoScaleMetricsCommand command = new GetAutoScaleMetricsCommand(router.getPrivateIpAddress(), network.getVpcId() != null, publicIpAddr.first(), publicIpAddr.second(), metrics); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); command.setWait(30); diff --git a/systemvm/debian/opt/cloud/bin/get_haproxy_stats.sh b/systemvm/debian/opt/cloud/bin/get_haproxy_stats.sh new file mode 100755 index 000000000000..06211cfe8dc6 --- /dev/null +++ b/systemvm/debian/opt/cloud/bin/get_haproxy_stats.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +publicIp=$1 +port=$2 + +poolName=$(echo $publicIp-$port | tr "." "_") + +# read "scur" from haproxy stats socket. The values are +# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime,agent_status,agent_code,agent_duration,check_desc,agent_desc,check_rise,check_fall,check_health,agent_rise,agent_fall,agent_health,addr,cookie,mode,algo,conn_rate,conn_rate_max,conn_tot,intercepted,dcon,dses,wrew,connect,reuse,cache_lookups,cache_hits,srv_icur,src_ilim,qtime_max,ctime_max,rtime_max,ttime_max,eint,idle_conn_cur,safe_conn_cur,used_conn_cur,need_conn_est, + +current_session=$(echo "show stat" | socat stdio unix-connect:/run/haproxy/admin.sock | grep "^$poolName,FRONTEND" | cut -d "," -f5) + +echo $current_session From 869fce1055435710decc04b0bb509ce6b1ac3eaa Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 13 Jun 2022 15:18:22 +0200 Subject: [PATCH 031/360] AS: handle aggregated data --- .../network/as/AutoScaleManagerImpl.java | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index bd427269b7ae..b1175828d185 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1977,7 +1977,7 @@ private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { Long conditionId = Long.parseLong(params.get("con" + counter_vm[1])); Double coVal = Double.parseDouble(counterVals[1]); - updateCountersMap(countersMap, countersNumberMap, asGroup, counterId, conditionId, coVal); + updateCountersMapWithInstantData(countersMap, countersNumberMap, asGroup, counterId, conditionId, coVal); } catch (Exception e) { e.printStackTrace(); @@ -2024,7 +2024,7 @@ private void setPerformanceMonitorCommandParams(AutoScaleVmGroupVO asGroup, Map< params.put("total_counter", String.valueOf(total_counter)); } - private void updateCountersMap(Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { + private void updateCountersMapWithInstantData(Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { // Summary of all counter by counterId key String key = generateKeyFromConditionAndCounter(conditionId, counterId); if (countersMap.get(key) == null) { @@ -2060,6 +2060,18 @@ private void updateCountersMap(Map countersMap, Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { + // Summary of all counter by counterId key + String key = generateKeyFromConditionAndCounter(conditionId, counterId); + CounterVO counter = _counterDao.findById(counterId); + if (counter == null) { + return; + } + s_logger.debug(String.format("Updating countersMap for conditionId = %s, counterId = %d to %f", conditionId, counterId, coVal)); + countersMap.put(key, coVal); + countersNumberMap.put(key, 1); + } + private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { // check minimum vm of group Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId()); @@ -2245,7 +2257,7 @@ private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map for (AutoScaleVmGroupStatisticsVO stat : stats) { if (AutoScaleValueType.INSTANT.equals(stat.getValueType())) { s_logger.debug(String.format("Updating countersMap with %s (%s): %f, created on %s", counter.getSource(), counter.getValue(), stat.getRawValue(), stat.getCreated())); - updateCountersMap(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, stat.getRawValue()); + updateCountersMapWithInstantData(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, stat.getRawValue()); } else if (AutoScaleValueType.AGGREGATED.equals(stat.getValueType())) { String key = stat.getCounterId() + "-" + stat.getResourceId(); if (incorrectRecords.contains(key)) { @@ -2272,8 +2284,18 @@ private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map } if (MapUtils.isNotEmpty(aggregatedRecords)) { s_logger.debug("Processing aggregated data"); - for (List records : aggregatedRecords.values()) { - //TODO + for (String recordKey : aggregatedRecords.keySet()) { + Long counterId = Long.valueOf(recordKey.split("-")[0]); + Long resourceId = Long.valueOf(recordKey.split("-")[1]); + List records = aggregatedRecords.get(recordKey); + if (records.size() <= 1) { + s_logger.info(String.format("Ignoring aggregated records, conditionId = %s, counterId = %s", conditionId, counterId)); + continue; + } + AutoScaleVmGroupStatisticsVO firstRecord = records.get(0); + AutoScaleVmGroupStatisticsVO lastRecord = records.get(records.size() - 1); + Double coVal = (lastRecord.getRawValue() - firstRecord.getRawValue()) * 1000 / (lastRecord.getCreated().getTime() - firstRecord.getCreated().getTime()); + updateCountersMapWithAggregatedData(countersMap, countersNumberMap, asGroup, conditionId, counterId, coVal); } } } From 507ed2174c659e16218586d122dc612b43a0c590 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 13 Jun 2022 17:37:02 +0200 Subject: [PATCH 032/360] AS: consider multiple autoscale policies with same condition --- .../network/as/AutoScaleManagerImpl.java | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index b1175828d185..956c516671ba 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1730,10 +1730,10 @@ private List getCounters(long groupId) { return counters; } - private AutoScalePolicy.Action getAutoscaleAction(Map countersMap, Map countersNumberMap, long groupId, Map params) { - s_logger.debug("[AutoScale] Getting autoscale action for group : " + groupId); + private AutoScalePolicy.Action getAutoscaleAction(Map countersMap, Map countersNumberMap, AutoScaleVmGroup asGroup, Map params) { + s_logger.debug("[AutoScale] Getting autoscale action for group : " + asGroup.getId()); - List listMap = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId); + List listMap = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(asGroup.getId()); if ((listMap == null) || (listMap.size() == 0)) return null; for (AutoScaleVmGroupPolicyMapVO asVmgPmap : listMap) { @@ -1772,7 +1772,11 @@ private AutoScalePolicy.Action getAutoscaleAction(Map countersMa counter_count++; } while (true); - String key = generateKeyFromConditionAndCounter(conditionVO.getId(), counterVO.getId()); + String key = generateKeyFromPolicyAndConditionAndCounter(policyVO.getId(), conditionVO.getId(), counterVO.getId()); + Network.Provider provider = getLoadBalancerServiceProvider(asGroup); + if (Network.Provider.Netscaler.equals(provider)) { + key = generateKeyFromPolicyAndConditionAndCounter(0L, conditionVO.getId(), counterVO.getId()); + } Double sum = countersMap.get(key); Integer number = countersNumberMap.get(key); s_logger.debug(String.format("policyId = %d, conditionId = %d, counter = %s, sum = %f, number = %s", policyVO.getId(), conditionVO.getId(), counterVO.getSource(), sum, number)); @@ -1975,16 +1979,19 @@ private void monitorNetScalerAsGroup(AutoScaleVmGroupVO asGroup) { Long counterId = Long.parseLong(counter_vm[1]); Long conditionId = Long.parseLong(params.get("con" + counter_vm[1])); + Integer duration = Integer.parseInt(params.get("duration" + counter_vm[1])); + Long policyId = 0L; + Double coVal = Double.parseDouble(counterVals[1]); - updateCountersMapWithInstantData(countersMap, countersNumberMap, asGroup, counterId, conditionId, coVal); + updateCountersMapWithInstantData(countersMap, countersNumberMap, asGroup, counterId, conditionId, policyId, coVal); } catch (Exception e) { e.printStackTrace(); } } - AutoScalePolicy.Action scaleAction = getAutoscaleAction(countersMap, countersNumberMap, asGroup.getId(), params); + AutoScalePolicy.Action scaleAction = getAutoscaleAction(countersMap, countersNumberMap, asGroup, params); if (scaleAction != null) { s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); if (AutoScalePolicy.Action.ScaleUp.equals(scaleAction)) { @@ -2024,9 +2031,9 @@ private void setPerformanceMonitorCommandParams(AutoScaleVmGroupVO asGroup, Map< params.put("total_counter", String.valueOf(total_counter)); } - private void updateCountersMapWithInstantData(Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { + private void updateCountersMapWithInstantData(Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Long policyId, Double coVal) { // Summary of all counter by counterId key - String key = generateKeyFromConditionAndCounter(conditionId, counterId); + String key = generateKeyFromPolicyAndConditionAndCounter(policyId, conditionId, counterId); if (countersMap.get(key) == null) { /* initialize if data is not set */ countersMap.put(key, Double.valueOf(0)); @@ -2060,9 +2067,9 @@ private void updateCountersMapWithInstantData(Map countersMap, M countersNumberMap.put(key, countersNumberMap.get(key) + 1); } - private void updateCountersMapWithAggregatedData(Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Double coVal) { + private void updateCountersMapWithAggregatedData(Map countersMap, Map countersNumberMap, AutoScaleVmGroupVO asGroup, Long counterId, Long conditionId, Long policyId, Double coVal) { // Summary of all counter by counterId key - String key = generateKeyFromConditionAndCounter(conditionId, counterId); + String key = generateKeyFromPolicyAndConditionAndCounter(policyId, conditionId, counterId); CounterVO counter = _counterDao.findById(counterId); if (counter == null) { return; @@ -2112,7 +2119,7 @@ private void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) { Map params = new HashMap(); setPerformanceMonitorCommandParams(asGroup, params); - AutoScalePolicy.Action scaleAction = getAutoscaleAction(countersMap, countersNumberMap, asGroup.getId(), params); + AutoScalePolicy.Action scaleAction = getAutoscaleAction(countersMap, countersNumberMap, asGroup, params); if (scaleAction != null) { s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId()); if (AutoScalePolicy.Action.ScaleUp.equals(scaleAction)) { @@ -2245,7 +2252,7 @@ private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map for (Long conditionId : conditionsMap.keySet()) { Date afterDate = new Date(System.currentTimeMillis() - ((long)vo.getDuration() << 10)); s_logger.debug(String.format("Updating countersMap for condition %d in policy %d in as group %d: ", conditionId, groupPolicyVO.getPolicyId(), asGroup.getId())); - s_logger.debug(String.format("Updating countersMap with stats in %d seconds : between %s and %s", vo.getDuration(), new Date(), afterDate)); + s_logger.debug(String.format("Updating countersMap with stats in %d seconds : between %s and %s", vo.getDuration(), afterDate, new Date())); CounterVO counter = conditionsMap.get(conditionId); List stats = _asGroupStatisticsDao.listByVmGroupIdAndCounterId(asGroup.getId(), counter.getId(), afterDate); if (CollectionUtils.isEmpty(stats)) { @@ -2257,7 +2264,7 @@ private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map for (AutoScaleVmGroupStatisticsVO stat : stats) { if (AutoScaleValueType.INSTANT.equals(stat.getValueType())) { s_logger.debug(String.format("Updating countersMap with %s (%s): %f, created on %s", counter.getSource(), counter.getValue(), stat.getRawValue(), stat.getCreated())); - updateCountersMapWithInstantData(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, stat.getRawValue()); + updateCountersMapWithInstantData(countersMap, countersNumberMap, asGroup, counter.getId(), conditionId, vo.getId(), stat.getRawValue()); } else if (AutoScaleValueType.AGGREGATED.equals(stat.getValueType())) { String key = stat.getCounterId() + "-" + stat.getResourceId(); if (incorrectRecords.contains(key)) { @@ -2285,6 +2292,7 @@ private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map if (MapUtils.isNotEmpty(aggregatedRecords)) { s_logger.debug("Processing aggregated data"); for (String recordKey : aggregatedRecords.keySet()) { + s_logger.debug("Processing aggregated data with recordKey = " + recordKey); Long counterId = Long.valueOf(recordKey.split("-")[0]); Long resourceId = Long.valueOf(recordKey.split("-")[1]); List records = aggregatedRecords.get(recordKey); @@ -2295,7 +2303,7 @@ private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map AutoScaleVmGroupStatisticsVO firstRecord = records.get(0); AutoScaleVmGroupStatisticsVO lastRecord = records.get(records.size() - 1); Double coVal = (lastRecord.getRawValue() - firstRecord.getRawValue()) * 1000 / (lastRecord.getCreated().getTime() - firstRecord.getCreated().getTime()); - updateCountersMapWithAggregatedData(countersMap, countersNumberMap, asGroup, conditionId, counterId, coVal); + updateCountersMapWithAggregatedData(countersMap, countersNumberMap, asGroup, counterId, conditionId, vo.getId(), coVal); } } } @@ -2303,8 +2311,8 @@ private void updateCountersMap(AutoScaleVmGroupVO asGroup, Map } } - private String generateKeyFromConditionAndCounter(Long conditionId, Long counterId) { - return conditionId + "-" + counterId; + private String generateKeyFromPolicyAndConditionAndCounter(Long policyId, Long conditionId, Long counterId) { + return policyId + "-" + conditionId + "-" + counterId; } private void cleanupAsVmGroupStatistics(AutoScaleVmGroup asGroup) { From 8c6eba56657df0b0113beafca779834c9b4cb02e Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 13 Jun 2022 19:40:59 +0200 Subject: [PATCH 033/360] AS: remove all vm group statistics when disable a vm group --- .../src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 956c516671ba..8fec6737e6b4 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1163,6 +1163,7 @@ public AutoScaleVmGroup disableAutoScaleVmGroup(Long id) { vmGroup.setState(AutoScaleVmGroup.State.Disabled); vmGroup = _autoScaleVmGroupDao.persist(vmGroup); success = configureAutoScaleVmGroup(id, AutoScaleVmGroup.State.Enabled); + _asGroupStatisticsDao.removeByGroupId(id); } catch (ResourceUnavailableException e) { vmGroup.setState(AutoScaleVmGroup.State.Enabled); _autoScaleVmGroupDao.persist(vmGroup); From a2b1240548592dae8543a267d5a7822ce3350769 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 14 Jun 2022 10:45:41 +0200 Subject: [PATCH 034/360] AS: support vmware and xenserver --- .../resource/LibvirtComputingResource.java | 9 +- .../vmware/resource/VmwareResource.java | 71 +++++++++++++++- .../resource/CitrixResourceBase.java | 34 ++++++++ ...er56GetAutoScaleMetricsCommandWrapper.java | 83 +++++++++++++++++++ 4 files changed, 187 insertions(+), 10 deletions(-) create mode 100644 plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xen56/XenServer56GetAutoScaleMetricsCommandWrapper.java diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index f43bfe64fcd7..cd0fe7111452 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -2305,15 +2305,12 @@ public String getHaproxyStats(final String privateIP, final String publicIp, fin return statsParser.getLine(); } - public long[] getNetworkLbStats(final String privateIP, final String publicIp, final Integer port) { - final String result = getHaproxyStats(privateIP, publicIp, port); + public long[] getNetworkLbStats(final String privateIp, final String publicIp, final Integer port) { + final String result = getHaproxyStats(privateIp, publicIp, port); final long[] stats = new long[1]; if (result != null) { final String[] splitResult = result.split(","); - int i = 0; - while (i < splitResult.length - 1) { - stats[0] += Long.parseLong(splitResult[i++]); - } + stats[0] += Long.parseLong(splitResult[0]); } return stats; } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index f224601dfc4e..00e1ea6c2300 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -183,6 +183,8 @@ import com.cloud.agent.api.VolumeStatsEntry; import com.cloud.agent.api.check.CheckSshAnswer; import com.cloud.agent.api.check.CheckSshCommand; +import com.cloud.agent.api.routing.GetAutoScaleMetricsAnswer; +import com.cloud.agent.api.routing.GetAutoScaleMetricsCommand; import com.cloud.agent.api.routing.IpAssocCommand; import com.cloud.agent.api.routing.IpAssocVpcCommand; import com.cloud.agent.api.routing.NetworkElementCommand; @@ -255,6 +257,7 @@ import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.TrafficType; import com.cloud.network.VmwareTrafficLabel; +import com.cloud.network.router.VirtualRouterAutoScale; import com.cloud.resource.ServerResource; import com.cloud.serializer.GsonHelper; import com.cloud.storage.Storage; @@ -599,6 +602,8 @@ public Answer executeRequest(Command cmd) { answer = execute((SetupPersistentNetworkCommand) cmd); } else if (clz == GetVmVncTicketCommand.class) { answer = execute((GetVmVncTicketCommand) cmd); + } else if (clz ==GetAutoScaleMetricsCommand.class) { + answer = execute((GetAutoScaleMetricsCommand) cmd); } else { answer = Answer.createUnsupportedCommandAnswer(cmd); } @@ -1104,13 +1109,18 @@ protected NetworkUsageAnswer VPCNetworkUsage(NetworkUsageCommand cmd) { String privateIp = cmd.getPrivateIP(); String option = cmd.getOption(); String publicIp = cmd.getGatewayIP(); + String vpcCIDR = cmd.getVpcCIDR(); + final long[] stats = getVPCNetworkStats(privateIp, publicIp, option, vpcCIDR); + return new NetworkUsageAnswer(cmd, "", stats[0], stats[1]); + } + + private long[] getVPCNetworkStats(String privateIp, String publicIp, String option, String vpcCIDR) { String args = "-l " + publicIp + " "; if (option.equals("get")) { args += "-g"; } else if (option.equals("create")) { args += "-c"; - String vpcCIDR = cmd.getVpcCIDR(); args += " -v " + vpcCIDR; } else if (option.equals("reset")) { args += "-r"; @@ -1119,7 +1129,7 @@ protected NetworkUsageAnswer VPCNetworkUsage(NetworkUsageCommand cmd) { } else if (option.equals("remove")) { args += "-d"; } else { - return new NetworkUsageAnswer(cmd, "success", 0L, 0L); + return new long[2]; } ExecutionResult callResult = executeInVR(privateIp, "vpc_netusage.sh", args); @@ -1141,10 +1151,63 @@ protected NetworkUsageAnswer VPCNetworkUsage(NetworkUsageCommand cmd) { stats[0] += Long.parseLong(splitResult[i++]); stats[1] += Long.parseLong(splitResult[i++]); } - return new NetworkUsageAnswer(cmd, "success", stats[0], stats[1]); + return stats; + } + } + return new long[2]; + } + + public long[] getNetworkLbStats(final String privateIp, final String publicIp, final Integer port) { + String args = publicIp + " " + port; + ExecutionResult callResult = executeInVR(privateIp, "get_haproxy_stats.sh", args); + + if (!callResult.isSuccess()) { + s_logger.error("Unable to execute GetAutoScaleMetricsCommand on DomR (" + privateIp + "), domR may not be ready yet. failure due to " + callResult.getDetails()); + } + String result = callResult.getDetails(); + if (result == null || result.isEmpty()) { + s_logger.error("Get autoscale metrics returns empty "); + } + final long[] stats = new long[1]; + if (result != null) { + final String[] splitResult = result.split(","); + stats[0] += Long.parseLong(splitResult[0]); + } + return stats; + } + + protected Answer execute(GetAutoScaleMetricsCommand cmd) { + Long bytesSent; + Long bytesReceived; + if (cmd.isForVpc()) { + final long[] stats = getVPCNetworkStats(cmd.getPrivateIP(), cmd.getPublicIP(), "get", null); + bytesSent = stats[0]; + bytesReceived = stats[1]; + } else { + final long [] stats = getNetworkStats(cmd.getPrivateIP()); + bytesSent = stats[0]; + bytesReceived = stats[1]; + } + final long [] lbStats = getNetworkLbStats(cmd.getPrivateIP(), cmd.getPublicIP(), cmd.getPort()); + final long lbConnections = lbStats[0]; + + List values = new ArrayList<>(); + + for (VirtualRouterAutoScale.AutoScaleMetrics metrics : cmd.getMetrics()) { + switch (metrics.getCounter()) { + case NetworkReceive: + values.add(new VirtualRouterAutoScale.AutoScaleMetricsValue(metrics, VirtualRouterAutoScale.AutoScaleValueType.AGGREGATED, Double.valueOf(bytesReceived))); + break; + case NetworkTransmit: + values.add(new VirtualRouterAutoScale.AutoScaleMetricsValue(metrics, VirtualRouterAutoScale.AutoScaleValueType.AGGREGATED, Double.valueOf(bytesSent))); + break; + case LbAverageConnections: + values.add(new VirtualRouterAutoScale.AutoScaleMetricsValue(metrics, VirtualRouterAutoScale.AutoScaleValueType.INSTANT, Double.valueOf(lbConnections))); + break; } } - return new NetworkUsageAnswer(cmd, "success", 0L, 0L); + + return new GetAutoScaleMetricsAnswer(cmd, true, values); } @Override diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index 6bed711b5f07..97c776ca5023 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -2991,6 +2991,40 @@ public long[] getNetworkStats(final Connection conn, final String privateIP) { return stats; } + public long[] getVPCNetworkStats(final Connection conn, final String privateIP, final String publicIp) { + String args = " -l " + publicIp + " -g"; + final ExecutionResult result = executeInVR(privateIP, "vpc_netusage.sh", args); + final String detail = result.getDetails(); + final long[] stats = new long[2]; + if (detail != null) { + final String[] splitResult = detail.split(":"); + int i = 0; + while (i < splitResult.length - 1) { + stats[0] += Long.parseLong(splitResult[i++]); + stats[1] += Long.parseLong(splitResult[i++]); + } + } + return stats; + } + + public long[] getNetworkLbStats(final Connection conn, final String privateIp, final String publicIp, final Integer port) { + String args = publicIp + " " + port; + ExecutionResult callResult = executeInVR(privateIp, "get_haproxy_stats.sh", args); + + if (!callResult.isSuccess()) { + s_logger.error("Unable to execute GetAutoScaleMetricsCommand on DomR (" + privateIp + "), domR may not be ready yet. failure due to " + callResult.getDetails()); + } + String result = callResult.getDetails(); + if (result == null || result.isEmpty()) { + s_logger.error("Get autoscale metrics returns empty "); + } + final long[] stats = new long[1]; + if (result != null) { + final String[] splitResult = result.split(","); + stats[0] += Long.parseLong(splitResult[0]); + } + return stats; + } public SR getNfsSR(final Connection conn, final String poolid, final String uuid, final String server, String serverpath, final String pooldesc) { final Map deviceConfig = new HashMap(); try { diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xen56/XenServer56GetAutoScaleMetricsCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xen56/XenServer56GetAutoScaleMetricsCommandWrapper.java new file mode 100644 index 000000000000..d46c59726bd0 --- /dev/null +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xen56/XenServer56GetAutoScaleMetricsCommandWrapper.java @@ -0,0 +1,83 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.xenserver.resource.wrapper.xen56; + +import com.cloud.network.router.VirtualRouterAutoScale; +import org.apache.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.routing.GetAutoScaleMetricsAnswer; +import com.cloud.agent.api.routing.GetAutoScaleMetricsCommand; +import com.cloud.hypervisor.xenserver.resource.XenServer56Resource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.xensource.xenapi.Connection; + +import java.util.ArrayList; +import java.util.List; + +@ResourceWrapper(handles = GetAutoScaleMetricsCommand.class) +public final class XenServer56GetAutoScaleMetricsCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(XenServer56NetworkUsageCommandWrapper.class); + + @Override + public Answer execute(final GetAutoScaleMetricsCommand command, final XenServer56Resource xenServer56) { + final Connection conn = xenServer56.getConnection(); + try { + Long bytesSent; + Long bytesReceived; + if (command.isForVpc()) { + final long[] stats = xenServer56.getVPCNetworkStats(conn, command.getPrivateIP(), command.getPublicIP()); + bytesSent = stats[0]; + bytesReceived = stats[1]; + } else { + final long[] stats = xenServer56.getNetworkStats(conn, command.getPrivateIP()); + bytesSent = stats[0]; + bytesReceived = stats[1]; + } + + final long [] lbStats = xenServer56.getNetworkLbStats(conn, command.getPrivateIP(), command.getPublicIP(), command.getPort()); + final long lbConnections = lbStats[0]; + + List values = new ArrayList<>(); + + for (VirtualRouterAutoScale.AutoScaleMetrics metrics : command.getMetrics()) { + switch (metrics.getCounter()) { + case NetworkReceive: + values.add(new VirtualRouterAutoScale.AutoScaleMetricsValue(metrics, VirtualRouterAutoScale.AutoScaleValueType.AGGREGATED, Double.valueOf(bytesReceived))); + break; + case NetworkTransmit: + values.add(new VirtualRouterAutoScale.AutoScaleMetricsValue(metrics, VirtualRouterAutoScale.AutoScaleValueType.AGGREGATED, Double.valueOf(bytesSent))); + break; + case LbAverageConnections: + values.add(new VirtualRouterAutoScale.AutoScaleMetricsValue(metrics, VirtualRouterAutoScale.AutoScaleValueType.INSTANT, Double.valueOf(lbConnections))); + break; + } + } + + return new GetAutoScaleMetricsAnswer(command, true, values); + } catch (final Exception ex) { + s_logger.warn("Failed to get autoscale metrics due to ", ex); + return new GetAutoScaleMetricsAnswer(command, false); + } + } + +} \ No newline at end of file From 63eca08beb5774b1793e5e3627cf90cfc135e713 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 15 Jun 2022 15:43:46 +0200 Subject: [PATCH 035/360] AS: add annotations for autoscale vm group --- .../cloudstack/annotation/AnnotationService.java | 3 ++- .../api/response/AutoScaleVmProfileResponse.java | 4 ++-- .../java/com/cloud/network/as/AutoScaleVmGroupVO.java | 3 ++- .../main/java/com/cloud/api/ApiResponseHelper.java | 3 +++ .../com/cloud/network/as/AutoScaleManagerImpl.java | 11 +++++++++++ .../cloudstack/annotation/AnnotationManagerImpl.java | 6 ++++++ ui/src/components/view/AnnotationsTab.vue | 1 + 7 files changed, 27 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java b/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java index 0aca007a44fd..d1ebff6f2cd5 100644 --- a/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java +++ b/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java @@ -44,7 +44,8 @@ enum EntityType { TEMPLATE(true), ISO(true), KUBERNETES_CLUSTER(true), SERVICE_OFFERING(false), DISK_OFFERING(false), NETWORK_OFFERING(false), ZONE(false), POD(false), CLUSTER(false), HOST(false), DOMAIN(false), - PRIMARY_STORAGE(false), SECONDARY_STORAGE(false), VR(false), SYSTEM_VM(false); + PRIMARY_STORAGE(false), SECONDARY_STORAGE(false), VR(false), SYSTEM_VM(false), + AUTOSCALE_VM_GROUP(true); private final boolean usersAllowed; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java index 412af1c27362..ed555780909b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java @@ -23,7 +23,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd.CommandType; -import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.BaseResponseWithAnnotations; import org.apache.cloudstack.api.EntityReference; import org.apache.cloudstack.api.Parameter; @@ -33,7 +33,7 @@ import com.google.gson.annotations.SerializedName; @EntityReference(value = AutoScaleVmProfile.class) -public class AutoScaleVmProfileResponse extends BaseResponse implements ControlledEntityResponse { +public class AutoScaleVmProfileResponse extends BaseResponseWithAnnotations implements ControlledEntityResponse { @SerializedName(ApiConstants.ID) @Param(description = "the autoscale vm profile ID") diff --git a/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupVO.java b/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupVO.java index 218e394ab1e9..4b4104d504e2 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupVO.java +++ b/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmGroupVO.java @@ -30,6 +30,7 @@ import javax.persistence.Temporal; import javax.persistence.TemporalType; +import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; import com.cloud.utils.db.GenericDao; @@ -37,7 +38,7 @@ @Entity @Table(name = "autoscale_vmgroups") @Inheritance(strategy = InheritanceType.JOINED) -public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity { +public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity, Identity { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 229db588a106..588105412cee 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -3341,6 +3341,9 @@ public AutoScaleVmProfileResponse createAutoScaleVmProfileResponse(AutoScaleVmPr } response.setObjectName("autoscalevmprofile"); + response.setHasAnnotation(annotationDao.hasAnnotations(profile.getUuid(), AnnotationService.EntityType.AUTOSCALE_VM_GROUP.name(), + _accountMgr.isRootAdmin(CallContext.current().getCallingAccount().getId()))); + // Populates the account information in the response populateOwner(response, profile); return response; diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 8fec6737e6b4..c2678cba7cce 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -39,6 +39,8 @@ import com.google.gson.reflect.TypeToken; import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.annotation.AnnotationService; +import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; @@ -239,6 +241,8 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScale @Inject private DomainRouterDao _routerDao; @Inject + private AnnotationDao _annotationDao; + @Inject protected RouterControlHelper _routerControlHelper; private long autoScaleStatsInterval = -1L; @@ -516,6 +520,13 @@ public boolean deleteAutoScaleVmProfile(long id) { if (_autoScaleVmGroupDao.isProfileInUse(id)) { throw new InvalidParameterValueException("Cannot delete AutoScale Vm Profile when it is in use by one more vm groups"); } + + // Remove comments (if any) + AutoScaleVmGroup group = _autoScaleVmGroupDao.findById(id); + if (group != null) { + _annotationDao.removeByEntityType(AnnotationService.EntityType.AUTOSCALE_VM_GROUP.name(), group.getUuid()); + } + boolean success = _autoScaleVmProfileDao.remove(id); if (success) { s_logger.info("Successfully deleted AutoScale Vm Profile with Id: " + id); diff --git a/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java b/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java index 1a60b66d9d87..a3b43e3b7ba5 100644 --- a/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java @@ -63,6 +63,7 @@ import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.kubernetes.cluster.KubernetesClusterHelper; +import com.cloud.network.as.dao.AutoScaleVmGroupDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; @@ -152,6 +153,8 @@ public final class AnnotationManagerImpl extends ManagerBase implements Annotati @Inject private NetworkOfferingDao networkOfferingDao; @Inject + private AutoScaleVmGroupDao autoScaleVmGroupDao; + @Inject EntityManager entityManager; private static final List adminRoles = Collections.singletonList(RoleType.Admin); @@ -184,6 +187,7 @@ public final class AnnotationManagerImpl extends ManagerBase implements Annotati s_typeMap.put(EntityType.SECONDARY_STORAGE, ApiCommandResourceType.ImageStore); s_typeMap.put(EntityType.VR, ApiCommandResourceType.DomainRouter); s_typeMap.put(EntityType.SYSTEM_VM, ApiCommandResourceType.SystemVm); + s_typeMap.put(EntityType.AUTOSCALE_VM_GROUP, ApiCommandResourceType.AutoScaleVmGroup); } public List getKubernetesClusterHelpers() { @@ -520,6 +524,8 @@ private ControlledEntity getEntityFromUuidAndType(String entityUuid, EntityType return templateDao.findByUuid(entityUuid); case KUBERNETES_CLUSTER: return kubernetesClusterHelpers.get(0).findByUuid(entityUuid); + case AUTOSCALE_VM_GROUP: + return autoScaleVmGroupDao.findByUuid(entityUuid); default: throw new CloudRuntimeException("Invalid entity type " + type); } diff --git a/ui/src/components/view/AnnotationsTab.vue b/ui/src/components/view/AnnotationsTab.vue index dc0282ead66c..47b7ae822463 100644 --- a/ui/src/components/view/AnnotationsTab.vue +++ b/ui/src/components/view/AnnotationsTab.vue @@ -190,6 +190,7 @@ export default { case 'SecondaryStorage': return 'SECONDARY_STORAGE' case 'SystemVm': return 'SYSTEM_VM' case 'VirtualRouter': return 'VR' + case 'AutoScaleVmGroup': return 'AUTOSCALE_VM_GROUP' default: return '' } }, From 078e43f9fef09574dd6ff0b04ed57b35d9dacbd3 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 15 Jun 2022 14:38:23 +0200 Subject: [PATCH 036/360] AS: update state to Scaling when scale up/down --- .../com/cloud/network/as/AutoScaleVmGroup.java | 4 +++- .../network/as/dao/AutoScaleVmGroupDao.java | 3 +++ .../as/dao/AutoScaleVmGroupDaoImpl.java | 14 ++++++++++++++ .../cloud/network/as/AutoScaleManagerImpl.java | 18 ++++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/com/cloud/network/as/AutoScaleVmGroup.java b/api/src/main/java/com/cloud/network/as/AutoScaleVmGroup.java index e6c434e5780f..1253c3192e8c 100644 --- a/api/src/main/java/com/cloud/network/as/AutoScaleVmGroup.java +++ b/api/src/main/java/com/cloud/network/as/AutoScaleVmGroup.java @@ -29,7 +29,7 @@ public interface AutoScaleVmGroup extends ControlledEntity, InternalIdentity, Displayable { enum State { - New, Revoke, Enabled, Disabled; + New, Revoke, Enabled, Disabled, Scaling; public static State fromValue(String state) { if (StringUtils.isBlank(state)) { @@ -42,6 +42,8 @@ public static State fromValue(String state) { return Enabled; } else if (state.equalsIgnoreCase("disabled")) { return Disabled; + } else if (state.equalsIgnoreCase("scaling")) { + return Scaling; } else { throw new InvalidParameterValueException("Unexpected AutoScale VM group state : " + state); } diff --git a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupDao.java b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupDao.java index 16d110410288..6aee5b5f4006 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupDao.java +++ b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupDao.java @@ -18,6 +18,7 @@ import java.util.List; +import com.cloud.network.as.AutoScaleVmGroup; import com.cloud.network.as.AutoScaleVmGroupVO; import com.cloud.utils.db.GenericDao; @@ -27,4 +28,6 @@ public interface AutoScaleVmGroupDao extends GenericDao 0; } + + @Override + public boolean updateState(long groupId, AutoScaleVmGroup.State oldState, AutoScaleVmGroup.State newState) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("id", SearchCriteria.Op.EQ, groupId); + sc.addAnd("state", SearchCriteria.Op.EQ, oldState); + AutoScaleVmGroupVO group = findOneBy(sc); + if (group == null) { + return false; + } + group.setState(newState); + return update(groupId, group); + } } diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index c2678cba7cce..0410f58d45ad 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -1567,6 +1567,12 @@ public void doScaleUp(long groupId, Integer numVm) { if (!checkConditionUp(asGroup, numVm)) { return; } + AutoScaleVmGroup.State oldState = asGroup.getState(); + AutoScaleVmGroup.State newState = AutoScaleVmGroup.State.Scaling; + if (!_autoScaleVmGroupDao.updateState(groupId, oldState, newState)) { + s_logger.error(String.format("Can not update vmgroup state from %s to %s, groupId: %s", oldState, newState, groupId)); + return; + } for (int i = 0; i < numVm; i++) { long vmId = createNewVM(asGroup); if (vmId == -1) { @@ -1602,6 +1608,9 @@ public void doScaleUp(long groupId, Integer numVm) { break; } } + if (!_autoScaleVmGroupDao.updateState(groupId, newState, oldState)) { + s_logger.error(String.format("Can not update vmgroup state from %s back to %s, groupId: %s", newState, oldState, groupId)); + } } @Override @@ -1614,6 +1623,12 @@ public void doScaleDown(final long groupId) { if (!checkConditionDown(asGroup)) { return; } + AutoScaleVmGroup.State oldState = asGroup.getState(); + AutoScaleVmGroup.State newState = AutoScaleVmGroup.State.Scaling; + if (!_autoScaleVmGroupDao.updateState(groupId, oldState, newState)) { + s_logger.error(String.format("Can not update vmgroup state from %s to %s, groupId: %s", oldState, newState, groupId)); + return; + } final long vmId = removeLBrule(asGroup); if (vmId != -1) { long profileId = asGroup.getProfileId(); @@ -1653,6 +1668,9 @@ public void run() { } else { s_logger.error("Can not remove LB rule for the VM being destroyed. Do nothing more."); } + if (!_autoScaleVmGroupDao.updateState(groupId, newState, oldState)) { + s_logger.error(String.format("Can not update vmgroup state from %s back to %s, groupId: %s", newState, oldState, groupId)); + } } @Override From 1a6e41a91da4a44e2a3f55e521b657e365b2452b Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 15 Jun 2022 15:44:14 +0200 Subject: [PATCH 037/360] AS UI: part1 --- ui/public/locales/en.json | 6 +- ui/src/components/view/ListView.vue | 5 + ui/src/config/section/compute.js | 64 + ui/src/core/lazy_lib/icons_use.js | 2 + .../views/compute/CreateAutoScaleVmGroup.vue | 2424 +++++++++++++++++ .../views/compute/UpdateAutoScaleVmGroup.vue | 2424 +++++++++++++++++ 6 files changed, 4924 insertions(+), 1 deletion(-) create mode 100644 ui/src/views/compute/CreateAutoScaleVmGroup.vue create mode 100644 ui/src/views/compute/UpdateAutoScaleVmGroup.vue diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index cf7bbe1cbb06..9e19e3d6a04c 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -567,6 +567,7 @@ "label.direct.attached.public.ip": "Direct attached public IP", "label.direct.ips": "Shared network IPs", "label.directdownload": "Direct download", +"label.disable.autoscale.vmgroup": "Disable AutoScale VM Group", "label.disable.host": "Disable host", "label.disable.network.offering": "Disable network offering", "label.disable.provider": "Disable provider", @@ -655,6 +656,7 @@ "label.egressdefaultpolicy": "Default egress policy", "label.elastic": "Elastic", "label.email": "Email", +"label.enable.autoscale.vmgroup": "Enable AutoScale VM Group", "label.enable.host": "Enable Host", "label.enable.network.offering": "Enable network offering", "label.enable.provider": "Enable provider", @@ -2054,11 +2056,13 @@ "message.confirm.delete.provider": "Please confirm that you would like to delete this provider?", "message.confirm.delete.srx": "Please confirm that you would like to delete SRX.", "message.confirm.destroy.router": "All services provided by this virtual router will be interrupted. Please confirm that you want to stop this router. Please confirm that you would like to destroy this router.", +"message.confirm.disable.autoscale.vmgroup": "Please confirm that you want to disable this autoscale vm group.", "message.confirm.disable.host": "Please confirm that you want to disable the host.", "message.confirm.disable.network.offering": "Are you sure you want to disable this network offering?", "message.confirm.disable.provider": "Please confirm that you would like to disable this provider.", "message.confirm.disable.storage": "Please confirm that you want to disable the storage pool.", "message.confirm.disable.vpc.offering": "Are you sure you want to disable this VPC offering?", +"message.confirm.enable.autoscale.vmgroup": "Please confirm that you want to enable this autoscale vm group.", "message.confirm.enable.host": "Please confirm that you want to enable the host.", "message.confirm.enable.network.offering": "Are you sure you want to enable this network offering?", "message.confirm.enable.provider": "Please confirm that you would like to enable this provider.", @@ -2619,4 +2623,4 @@ "state.suspended": "Suspended", "user.login": "Login", "user.logout": "Logout" -} \ No newline at end of file +} diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 292460541a6e..76f0bbe50034 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -70,6 +70,11 @@ --> + + + + +
+
+
{{ $t('label.duration') }}
+ +
+
+
{{ $t('label.quiettime') }}
+ +
+
+ + + {{ $t('label.edit') }} + +
+
+ +
{{ $t('label.counterid') }}
@@ -95,7 +115,7 @@ type="primary" :danger="true" icon="delete-outlined" - @onClick="deleteConditionFromScalePolicy(record.id)" /> + @onClick="deleteConditionFromAutoScalePolicy(record.id)" />
@@ -123,6 +143,8 @@ export default { filterColumns: ['Action'], loading: true, policyid: null, + duration: null, + quiettime: null, conditions: [], newCondition: { counterid: null, @@ -173,6 +195,8 @@ export default { id: this.resource.id }).then(response => { this.policyid = response.listautoscalevmgroupsresponse?.autoscalevmgroup?.[0]?.scaleuppolicies?.[0]?.id + this.duration = response.listautoscalevmgroupsresponse?.autoscalevmgroup?.[0]?.scaleuppolicies?.[0]?.duration + this.quiettime = response.listautoscalevmgroupsresponse?.autoscalevmgroup?.[0]?.scaleuppolicies?.[0]?.quiettime this.conditions = response.listautoscalevmgroupsresponse?.autoscalevmgroup?.[0]?.scaleuppolicies?.[0]?.conditions || [] const lbruleid = response.listautoscalevmgroupsresponse?.autoscalevmgroup?.[0]?.lbruleid api('listLoadBalancerRules', { @@ -246,8 +270,8 @@ export default { this.loading = false }) }, - deleteConditionFromScalePolicy (conditionId) { - this.updateScalePolicy(null, conditionId) + deleteConditionFromAutoScalePolicy (conditionId) { + this.updateAutoScalePolicy(null, conditionId) }, addCondition () { if (this.loading) return @@ -266,7 +290,7 @@ export default { jobId: response.conditionresponse.jobid, successMethod: (result) => { const newConditionId = result.jobresult.condition?.id - this.updateScalePolicy(newConditionId, null) + this.updateAutoScalePolicy(newConditionId, null) }, errorMessage: this.$t('message.create.condition.failed'), errorMethod: () => { @@ -278,7 +302,7 @@ export default { this.loading = false }) }, - updateScalePolicy (conditionIdToAdd, conditionIdToRemove) { + updateAutoScalePolicy (conditionIdToAdd, conditionIdToRemove) { if (this.loading) return this.loading = true @@ -291,6 +315,8 @@ export default { api('updateAutoScalePolicy', { id: this.policyid, + duration: this.duration, + quiettime: this.quiettime, conditionids: newConditionIds }).then(response => { this.$pollJob({ diff --git a/ui/src/views/compute/UpdateAutoScaleVmGroup.vue b/ui/src/views/compute/UpdateAutoScaleVmGroup.vue deleted file mode 100644 index b217ba757be4..000000000000 --- a/ui/src/views/compute/UpdateAutoScaleVmGroup.vue +++ /dev/null @@ -1,2424 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - - - - - - - From b8b4dfd26baab212140515c450f8ed82dd99df78 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 16 Jun 2022 15:31:53 +0200 Subject: [PATCH 046/360] AS UI: add ScaleDownPolicyTab.vue --- ui/src/config/section/compute.js | 4 + ui/src/views/compute/ScaleDownPolicyTab.vue | 437 ++++++++++++++++++++ 2 files changed, 441 insertions(+) create mode 100644 ui/src/views/compute/ScaleDownPolicyTab.vue diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 349a3e050ecb..671640733890 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -529,6 +529,10 @@ export default { name: 'scaleup.policy', component: shallowRef(defineAsyncComponent(() => import('@/views/compute/ScaleUpPolicyTab.vue'))) }, + { + name: 'scaledown.policy', + component: shallowRef(defineAsyncComponent(() => import('@/views/compute/ScaleDownPolicyTab.vue'))) + }, { name: 'comments', component: shallowRef(defineAsyncComponent(() => import('@/components/view/AnnotationsTab.vue'))) diff --git a/ui/src/views/compute/ScaleDownPolicyTab.vue b/ui/src/views/compute/ScaleDownPolicyTab.vue new file mode 100644 index 000000000000..d0f1e9174026 --- /dev/null +++ b/ui/src/views/compute/ScaleDownPolicyTab.vue @@ -0,0 +1,437 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + From 5434326542351da00e12b906f85e23926c99bbe3 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 16 Jun 2022 18:48:19 +0200 Subject: [PATCH 047/360] AS UI: update LoadBalancing.vue to view/create AS vm group --- ui/public/locales/en.json | 1 + ui/src/views/network/LoadBalancing.vue | 42 ++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index d216f67f3ec4..446f42afc5dc 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1153,6 +1153,7 @@ "label.networktype": "Network type", "label.networkwrite": "Network write", "label.new": "New", +"label.new.autoscale.vmgroup": "New AutoScale VM Group", "label.new.instance.group": "New instance group", "label.new.password": "New password", "label.new.project": "New project", diff --git a/ui/src/views/network/LoadBalancing.vue b/ui/src/views/network/LoadBalancing.vue index 63d7f35e92ce..17b8cf52dfca 100644 --- a/ui/src/views/network/LoadBalancing.vue +++ b/ui/src/views/network/LoadBalancing.vue @@ -104,8 +104,18 @@ {{ returnStickinessLabel(record.id) }} + - - -
- - - + + + 0 && this.networks[0].type === 'Isolated') + }, showSecurityGroupSection () { return (this.networks.length > 0 && this.zone.securitygroupsenabled) || (this.zone && this.zone.networktype === 'Basic') }, @@ -922,7 +940,7 @@ export default { }, watch: { '$route' (to, from) { - if (to.name === 'deployVirtualMachine') { + if (to.name === 'createAutoScaleVmGroup') { this.resetData() } }, @@ -1420,6 +1438,13 @@ export default { escapePropertyKey (key) { return key.split('.').join('\\002E') }, + updateLoadBalancers (id) { + if (id === '0') { + this.form.lbruleid = undefined + return + } + this.form.lbruleid = id + }, updateSecurityGroups (securitygroupids) { this.securitygroupids = securitygroupids || [] }, diff --git a/ui/src/views/compute/wizard/LoadBalancerSelection.vue b/ui/src/views/compute/wizard/LoadBalancerSelection.vue new file mode 100644 index 000000000000..f69fe652253e --- /dev/null +++ b/ui/src/views/compute/wizard/LoadBalancerSelection.vue @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/compute/wizard/NetworkSelection.vue b/ui/src/views/compute/wizard/NetworkSelection.vue index a164b0e9dc8f..b8d18448b8ae 100644 --- a/ui/src/views/compute/wizard/NetworkSelection.vue +++ b/ui/src/views/compute/wizard/NetworkSelection.vue @@ -133,7 +133,7 @@ export default { }, autoscale: { type: Boolean, - default: false + default: () => false }, preFillContent: { type: Object, From a20c23f045eb14954d4dcede78ae44300a1a61b5 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 21 Jun 2022 16:01:21 +0200 Subject: [PATCH 059/360] AS UI: remove keyboard/boot/startvm/affinity/instanceGroups/userdata --- .../views/compute/CreateAutoScaleVmGroup.vue | 307 +----------------- 1 file changed, 5 insertions(+), 302 deletions(-) diff --git a/ui/src/views/compute/CreateAutoScaleVmGroup.vue b/ui/src/views/compute/CreateAutoScaleVmGroup.vue index e085daf03225..cb69b32dddfe 100644 --- a/ui/src/views/compute/CreateAutoScaleVmGroup.vue +++ b/ui/src/views/compute/CreateAutoScaleVmGroup.vue @@ -429,72 +429,6 @@ - - - @@ -504,24 +438,6 @@ - - - - - - - - - @@ -591,7 +507,7 @@ diff --git a/ui/src/views/compute/CreateAutoScaleVmGroup.vue b/ui/src/views/compute/CreateAutoScaleVmGroup.vue index 6bab0b016f08..f65eea81d2ef 100644 --- a/ui/src/views/compute/CreateAutoScaleVmGroup.vue +++ b/ui/src/views/compute/CreateAutoScaleVmGroup.vue @@ -440,15 +440,45 @@