Skip to content

Commit 4e2f461

Browse files
soreanasoreana
authored
Prometheus exporter enhancement (#4438)
* Export count of total/up/down hosts by tags * Export count of vms by state and host tag. * Add host tags to host cpu/cores/memory usage in Prometheus exporter * Cloudstack Prometheus exporter: Add allocated capacity group by host tag. * Show count of Active domains on grafana. * Show count of Active accounts and vms by size on grafana * Use prepared statement to query database for a number of VM who use a specific tag. * Extract repeated codes to new methods.
1 parent 784578d commit 4e2f461

File tree

11 files changed

+376
-39
lines changed

11 files changed

+376
-39
lines changed

‎engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.cloud.capacity.CapacityVO;
2323
import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity;
2424
import com.cloud.utils.Pair;
25+
import com.cloud.utils.Ternary;
2526
import com.cloud.utils.db.GenericDao;
2627

2728
public interface CapacityDao extends GenericDao<CapacityVO, Long> {
@@ -39,6 +40,8 @@ public interface CapacityDao extends GenericDao<CapacityVO, Long> {
3940

4041
Pair<List<Long>, Map<Long, Double>> orderClustersByAggregateCapacity(long id, long vmId, short capacityType, boolean isZone);
4142

43+
Ternary<Long, Long, Long> findCapacityByZoneAndHostTag(Long zoneId, String hostTag);
44+
4245
List<SummedCapacity> findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId);
4346

4447
List<Long> listPodsByHostCapacities(long zoneId, int requiredCpu, long requiredRam, short capacityType);

‎engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import javax.inject.Inject;
2828

2929
import org.apache.log4j.Logger;
30+
import org.apache.commons.lang3.StringUtils;
3031
import org.springframework.stereotype.Component;
3132

3233
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
@@ -37,6 +38,7 @@
3738
import com.cloud.dc.ClusterDetailsDao;
3839
import com.cloud.storage.Storage;
3940
import com.cloud.utils.Pair;
41+
import com.cloud.utils.Ternary;
4042
import com.cloud.utils.db.GenericDaoBase;
4143
import com.cloud.utils.db.GenericSearchBuilder;
4244
import com.cloud.utils.db.JoinBuilder.JoinType;
@@ -203,11 +205,15 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
203205
"FROM (SELECT vi.data_center_id, (CASE WHEN ISNULL(service_offering.cpu) THEN custom_cpu.value ELSE service_offering.cpu end) AS cpu, " +
204206
"(CASE WHEN ISNULL(service_offering.speed) THEN custom_speed.value ELSE service_offering.speed end) AS speed, " +
205207
"(CASE WHEN ISNULL(service_offering.ram_size) THEN custom_ram_size.value ELSE service_offering.ram_size end) AS ram_size " +
206-
"FROM (((vm_instance vi LEFT JOIN service_offering ON(((vi.service_offering_id = service_offering.id))) " +
207-
"LEFT JOIN user_vm_details custom_cpu ON(((custom_cpu.vm_id = vi.id) AND (custom_cpu.name = 'CpuNumber')))) " +
208-
"LEFT JOIN user_vm_details custom_speed ON(((custom_speed.vm_id = vi.id) AND (custom_speed.name = 'CpuSpeed')))) " +
209-
"LEFT JOIN user_vm_details custom_ram_size ON(((custom_ram_size.vm_id = vi.id) AND (custom_ram_size.name = 'memory')))) " +
210-
"WHERE ISNULL(vi.removed) AND vi.state NOT IN ('Destroyed', 'Error', 'Expunging')";
208+
"FROM vm_instance vi LEFT JOIN service_offering ON(((vi.service_offering_id = service_offering.id))) " +
209+
"LEFT JOIN user_vm_details custom_cpu ON(((custom_cpu.vm_id = vi.id) AND (custom_cpu.name = 'CpuNumber'))) " +
210+
"LEFT JOIN user_vm_details custom_speed ON(((custom_speed.vm_id = vi.id) AND (custom_speed.name = 'CpuSpeed'))) " +
211+
"LEFT JOIN user_vm_details custom_ram_size ON(((custom_ram_size.vm_id = vi.id) AND (custom_ram_size.name = 'memory'))) ";
212+
213+
private static final String WHERE_STATE_IS_NOT_DESTRUCTIVE =
214+
"WHERE ISNULL(vi.removed) AND vi.state NOT IN ('Destroyed', 'Error', 'Expunging')";
215+
216+
private static final String LEFT_JOIN_VM_TEMPLATE = "LEFT JOIN vm_template ON vm_template.id = vi.vm_template_id ";
211217

212218
public CapacityDaoImpl() {
213219
_hostIdTypeSearch = createSearchBuilder();
@@ -422,6 +428,41 @@ public List<SummedCapacity> listCapacitiesGroupedByLevelAndType(Integer capacity
422428

423429
}
424430

431+
@Override
432+
public Ternary<Long, Long, Long> findCapacityByZoneAndHostTag(Long zoneId, String hostTag) {
433+
TransactionLegacy txn = TransactionLegacy.currentTxn();
434+
PreparedStatement pstmt;
435+
436+
StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE);
437+
if (StringUtils.isNotEmpty(hostTag)) {
438+
allocatedSql.append(LEFT_JOIN_VM_TEMPLATE);
439+
}
440+
allocatedSql.append(WHERE_STATE_IS_NOT_DESTRUCTIVE);
441+
if (zoneId != null) {
442+
allocatedSql.append(" AND vi.data_center_id = ?");
443+
}
444+
if (StringUtils.isNotEmpty(hostTag)) {
445+
allocatedSql.append(" AND (vm_template.template_tag = '").append(hostTag).append("'");
446+
allocatedSql.append(" OR service_offering.host_tag = '").append(hostTag).append("')");
447+
}
448+
allocatedSql.append(" ) AS v GROUP BY v.data_center_id");
449+
450+
try {
451+
// add allocated capacity of zone in result
452+
pstmt = txn.prepareAutoCloseStatement(allocatedSql.toString());
453+
if (zoneId != null) {
454+
pstmt.setLong(1, zoneId);
455+
}
456+
ResultSet rs = pstmt.executeQuery();
457+
if (rs.next()) {
458+
return new Ternary<>(rs.getLong(2), rs.getLong(3), rs.getLong(4)); // cpu cores, cpu, memory
459+
}
460+
return new Ternary<>(0L, 0L, 0L);
461+
} catch (SQLException e) {
462+
throw new CloudRuntimeException("DB Exception on: " + allocatedSql, e);
463+
}
464+
}
465+
425466
@Override
426467
public List<SummedCapacity> findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId) {
427468

@@ -430,6 +471,7 @@ public List<SummedCapacity> findCapacityBy(Integer capacityType, Long zoneId, Lo
430471
List<SummedCapacity> results = new ArrayList<SummedCapacity>();
431472

432473
StringBuilder allocatedSql = new StringBuilder(LIST_ALLOCATED_CAPACITY_GROUP_BY_CAPACITY_AND_ZONE);
474+
allocatedSql.append(WHERE_STATE_IS_NOT_DESTRUCTIVE);
433475

434476
HashMap<Long, Long> sumCpuCore = new HashMap<Long, Long>();
435477
HashMap<Long, Long> sumCpu = new HashMap<Long, Long>();

‎engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,4 +78,5 @@ public interface AccountDao extends GenericDao<AccountVO, Long> {
7878
*/
7979
long getDomainIdForGivenAccountId(long id);
8080

81+
int getActiveDomains();
8182
}

‎engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import com.cloud.utils.db.GenericSearchBuilder;
2929
import com.cloud.utils.db.SearchBuilder;
3030
import com.cloud.utils.db.SearchCriteria;
31+
import com.cloud.utils.db.SearchCriteria.Func;
3132
import com.cloud.utils.db.SearchCriteria.Op;
3233
import org.apache.commons.lang3.StringUtils;
3334
import com.cloud.utils.db.TransactionLegacy;
@@ -54,6 +55,7 @@ public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements A
5455
protected final SearchBuilder<AccountVO> NonProjectAccountSearch;
5556
protected final SearchBuilder<AccountVO> AccountByRoleSearch;
5657
protected final GenericSearchBuilder<AccountVO, Long> AccountIdsSearch;
58+
protected final GenericSearchBuilder<AccountVO, Long> ActiveDomainCount;
5759

5860
public AccountDaoImpl() {
5961
AllFieldsSearch = createSearchBuilder();
@@ -101,6 +103,13 @@ public AccountDaoImpl() {
101103
AccountByRoleSearch = createSearchBuilder();
102104
AccountByRoleSearch.and("roleId", AccountByRoleSearch.entity().getRoleId(), SearchCriteria.Op.EQ);
103105
AccountByRoleSearch.done();
106+
107+
ActiveDomainCount = createSearchBuilder(Long.class);
108+
ActiveDomainCount.select(null, Func.COUNT, null);
109+
ActiveDomainCount.and("domain", ActiveDomainCount.entity().getDomainId(), SearchCriteria.Op.EQ);
110+
ActiveDomainCount.and("state", ActiveDomainCount.entity().getState(), SearchCriteria.Op.EQ);
111+
ActiveDomainCount.groupBy(ActiveDomainCount.entity().getDomainId());
112+
ActiveDomainCount.done();
104113
}
105114

106115
@Override
@@ -318,5 +327,10 @@ public long getDomainIdForGivenAccountId(long id) {
318327
}
319328
}
320329

321-
330+
@Override
331+
public int getActiveDomains() {
332+
SearchCriteria<Long> sc = ActiveDomainCount.create();
333+
sc.setParameters("state", "enabled");
334+
return customSearch(sc, null).size();
335+
}
322336
}

‎engine/schema/src/main/java/com/cloud/vm/dao/UserVmDao.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.util.Set;
2323

2424
import com.cloud.utils.Pair;
25+
import com.cloud.utils.Ternary;
2526
import com.cloud.utils.db.GenericDao;
2627
import com.cloud.vm.UserVmVO;
2728
import com.cloud.vm.VirtualMachine;
@@ -92,4 +93,8 @@ public interface UserVmDao extends GenericDao<UserVmVO, Long> {
9293
List<UserVmVO> listByIsoId(Long isoId);
9394

9495
List<Pair<Pair<String, VirtualMachine.Type>, Pair<Long, String>>> getVmsDetailByNames(Set<String> vmNames, String detail);
96+
97+
List<Ternary<Integer, Integer, Integer>> countVmsBySize(long dcId, int limit);
98+
99+
int getActiveAccounts(final long dcId);
95100
}

‎engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import com.cloud.tags.dao.ResourceTagDao;
4141
import com.cloud.user.Account;
4242
import com.cloud.utils.Pair;
43+
import com.cloud.utils.Ternary;
4344
import com.cloud.utils.db.Attribute;
4445
import com.cloud.utils.db.GenericDaoBase;
4546
import com.cloud.utils.db.GenericSearchBuilder;
@@ -75,6 +76,7 @@ public class UserVmDaoImpl extends GenericDaoBase<UserVmVO, Long> implements Use
7576
protected SearchBuilder<UserVmVO> AccountDataCenterVirtualSearch;
7677
protected GenericSearchBuilder<UserVmVO, Long> CountByAccountPod;
7778
protected GenericSearchBuilder<UserVmVO, Long> CountByAccount;
79+
protected GenericSearchBuilder<UserVmVO, Long> CountActiveAccount;
7880
protected GenericSearchBuilder<UserVmVO, Long> PodsHavingVmsForAccount;
7981

8082
protected SearchBuilder<UserVmVO> UserVmSearch;
@@ -192,6 +194,15 @@ void init() {
192194
CountByAccount.and("displayVm", CountByAccount.entity().isDisplayVm(), SearchCriteria.Op.EQ);
193195
CountByAccount.done();
194196

197+
CountActiveAccount = createSearchBuilder(Long.class);
198+
CountActiveAccount.select(null, Func.COUNT, null);
199+
CountActiveAccount.and("account", CountActiveAccount.entity().getAccountId(), SearchCriteria.Op.EQ);
200+
CountActiveAccount.and("type", CountActiveAccount.entity().getType(), SearchCriteria.Op.EQ);
201+
CountActiveAccount.and("dataCenterId", CountActiveAccount.entity().getDataCenterId(), SearchCriteria.Op.EQ);
202+
CountActiveAccount.and("state", CountActiveAccount.entity().getState(), SearchCriteria.Op.NIN);
203+
CountActiveAccount.groupBy(CountActiveAccount.entity().getAccountId());
204+
CountActiveAccount.done();
205+
195206
SearchBuilder<NicVO> nicSearch = _nicDao.createSearchBuilder();
196207
nicSearch.and("networkId", nicSearch.entity().getNetworkId(), SearchCriteria.Op.EQ);
197208
nicSearch.and("ip4Address", nicSearch.entity().getIPv4Address(), SearchCriteria.Op.NNULL);
@@ -721,4 +732,35 @@ public List<Pair<Pair<String, VirtualMachine.Type>, Pair<Long, String>>> getVmsD
721732

722733
return vmsDetailByNames;
723734
}
735+
736+
@Override
737+
public List<Ternary<Integer, Integer, Integer>> countVmsBySize(long dcId, int limit) {
738+
TransactionLegacy txn = TransactionLegacy.currentTxn();
739+
String sql = "SELECT cpu,ram_size,count(1) AS count FROM (SELECT * FROM user_vm_view WHERE data_center_id = ? AND state NOT IN ('Destroyed', 'Error', 'Expunging') GROUP BY id) AS uvv GROUP BY cpu,ram_size ORDER BY count DESC ";
740+
if (limit >= 0)
741+
sql = sql + "limit " + limit;
742+
PreparedStatement pstmt = null;
743+
List<Ternary<Integer, Integer, Integer>> result = new ArrayList<>();
744+
try {
745+
pstmt = txn.prepareAutoCloseStatement(sql);
746+
pstmt.setLong(1, dcId);
747+
ResultSet rs = pstmt.executeQuery();
748+
while (rs.next()) {
749+
result.add(new Ternary<Integer, Integer, Integer>(rs.getInt(1), rs.getInt(2), rs.getInt(3)));
750+
}
751+
} catch (Exception e) {
752+
s_logger.warn("Error counting vms by size for dcId= " + dcId, e);
753+
}
754+
return result;
755+
}
756+
757+
@Override
758+
public int getActiveAccounts(final long dcId) {
759+
SearchCriteria<Long> sc = CountActiveAccount.create();
760+
sc.setParameters("type", VirtualMachine.Type.User);
761+
sc.setParameters("state", State.Destroyed, State.Error, State.Expunging, State.Stopped);
762+
sc.setParameters("dataCenterId", dcId);
763+
764+
return customSearch(sc, null).size();
765+
}
724766
}

‎engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
133133

134134
Long countByZoneAndState(long zoneId, State state);
135135

136+
Long countByZoneAndStateAndHostTag(long dcId, State state, String hostTag);
137+
136138
List<VMInstanceVO> listNonRemovedVmsByTypeAndNetwork(long networkId, VirtualMachine.Type... types);
137139

138140
/**

‎engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,9 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
129129

130130
private static final String UPDATE_SYSTEM_VM_TEMPLATE_ID_FOR_HYPERVISOR = "UPDATE `cloud`.`vm_instance` SET vm_template_id = ? WHERE type <> 'User' AND hypervisor_type = ? AND removed is NULL";
131131

132+
private static final String COUNT_VMS_BY_ZONE_AND_STATE_AND_HOST_TAG = "SELECT COUNT(1) FROM vm_instance vi JOIN service_offering so ON vi.service_offering_id=so.id " +
133+
"JOIN vm_template vt ON vi.vm_template_id = vt.id WHERE vi.data_center_id = ? AND vi.state = ? AND vi.removed IS NULL AND (so.host_tag = ? OR vt.template_tag = ?)";
134+
132135
@Inject
133136
protected HostDao _hostDao;
134137

@@ -807,6 +810,28 @@ public Long countByZoneAndState(long zoneId, State state) {
807810
return customSearch(sc, null).get(0);
808811
}
809812

813+
@Override
814+
public Long countByZoneAndStateAndHostTag(long dcId, State state, String hostTag) {
815+
TransactionLegacy txn = TransactionLegacy.currentTxn();
816+
PreparedStatement pstmt = null;
817+
try {
818+
pstmt = txn.prepareAutoCloseStatement(COUNT_VMS_BY_ZONE_AND_STATE_AND_HOST_TAG);
819+
820+
pstmt.setLong(1, dcId);
821+
pstmt.setString(2, String.valueOf(state));
822+
pstmt.setString(3, hostTag);
823+
pstmt.setString(4, hostTag);
824+
825+
ResultSet rs = pstmt.executeQuery();
826+
if (rs.next()) {
827+
return rs.getLong(1);
828+
}
829+
} catch (Exception e) {
830+
s_logger.warn(String.format("Error counting vms by host tag for dcId= %s, hostTag= %s", dcId, hostTag), e);
831+
}
832+
return 0L;
833+
}
834+
810835
@Override
811836
public List<VMInstanceVO> listNonRemovedVmsByTypeAndNetwork(long networkId, VirtualMachine.Type... types) {
812837
if (NetworkTypeSearch == null) {

0 commit comments

Comments
 (0)