RHINENG-21445: metrics updates

Author: TenSt

dashboards/app-sre/grafana-dashboard-insights-patchman-engine-general.configmap.yaml

@@ -5694,7 +5694,7 @@ data:
                         "datasource": {
                             "uid": "$datasource"
                         },
-                        "expr": "avg(sum(patchman_engine_vmaas_sync_systems{last_upload=~\"last.*|all\"}) by (last_upload, pod)) by (last_upload) - avg(sum(label_replace(patchman_engine_vmaas_sync_cyndi_systems{type=~\"last.*|all\"}, \"last_upload\", \"$1\", \"type\", \"(.*)\")) by (last_upload, pod)) by (last_upload)",
+                        "expr": "avg(sum(patchman_engine_vmaas_sync_systems{last_upload=~\"last.*|all\"}) by (last_upload, pod)) by (last_upload) - avg(sum(label_replace(patchman_engine_vmaas_sync_systems_count_by_type{type=~\"last.*|all\"}, \"last_upload\", \"$1\", \"type\", \"(.*)\")) by (last_upload, pod)) by (last_upload)",
                         "format": "time_series",
                         "hide": false,
                         "intervalFactor": 1,
@@ -6064,14 +6064,14 @@ data:
                         "datasource": {
                             "uid": "$datasource"
                         },
-                        "expr": "avg(sum(patchman_engine_vmaas_sync_cyndi_tags_count) by (type, pod)) by (type)",
+                        "expr": "avg(sum(patchman_engine_vmaas_sync_tags_count_by_type) by (type, pod)) by (type)",
                         "format": "time_series",
                         "intervalFactor": 1,
                         "legendFormat": "{{type}}",
                         "refId": "A"
                     }
                 ],
-                "title": "Database items - Cyndi data",
+                "title": "Database items - System inventory data",
                 "type": "timeseries"
             },
             {

tasks/config.go

@@ -14,9 +14,7 @@ var (
 	DeleteUnusedDataLimit = utils.PodConfig.GetInt("delete_unused_data_limit", 1000)
 	// Remove only LIMIT systems in a run, useful to avoid complete wipe in case of error
 	DeleteCulledSystemsLimit = utils.PodConfig.GetInt("delete_culled_systems_limit", 1000)
-	// Toggle cyndi metrics reporting
-	EnableCyndiMetrics = utils.PodConfig.GetBool("enable_cyndi_metrics", true)
-	UseTraceLevel      = log.IsLevelEnabled(log.TraceLevel)
+	UseTraceLevel            = log.IsLevelEnabled(log.TraceLevel)
 	// Toggle system reevaluation base on changed repos
 	EnabledRepoBasedReeval = utils.PodConfig.GetBool("repo_based_re_evaluation", true)
 	// Send recalc messages for systems with modified repos

tasks/vmaas_sync/metrics.go

@@ -103,7 +103,7 @@ func Metrics() *push.Pusher {
 	registry := prometheus.NewRegistry()
 	registry.MustRegister(vmaasCallCnt, storeAdvisoriesCnt, storePackagesCnt,
 		systemsCnt, advisoriesCnt, systemAdvisoriesStats, syncDuration, messageSendDuration, packageCnt, packageNameCnt,
-		databaseSizeBytesGaugeVec, databaseProcessesGaugeVec, cyndiSystemsCnt, cyndiTagsCnt,
+		databaseSizeBytesGaugeVec, databaseProcessesGaugeVec, systemsCntByType, tagsCntByType,
 		advisoriesCountMismatch)
 
 	// update advanced metrics
@@ -118,10 +118,7 @@ func update() {
 	updatePackageMetrics()
 	updateSystemAdvisoriesStats()
 	updateDBMetrics()
-
-	if tasks.EnableCyndiMetrics {
-		updateCyndiData()
-	}
+	updateSystemInventoryData()
 }
 
 func updateSystemMetrics() {

tasks/vmaas_sync/metrics_cyndi.go

@@ -17,13 +17,12 @@ func TestTableSizes(t *testing.T) {
 	for _, item := range tableSizes {
 		uniqueTables[item.Key] = true
 	}
-	assert.Equal(t, 231, len(tableSizes))
-	assert.Equal(t, 231, len(uniqueTables))
+	assert.Equal(t, 230, len(tableSizes))
+	assert.Equal(t, 230, len(uniqueTables))
 	assert.True(t, uniqueTables["public.system_inventory"]) // check whether table names were loaded
 	assert.True(t, uniqueTables["public.system_patch"])     // check whether table names were loaded
 	assert.True(t, uniqueTables["public.package"])
 	assert.True(t, uniqueTables["public.repo"])
-	assert.True(t, uniqueTables["inventory.hosts_v1_0"])
 }
 
 func TestDatabaseSize(t *testing.T) {

tasks/vmaas_sync/metrics_cyndi_test.go

@@ -0,0 +1,78 @@
+package vmaas_sync
+
+import (
+	"app/base/database"
+	"app/base/utils"
+	"app/tasks"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	allSystemCount        = "allSystem"
+	systemsSapSystemCount = "systemsSapSystem"
+	systemsWithTagsCount  = "systemsWithTag"
+)
+
+var (
+	systemsCntByType = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Help:      "How many systems are stored and how up-to-date they are",
+		Namespace: "patchman_engine",
+		Subsystem: "vmaas_sync",
+		Name:      "systems_count_by_type",
+	}, []string{"type"})
+
+	tagsCntByType = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Help:      "How many inventory hosts and which tags are there",
+		Namespace: "patchman_engine",
+		Subsystem: "vmaas_sync",
+		Name:      "tags_count_by_type",
+	}, []string{"type"})
+)
+
+// nolint: lll
+type systemInventoryMetricColumns struct {
+	AllSystems    int64 `query:"count(*)" gorm:"column:all_systems"`
+	SapSystems    int64 `query:"count(*) filter (where sap_workload = 'true')" gorm:"column:sap_systems"`
+	TaggedSystems int64 `query:"count(*) filter (where jsonb_array_length(tags) > 0)" gorm:"column:tagged_systems"`
+	Updated1D     int64 `query:"count(*) filter (where last_updated > (now() - interval '1 day'))" gorm:"column:updated1d"`
+	Updated7D     int64 `query:"count(*) filter (where last_updated > (now() - interval '7 day'))" gorm:"column:updated7d"`
+	Updated30D    int64 `query:"count(*) filter (where last_updated > (now() - interval '30 day'))" gorm:"column:updated30d"`
+}
+
+var querySystemInventoryMetricColumns = database.MustGetSelect(&systemInventoryMetricColumns{})
+
+func updateSystemInventoryData() {
+	tagStats, systemStats, err := getSystemInventoryData()
+	if err != nil {
+		utils.LogError("err", err.Error(), "unable to update system inventory metrics")
+	}
+
+	for label, count := range tagStats {
+		tagsCntByType.WithLabelValues(label).Set(float64(count))
+	}
+	for label, count := range systemStats {
+		systemsCntByType.WithLabelValues(label).Set(float64(count))
+	}
+}
+
+func getSystemInventoryData() (tagStats map[string]int64, systemStats map[string]int64, err error) {
+	var row systemInventoryMetricColumns
+	err = tasks.CancelableDB().Table("system_inventory").
+		Select(querySystemInventoryMetricColumns).Take(&row).Error
+	if err != nil {
+		return tagStats, systemStats, err
+	}
+	tagStats = map[string]int64{
+		allSystemCount:        row.AllSystems,
+		systemsSapSystemCount: row.SapSystems,
+		systemsWithTagsCount:  row.TaggedSystems,
+	}
+	systemStats = map[string]int64{
+		lastUploadLast1D:  row.Updated1D,
+		lastUploadLast7D:  row.Updated7D,
+		lastUploadLast30D: row.Updated30D,
+		lastUploadAll:     row.AllSystems,
+	}
+	return tagStats, systemStats, nil
+}

tasks/vmaas_sync/metrics_db_test.go

@@ -37,6 +37,27 @@ func TestSystemsCounts(t *testing.T) {
 	assert.Nil(t, shiftSystemsLastUpload(-timeshift))
 }
 
+func TestGetSystemInventoryData(t *testing.T) {
+	utils.SkipWithoutDB(t)
+	core.SetupTestEnvironment()
+
+	tagStats, systemStats, err := getSystemInventoryData()
+	assert.Nil(t, err)
+	assert.NotNil(t, tagStats)
+	assert.NotNil(t, systemStats)
+
+	// Tag stats: all systems, SAP systems, systems with tags (from test data: 18 total, 15 SAP, 16 with tags)
+	assert.Equal(t, int64(18), tagStats[allSystemCount])
+	assert.Equal(t, int64(15), tagStats[systemsSapSystemCount])
+	assert.Equal(t, int64(16), tagStats[systemsWithTagsCount])
+
+	// System stats: updated in last 1D/7D/30D and total
+	assert.GreaterOrEqual(t, systemStats[lastUploadLast1D], int64(0))
+	assert.GreaterOrEqual(t, systemStats[lastUploadLast7D], int64(0))
+	assert.GreaterOrEqual(t, systemStats[lastUploadLast30D], int64(0))
+	assert.Equal(t, int64(18), systemStats[lastUploadAll])
+}
+
 func TestAdvisoryCounts(t *testing.T) {
 	utils.SkipWithoutDB(t)
 	core.SetupTestEnvironment()