Skip to content

Commit fc5e29e

Browse files
askyrieaskyrie
committed
feat: get custom metrics
1 parent 8be2c9a commit fc5e29e

5 files changed

Lines changed: 87 additions & 7 deletions

File tree

‎server/querier/engine/clickhouse/clickhouse.go‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1329,6 +1329,11 @@ func (e *CHEngine) TransFrom(froms sqlparser.TableExprs) error {
13291329
e.Table = table
13301330
// native field
13311331
if config.ControllerCfg.DFWebService.Enabled && (slices.Contains([]string{chCommon.DB_NAME_DEEPFLOW_ADMIN, chCommon.DB_NAME_DEEPFLOW_TENANT, chCommon.DB_NAME_APPLICATION_LOG, chCommon.DB_NAME_EXT_METRICS}, e.DB) || slices.Contains([]string{chCommon.TABLE_NAME_L7_FLOW_LOG, chCommon.TABLE_NAME_EVENT, chCommon.TABLE_NAME_FILE_EVENT}, e.Table)) {
1332+
// get custom-metrics
1333+
customMetrics, err := chCommon.GetCustomMetrics(e.ORGID)
1334+
if err != nil {
1335+
log.Error(err.Error())
1336+
}
13321337
e.NativeField = map[string]*metrics.Metrics{}
13331338
getNativeUrl := fmt.Sprintf("http://localhost:%d/v1/native-fields/?db=%s&table_name=%s", config.ControllerCfg.ListenPort, e.DB, e.Table)
13341339
resp, err := ctlcommon.CURLPerform("GET", getNativeUrl, nil, ctlcommon.WithHeader(ctlcommon.HEADER_KEY_X_ORG_ID, e.ORGID))
@@ -1346,9 +1351,18 @@ func (e *CHEngine) TransFrom(froms sqlparser.TableExprs) error {
13461351
continue
13471352
}
13481353
if fieldType == chCommon.NATIVE_FIELD_TYPE_METRIC {
1354+
var mUnit string
1355+
mType := metrics.METRICS_TYPE_COUNTER
1356+
customMetric, ok := customMetrics[fmt.Sprintf("%s.%s.%s", e.DB, e.Table, nativeMetric)]
1357+
if ok {
1358+
mType = customMetric.Get("TYPE").MustInt()
1359+
mUnit = customMetric.Get("UNIT").MustString()
1360+
displayName = customMetric.Get("DISPLAY_NAME").MustString()
1361+
description = customMetric.Get("DESCRIPTION").MustString()
1362+
}
13491363
metric := metrics.NewMetrics(
13501364
0, nativeMetric,
1351-
displayName, displayName, displayName, "", "", "", metrics.METRICS_TYPE_COUNTER,
1365+
displayName, displayName, displayName, mUnit, mUnit, mUnit, mType,
13521366
chCommon.NATIVE_FIELD_CATEGORY_METRICS, []bool{true, true, true}, "", table, description, description, description, "", "",
13531367
)
13541368
e.NativeField[nativeMetric] = metric

‎server/querier/engine/clickhouse/common/const.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ const (
5757
NATIVE_FIELD_STATE_NORMAL = 1
5858
)
5959

60+
const (
61+
API_CUSTOM_METRICS_FORMAT = "http://localhost:%d/v1/custom-metrics"
62+
)
63+
6064
var DB_TABLE_MAP = map[string][]string{
6165
DB_NAME_FLOW_LOG: []string{"l4_flow_log", "l7_flow_log", "l4_packet", "l7_packet"},
6266
DB_NAME_FLOW_METRICS: []string{"network", "network_map", "application", "application_map", "traffic_policy"},

‎server/querier/engine/clickhouse/common/utils.go‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,13 @@ import (
2828
"strconv"
2929
"strings"
3030

31+
"github.com/bitly/go-simplejson"
32+
"github.com/xwb1989/sqlparser"
33+
34+
ctlcommon "github.com/deepflowio/deepflow/server/controller/common"
3135
"github.com/deepflowio/deepflow/server/querier/config"
3236
"github.com/deepflowio/deepflow/server/querier/engine/clickhouse/client"
3337
logging "github.com/op/go-logging"
34-
"github.com/xwb1989/sqlparser"
3538
)
3639

3740
var log = logging.MustGetLogger("common")
@@ -252,3 +255,25 @@ func GetExtTables(db, where, queryCacheTTL, orgID string, useQueryCache bool, ct
252255
}
253256
return values
254257
}
258+
259+
func GetCustomMetrics(orgID string) (map[string]*simplejson.Json, error) {
260+
result := make(map[string]*simplejson.Json)
261+
url := fmt.Sprintf(API_CUSTOM_METRICS_FORMAT, config.ControllerCfg.ListenPort)
262+
resp, err := ctlcommon.CURLPerform("GET", url, nil,
263+
ctlcommon.WithHeader(ctlcommon.HEADER_KEY_X_ORG_ID, orgID),
264+
ctlcommon.WithHeader(ctlcommon.HEADER_KEY_X_USER_ID, strconv.Itoa(ctlcommon.USER_ID_SUPER_ADMIN)),
265+
ctlcommon.WithHeader(ctlcommon.HEADER_KEY_X_USER_TYPE, strconv.Itoa(ctlcommon.USER_TYPE_SUPER_ADMIN)),
266+
)
267+
if err != nil {
268+
return result, fmt.Errorf("request controller url (%s) failed: %s", url, err.Error())
269+
}
270+
resultArray := resp.Get("DATA").MustArray()
271+
for i := range resultArray {
272+
item := resp.Get("DATA").GetIndex(i)
273+
db := item.Get("DB").MustString()
274+
table := item.Get("TABLE").MustString()
275+
name := item.Get("NAME").MustString()
276+
result[fmt.Sprintf("%s.%s.%s", db, table, name)] = item
277+
}
278+
return result, nil
279+
}

‎server/querier/engine/clickhouse/metrics/ext_common.go‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,16 +59,29 @@ func GetExtMetrics(db, table, where, queryCacheTTL, orgID string, useQueryCache
5959
log.Error(err)
6060
return nil, err
6161
}
62+
customMetrics, err := common.GetCustomMetrics(orgID)
63+
if err != nil {
64+
log.Error(err.Error())
65+
}
6266
for i, value := range externalMetricFloatRst.Values {
6367
tagName := value.([]interface{})[0]
6468
tableName := value.([]interface{})[1].(string)
6569
externalTag := tagName.(string)
6670
metrics_names_field, metrics_values_field := METRICS_ARRAY_NAME_MAP[db][0], METRICS_ARRAY_NAME_MAP[db][1]
6771
dbField := fmt.Sprintf("if(indexOf(%s, '%s')=0, null, %s[indexOf(%s, '%s')])", metrics_names_field, externalTag, metrics_values_field, metrics_names_field, externalTag)
6872
metricName := fmt.Sprintf("metrics.%s", externalTag)
73+
var mUnit, description string
74+
mType := METRICS_TYPE_COUNTER
75+
customMetric, ok := customMetrics[fmt.Sprintf("%s.%s.%s", db, tableName, externalTag)]
76+
if ok {
77+
mType = customMetric.Get("TYPE").MustInt()
78+
mUnit = customMetric.Get("UNIT").MustString()
79+
metricName = customMetric.Get("DISPLAY_NAME").MustString()
80+
description = customMetric.Get("DESCRIPTION").MustString()
81+
}
6982
lm := NewMetrics(
70-
i, dbField, metricName, metricName, metricName, "", "", "", METRICS_TYPE_COUNTER,
71-
common.NATIVE_FIELD_CATEGORY_METRICS, []bool{true, true, true}, "", tableName, "", "", "", "", "",
83+
i, dbField, metricName, metricName, metricName, mUnit, mUnit, mUnit, mType,
84+
common.NATIVE_FIELD_CATEGORY_METRICS, []bool{true, true, true}, "", tableName, description, description, description, "", "",
7285
)
7386
loadMetrics[fmt.Sprintf("%s-%s", metricName, tableName)] = lm
7487
}
@@ -101,8 +114,17 @@ func GetExtMetrics(db, table, where, queryCacheTTL, orgID string, useQueryCache
101114
if fieldType != common.NATIVE_FIELD_TYPE_METRIC {
102115
continue
103116
}
117+
var mUnit string
118+
mType := METRICS_TYPE_COUNTER
119+
customMetric, ok := customMetrics[fmt.Sprintf("%s.%s.%s", db, table, nativeMetric)]
120+
if ok {
121+
mType = customMetric.Get("TYPE").MustInt()
122+
mUnit = customMetric.Get("UNIT").MustString()
123+
displayName = customMetric.Get("DISPLAY_NAME").MustString()
124+
description = customMetric.Get("DESCRIPTION").MustString()
125+
}
104126
lm := NewMetrics(
105-
len(loadMetrics), nativeMetric, displayName, displayName, displayName, "", "", "", METRICS_TYPE_COUNTER,
127+
len(loadMetrics), nativeMetric, displayName, displayName, displayName, mUnit, mUnit, mUnit, mType,
106128
common.NATIVE_FIELD_CATEGORY_METRICS, []bool{true, true, true}, "", table, description, description, description, "", "",
107129
)
108130
loadMetrics[fmt.Sprintf("%s-%s", nativeMetric, table)] = lm

‎server/querier/engine/clickhouse/metrics/metrics.go‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -241,17 +241,32 @@ func GetMetrics(field, db, table, orgID string, nativeField map[string]*Metrics)
241241
)
242242
return metric, true
243243
}
244+
244245
// dynamic metrics
245246
if slices.Contains([]string{ckcommon.DB_NAME_DEEPFLOW_ADMIN, ckcommon.DB_NAME_DEEPFLOW_TENANT, ckcommon.DB_NAME_APPLICATION_LOG, ckcommon.DB_NAME_EXT_METRICS}, db) || slices.Contains([]string{ckcommon.TABLE_NAME_L7_FLOW_LOG, ckcommon.TABLE_NAME_EVENT, ckcommon.TABLE_NAME_FILE_EVENT}, table) {
246247
fieldSplit := strings.Split(field, ".")
247248
if len(fieldSplit) > 1 {
248249
if fieldSplit[0] == "metrics" {
249250
fieldName := strings.Replace(field, "metrics.", "", 1)
251+
var mUnit, description string
252+
mType := METRICS_TYPE_COUNTER
253+
customMetrics, err := ckcommon.GetCustomMetrics(orgID)
254+
if err != nil {
255+
log.Error(err.Error())
256+
} else {
257+
customMetric, ok := customMetrics[fmt.Sprintf("%s.%s.%s", db, table, fieldName)]
258+
if ok {
259+
mType = customMetric.Get("TYPE").MustInt()
260+
mUnit = customMetric.Get("UNIT").MustString()
261+
field = customMetric.Get("DISPLAY_NAME").MustString()
262+
description = customMetric.Get("DESCRIPTION").MustString()
263+
}
264+
}
250265
metrics_names_field, metrics_values_field := METRICS_ARRAY_NAME_MAP[db][0], METRICS_ARRAY_NAME_MAP[db][1]
251266
metric := NewMetrics(
252267
0, fmt.Sprintf("if(indexOf(%s, '%s')=0,null,%s[indexOf(%s, '%s')])", metrics_names_field, fieldName, metrics_values_field, metrics_names_field, fieldName),
253-
field, field, field, "", "", "", METRICS_TYPE_COUNTER,
254-
ckcommon.NATIVE_FIELD_CATEGORY_METRICS, []bool{true, true, true}, "", table, "", "", "", "", "",
268+
field, field, field, mUnit, mUnit, mUnit, mType,
269+
ckcommon.NATIVE_FIELD_CATEGORY_METRICS, []bool{true, true, true}, "", table, description, description, description, "", "",
255270
)
256271
return metric, true
257272
} else if fieldSplit[0] == "tag" {

0 commit comments

Comments
 (0)