exporter: parallelize metrics generation and refresh in background

Author: te.li

charts/hami-webui/templates/configmap.yaml

@@ -15,7 +15,9 @@ data:
     prometheus:
       address: {{ ternary .Values.externalPrometheus.address (printf "http://%s-kube-prometh-prometheus.%s.svc.cluster.local:9090" (include "hami-webui.fullname" .) (include "hami-webui.namespace" .)) .Values.externalPrometheus.enabled }}
       timeout: 1m
+    exporter_concurrency_limit: {{ .Values.exporterConcurrencyLimit }}
+    metrics_generate_interval: {{ .Values.metricsGenerateInterval }}
     node_selectors:
     {{- range $key, $value := .Values.vendorNodeSelectors }}
       {{ $key }}: {{ $value }}
-    {{- end }}
+    {{- end }}

charts/hami-webui/values.yaml

@@ -126,6 +126,9 @@ tolerations: []
 
 affinity: {}
 
+exporterConcurrencyLimit: 16
+metricsGenerateInterval: 15s
+
 dcgm-exporter:
   enabled: true
   serviceMonitor:
@@ -162,4 +165,4 @@ kube-prometheus-stack:
 externalPrometheus:
   enabled: false
   # If externalPrometheus.enabled is true, this address will be used
-  address: "http://prometheus-kube-prometheus-prometheus.prometheus.svc.cluster.local:9090"
+  address: "http://prometheus-kube-prometheus-prometheus.prometheus.svc.cluster.local:9090"

server/config/config.yaml

@@ -13,4 +13,6 @@ node_selectors:
   Ascend: ascend=on
   DCU: dcu=on
   MLU: mlu=on
-  Metax: metax-tech.com/gpu.installed=true
+  Metax: metax-tech.com/gpu.installed=true
+exporter_concurrency_limit: 16
+metrics_generate_interval: 15s

server/go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/prometheus/client_golang v1.19.1
 	github.com/prometheus/common v0.55.0
 	go.uber.org/automaxprocs v1.5.3
+	golang.org/x/sync v0.7.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157
 	google.golang.org/grpc v1.64.0
 	google.golang.org/protobuf v1.34.2
@@ -63,7 +64,6 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/term v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect

server/internal/conf/conf.proto

@@ -9,6 +9,8 @@ message Bootstrap {
   Server server = 1;
   Prometheus prometheus = 2;
   map<string, string> node_selectors = 3;
+  int32 exporter_concurrency_limit = 5;
+  string metrics_generate_interval = 6;
 }
 
 message Server {