diff --git a/pkg/microservice/aslan/server/rest/router.go b/pkg/microservice/aslan/server/rest/router.go index 5e0c10afd3..ea7ce2bc4b 100644 --- a/pkg/microservice/aslan/server/rest/router.go +++ b/pkg/microservice/aslan/server/rest/router.go @@ -18,8 +18,6 @@ package rest import ( "github.com/gin-gonic/gin" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" swaggerfiles "github.com/swaggo/files" ginswagger "github.com/swaggo/gin-swagger" @@ -29,13 +27,11 @@ import ( codehosthandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/code/handler" collaborationhandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/collaboration/handler" commonhandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/handler" - "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/service/workflowcontroller" cronhandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/cron/handler" deliveryhandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/delivery/handler" environmenthandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/environment/handler" loghandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/log/handler" multiclusterhandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/multicluster/handler" - clusterservice "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/multicluster/service" pluginhandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/plugin/handler" projecthandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/project/handler" releaseplanhandler "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/release_plan/handler" @@ -56,29 +52,10 @@ import ( connectorHandler "github.com/koderover/zadig/v2/pkg/microservice/systemconfig/core/connector/handler" emailHandler "github.com/koderover/zadig/v2/pkg/microservice/systemconfig/core/email/handler" featuresHandler "github.com/koderover/zadig/v2/pkg/microservice/systemconfig/core/features/handler" - "github.com/koderover/zadig/v2/pkg/tool/metrics" // Note: have to load docs for swagger to work. See https://blog.csdn.net/weixin_43249914/article/details/103035711 // _ "github.com/koderover/zadig/v2/pkg/microservice/aslan/server/rest/doc" ) -func init() { - // initialization for prometheus metrics - metrics.Metrics = prometheus.NewRegistry() - - metrics.Metrics.MustRegister(metrics.RunningWorkflows) - metrics.Metrics.MustRegister(metrics.PendingWorkflows) - metrics.Metrics.MustRegister(metrics.RequestTotal) - metrics.Metrics.MustRegister(metrics.CPU) - metrics.Metrics.MustRegister(metrics.Memory) - metrics.Metrics.MustRegister(metrics.CPUPercentage) - metrics.Metrics.MustRegister(metrics.MemoryPercentage) - metrics.Metrics.MustRegister(metrics.Healthy) - metrics.Metrics.MustRegister(metrics.Cluster) - metrics.Metrics.MustRegister(metrics.ResponseTime) - - metrics.UpdatePodMetrics() -} - // @title Zadig aslan service REST APIs // @version 1.0 // @description The API doc is targeting for Zadig developers rather than Zadig users. @@ -187,26 +164,6 @@ func (s *engine) injectRouterGroup(router *gin.RouterGroup) { } router.GET("/api/apidocs/*any", ginswagger.WrapHandler(swaggerfiles.Handler)) - - // prometheus metrics API - handlefunc := func(c *gin.Context) { - metrics.UpdatePodMetrics() - - runningCustomQueue := workflowcontroller.RunningTasks() - pendingCustomQueue := workflowcontroller.PendingTasks() - - metrics.SetRunningWorkflows(int64(len(runningCustomQueue))) - metrics.SetPendingWorkflows(int64(len(pendingCustomQueue))) - - metrics.Cluster.Reset() - clusterStatusMap := clusterservice.GetClusterStatus() - for clusterName, status := range clusterStatusMap { - metrics.SetClusterStatus(clusterName, status) - } - - promhttp.HandlerFor(metrics.Metrics, promhttp.HandlerOpts{}).ServeHTTP(c.Writer, c.Request) - } - router.GET("/api/metrics", handlefunc) } type injector interface { diff --git a/pkg/microservice/aslan/server/rest/server.go b/pkg/microservice/aslan/server/rest/server.go index 5a02deed80..98c54c8e00 100644 --- a/pkg/microservice/aslan/server/rest/server.go +++ b/pkg/microservice/aslan/server/rest/server.go @@ -52,7 +52,6 @@ func (s *engine) injectMiddlewares() { return } g.Use(ginmiddleware.ProcessLicense()) - g.Use(ginmiddleware.RegisterRequest()) g.Use(ginmiddleware.OperationLogStatus()) g.Use(ginmiddleware.Response()) g.Use(ginmiddleware.RequestID()) diff --git a/pkg/middleware/gin/request_metrics.go b/pkg/middleware/gin/request_metrics.go deleted file mode 100644 index 46c90c8dd2..0000000000 --- a/pkg/middleware/gin/request_metrics.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2023 The KodeRover Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gin - -import ( - "time" - - "github.com/gin-gonic/gin" - - "github.com/koderover/zadig/v2/pkg/tool/metrics" -) - -func RegisterRequest() gin.HandlerFunc { - return func(c *gin.Context) { - if c.Request.Header.Get("Accept") == "text/event-stream" { - return - } - startTime := time.Now().UnixMilli() - path := c.Request.URL.Path - c.Next() - - metrics.RegisterRequest(startTime, c.Request.Method, path, c.Writer.Status()) - } -} diff --git a/pkg/tool/metrics/prometheus.go b/pkg/tool/metrics/prometheus.go deleted file mode 100644 index 206e19ab44..0000000000 --- a/pkg/tool/metrics/prometheus.go +++ /dev/null @@ -1,249 +0,0 @@ -package metrics - -import ( - "context" - "fmt" - "strings" - "time" - - "github.com/koderover/zadig/v2/pkg/tool/clientmanager" - "github.com/prometheus/client_golang/prometheus" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/metrics/pkg/apis/metrics/v1beta1" - - "github.com/koderover/zadig/v2/pkg/microservice/aslan/config" - "github.com/koderover/zadig/v2/pkg/setting" - "github.com/koderover/zadig/v2/pkg/shared/kube/wrapper" - "github.com/koderover/zadig/v2/pkg/tool/kube/getter" -) - -var ( - Metrics *prometheus.Registry - - serviceList = []string{ - "aslan", - "cron", - "dex", - "discovery", - "dind", - "gateway-proxy", - "gateway", - "gloo", - "hub-server", - "kr-minio", - "kr-mysql", - "kr-mongodb", - "nsqlookup", - "opa", - "plutus-vendor", - "user", - "vendor-portal", - "warpdrive", - "zadig-portal", - "time-nlp", - } - - RunningWorkflows = prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "running_workflows", - Help: "Number of currently running workflows", - }, - ) - - PendingWorkflows = prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "pending_workflows", - Help: "Number of currently pending workflows", - }, - ) - - RequestTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "request_total", - Help: "Number of requests", - }, - []string{"method", "handler", "status"}, - ) - - CPU = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "cpu", - Help: "CPU usage", - }, - []string{"service", "pod"}, - ) - - Memory = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "memory", - Help: "Memory usage", - }, - []string{"service", "pod"}, - ) - - CPUPercentage = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "cpu_percentage", - Help: "CPU usage percentage", - }, - []string{"service", "pod"}, - ) - - MemoryPercentage = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "memory_percentage", - Help: "Memory usage percentage", - }, - []string{"service", "pod"}, - ) - - Healthy = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "healthy", - Help: "service healthy status", - }, - []string{"service", "pod"}, - ) - - Cluster = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "cluster", - Help: "cluster status", - }, - []string{"cluster"}, - ) - - ResponseTime = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "api_response_time", - Help: "The API response time in seconds", - Buckets: prometheus.LinearBuckets(0.2, 0.2, 10), - }, - []string{"method", "handler", "status"}, - ) -) - -func SetRunningWorkflows(value int64) { - RunningWorkflows.Set(float64(value)) -} - -func SetPendingWorkflows(value int64) { - PendingWorkflows.Set(float64(value)) -} - -func RegisterRequest(startTime int64, method, handler string, status int) { - RequestTotal.WithLabelValues(method, handler, fmt.Sprintf("%d", status)).Inc() - ResponseTime.WithLabelValues(method, handler, fmt.Sprintf("%d", status)).Observe(float64(time.Now().UnixMilli()-startTime) / 1000) -} - -func SetCPUUsage(serviceName, podName string, value int64) { - // convert to full core - CPU.WithLabelValues(serviceName, podName).Set(float64(value) / 1000) -} - -func SetMemoryUsage(serviceName, podName string, value int64) { - // convert to MB - Memory.WithLabelValues(serviceName, podName).Set(float64(value) / 1024 / 1024) -} - -func SetCPUUsagePercentage(serviceName, podName string, usageValue, limitValue int64) { - CPUPercentage.WithLabelValues(serviceName, podName).Set(float64(usageValue) / float64(limitValue)) -} - -func SetMemoryUsagePercentage(serviceName, podName string, usageValue, limitValue int64) { - MemoryPercentage.WithLabelValues(serviceName, podName).Set(float64(usageValue) / float64(limitValue)) -} - -func SetHealthyStatus(serviceName, podName string, ready bool) { - if ready { - Healthy.WithLabelValues(serviceName, podName).Set(1.0) - } else { - Healthy.WithLabelValues(serviceName, podName).Set(0.0) - } -} - -func SetClusterStatus(clusterName string, status float64) { - Cluster.WithLabelValues(clusterName).Set(status) -} - -func UpdatePodMetrics() error { - CPU.Reset() - Memory.Reset() - CPUPercentage.Reset() - MemoryPercentage.Reset() - Healthy.Reset() - - metricsClient, err := clientmanager.NewKubeClientManager().GetKubernetesMetricsClient(setting.LocalClusterID) - if err != nil { - fmt.Printf("failed to get metrics client, err: %v\n", err) - return err - } - - podMetriceList, err := metricsClient.PodMetricses(config.Namespace()).List(context.TODO(), v1.ListOptions{}) - if err != nil { - fmt.Printf("failed to get pod metrics, err: %v\n", err) - return err - } - - kubeClient, err := clientmanager.NewKubeClientManager().GetControllerRuntimeClient(setting.LocalClusterID) - if err != nil { - return err - } - - pods, err := getter.ListPods(config.Namespace(), labels.Everything(), kubeClient) - if err != nil { - return err - } - podMap := make(map[string]*corev1.Pod) - for _, pod := range pods { - podMap[pod.Name] = pod - } - podMetricsMap := make(map[string]*v1beta1.PodMetrics) - for _, podMetrics := range podMetriceList.Items { - tmpPodMetrics := podMetrics - podMetricsMap[podMetrics.Name] = &tmpPodMetrics - } - - for _, pod := range pods { - for _, service := range serviceList { - if strings.Contains(pod.Name, service) { - podMetric := podMetricsMap[pod.Name] - updateResourceMetrics(service, podMetric, pod) - break - } - } - } - - return nil -} - -func updateResourceMetrics(serviceName string, podMetrics *v1beta1.PodMetrics, pod *corev1.Pod) { - containterMetricsMap := make(map[string]*v1beta1.ContainerMetrics) - if podMetrics != nil { - for _, c := range podMetrics.Containers { - containterMetricsMap[c.Name] = &c - } - } - - for _, containter := range pod.Spec.Containers { - if containter.Name == serviceName { - containterMetrics := containterMetricsMap[serviceName] - if containterMetrics == nil { - SetHealthyStatus(serviceName, pod.Name, false) - break - } - - SetHealthyStatus(serviceName, pod.Name, wrapper.Pod(pod).Resource().Ready) - - SetCPUUsage(serviceName, pod.Name, containterMetrics.Usage.Cpu().MilliValue()) - SetMemoryUsage(serviceName, pod.Name, containterMetrics.Usage.Memory().Value()) - - SetCPUUsagePercentage(serviceName, podMetrics.Name, containterMetrics.Usage.Cpu().MilliValue(), containter.Resources.Limits.Cpu().MilliValue()) - SetMemoryUsagePercentage(serviceName, podMetrics.Name, containterMetrics.Usage.Memory().Value(), containter.Resources.Limits.Memory().Value()) - - break - } - } -}