Refactored Prometheus integration

Co-authored-by: Matteo Nardelli <matnar@gmail.com>

Author: grussorusso

cmd/serverledge/main.go

@@ -59,7 +59,7 @@ func main() {
 	}
 	node.NodeIdentifier = myKey
 
-	go metrics.Init()
+	metrics.Init()
 
 	if config.GetBool(config.TRACING_ENABLED, false) {
 		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)

internal/api/server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/serverledge-faas/serverledge/internal/metrics"
 	"log"
 	"net/http"
 	"os"
@@ -29,6 +30,14 @@ func StartAPIServer(e *echo.Echo) {
 	e.GET("/function", GetFunctions)
 	e.GET("/poll/:reqId", PollAsyncResult)
 	e.GET("/status", GetServerStatus)
+
+	if config.GetBool(config.METRICS_ENABLED, false) {
+		e.GET("/metrics", func(c echo.Context) error {
+			metrics.ScrapingHandler.ServeHTTP(c.Response(), c.Request())
+			return nil
+		})
+	}
+
 	// Workflow routes
 	e.POST("/workflow/invoke/:workflow", InvokeWorkflow)
 	e.POST("/workflow/resume/:workflow", ResumeWorkflow)

internal/config/keys.go

@@ -59,7 +59,7 @@ const LISTEN_UDP_PORT = "registry.udp.port"
 // enable metrics system
 const METRICS_ENABLED = "metrics.enabled"
 
-const METRICS_PROMETHEUS_HOST = "metrics.prometheus.host"
+// HTTP port used for Prometheus scraping
 const METRICS_PROMETHEUS_PORT = "metrics.prometheus.port"
 
 // Scheduling policy to use

internal/metrics/metrics.go

@@ -15,7 +15,20 @@ import (
 
 var Enabled bool
 var registry = prometheus.NewRegistry()
-var nodeIdentifier string
+var ScrapingHandler http.Handler = nil
+var durationBuckets = []float64{0.002, 0.005, 0.010, 0.02, 0.03, 0.05, 0.1, 0.15, 0.3, 0.6, 1.0}
+
+var (
+	CompletedInvocations = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "completed_total",
+		Help: "Number of completed function invocations",
+	}, []string{"node", "function"})
+	ExecutionTimes = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "execution_time",
+		Help:    "Function duration",
+		Buckets: durationBuckets,
+	}, []string{"node", "function"})
+)
 
 func Init() {
 	if config.GetBool(config.METRICS_ENABLED, false) {
@@ -26,43 +39,16 @@ func Init() {
 		return
 	}
 
-	nodeIdentifier = node.NodeIdentifier
-	registerGlobalMetrics()
+	registry.MustRegister(CompletedInvocations)
+	registry.MustRegister(ExecutionTimes)
 
-	handler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{
+	ScrapingHandler = promhttp.HandlerFor(registry, promhttp.HandlerOpts{
 		EnableOpenMetrics: true})
-	http.Handle("/metrics", handler)
-	err := http.ListenAndServe(":2112", nil)
-	if err != nil {
-		log.Printf("Listen and serve terminated with error: %s\n", err)
-		return
-	}
 }
 
-// Global metrics
-var (
-	CompletedInvocations = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "sedge_completed_total",
-		Help: "The total number of completed function invocations",
-	}, []string{"node", "function"})
-	ExecutionTimes = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Name:    "sedge_exectime",
-		Help:    "Function duration",
-		Buckets: durationBuckets,
-	},
-		[]string{"node", "function"})
-)
-
-var durationBuckets = []float64{0.002, 0.005, 0.010, 0.02, 0.03, 0.05, 0.1, 0.15, 0.3, 0.6, 1.0}
-
 func AddCompletedInvocation(funcName string) {
-	CompletedInvocations.With(prometheus.Labels{"function": funcName, "node": nodeIdentifier}).Inc()
+	CompletedInvocations.With(prometheus.Labels{"function": funcName, "node": node.NodeIdentifier}).Inc()
 }
 func AddFunctionDurationValue(funcName string, duration float64) {
-	ExecutionTimes.With(prometheus.Labels{"function": funcName, "node": nodeIdentifier}).Observe(duration)
-}
-
-func registerGlobalMetrics() {
-	registry.MustRegister(CompletedInvocations)
-	registry.MustRegister(ExecutionTimes)
+	ExecutionTimes.With(prometheus.Labels{"function": funcName, "node": node.NodeIdentifier}).Observe(duration)
 }

internal/test/main_test.go

@@ -64,7 +64,7 @@ func testStartServerledge(isInCloud bool, outboundIp string) (*registration.Regi
 
 	node.NodeIdentifier = myKey
 
-	go metrics.Init()
+	metrics.Init()
 
 	e := echo.New()
 

prometheus.yml

@@ -11,6 +11,6 @@ scrape_configs:
     # scheme defaults to 'http'.
 
     static_configs:
-      - targets: ["172.17.0.1:2112"]
+      - targets: ["172.17.0.1:1323"]