feat: Add endpoints_per_pool Prometheus metric for hash-based routing

src/code.cloudfoundry.org/gorouter/metrics/compositereporter.go

@@ -47,6 +47,8 @@ type MetricReporter interface {
 	CaptureNATSBufferedMessages(messages int)
 	CaptureNATSDroppedMessages(messages int)
 	UnmuzzleRouteRegistrationLatency()
+	CaptureEndpointsPerPool(endpoints int, route string, lbAlgo string)
+	UncaptureEndpointsPerPool(route, lbAlgo string)
 }
 
 type ComponentTagged interface {
@@ -230,6 +232,18 @@ func (m MultiMetricReporter) CaptureNATSDroppedMessages(messages int) {
 	}
 }
 
+func (m MultiMetricReporter) CaptureEndpointsPerPool(endpoints int, route string, lbAlgo string) {
+	for _, r := range m {
+		r.CaptureEndpointsPerPool(endpoints, route, lbAlgo)
+	}
+}
+
+func (m MultiMetricReporter) UncaptureEndpointsPerPool(route, lbAlgo string) {
+	for _, r := range m {
+		r.UncaptureEndpointsPerPool(route, lbAlgo)
+	}
+}
+
 func (c *CompositeReporter) CaptureBadRequest() {
 	c.VarzReporter.CaptureBadRequest()
 	c.MetricReporter.CaptureBadRequest()
@@ -257,3 +271,11 @@ func (c *CompositeReporter) CaptureRoutingResponseLatency(b *route.Endpoint, sta
 func (c *CompositeReporter) CaptureHTTPLatency(d time.Duration, sourceID string) {
 	c.MetricReporter.CaptureHTTPLatency(d, sourceID)
 }
+
+func (c *CompositeReporter) CaptureEndpointsPerPool(endpoints int, route string, lbAlgo string) {
+	c.MetricReporter.CaptureEndpointsPerPool(endpoints, route, lbAlgo)
+}
+
+func (c *CompositeReporter) UncaptureEndpointsPerPool(route, lbAlgo string) {
+	c.MetricReporter.UncaptureEndpointsPerPool(route, lbAlgo)
+}

src/code.cloudfoundry.org/gorouter/metrics/metricsreporter.go

@@ -201,6 +201,14 @@ func (m *Metrics) CaptureNATSDroppedMessages(messages int) {
 func (m *Metrics) CaptureHTTPLatency(_ time.Duration, _ string) {
 }
 
+// Empty implementation here is to fulfil interface
+func (m *Metrics) CaptureEndpointsPerPool(endpoints int, route string, lbAlgo string) {
+}
+
+// Empty implementation here is to fulfil interface
+func (m *Metrics) UncaptureEndpointsPerPool(route string, lbAlgo string) {
+}
+
 func getResponseCounterName(statusCode int) string {
 	statusCode = statusCode / 100
 	if statusCode >= 2 && statusCode <= 5 {

src/code.cloudfoundry.org/gorouter/metrics_prometheus/metrics.go

@@ -43,6 +43,7 @@ type Metrics struct {
 	NATSDroppedMessages         mr.Gauge
 	HTTPLatency                 mr.GaugeVec
 	perRequestMetricsReporting  bool
+	EndpointsPerPool            mr.GaugeVec
 }
 
 func NewMetricsRegistry(config config.PrometheusConfig) *mr.Registry {
@@ -91,6 +92,7 @@ func NewMetrics(registry *mr.Registry, perRequestMetricsReporting bool) *Metrics
 		NATSDroppedMessages:         registry.NewGauge("total_dropped_messages", "number of total dropped messages in NATS"),
 		HTTPLatency:                 registry.NewGaugeVec("http_latency_seconds", "the latency of http requests from gorouter and back in sec", []string{"source_id"}),
 		perRequestMetricsReporting:  perRequestMetricsReporting,
+		EndpointsPerPool:            registry.NewGaugeVec("endpoints_per_pool", "number of endpoints per pool", []string{"route", "lb_algorithm"}),
 	}
 }
 
@@ -220,6 +222,15 @@ func (metrics *Metrics) CaptureHTTPLatency(d time.Duration, sourceID string) {
 	metrics.HTTPLatency.Set(float64(d)/float64(time.Second), []string{sourceID})
 }
 
+// CaptureEndpointsPerPool sets the number of endpoints for a given route and load balancing algorithm
+func (metrics *Metrics) CaptureEndpointsPerPool(count int, route string, loadBalancingAlgo string) {
+	metrics.EndpointsPerPool.Set(float64(count), []string{route, loadBalancingAlgo})
+}
+
+func (metrics *Metrics) UncaptureEndpointsPerPool(route string, lbAlgo string) {
+	metrics.EndpointsPerPool.Delete([]string{route, lbAlgo})
+}
+
 func statusGroupName(statusCode int) string {
 	statusGroupNum := statusCode / 100
 	if statusGroupNum >= 2 && statusGroupNum <= 5 {

src/code.cloudfoundry.org/gorouter/metrics_prometheus/metrics_test.go

@@ -450,6 +450,54 @@ var _ = Describe("Metrics", func() {
 			Expect(getMetrics(r.Port())).To(ContainSubstring("http_latency_seconds{source_id=\"some-source\"} 0.63"))
 		})
 	})
+
+	Context("endpoints_per_pool metric", func() {
+		BeforeEach(func() {
+			var config = config.PrometheusConfig{Port: 0}
+			r = NewMetricsRegistry(config)
+			m = NewMetrics(r, true)
+		})
+
+		It("reports the number of endpoints per pool with correct labels", func() {
+			m.CaptureEndpointsPerPool(5, "routeA", "round_robin")
+			metricsOutput := getMetrics(r.Port())
+			expected := "endpoints_per_pool{lb_algorithm=\"round_robin\",route=\"routeA\"} 5"
+			Expect(metricsOutput).To(ContainSubstring(expected))
+		})
+
+		It("updates the value for the same label combination", func() {
+			m.CaptureEndpointsPerPool(5, "routeA", "round_robin")
+			m.CaptureEndpointsPerPool(7, "routeA", "round_robin")
+			metricsOutput := getMetrics(r.Port())
+			expected := "endpoints_per_pool{lb_algorithm=\"round_robin\",route=\"routeA\"} 7"
+			Expect(metricsOutput).To(ContainSubstring(expected))
+		})
+
+		It("reports multiple values for different label combinations", func() {
+			m.CaptureEndpointsPerPool(5, "routeA", "round_robin")
+			m.CaptureEndpointsPerPool(3, "routeB", "least_conn")
+			metricsOutput := getMetrics(r.Port())
+			expectedA := "endpoints_per_pool{lb_algorithm=\"round_robin\",route=\"routeA\"} 5"
+			expectedB := "endpoints_per_pool{lb_algorithm=\"least_conn\",route=\"routeB\"} 3"
+			Expect(metricsOutput).To(ContainSubstring(expectedA))
+			Expect(metricsOutput).To(ContainSubstring(expectedB))
+		})
+
+		It("deletes the metric for a given route and LB algorithm", func() {
+			m.CaptureEndpointsPerPool(5, "routeA", "round_robin")
+			Expect(getMetrics(r.Port())).To(ContainSubstring("endpoints_per_pool{lb_algorithm=\"round_robin\",route=\"routeA\"} 5"))
+
+			m.UncaptureEndpointsPerPool("routeA", "round_robin")
+			Expect(getMetrics(r.Port())).NotTo(ContainSubstring("endpoints_per_pool{lb_algorithm=\"round_robin\",route=\"routeA\"}"))
+		})
+
+		It("does nothing when deleting a non-existent label combination", func() {
+			m.CaptureEndpointsPerPool(5, "routeA", "round_robin")
+
+			m.UncaptureEndpointsPerPool("routeX", "round_robin")
+			Expect(getMetrics(r.Port())).To(ContainSubstring("endpoints_per_pool{lb_algorithm=\"round_robin\",route=\"routeA\"} 5"))
+		})
+	})
 })
 
 func getMetrics(port string) string {