Skip to content

Commit 61c8b88

Browse files
EXPEbdodlaBhargav Dodla
EXPEbdodla
and
Bhargav Dodla
authored
feat: Expose Redis connection pool metrics using prometheus (#158)
* chore: Update go-redis dependency to v9.7.0 * feat: Added prometheus redis connection metrics * feat: Integrate Prometheus metrics for gRPC server and add metrics endpoint --------- Co-authored-by: Bhargav Dodla <bdodla@expediagroup.com>
1 parent 2a19428 commit 61c8b88

5 files changed

Lines changed: 62 additions & 8 deletions

File tree

‎go.mod‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,12 @@ require (
99
github.com/ghodss/yaml v1.0.0
1010
github.com/golang/protobuf v1.5.4
1111
github.com/google/uuid v1.6.0
12+
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1
1213
github.com/mattn/go-sqlite3 v1.14.23
1314
github.com/pkg/errors v0.9.1
14-
github.com/redis/go-redis/v9 v9.6.1
15+
github.com/prometheus/client_golang v1.20.5
16+
github.com/redis/go-redis/extra/redisprometheus/v9 v9.7.0
17+
github.com/redis/go-redis/v9 v9.7.0
1518
github.com/rs/zerolog v1.33.0
1619
github.com/spaolacci/murmur3 v1.1.0
1720
github.com/stretchr/testify v1.9.0
@@ -44,13 +47,15 @@ require (
4447
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
4548
github.com/andybalholm/brotli v1.1.0 // indirect
4649
github.com/apache/thrift v0.21.0 // indirect
50+
github.com/beorn7/perks v1.0.1 // indirect
4751
github.com/cespare/xxhash/v2 v2.3.0 // indirect
4852
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
4953
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
5054
github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect
5155
github.com/goccy/go-json v0.10.3 // indirect
5256
github.com/golang/snappy v0.0.4 // indirect
5357
github.com/google/flatbuffers v24.3.25+incompatible // indirect
58+
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect
5459
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 // indirect
5560
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
5661
github.com/hashicorp/go-sockaddr v1.0.6 // indirect
@@ -62,8 +67,12 @@ require (
6267
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
6368
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
6469
github.com/mitchellh/mapstructure v1.5.0 // indirect
70+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
6571
github.com/pierrec/lz4/v4 v4.1.21 // indirect
6672
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
73+
github.com/prometheus/client_model v0.6.1 // indirect
74+
github.com/prometheus/common v0.55.0 // indirect
75+
github.com/prometheus/procfs v0.15.1 // indirect
6776
github.com/ryanuber/go-glob v1.0.0 // indirect
6877
github.com/stretchr/objx v0.5.2 // indirect
6978
github.com/zeebo/xxh3 v1.0.2 // indirect

‎go.sum‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ github.com/apache/arrow/go/v17 v17.0.0 h1:RRR2bdqKcdbss9Gxy2NS/hK8i4LDMh23L6BbkN
2727
github.com/apache/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc=
2828
github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
2929
github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
30+
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
31+
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
3032
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
3133
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
3234
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@@ -70,6 +72,10 @@ github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBB
7072
github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
7173
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
7274
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
75+
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA=
76+
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU=
77+
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk=
78+
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI=
7379
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 h1:iBt4Ew4XEGLfh6/bPk4rSYmuZJGizr6/x/AEizP0CQc=
7480
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8/go.mod h1:aiJI+PIApBRQG7FZTEBx5GiiX+HbOHilUdNxUZi4eV0=
7581
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
@@ -85,10 +91,12 @@ github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2
8591
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
8692
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
8793
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
88-
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
89-
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
94+
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
95+
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
9096
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
9197
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
98+
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
99+
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
92100
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
93101
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
94102
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -103,6 +111,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
103111
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
104112
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
105113
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
114+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
115+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
106116
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
107117
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
108118
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
@@ -120,12 +130,24 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
120130
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
121131
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
122132
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
123-
github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
124-
github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
133+
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
134+
github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
135+
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
136+
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
137+
github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
138+
github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
139+
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
140+
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
141+
github.com/redis/go-redis/extra/redisprometheus/v9 v9.7.0 h1:SFokIgXIMhDimi19mI5bECkCzwTMKSWcQbrwLYb0DVs=
142+
github.com/redis/go-redis/extra/redisprometheus/v9 v9.7.0/go.mod h1:HONOQyhzmL31tCcGdJOqpth+byLt/unAuCiVcQDiCmg=
143+
github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
144+
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
125145
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
126146
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
127147
github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 h1:4+LEVOB87y175cLJC/mbsgKmoDOjrBldtXvioEy96WY=
128148
github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3/go.mod h1:vl5+MqJ1nBINuSsUI2mGgH79UweUT/B5Fy8857PqyyI=
149+
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
150+
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
129151
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
130152
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
131153
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=

‎go/internal/feast/onlinestore/redisonlinestore.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,15 @@ import (
1414
"github.com/feast-dev/feast/go/internal/feast/registry"
1515
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
1616

17-
"github.com/redis/go-redis/v9"
17+
redisprometheus "github.com/redis/go-redis/extra/redisprometheus/v9"
18+
redis "github.com/redis/go-redis/v9"
1819
"github.com/spaolacci/murmur3"
1920
"google.golang.org/protobuf/proto"
2021
"google.golang.org/protobuf/types/known/timestamppb"
2122

2223
"github.com/feast-dev/feast/go/protos/feast/serving"
2324
"github.com/feast-dev/feast/go/protos/feast/types"
25+
"github.com/prometheus/client_golang/prometheus"
2426
"github.com/rs/zerolog/log"
2527
redistrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/redis/go-redis.v9"
2628
)
@@ -122,6 +124,8 @@ func NewRedisOnlineStore(project string, config *registry.RepoConfig, onlineStor
122124
})
123125
if strings.ToLower(os.Getenv("ENABLE_DATADOG_REDIS_TRACING")) == "true" {
124126
redistrace.WrapClient(store.client, redistrace.WithServiceName(redisTraceServiceName))
127+
collector := redisprometheus.NewCollector("mlpfs", "redis", store.client)
128+
prometheus.MustRegister(collector)
125129
}
126130
} else if redisStoreType == redisCluster {
127131
log.Info().Msgf("Using Redis Cluster: %s", address)
@@ -133,6 +137,8 @@ func NewRedisOnlineStore(project string, config *registry.RepoConfig, onlineStor
133137
})
134138
if strings.ToLower(os.Getenv("ENABLE_DATADOG_REDIS_TRACING")) == "true" {
135139
redistrace.WrapClient(store.clusterClient, redistrace.WithServiceName(redisTraceServiceName))
140+
collector := redisprometheus.NewCollector("mlpfs", "redis", store.clusterClient)
141+
prometheus.MustRegister(collector)
136142
}
137143
}
138144

‎go/internal/feast/server/http_server.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/feast-dev/feast/go/protos/feast/serving"
1919
prototypes "github.com/feast-dev/feast/go/protos/feast/types"
2020
"github.com/feast-dev/feast/go/types"
21+
"github.com/prometheus/client_golang/prometheus/promhttp"
2122
"github.com/rs/zerolog/log"
2223
httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http"
2324
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
@@ -345,6 +346,7 @@ func (s *httpServer) Serve(host string, port int) error {
345346
}
346347
mux := httptrace.NewServeMux()
347348
mux.Handle("/get-online-features", recoverMiddleware(http.HandlerFunc(s.getOnlineFeatures)))
349+
mux.Handle("/metrics", promhttp.Handler())
348350
mux.HandleFunc("/health", healthCheckHandler)
349351
s.server = &http.Server{Addr: fmt.Sprintf("%s:%d", host, port), Handler: mux, ReadTimeout: 5 * time.Second, WriteTimeout: 10 * time.Second, IdleTimeout: 15 * time.Second}
350352
err := s.server.ListenAndServe()

‎go/main.go‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"flag"
55
"fmt"
66
"net"
7+
"net/http"
78
"os"
89
"os/signal"
910
"strings"
@@ -14,13 +15,16 @@ import (
1415
"github.com/feast-dev/feast/go/internal/feast/server"
1516
"github.com/feast-dev/feast/go/internal/feast/server/logging"
1617
"github.com/feast-dev/feast/go/protos/feast/serving"
18+
"github.com/prometheus/client_golang/prometheus"
1719
"github.com/rs/zerolog/log"
1820
"google.golang.org/grpc"
1921
"google.golang.org/grpc/health"
2022
"google.golang.org/grpc/health/grpc_health_v1"
2123

24+
grpcPrometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
25+
"github.com/prometheus/client_golang/prometheus/promhttp"
2226
_ "go.uber.org/automaxprocs"
23-
grpctrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/google.golang.org/grpc"
27+
grpcTrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/google.golang.org/grpc"
2428
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
2529
)
2630

@@ -127,11 +131,22 @@ func StartGrpcServer(fs *feast.FeatureStore, host string, port int, writeLoggedF
127131
return err
128132
}
129133

130-
grpcServer := grpc.NewServer(grpc.UnaryInterceptor(grpctrace.UnaryServerInterceptor()))
134+
grpcPromMetrics := grpcPrometheus.NewServerMetrics()
135+
prometheus.MustRegister(grpcPromMetrics)
136+
grpcServer := grpc.NewServer(
137+
grpc.ChainUnaryInterceptor(grpcTrace.UnaryServerInterceptor(), grpcPromMetrics.UnaryServerInterceptor()),
138+
)
131139
serving.RegisterServingServiceServer(grpcServer, ser)
132140
healthService := health.NewServer()
133141
grpc_health_v1.RegisterHealthServer(grpcServer, healthService)
134142

143+
// Running Prometheus metrics endpoint on a separate goroutine
144+
go func() {
145+
http.Handle("/metrics", promhttp.Handler())
146+
log.Info().Msg("Starting metrics endpoint on port 8080")
147+
log.Fatal().Stack().Err(http.ListenAndServe(":8080", nil))
148+
}()
149+
135150
stop := make(chan os.Signal, 1)
136151
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
137152

0 commit comments

Comments
 (0)