Merge pull request #11 from mozilla-services/ajvb/statsd

Add statsd timer collection for http, get reputation, and redis commands

Author: ajvb

http.go

@@ -32,6 +32,10 @@ func (v *ViolationRequest) Validate() error {
 
 func mwHandler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		s := time.Now()
+		defer func() {
+			sruntime.statsd.Timing("http.timing", time.Since(s))
+		}()
 		w.Header().Add("X-Frame-Options", "DENY")
 		w.Header().Add("X-Content-Type-Options", "nosniff")
 		w.Header().Add("Content-Security-Policy",
@@ -111,6 +115,10 @@ func httpGetAllReputation(w http.ResponseWriter, r *http.Request) {
 }
 
 func httpGetReputation(w http.ResponseWriter, r *http.Request) {
+	s := time.Now()
+	defer func() {
+		sruntime.statsd.Timing("http.get_reputation.timing", time.Since(s))
+	}()
 	ipstr := mux.Vars(r)["ip"]
 	if net.ParseIP(ipstr) == nil {
 		w.WriteHeader(http.StatusBadRequest)

iprepd.go

@@ -15,6 +15,7 @@ type serverRuntime struct {
 	redis            redisLink
 	versionResponse  []byte
 	exceptionsLoaded chan bool
+	statsd           *statsdClient
 }
 
 type serverCfg struct {
@@ -43,6 +44,9 @@ type serverCfg struct {
 		AWS  bool
 	}
 	VersionResponse string
+	Statsd          struct {
+		Addr string
+	}
 }
 
 func (cfg *serverCfg) validate() error {
@@ -102,6 +106,10 @@ func StartDaemon(confpath string) {
 	if err != nil {
 		log.Fatalf(err.Error())
 	}
+	sruntime.statsd, err = newStatsdClient(sruntime.cfg)
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
 	sruntime.redis, err = newRedisLink(sruntime.cfg)
 	if err != nil {
 		log.Fatalf(err.Error())

iprepd.yaml.sample

@@ -78,3 +78,6 @@ exceptions:
 # request to the /__version__ endpoint. If the file isn't found a warning will be printed
 # in the log and the daemon will not return any data at this endpoint.
 versionresponse: ./version.json
+# statsd address for metrics. To disable metrics collection, simply delete the below section.
+statsd:
+  addr: "127.0.0.1:8125"

iprepd_test.go

@@ -55,6 +55,11 @@ func TestMain(m *testing.M) {
 	if renv != "" {
 		tcfg.Redis.Addr = renv
 	}
+	sruntime.statsd, err = newStatsdClient(tcfg)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
 	sruntime.redis, err = newRedisLink(tcfg)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)

redis.go

@@ -1,6 +1,7 @@
 package iprepd
 
 import (
+	"fmt"
 	"math/rand"
 	"runtime"
 	"time"
@@ -48,6 +49,15 @@ func (r *redisLink) set(k string, v interface{}, e time.Duration) *redis.StatusC
 	return r.master.Set(k, v, e)
 }
 
+func instrumentRedisCmd(old func(cmd redis.Cmder) error) func(cmd redis.Cmder) error {
+	return func(cmd redis.Cmder) error {
+		s := time.Now()
+		err := old(cmd)
+		sruntime.statsd.Timing(fmt.Sprintf("redis.%s.timing", cmd.Name()), time.Since(s))
+		return err
+	}
+}
+
 func newRedisLink(cfg serverCfg) (ret redisLink, err error) {
 	minIdleConns := cfg.Redis.MinIdleConn
 	if cfg.Redis.MaxPoolSize != 0 && cfg.Redis.MaxPoolSize < 20 {
@@ -65,6 +75,7 @@ func newRedisLink(cfg serverCfg) (ret redisLink, err error) {
 		PoolSize:     cfg.Redis.MaxPoolSize,
 		MinIdleConns: minIdleConns,
 	})
+	ret.master.WrapProcess(instrumentRedisCmd)
 	_, err = ret.ping().Result()
 	if err != nil {
 		return
@@ -84,6 +95,7 @@ func newRedisLink(cfg serverCfg) (ret redisLink, err error) {
 			PoolSize:     cfg.Redis.MaxPoolSize,
 			MinIdleConns: minIdleConns,
 		})
+		y.WrapProcess(instrumentRedisCmd)
 		ret.readClients = append(ret.readClients, y)
 	}
 	// Also use the master for reads

statsd.go

@@ -0,0 +1,31 @@
+package iprepd
+
+import (
+	"time"
+
+	"github.com/DataDog/datadog-go/statsd"
+)
+
+type statsdClient struct {
+	client *statsd.Client
+}
+
+func newStatsdClient(cfg serverCfg) (*statsdClient, error) {
+	if cfg.Statsd.Addr == "" {
+		return &statsdClient{client: nil}, nil
+	}
+	c, err := statsd.New(cfg.Statsd.Addr)
+	if err != nil {
+		return nil, err
+	}
+	c.Namespace = "iprepd_server."
+
+	return &statsdClient{client: c}, nil
+}
+
+func (sc statsdClient) Timing(name string, value time.Duration) error {
+	if sc.client == nil {
+		return nil
+	}
+	return sc.client.Timing(name, value, []string{}, 1)
+}