Merge pull request #9 from mozilla-services/ajvb/dump-reputation

Endpoint to dump reputation entires

Author: Aaron Meihm

README.md

@@ -131,6 +131,23 @@ error will be logged.
 ]
 ```
 
+#### GET /dump
+
+Returns all reputation entries.
+
+**Note: This makes use of the [KEYS](https://redis.io/commands/keys) redis command, which is known to be very slow. Use with care.**
+
+##### Response body
+
+```json
+[
+  {"ip": "10.0.0.1", "reputation": 75, "reviewed": false, "lastupdated": "2018-04-23T18:25:43.511Z"},
+  {"ip": "10.0.0.2", "reputation": 50, "reviewed": false, "lastupdated": "2018-04-23T18:31:27.457Z"},
+  {"ip": "10.0.20.2", "reputation": 25, "reviewed": false, "lastupdated": "2018-04-23T17:22:42.230Z"},
+]
+```
+
+
 #### GET /\_\_heartbeat\_\_
 
 Service heartbeat endpoint.

http.go

@@ -56,6 +56,7 @@ func newRouter() *mux.Router {
 	r.HandleFunc("/{ip:(?:[0-9]{1,3}\\.){3}[0-9]{1,3}}", auth(httpDeleteReputation)).Methods("DELETE")
 	r.HandleFunc("/violations/{ip:(?:[0-9]{1,3}\\.){3}[0-9]{1,3}}", auth(httpPutViolation)).Methods("PUT")
 	r.HandleFunc("/violations", auth(httpPutViolations)).Methods("PUT")
+	r.HandleFunc("/dump", auth(httpGetAllReputation)).Methods("GET")
 
 	return r
 }
@@ -88,6 +89,27 @@ func httpGetViolations(w http.ResponseWriter, r *http.Request) {
 	w.Write(buf)
 }
 
+func httpGetAllReputation(w http.ResponseWriter, r *http.Request) {
+	allRep, err := repDump()
+	if err != nil {
+		if err == redis.Nil {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		log.Warnf(err.Error())
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	buf, err := json.Marshal(allRep)
+	if err != nil {
+		log.Warnf(err.Error())
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(buf)
+}
+
 func httpGetReputation(w http.ResponseWriter, r *http.Request) {
 	ipstr := mux.Vars(r)["ip"]
 	if net.ParseIP(ipstr) == nil {

http_test.go

@@ -110,6 +110,35 @@ func TestHandlers(t *testing.T) {
 	h.ServeHTTP(recorder, req)
 	assert.Equal(t, http.StatusBadRequest, recorder.Code)
 
+	// dump reputation
+	recorder = httptest.NewRecorder()
+	req = httptest.NewRequest("GET", "/dump", nil)
+	h.ServeHTTP(recorder, req)
+	assert.Equal(t, http.StatusOK, recorder.Code)
+	res = recorder.Result()
+	assert.Equal(t, "application/json", res.Header.Get("Content-Type"))
+	buf, err = ioutil.ReadAll(res.Body)
+	assert.Nil(t, err)
+	var reputations []Reputation
+	err = json.Unmarshal(buf, &reputations)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(reputations))
+	c := 0
+	for _, rep := range reputations {
+		if rep.IP == "192.168.2.20" {
+			c += 1
+			assert.Equal(t, "192.168.2.20", rep.IP)
+			assert.Equal(t, 25, rep.Reputation)
+		}
+		if rep.IP == "192.168.0.1" {
+			c += 1
+			assert.Equal(t, "192.168.0.1", rep.IP)
+			assert.Equal(t, 50, rep.Reputation)
+		}
+		assert.Equal(t, false, rep.Reviewed)
+	}
+	assert.Equal(t, 2, c)
+
 	// delete entry
 	recorder = httptest.NewRecorder()
 	req = httptest.NewRequest("DELETE", "/192.168.2.20", nil)

redis.go

@@ -13,6 +13,10 @@ type redisLink struct {
 	readClients []*redis.Client
 }
 
+func (r *redisLink) keys(pattern string) *redis.StringSliceCmd {
+	return r.master.Keys(pattern)
+}
+
 func (r *redisLink) del(k ...string) *redis.IntCmd {
 	return r.master.Del(k...)
 }

score.go

@@ -126,3 +126,25 @@ func repDelete(ipstr string) (err error) {
 	_, err = sruntime.redis.del(ipstr).Result()
 	return
 }
+
+func repDump() (ret []Reputation, err error) {
+	keys, err := sruntime.redis.keys("*").Result()
+	if err != nil {
+		return
+	}
+
+	for _, ip := range keys {
+		buf, err := sruntime.redis.get(ip)
+		if err != nil {
+			return ret, err
+		}
+		reputation := Reputation{}
+		err = json.Unmarshal(buf, &reputation)
+		if err != nil {
+			return ret, err
+		}
+		ret = append(ret, reputation)
+	}
+
+	return
+}