Metrics: add daily top 10

Add export of current daily download stats for all files that have been
downloaded in the current day.
With this is also added a retention period for these stats as they can
be heavy for the database and should be removed after use.

Author: Skantes

config/config.go

@@ -54,11 +54,12 @@ func defaultConfig() Configuration {
 			SHA256: true,
 			MD5:    false,
 		},
-		DisallowRedirects:       false,
-		WeightDistributionRange: 1.5,
-		DisableOnMissingFile:    false,
-		RPCListenAddress:        "localhost:3390",
-		RPCPassword:             "",
+		DisallowRedirects:        false,
+		WeightDistributionRange:  1.5,
+		DisableOnMissingFile:     false,
+		RPCListenAddress:         "localhost:3390",
+		RPCPassword:              "",
+		MetricsTopFilesRetention: 1,
 	}
 }
 
@@ -93,6 +94,8 @@ type Configuration struct {
 
 	RPCListenAddress string `yaml:"RPCListenAddress"`
 	RPCPassword      string `yaml:"RPCPassword"`
+
+	MetricsTopFilesRetention int `yaml:"MetricsTopFilesRetention"`
 }
 
 type fallback struct {
@@ -165,6 +168,9 @@ func ReloadConfig() error {
 	if c.RepositoryScanInterval < 0 {
 		c.RepositoryScanInterval = 0
 	}
+	if c.MetricsTopFilesRetention < 1 {
+		return fmt.Errorf("Invalid retention duration in days")
+	}
 
 	if config != nil &&
 		(c.RedisAddress != config.RedisAddress ||

http/metrics.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/etix/mirrorbits/config"
 	"github.com/gomodule/redigo/redis"
 )
 
@@ -233,6 +234,42 @@ func (h *HTTP) metricsHandler(w http.ResponseWriter, r *http.Request) {
 		output += "\n"
 	}
 
+	// Get daily stats for top 10
+	retention := config.GetConfig().MetricsTopFilesRetention
+	for nbDays := 0; nbDays < retention; nbDays++ {
+		day := today.AddDate(0, 0, nbDays*-1)
+		rconn.Send("MULTI")
+		for _, file := range fileList {
+			mkey := fmt.Sprintf("STATS_TOP_%s_%s", file,
+				day.Format("2006_01_02"))
+			rconn.Send("HGETALL", mkey)
+		}
+		stats, err = redis.Values(rconn.Do("EXEC"))
+		if err != nil {
+			log.Error("Cannot fetch file per country stats: " + err.Error())
+			return
+		}
+		for index, stat := range stats {
+			hashSlice, _ := redis.ByteSlices(stat, err)
+			for i := 0; i < len(hashSlice); i += 2 {
+				field, _ := redis.String(hashSlice[i], err)
+				value, _ := redis.Int(hashSlice[i+1], err)
+				sep := strings.Index(field, "_")
+				country := field[:sep]
+				mirrorID, err := strconv.Atoi(field[sep+1:])
+				if err != nil {
+					log.Error("Failed to convert mirror ID: ", err)
+					return
+				}
+				mirror := mirrorsMap[mirrorID]
+				output += fmt.Sprintf("stats_top"+
+					"{file=\"%s\",country=\"%s\",mirror=\"%s\"} %d\n",
+					fileList[index], country, mirror, value,
+				)
+			}
+		}
+	}
+
 	w.Write([]byte(output))
 }
 

http/stats.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/etix/mirrorbits/config"
 	"github.com/etix/mirrorbits/database"
 	"github.com/etix/mirrorbits/filesystem"
 	"github.com/etix/mirrorbits/mirrors"
@@ -140,8 +141,8 @@ func (s *Stats) pushStats() {
 		return
 	}
 
+	topFileRetention := config.GetConfig().MetricsTopFilesRetention
 	rconn.Send("MULTI")
-
 	for k, v := range s.mapStats {
 		if v == 0 {
 			continue
@@ -216,6 +217,16 @@ func (s *Stats) pushStats() {
 				rconn.Send("HINCRBY", mkey, key, v)
 				mkey = mkey[:strings.LastIndex(mkey, "_")]
 			}
+			mkey = fmt.Sprintf("STATS_TOP_%s_%s", file, date)
+			rconn.Send("HINCRBY", mkey, key, v)
+			t, err := time.Parse("2006_01_02", date)
+			if err != nil {
+				log.Error("Failed to parse date: ", err.Error())
+				return
+			}
+			t = t.AddDate(0, 0, topFileRetention)
+			topExpireDate := t.Format("2006_01_02")
+			rconn.Send("EXPIREAT", mkey, topExpireDate)
 		} else {
 			log.Warning("Stats: unknown type", typ)
 		}

mirrorbits.conf

@@ -128,3 +128,12 @@
 #     - URL: http://fallback2.mirror/repo/
 #       CountryCode: us
 #       ContinentCode: na
+
+###################
+##### METRICS #####
+###################
+
+## Number of days to keep the daily top 10 metrics in database
+## Keep in mind that this will have an impact on the database's RAM usage
+## It is not recommended to keep more than a few days of retention.
+## MetricsTopFilesRetention: 1