Metrics: add detailed metrics for selected files

This adds a more detailed metrics category for files served by
mirrorbits.
These tracked files will detail their downloads per country, but due to
the increased number of fields required for this, only files selected
through the command line will be tracked.
This also adds command line arguments to add, delete, and list the
files to monitore closely.

Author: Skantes

cli/commands.go

@@ -99,11 +99,14 @@ func (c *cli) CmdHelp() error {
 	help += fmt.Sprintf("CLI commands:\n")
 	for _, command := range [][]string{
 		{"add", "Add a new mirror"},
+		{"addMetric", "Add a tracked file to the metrics route"},
 		{"disable", "Disable a mirror"},
+		{"delMetric", "Delete a tracked file from the metrics route"},
 		{"edit", "Edit a mirror"},
 		{"enable", "Enable a mirror"},
 		{"export", "Export the mirror database"},
 		{"list", "List all mirrors"},
+		{"listMetrics", "List all tracked files from the metrics route"},
 		{"logs", "Print logs of a mirror"},
 		{"refresh", "Refresh the local repository"},
 		{"reload", "Reload configuration"},
@@ -349,6 +352,99 @@ func (c *cli) CmdAdd(args ...string) error {
 	return nil
 }
 
+func (c *cli) CmdAddmetric(args ...string) error {
+	cmd := SubCmd("addMetric", "FILE_PATH", "Add a file to the metrics route")
+
+	if err := cmd.Parse(args); err != nil {
+		return nil
+	}
+	if cmd.NArg() != 1 {
+		cmd.Usage()
+		return nil
+	}
+	file := cmd.Arg(0)
+
+	client := c.GetRPC()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout)
+	defer cancel()
+	_, err := client.AddMetric(ctx, &rpc.Metric{
+		Filename: string(file),
+	})
+	if err != nil {
+		log.Fatal("Error while adding metric: ", err)
+	}
+
+	log.Info("File ", file, " successfully added to metrics.")
+	return nil
+}
+
+func (c *cli) CmdDelmetric(args ...string) error {
+	cmd := SubCmd("delMetric", "FILE_PATH", "Delete a file from the metrics route")
+
+	if err := cmd.Parse(args); err != nil {
+		return nil
+	}
+	if cmd.NArg() != 1 {
+		cmd.Usage()
+		return nil
+	}
+	file := cmd.Arg(0)
+
+	client := c.GetRPC()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout)
+	defer cancel()
+	_, err := client.DelMetric(ctx, &rpc.Metric{
+		Filename: string(file),
+	})
+	if err != nil {
+		log.Fatal("Error while deleting metric: ", err)
+	}
+
+	log.Info("File ", file, " successfully deleted from metrics.")
+	return nil
+}
+
+func (c *cli) CmdListmetrics(args ...string) error {
+	cmd := SubCmd("listMetric", "[OPTIONNAL FILTER PATTERN]",
+		"Optionnal pattern to filter results")
+
+	filterPattern := "*"
+	if err := cmd.Parse(args); err != nil {
+		return nil
+	}
+	nArg := cmd.NArg()
+	if nArg > 1 {
+		cmd.Usage()
+		return nil
+	} else if nArg == 1 {
+		filterPattern = "*" + cmd.Arg(0) + "*"
+	}
+
+	client := c.GetRPC()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout)
+	defer cancel()
+	fileList, err := client.ListMetrics(ctx, &rpc.Metric{
+		Filename: string(filterPattern),
+	})
+	if err != nil {
+		log.Fatal("Error while listing metrics: ", err)
+	}
+
+	if len(fileList.Filename) == 0 {
+		if nArg == 1 {
+			log.Info("There are no tracked files matching your request.")
+		} else {
+			log.Info("There are no tracked files.")
+		}
+	} else {
+		for _, file := range fileList.Filename {
+			log.Info(file)
+		}
+	}
+
+	return nil
+}
+
 func (c *cli) CmdRemove(args ...string) error {
 	cmd := SubCmd("remove", "IDENTIFIER", "Remove an existing mirror")
 	force := cmd.Bool("f", false, "Never prompt for confirmation")

database/utils.go

@@ -65,6 +65,43 @@ func (r *Redis) GetListOfFiles() ([]string, error) {
 	return files, nil
 }
 
+func (r *Redis) IsFileTracked(file string) (bool, error) {
+	trackedFileList, err := r.GetListOfTrackedFiles()
+	if err != nil {
+		return false, nil
+	}
+	for _, v := range trackedFileList {
+		if v == file {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (r *Redis) GetListOfTrackedFiles() ([]string, error) {
+	conn, err := r.Connect()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	values, err := redis.Values(conn.Do("SMEMBERS", "TRACKED_FILES"))
+	if err != nil {
+		return nil, err
+	}
+
+	files := make([]string, len(values))
+	for i, v := range values {
+		value, okValue := v.([]byte)
+		if !okValue {
+			return nil, errors.New("invalid type for file")
+		}
+		files[i] = string(value)
+	}
+
+	return files, nil
+}
+
 func (r *Redis) GetListOfCountries() ([]string, error) {
 	conn, err := r.Connect()
 	if err != nil {

http/http.go

@@ -331,10 +331,15 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex
 		http.Error(w, err.Error(), status)
 	}
 
+	isTracked, err := h.redis.IsFileTracked(fileInfo.Path)
+	if err != nil {
+		log.Error("There was a problem fetching the tracked file list: ", err)
+	}
+
 	if !ctx.IsMirrorlist() {
 		logs.LogDownload(resultRenderer.Type(), status, results, err)
 		if len(mlist) > 0 {
-			h.stats.CountDownload(mlist[0], fileInfo, clientInfo)
+			h.stats.CountDownload(mlist[0], fileInfo, clientInfo, isTracked)
 		}
 	}
 

http/metrics.go

@@ -3,6 +3,9 @@ package http
 import (
 	"fmt"
 	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/gomodule/redigo/redis"
@@ -15,17 +18,18 @@ type metrics struct {
 	Total int
 }
 
-func statsToPrometheusFormat(metrics metrics, labelName string, labelValue string) string {
+func statsToPrometheusFormat(metrics metrics, label_name string,
+	label_value string) string {
 	var output string
 
 	output += fmt.Sprintf("%s_total{%s=\"%s\"} %d\n",
-		labelName, labelName, labelValue, metrics.Total)
+		label_name, label_name, label_value, metrics.Total)
 	output += fmt.Sprintf("%s_day{%s=\"%s\"} %d\n",
-		labelName, labelName, labelValue, metrics.Day)
+		label_name, label_name, label_value, metrics.Day)
 	output += fmt.Sprintf("%s_month{%s=\"%s\"} %d\n",
-		labelName, labelName, labelValue, metrics.Month)
+		label_name, label_name, label_value, metrics.Month)
 	output += fmt.Sprintf("%s_year{%s=\"%s\"} %d\n\n",
-		labelName, labelName, labelValue, metrics.Year)
+		label_name, label_name, label_value, metrics.Year)
 
 	return output
 }
@@ -141,5 +145,82 @@ func (h *HTTP) metricsHandler(w http.ResponseWriter, r *http.Request) {
 		index += 4
 	}
 
+	// Get all tracked files
+	var trackedFilesList []string
+	mkeyLenMap := make(map[string]int)
+	mkeyFileMap := make(map[string]string)
+	trackedFilesList, err = h.redis.GetListOfTrackedFiles()
+	if err != nil {
+		log.Error("Cannot fetch list of files: " + err.Error())
+		return
+	}
+
+	// Get tracked files downloads per country and mirror
+	// rconn.Send("MULTI")
+	for _, file := range trackedFilesList {
+		mkey := "STATS_TRACKED_" + file + "_" + today.Format("2006_01_02")
+		for i := 0; i < 4; i++ {
+			exists, _ := redis.Bool(rconn.Do("EXISTS", mkey))
+			if exists {
+				mkeyLenMap[mkey], _ = redis.Int(rconn.Do("HLEN", mkey))
+				log.Info("mKey: ", mkey)
+				log.Info(file, " len: ", mkeyLenMap[mkey])
+				mkeyFileMap[mkey] = file
+				// rconn.Send("HGETALL", mkey)
+				mkey = mkey[:strings.LastIndex(mkey, "_")]
+			}
+		}
+	}
+
+	rconn.Send("MULTI")
+	for mkey := range mkeyLenMap {
+		rconn.Send("HGETALL", mkey)
+	}
+	stats, err = redis.Values(rconn.Do("EXEC"))
+	if err != nil {
+		log.Error("Cannot fetch file per country stats: " + err.Error())
+		return
+	}
+
+	pattern := regexp.MustCompile("_")
+	index = 0
+	for mkey, hashLen := range mkeyLenMap {
+		for i := 0; i < 2*hashLen; i += 2 {
+			hashSlice, _ := redis.ByteSlices(stats[index], err)
+			key, _ := redis.String(hashSlice[i], err)
+			value, _ := redis.Int(hashSlice[i+1], err)
+			log.Info("mkey: ", mkey)
+			log.Info("File: ", mkeyFileMap[mkey])
+			log.Info("Key: ", key)
+			log.Info("Value: ", value)
+			sep := strings.Index(key, "_")
+			country := key[:sep]
+			mirrorID, err := strconv.Atoi(key[sep+1:])
+			if err != nil {
+				log.Error("Failed to convert mirror ID: ", err)
+				return
+			}
+			mirror := mirrorsMap[mirrorID]
+			duration := len(pattern.FindAllStringIndex(mkey, -1))
+			var durationStr string
+			switch duration {
+			case 2:
+				durationStr = "total"
+			case 3:
+				durationStr = "year"
+			case 4:
+				durationStr = "month"
+			case 5:
+				durationStr = "day"
+			}
+			output += fmt.Sprintf("stats_tracked_"+
+				"%s{file=\"%s\",country=\"%s\",mirror=\"%s\"} %d\n",
+				durationStr, mkeyFileMap[mkey], country, mirror, value,
+			)
+		}
+		output += "\n"
+		index++
+	}
+
 	w.Write([]byte(output))
 }

http/stats.go

@@ -56,6 +56,7 @@ type countItem struct {
 	country  string
 	size     int64
 	time     time.Time
+	tracked  bool
 }
 
 // NewStats returns an instance of the stats counter
@@ -79,7 +80,7 @@ func (s *Stats) Terminate() {
 
 // CountDownload is a lightweight method used to count a new download for a specific file and mirror
 func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo,
-	clientInfo network.GeoIPRecord) error {
+	clientInfo network.GeoIPRecord, isTracked bool) error {
 	if m.Name == "" {
 		return errUnknownMirror
 	}
@@ -91,7 +92,7 @@ func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo,
 	}
 
 	s.countChan <- countItem{m.ID, fileinfo.Path, clientInfo.Country,
-		fileinfo.Size, time.Now().UTC()}
+		fileinfo.Size, time.Now().UTC(), isTracked}
 	return nil
 }
 
@@ -114,6 +115,9 @@ func (s *Stats) processCountDownload() {
 			s.mapStats["s"+date+mirrorID] += c.size
 			s.mapStats["c"+date+c.country]++
 			s.mapStats["S"+date+c.country] += c.size
+			if c.tracked {
+				s.mapStats["F"+date+c.filepath+"|"+c.country+"_"+mirrorID]++
+			}
 		case <-pushTicker.C:
 			s.pushStats()
 		}
@@ -202,6 +206,18 @@ func (s *Stats) pushStats() {
 				rconn.Send("HINCRBY", mkey, object, v)
 				mkey = mkey[:strings.LastIndex(mkey, "_")]
 			}
+		} else if typ == "F" {
+			// File downloads per country
+
+			sep := strings.LastIndex(object, "|")
+			file := object[:sep]
+			key := object[sep+1:]
+			mkey := fmt.Sprintf("STATS_TRACKED_%s_%s", file, date)
+
+			for i := 0; i < 4; i++ {
+				rconn.Send("HINCRBY", mkey, key, v)
+				mkey = mkey[:strings.LastIndex(mkey, "_")]
+			}
 		} else {
 			log.Warning("Stats: unknown type", typ)
 		}