Metrics: add detailed metrics for selected files

This adds a more detailed metrics category for files served by
mirrorbits.
These tracked files will detail their downloads per country, but due to
the increased number of fields required for this, only files selected
through the command line will be tracked.
This also adds command line arguments to add, delete, and list the
files to monitore closely.

Author: Skantes

cli/commands.go

@@ -99,11 +99,14 @@ func (c *cli) CmdHelp() error {
 	help += fmt.Sprintf("CLI commands:\n")
 	for _, command := range [][]string{
 		{"add", "Add a new mirror"},
+		{"addMetric", "Add a tracked file to the metrics route"},
 		{"disable", "Disable a mirror"},
+		{"delMetric", "Delete a tracked file from the metrics route"},
 		{"edit", "Edit a mirror"},
 		{"enable", "Enable a mirror"},
 		{"export", "Export the mirror database"},
 		{"list", "List all mirrors"},
+		{"listMetrics", "List all tracked files from the metrics route"},
 		{"logs", "Print logs of a mirror"},
 		{"refresh", "Refresh the local repository"},
 		{"reload", "Reload configuration"},
@@ -114,7 +117,7 @@ func (c *cli) CmdHelp() error {
 		{"upgrade", "Seamless binary upgrade"},
 		{"version", "Print version information"},
 	} {
-		help += fmt.Sprintf("    %-10.10s%s\n", command[0], command[1])
+		help += fmt.Sprintf("    %-15.15s%s\n", command[0], command[1])
 	}
 	fmt.Fprintf(os.Stderr, "%s\n", help)
 	return nil
@@ -349,6 +352,98 @@ func (c *cli) CmdAdd(args ...string) error {
 	return nil
 }
 
+func (c *cli) CmdAddmetric(args ...string) error {
+	cmd := SubCmd("addMetric", "FILE_PATH", "Add a file to the metrics route")
+
+	if err := cmd.Parse(args); err != nil {
+		return nil
+	}
+	if cmd.NArg() != 1 {
+		cmd.Usage()
+		return nil
+	}
+	file := cmd.Arg(0)
+
+	client := c.GetRPC()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout)
+	defer cancel()
+	_, err := client.AddMetric(ctx, &rpc.Metric{
+		Filename: string(file),
+	})
+	if err != nil {
+		log.Fatal("Error while adding metric: ", err)
+	}
+
+	log.Info("File ", file, " successfully added to metrics.")
+	return nil
+}
+
+func (c *cli) CmdDelmetric(args ...string) error {
+	cmd := SubCmd("delMetric", "FILE_PATH", "Delete a file from the metrics route")
+
+	if err := cmd.Parse(args); err != nil {
+		return nil
+	}
+	if cmd.NArg() != 1 {
+		cmd.Usage()
+		return nil
+	}
+	file := cmd.Arg(0)
+
+	client := c.GetRPC()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout)
+	defer cancel()
+	_, err := client.DelMetric(ctx, &rpc.Metric{
+		Filename: string(file),
+	})
+	if err != nil {
+		log.Fatal("Error while deleting metric: ", err)
+	}
+
+	log.Info("File ", file, " successfully deleted from metrics.")
+	return nil
+}
+
+func (c *cli) CmdListmetrics(args ...string) error {
+	cmd := SubCmd("listMetrics", "[OPTIONNAL FILTER PATTERN]", "Optionnal pattern to filter results")
+
+	filterPattern := "*"
+	if err := cmd.Parse(args); err != nil {
+		return nil
+	}
+	nArg := cmd.NArg()
+	if nArg > 1 {
+		cmd.Usage()
+		return nil
+	} else if nArg == 1 {
+		filterPattern = "*" + cmd.Arg(0) + "*"
+	}
+
+	client := c.GetRPC()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout)
+	defer cancel()
+	fileList, err := client.ListMetrics(ctx, &rpc.Metric{
+		Filename: string(filterPattern),
+	})
+	if err != nil {
+		log.Fatal("Error while listing metrics: ", err)
+	}
+
+	if len(fileList.Filename) == 0 {
+		if nArg == 1 {
+			log.Info("There are no tracked files matching your request.")
+		} else {
+			log.Info("There are no tracked files.")
+		}
+	} else {
+		for _, file := range fileList.Filename {
+			log.Info(file)
+		}
+	}
+
+	return nil
+}
+
 func (c *cli) CmdRemove(args ...string) error {
 	cmd := SubCmd("remove", "IDENTIFIER", "Remove an existing mirror")
 	force := cmd.Bool("f", false, "Never prompt for confirmation")

database/utils.go

@@ -65,6 +65,43 @@ func (r *Redis) GetListOfFiles() ([]string, error) {
 	return files, nil
 }
 
+func (r *Redis) IsFileTracked(file string) (bool, error) {
+	trackedFileList, err := r.GetListOfTrackedFiles()
+	if err != nil {
+		return false, nil
+	}
+	for _, v := range trackedFileList {
+		if v == file {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (r *Redis) GetListOfTrackedFiles() ([]string, error) {
+	conn, err := r.Connect()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	values, err := redis.Values(conn.Do("SMEMBERS", "TRACKED_FILES"))
+	if err != nil {
+		return nil, err
+	}
+
+	files := make([]string, len(values))
+	for i, v := range values {
+		value, okValue := v.([]byte)
+		if !okValue {
+			return nil, errors.New("invalid type for file")
+		}
+		files[i] = string(value)
+	}
+
+	return files, nil
+}
+
 func (r *Redis) GetListOfCountries() ([]string, error) {
 	conn, err := r.Connect()
 	if err != nil {
@@ -89,6 +126,30 @@ func (r *Redis) GetListOfCountries() ([]string, error) {
 	return files, nil
 }
 
+func (r *Redis) GetListOfTrackFilesFields(file string) ([]string, error) {
+	conn, err := r.Connect()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	values, err := redis.Values(conn.Do("HKEYS", "STATS_TRACKED_"+file))
+	if err != nil {
+		return nil, err
+	}
+
+	files := make([]string, len(values))
+	for i, v := range values {
+		value, okValue := v.([]byte)
+		if !okValue {
+			return nil, errors.New("invalid type for file")
+		}
+		files[i] = string(value)
+	}
+
+	return files, nil
+}
+
 func (r *Redis) AddCountry(country string) error {
 	conn, err := r.Connect()
 	if err != nil {

http/http.go

@@ -331,10 +331,15 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex
 		http.Error(w, err.Error(), status)
 	}
 
+	isTracked, err := h.redis.IsFileTracked(fileInfo.Path)
+	if err != nil {
+		log.Error("There was a problem fetching the tracked file list: ", err)
+	}
+
 	if !ctx.IsMirrorlist() {
 		logs.LogDownload(resultRenderer.Type(), status, results, err)
 		if len(mlist) > 0 {
-			h.stats.CountDownload(mlist[0], fileInfo, clientInfo)
+			h.stats.CountDownload(mlist[0], fileInfo, clientInfo, isTracked)
 		}
 	}