Skip to content

Commit 052e44c

Browse files
SkantesSkantes
committed
Metrics: add detailed metrics for selected files
This adds a more detailed metrics category for files served by mirrorbits. These tracked files will detail their downloads per country, but due to the increased number of fields required for this, only files selected through the command line will be tracked. This also adds command line arguments to add, delete, and list the files to monitore closely.
1 parent b63a4ae commit 052e44c

8 files changed

Lines changed: 608 additions & 98 deletions

File tree

‎cli/commands.go‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,14 @@ func (c *cli) CmdHelp() error {
9999
help += fmt.Sprintf("CLI commands:\n")
100100
for _, command := range [][]string{
101101
{"add", "Add a new mirror"},
102+
{"addMetric", "Add a tracked file to the metrics route"},
102103
{"disable", "Disable a mirror"},
104+
{"delMetric", "Delete a tracked file from the metrics route"},
103105
{"edit", "Edit a mirror"},
104106
{"enable", "Enable a mirror"},
105107
{"export", "Export the mirror database"},
106108
{"list", "List all mirrors"},
109+
{"listMetrics", "List all tracked files from the metrics route"},
107110
{"logs", "Print logs of a mirror"},
108111
{"refresh", "Refresh the local repository"},
109112
{"reload", "Reload configuration"},
@@ -349,6 +352,99 @@ func (c *cli) CmdAdd(args ...string) error {
349352
return nil
350353
}
351354

355+
func (c *cli) CmdAddmetric(args ...string) error {
356+
cmd := SubCmd("addMetric", "FILE_PATH", "Add a file to the metrics route")
357+
358+
if err := cmd.Parse(args); err != nil {
359+
return nil
360+
}
361+
if cmd.NArg() != 1 {
362+
cmd.Usage()
363+
return nil
364+
}
365+
file := cmd.Arg(0)
366+
367+
client := c.GetRPC()
368+
ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout)
369+
defer cancel()
370+
_, err := client.AddMetric(ctx, &rpc.Metric{
371+
Filename: string(file),
372+
})
373+
if err != nil {
374+
log.Fatal("Error while adding metric: ", err)
375+
}
376+
377+
log.Info("File ", file, " successfully added to metrics.")
378+
return nil
379+
}
380+
381+
func (c *cli) CmdDelmetric(args ...string) error {
382+
cmd := SubCmd("delMetric", "FILE_PATH", "Delete a file from the metrics route")
383+
384+
if err := cmd.Parse(args); err != nil {
385+
return nil
386+
}
387+
if cmd.NArg() != 1 {
388+
cmd.Usage()
389+
return nil
390+
}
391+
file := cmd.Arg(0)
392+
393+
client := c.GetRPC()
394+
ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout)
395+
defer cancel()
396+
_, err := client.DelMetric(ctx, &rpc.Metric{
397+
Filename: string(file),
398+
})
399+
if err != nil {
400+
log.Fatal("Error while deleting metric: ", err)
401+
}
402+
403+
log.Info("File ", file, " successfully deleted from metrics.")
404+
return nil
405+
}
406+
407+
func (c *cli) CmdListmetrics(args ...string) error {
408+
cmd := SubCmd("listMetric", "[OPTIONNAL FILTER PATTERN]",
409+
"Optionnal pattern to filter results")
410+
411+
filterPattern := "*"
412+
if err := cmd.Parse(args); err != nil {
413+
return nil
414+
}
415+
nArg := cmd.NArg()
416+
if nArg > 1 {
417+
cmd.Usage()
418+
return nil
419+
} else if nArg == 1 {
420+
filterPattern = "*" + cmd.Arg(0) + "*"
421+
}
422+
423+
client := c.GetRPC()
424+
ctx, cancel := context.WithTimeout(context.Background(), defaultRPCTimeout)
425+
defer cancel()
426+
fileList, err := client.ListMetrics(ctx, &rpc.Metric{
427+
Filename: string(filterPattern),
428+
})
429+
if err != nil {
430+
log.Fatal("Error while listing metrics: ", err)
431+
}
432+
433+
if len(fileList.Filename) == 0 {
434+
if nArg == 1 {
435+
log.Info("There are no tracked files matching your request.")
436+
} else {
437+
log.Info("There are no tracked files.")
438+
}
439+
} else {
440+
for _, file := range fileList.Filename {
441+
log.Info(file)
442+
}
443+
}
444+
445+
return nil
446+
}
447+
352448
func (c *cli) CmdRemove(args ...string) error {
353449
cmd := SubCmd("remove", "IDENTIFIER", "Remove an existing mirror")
354450
force := cmd.Bool("f", false, "Never prompt for confirmation")

‎database/utils.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,43 @@ func (r *Redis) GetListOfFiles() ([]string, error) {
6565
return files, nil
6666
}
6767

68+
func (r *Redis) IsFileTracked(file string) (bool, error) {
69+
trackedFileList, err := r.GetListOfTrackedFiles()
70+
if err != nil {
71+
return false, nil
72+
}
73+
for _, v := range trackedFileList {
74+
if v == file {
75+
return true, nil
76+
}
77+
}
78+
return false, nil
79+
}
80+
81+
func (r *Redis) GetListOfTrackedFiles() ([]string, error) {
82+
conn, err := r.Connect()
83+
if err != nil {
84+
return nil, err
85+
}
86+
defer conn.Close()
87+
88+
values, err := redis.Values(conn.Do("SMEMBERS", "TRACKED_FILES"))
89+
if err != nil {
90+
return nil, err
91+
}
92+
93+
files := make([]string, len(values))
94+
for i, v := range values {
95+
value, okValue := v.([]byte)
96+
if !okValue {
97+
return nil, errors.New("invalid type for file")
98+
}
99+
files[i] = string(value)
100+
}
101+
102+
return files, nil
103+
}
104+
68105
func (r *Redis) GetListOfCountries() ([]string, error) {
69106
conn, err := r.Connect()
70107
if err != nil {

‎http/http.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,10 +331,15 @@ func (h *HTTP) mirrorHandler(w http.ResponseWriter, r *http.Request, ctx *Contex
331331
http.Error(w, err.Error(), status)
332332
}
333333

334+
isTracked, err := h.redis.IsFileTracked(fileInfo.Path)
335+
if err != nil {
336+
log.Error("There was a problem fetching the tracked file list: ", err)
337+
}
338+
334339
if !ctx.IsMirrorlist() {
335340
logs.LogDownload(resultRenderer.Type(), status, results, err)
336341
if len(mlist) > 0 {
337-
h.stats.CountDownload(mlist[0], fileInfo, clientInfo)
342+
h.stats.CountDownload(mlist[0], fileInfo, clientInfo, isTracked)
338343
}
339344
}
340345

‎http/metrics.go‎

Lines changed: 87 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ package http
33
import (
44
"fmt"
55
"net/http"
6+
"regexp"
7+
"strconv"
8+
"strings"
69
"time"
710

811
"github.com/gomodule/redigo/redis"
@@ -15,17 +18,18 @@ type metrics struct {
1518
Total int
1619
}
1720

18-
func statsToPrometheusFormat(metrics metrics, labelName string, labelValue string) string {
21+
func statsToPrometheusFormat(metrics metrics, label_name string,
22+
label_value string) string {
1923
var output string
2024

2125
output += fmt.Sprintf("%s_total{%s=\"%s\"} %d\n",
22-
labelName, labelName, labelValue, metrics.Total)
26+
label_name, label_name, label_value, metrics.Total)
2327
output += fmt.Sprintf("%s_day{%s=\"%s\"} %d\n",
24-
labelName, labelName, labelValue, metrics.Day)
28+
label_name, label_name, label_value, metrics.Day)
2529
output += fmt.Sprintf("%s_month{%s=\"%s\"} %d\n",
26-
labelName, labelName, labelValue, metrics.Month)
30+
label_name, label_name, label_value, metrics.Month)
2731
output += fmt.Sprintf("%s_year{%s=\"%s\"} %d\n\n",
28-
labelName, labelName, labelValue, metrics.Year)
32+
label_name, label_name, label_value, metrics.Year)
2933

3034
return output
3135
}
@@ -141,5 +145,83 @@ func (h *HTTP) metricsHandler(w http.ResponseWriter, r *http.Request) {
141145
index += 4
142146
}
143147

148+
// Get all tracked files
149+
var trackedFilesList []string
150+
mkeyLenMap := make(map[string]int)
151+
mkeyFileMap := make(map[string]string)
152+
trackedFilesList, err = h.redis.GetListOfTrackedFiles()
153+
if err != nil {
154+
log.Error("Cannot fetch list of files: " + err.Error())
155+
return
156+
}
157+
158+
// Get tracked files downloads per country and mirror
159+
// rconn.Send("MULTI")
160+
for _, file := range trackedFilesList {
161+
log.Info("Tracked files: ", file)
162+
mkey := "STATS_TRACKED_" + file + "_" + today.Format("2006_01_02")
163+
for i := 0; i < 4; i++ {
164+
exists, _ := redis.Bool(rconn.Do("EXISTS", mkey))
165+
if exists {
166+
mkeyLenMap[mkey], _ = redis.Int(rconn.Do("HLEN", mkey))
167+
log.Info("mKey: ", mkey)
168+
log.Info(file, " len: ", mkeyLenMap[mkey])
169+
mkeyFileMap[mkey] = file
170+
// rconn.Send("HGETALL", mkey)
171+
mkey = mkey[:strings.LastIndex(mkey, "_")]
172+
}
173+
}
174+
}
175+
176+
rconn.Send("MULTI")
177+
for mkey := range mkeyLenMap {
178+
rconn.Send("HGETALL", mkey)
179+
}
180+
stats, err = redis.Values(rconn.Do("EXEC"))
181+
if err != nil {
182+
log.Error("Cannot fetch file per country stats: " + err.Error())
183+
return
184+
}
185+
186+
pattern := regexp.MustCompile("_")
187+
index = 0
188+
for mkey, hashLen := range mkeyLenMap {
189+
for i := 0; i < 2*hashLen; i += 2 {
190+
hashSlice, _ := redis.ByteSlices(stats[index], err)
191+
key, _ := redis.String(hashSlice[i], err)
192+
value, _ := redis.Int(hashSlice[i+1], err)
193+
log.Info("mkey: ", mkey)
194+
log.Info("File: ", mkeyFileMap[mkey])
195+
log.Info("Key: ", key)
196+
log.Info("Value: ", value)
197+
sep := strings.Index(key, "_")
198+
country := key[:sep]
199+
mirrorID, err := strconv.Atoi(key[sep+1:])
200+
if err != nil {
201+
log.Error("Failed to convert mirror ID: ", err)
202+
return
203+
}
204+
mirror := mirrorsMap[mirrorID]
205+
duration := len(pattern.FindAllStringIndex(mkey, -1))
206+
var durationStr string
207+
switch duration {
208+
case 2:
209+
durationStr = "total"
210+
case 3:
211+
durationStr = "year"
212+
case 4:
213+
durationStr = "month"
214+
case 5:
215+
durationStr = "day"
216+
}
217+
output += fmt.Sprintf("stats_tracked_"+
218+
"%s{file=\"%s\",country=\"%s\",mirror=\"%s\"} %d\n",
219+
durationStr, mkeyFileMap[mkey], country, mirror, value,
220+
)
221+
}
222+
output += "\n"
223+
index++
224+
}
225+
144226
w.Write([]byte(output))
145227
}

‎http/stats.go‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ type countItem struct {
5656
country string
5757
size int64
5858
time time.Time
59+
tracked bool
5960
}
6061

6162
// NewStats returns an instance of the stats counter
@@ -79,7 +80,7 @@ func (s *Stats) Terminate() {
7980

8081
// CountDownload is a lightweight method used to count a new download for a specific file and mirror
8182
func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo,
82-
clientInfo network.GeoIPRecord) error {
83+
clientInfo network.GeoIPRecord, isTracked bool) error {
8384
if m.Name == "" {
8485
return errUnknownMirror
8586
}
@@ -91,7 +92,7 @@ func (s *Stats) CountDownload(m mirrors.Mirror, fileinfo filesystem.FileInfo,
9192
}
9293

9394
s.countChan <- countItem{m.ID, fileinfo.Path, clientInfo.Country,
94-
fileinfo.Size, time.Now().UTC()}
95+
fileinfo.Size, time.Now().UTC(), isTracked}
9596
return nil
9697
}
9798

@@ -114,6 +115,9 @@ func (s *Stats) processCountDownload() {
114115
s.mapStats["s"+date+mirrorID] += c.size
115116
s.mapStats["c"+date+c.country]++
116117
s.mapStats["S"+date+c.country] += c.size
118+
if c.tracked {
119+
s.mapStats["F"+date+c.filepath+"|"+c.country+"_"+mirrorID]++
120+
}
117121
case <-pushTicker.C:
118122
s.pushStats()
119123
}
@@ -202,6 +206,18 @@ func (s *Stats) pushStats() {
202206
rconn.Send("HINCRBY", mkey, object, v)
203207
mkey = mkey[:strings.LastIndex(mkey, "_")]
204208
}
209+
} else if typ == "F" {
210+
// File downloads per country
211+
212+
sep := strings.LastIndex(object, "|")
213+
file := object[:sep]
214+
key := object[sep+1:]
215+
mkey := fmt.Sprintf("STATS_TRACKED_%s_%s", file, date)
216+
217+
for i := 0; i < 4; i++ {
218+
rconn.Send("HINCRBY", mkey, key, v)
219+
mkey = mkey[:strings.LastIndex(mkey, "_")]
220+
}
205221
} else {
206222
log.Warning("Stats: unknown type", typ)
207223
}

0 commit comments

Comments
 (0)