Merge pull request #306 from coroot/cap_fqdn_per_container_cardinality

cap unique FQDN labels per container in `container_dns_requests_total`

Author: def

containers/container.go

@@ -130,8 +130,9 @@ type Container struct {
 	activeConnections        map[ConnectionKey]*ActiveConnection
 	connectionsByPidFd       map[PidFd]*ActiveConnection
 
-	l7Stats  L7Stats
-	dnsStats *L7Metrics
+	l7Stats   L7Stats
+	dnsStats  *L7Metrics
+	seenFQDNs map[string]struct{}
 
 	gpuStats map[string]*GpuUsage
 
@@ -187,6 +188,7 @@ func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, pid
 		connectionsByPidFd:       map[PidFd]*ActiveConnection{},
 		l7Stats:                  L7Stats{},
 		dnsStats:                 &L7Metrics{},
+		seenFQDNs:                map[string]struct{}{},
 
 		gpuStats: map[string]*GpuUsage{},
 
@@ -677,6 +679,8 @@ func (c *Container) updateConnectionTrafficStats(ac *ActiveConnection, sent, rec
 	ac.BytesReceived = received
 }
 
+const fqdnOverflowLabel = "~other"
+
 func (c *Container) onDNSRequest(r *l7.RequestData) map[netaddr.IP]*common.Domain {
 	status := r.Status.DNS()
 	if status == "" {
@@ -702,7 +706,17 @@ func (c *Container) onDNSRequest(r *l7.RequestData) map[netaddr.IP]*common.Domai
 			[]string{"request_type", "domain", "status"},
 		)
 	}
-	if m, _ := c.dnsStats.Requests.GetMetricWithLabelValues(t, fqdn, status); m != nil {
+	metricFQDN := fqdn
+	if fqdn != "" {
+		if _, ok := c.seenFQDNs[fqdn]; !ok {
+			if len(c.seenFQDNs) < *flags.MaxFQDNsPerContainer {
+				c.seenFQDNs[fqdn] = struct{}{}
+			} else {
+				metricFQDN = fqdnOverflowLabel
+			}
+		}
+	}
+	if m, _ := c.dnsStats.Requests.GetMetricWithLabelValues(t, metricFQDN, status); m != nil {
 		m.Inc()
 	}
 	if r.Duration != 0 {

flags/flags.go

@@ -43,6 +43,7 @@ var (
 	LogPerSecond            = kingpin.Flag("log-per-second", "The number of logs per second").Default("10.0").Envar("LOG_PER_SECOND").Float64()
 	LogBurst                = kingpin.Flag("log-burst", "The maximum number of tokens that can be consumed in a single call to allow").Default("100").Envar("LOG_BURST").Int()
 	LogPatternsPerContainer = kingpin.Flag("log-patterns-per-container", "Max unique log patterns per container per level").Default("256").Envar("LOG_PATTERNS_PER_CONTAINER").Int()
+	MaxFQDNsPerContainer    = kingpin.Flag("max-fqdns-per-container", "Max unique FQDN values per container, extras are bucketed under '~other'").Default("50").Envar("MAX_FQDNS_PER_CONTAINER").Int()
 
 	MaxLabelLength = kingpin.Flag("max-label-length", "Maximum length of a metric label value").Default("4096").Envar("MAX_LABEL_LENGTH").Int()