Merge pull request #129 from NETWAYS/label-regex

Support regex in include-exclude labels

Author: martialblog

README.md

@@ -233,6 +233,8 @@ OK - 1 Alerts: 0 Firing - 0 Pending - 1 Inactive
  \_[OK] [ApacheDown] is inactive
 ```
 
+The `--include-label` and `--exclude-label` flag support regular expressions in the values (e.g. `severity=warn.+`).
+
 #### Checking watchdog alerts
 
 In Prometheus a "watchdog" or "dead man's switch" is an alert that is always firing to ensure alerting pipeline is working. The `-W, --watchdog` flag can be used to flip/negate the exit state of the plugin for these kind of alerts:

cmd/alert.go

@@ -120,13 +120,6 @@ inactive = 0`,
 				}
 			}
 
-			labelsMatchedInclude := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.IncludeLabels)
-
-			if len(cliAlertConfig.IncludeLabels) > 0 && !labelsMatchedInclude {
-				// If the alert labels don't match here we can skip it.
-				continue
-			}
-
 			// Skip inactive alerts if flag is set
 			if len(rl.AlertingRule.Alerts) == 0 && cliAlertConfig.ProblemsOnly {
 				continue
@@ -143,7 +136,12 @@ inactive = 0`,
 				continue
 			}
 
-			labelsMatchedExclude := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.ExcludeLabels)
+			// Check if the alert group should be excluded
+			labelsMatchedExclude, regexErr := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.ExcludeLabels)
+
+			if regexErr != nil {
+				check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error())
+			}
 
 			if len(cliAlertConfig.ExcludeLabels) > 0 && labelsMatchedExclude {
 				// If the alert labels matches here we can skip it.
@@ -191,6 +189,28 @@ inactive = 0`,
 						counterFiring++
 					}
 
+					labelsMatchedInclude, regexErr := matchesLabel(alert.Labels, cliAlertConfig.IncludeLabels)
+
+					if regexErr != nil {
+						check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error())
+					}
+
+					if len(cliAlertConfig.IncludeLabels) > 0 && !labelsMatchedInclude {
+						// If the alert labels don't match here we can skip it.
+						continue
+					}
+
+					labelsMatchedExclude, regexErr := matchesLabel(alert.Labels, cliAlertConfig.ExcludeLabels)
+
+					if regexErr != nil {
+						check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error())
+					}
+
+					if len(cliAlertConfig.ExcludeLabels) > 0 && labelsMatchedExclude {
+						// If the alert labels matches here we can skip it.
+						continue
+					}
+
 					sc := result.NewPartialResult()
 
 					rlStatus := rl.GetStatus(cliAlertConfig.StateLabelKey)
@@ -261,12 +281,12 @@ func init() {
 
 	fs.StringArrayVar(&cliAlertConfig.IncludeLabels, "include-label", []string{},
 		"The label of one or more specific alerts to include. "+
-			"\nThis parameter can be repeated e.g.: '--include-label prio=high --include-label another=example'"+
+			"\nThis parameter can be repeated e.g.: '--include-label prio=high --include-label another=example'. Supports regex for values"+
 			"\nNote that repeated --include-label are combined using a union.")
 
 	fs.StringArrayVar(&cliAlertConfig.ExcludeLabels, "exclude-label", []string{},
 		"The label of one or more specific alerts to exclude."+
-			"\nThis parameter can be repeated e.g.: '--exclude-label prio=high --exclude-label another=example'")
+			"\nThis parameter can be repeated e.g.: '--exclude-label prio=high --exclude-label another=example'. Supports regex for values")
 
 	fs.BoolVarP(&cliAlertConfig.ProblemsOnly, "problems", "P", false,
 		"Display only alerts which status is not inactive/OK. Note that in combination with the --name flag this might result in no alerts being displayed")
@@ -314,22 +334,33 @@ func matches(input string, regexToExclude []string) (bool, error) {
 }
 
 // Matches a list of labels against a list of labels
-func matchesLabel(labels model.LabelSet, labelsToMatch []string) bool {
+func matchesLabel(labels model.LabelSet, labelsToMatch []string) (bool, error) {
 	for _, lb := range labelsToMatch {
-		kv := strings.SplitN(lb, "=", 2)
+		expectedLabelSet := strings.SplitN(lb, "=", 2)
 
-		if len(kv) != 2 {
+		if len(expectedLabelSet) != 2 {
 			continue
 		}
 
-		key, value := model.LabelName(kv[0]), model.LabelValue(kv[1])
+		// Do we have a value for the expected key?
+		actualValue, ok := labels[model.LabelName(expectedLabelSet[0])]
 
-		if val, ok := labels[key]; ok && val == value {
-			return true
+		if !ok {
+			return false, nil
+		}
+
+		re, err := regexp.Compile(expectedLabelSet[1])
+		if err != nil {
+			return false, err
+		}
+
+		// Does the values match the expected label regex?
+		if re.MatchString(string(actualValue)) {
+			return true, nil
 		}
 	}
 
-	return false
+	return false, nil
 }
 
 // negateStatus turns an OK state into critical and a warning/critical state into OK

cmd/alert_test.go

@@ -39,6 +39,8 @@ func TestAlertCmd(t *testing.T) {
 
 	alertTestDataSet4 := "../testdata/unittest/alertDataset4.json"
 
+	alertTestDataSet5 := "../testdata/unittest/alertDataset5.json"
+
 	tests := []AlertTest{
 		{
 			name: "alert-none",
@@ -85,8 +87,8 @@ func TestAlertCmd(t *testing.T) {
 			args: []string{"run", "../main.go", "alert"},
 			expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive
 \_ [OK] [HostOutOfMemory] is inactive
-\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"}
-\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
+\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"}
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
 |total=3 firing=1 pending=1 inactive=1
 
 exit status 2
@@ -100,8 +102,8 @@ exit status 2
 			})),
 			args: []string{"run", "../main.go", "alert", "--problems"},
 			expected: `[CRITICAL] - 2 Alerts: 1 Firing - 1 Pending - 0 Inactive
-\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"}
-\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
+\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"}
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
 |total=2 firing=1 pending=1 inactive=0
 
 exit status 2
@@ -115,7 +117,7 @@ exit status 2
 			})),
 			args: []string{"run", "../main.go", "alert", "--problems", "-g", "TLS"},
 			expected: `[CRITICAL] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive
-\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
 |total=1 firing=1 pending=0 inactive=0
 
 exit status 2
@@ -129,8 +131,8 @@ exit status 2
 			})),
 			args: []string{"run", "../main.go", "alert", "--problems", "-g", "SQL", "-g", "TLS"},
 			expected: `[CRITICAL] - 2 Alerts: 1 Firing - 1 Pending - 0 Inactive
-\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"}
-\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
+\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"}
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
 |total=2 firing=1 pending=1 inactive=0
 
 exit status 2
@@ -144,7 +146,7 @@ exit status 2
 			})),
 			args: []string{"run", "../main.go", "alert", "--problems", "--exclude-alert", "Sql.*DeniedRate"},
 			expected: `[CRITICAL] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive
-\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
 |total=1 firing=1 pending=0 inactive=0
 
 exit status 2
@@ -196,7 +198,7 @@ exit status 3
 			args: []string{"run", "../main.go", "alert", "--name", "HostOutOfMemory", "--name", "BlackboxTLS"},
 			expected: `[CRITICAL] - 2 Alerts: 1 Firing - 0 Pending - 1 Inactive
 \_ [OK] [HostOutOfMemory] is inactive
-\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
 |total=2 firing=1 pending=0 inactive=1
 
 exit status 2
@@ -210,7 +212,7 @@ exit status 2
 			})),
 			args: []string{"run", "../main.go", "alert", "--name", "HostOutOfMemory", "--name", "BlackboxTLS", "--problems"},
 			expected: `[CRITICAL] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive
-\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
 |total=1 firing=1 pending=0 inactive=0
 
 exit status 2
@@ -250,10 +252,10 @@ exit status 2
 				w.Write(loadTestdata(alertTestDataSet1))
 			})),
 			args: []string{"run", "../main.go", "alert", "--include-label", "severity=critical"},
-			expected: `[CRITICAL] - 2 Alerts: 1 Firing - 0 Pending - 1 Inactive
+			expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive
 \_ [OK] [HostOutOfMemory] is inactive
-\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
-|total=2 firing=1 pending=0 inactive=1
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
+|total=3 firing=1 pending=1 inactive=1
 
 exit status 2
 `,
@@ -266,7 +268,21 @@ exit status 2
 			})),
 			args: []string{"run", "../main.go", "alert", "--exclude-label", "severity=critical"},
 			expected: `[WARNING] - 1 Alerts: 0 Firing - 1 Pending - 0 Inactive
-\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"}
+\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"}
+|total=1 firing=0 pending=1 inactive=0
+
+exit status 1
+`,
+		},
+		{
+			name: "alert-exclude-label-regex",
+			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				w.Write(loadTestdata(alertTestDataSet1))
+			})),
+			args: []string{"run", "../main.go", "alert", "--exclude-label", "severity=crit.*"},
+			expected: `[WARNING] - 1 Alerts: 0 Firing - 1 Pending - 0 Inactive
+\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"}
 |total=1 firing=0 pending=1 inactive=0
 
 exit status 1
@@ -281,8 +297,24 @@ exit status 1
 			args: []string{"run", "../main.go", "alert", "--include-label", "team=database", "--include-label", "severity=critical"},
 			expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive
 \_ [OK] [HostOutOfMemory] is inactive
-\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"}
-\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
+\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"}
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
+|total=3 firing=1 pending=1 inactive=1
+
+exit status 2
+`,
+		},
+		{
+			name: "alert-include-label-multiple-regex",
+			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				w.Write(loadTestdata(alertTestDataSet1))
+			})),
+			args: []string{"run", "../main.go", "alert", "--include-label", "team=data.+", "--include-label", "severity=critical"},
+			expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive
+\_ [OK] [HostOutOfMemory] is inactive
+\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"}
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
 |total=3 firing=1 pending=1 inactive=1
 
 exit status 2
@@ -297,8 +329,8 @@ exit status 2
 			args: []string{"run", "../main.go", "alert", "--include-label", "severity=warning", "--include-label", "severity=critical"},
 			expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive
 \_ [OK] [HostOutOfMemory] is inactive
-\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"}
-\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
+\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"}
+\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
 |total=3 firing=1 pending=1 inactive=1
 
 exit status 2
@@ -322,11 +354,26 @@ exit status 2
 			args: []string{"run", "../main.go", "alert", "--label-key-state=icinga"},
 			expected: `[WARNING] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive
 \_ [OK] [HostOutOfMemory] is inactive
-\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"}
-\_ [OK] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"}
+\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"}
+\_ [OK] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"}
 |total=3 firing=1 pending=1 inactive=1
 
 exit status 1
+`,
+		},
+		{
+			name: "alert-include-with-name-and-regex",
+			server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+				w.Write(loadTestdata(alertTestDataSet5))
+			})),
+			args: []string{"run", "../main.go", "alert", "--name", "ContainerKilled", "--include-label", "name=(mosquitto|nodered)"},
+			expected: `[CRITICAL] - 3 Alerts: 3 Firing - 0 Pending - 0 Inactive
+\_ [CRITICAL] [ContainerKilled] - Job: [cadvisor] on Instance: [example:8888] is firing - value: 123.40 - {"alertname":"ContainerKilled","instance":"example:8888","job":"cadvisor","name":"nodered","severity":"warning"}
+\_ [CRITICAL] [ContainerKilled] - Job: [cadvisor] on Instance: [example:8888] is firing - value: 123.40 - {"alertname":"ContainerKilled","instance":"example:8888","job":"cadvisor","name":"mosquitto","severity":"warning"}
+|total=3 firing=3 pending=0 inactive=0
+
+exit status 2
 `,
 		},
 	}