|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "fmt" |
| 5 | + "github.com/pkg/errors" |
| 6 | + "gopkg.in/yaml.v3" |
| 7 | + "io" |
| 8 | + "os" |
| 9 | + "regexp" |
| 10 | +) |
| 11 | + |
| 12 | +// flakeDetectionPolicyConfig represents configuration used by flakechecker to evaluate failed tests. |
| 13 | +type flakeDetectionPolicyConfig struct { |
| 14 | + // JobNameRegex is a regular expression for the name of the CI job that should be evaluated by flakechecker. |
| 15 | + // (i.e. CI jobs for PRs should be evaluated, but not CI jobs for commits already merged to "main" branch) |
| 16 | + JobNameRegex string `yaml:"jobNameRegex"` |
| 17 | + // ClassName is class name of the test that should be isolated. Usually class name for Groovy tests, |
| 18 | + // package name for golang tests, etc. |
| 19 | + ClassName string `yaml:"className"` |
| 20 | + // TestNameRegex is a regular expression used to match test names. Some test names contain detailed information |
| 21 | + // (i.e. version 4.4.4), but we want to use ratio for all tests in that group (i.e. 4.4.z). |
| 22 | + // Using a regex allow us to group tests as needed. |
| 23 | + TestNameRegex string `yaml:"testNameRegex"` |
| 24 | + // TestNameRegex is CI job name that should be used for ratio calculation. |
| 25 | + // i.e. we take CI runs for commits on "main" branch as input for evaluation of flake ratio. |
| 26 | + RatioJobName string `yaml:"ratioJobName"` |
| 27 | + // RatioThreshold is the maximum failure percentage that is used to distinguish a flaky test from |
| 28 | + // a completely broken test. This information is usually fetched from historical executions and data |
| 29 | + // collected in DB. If measured flakiness exceeds this threshold, we no longer want to suppress test failure, |
| 30 | + // because we suspect it might have regressed above what we consider acceptable. |
| 31 | + RatioThreshold int `yaml:"ratioThreshold"` |
| 32 | +} |
| 33 | + |
| 34 | +type flakeDetectionPolicy struct { |
| 35 | + config flakeDetectionPolicyConfig |
| 36 | + compiledJobNameRegex *regexp.Regexp |
| 37 | + compiledTestNameRegex *regexp.Regexp |
| 38 | +} |
| 39 | + |
| 40 | +func newFlakeDetectionPolicy(config flakeDetectionPolicyConfig) (*flakeDetectionPolicy, error) { |
| 41 | + compiledJobNameRegex, err := regexp.Compile(fmt.Sprintf("^%s$", config.JobNameRegex)) |
| 42 | + if err != nil { |
| 43 | + return nil, errors.Wrap(err, fmt.Sprintf("invalid flake config match job regex: %v", config.JobNameRegex)) |
| 44 | + } |
| 45 | + |
| 46 | + compiledTestNameRegex, err := regexp.Compile(fmt.Sprintf("^%s$", config.TestNameRegex)) |
| 47 | + if err != nil { |
| 48 | + return nil, errors.Wrap(err, fmt.Sprintf("invalid flake config test name regex: %v", config.TestNameRegex)) |
| 49 | + } |
| 50 | + |
| 51 | + return &flakeDetectionPolicy{ |
| 52 | + config: config, |
| 53 | + compiledJobNameRegex: compiledJobNameRegex, |
| 54 | + compiledTestNameRegex: compiledTestNameRegex, |
| 55 | + }, nil |
| 56 | +} |
| 57 | + |
| 58 | +func (r *flakeDetectionPolicy) matchJobName(jobName string) bool { |
| 59 | + return r.compiledJobNameRegex.MatchString(jobName) |
| 60 | +} |
| 61 | + |
| 62 | +func (r *flakeDetectionPolicy) matchClassName(classname string) bool { |
| 63 | + return classname == r.config.ClassName |
| 64 | +} |
| 65 | + |
| 66 | +func (r *flakeDetectionPolicy) matchTestName(testName string) bool { |
| 67 | + return r.compiledTestNameRegex.MatchString(testName) |
| 68 | +} |
| 69 | + |
| 70 | +func findFlakeConfigForTest(flakeCheckerRecs []*flakeDetectionPolicy, jobName string, className string, testName string) (*flakeDetectionPolicy, error) { |
| 71 | + for _, flakeCheckerRec := range flakeCheckerRecs { |
| 72 | + if flakeCheckerRec.matchJobName(jobName) && flakeCheckerRec.matchClassName(className) && flakeCheckerRec.matchTestName(testName) { |
| 73 | + return flakeCheckerRec, nil |
| 74 | + } |
| 75 | + } |
| 76 | + |
| 77 | + return nil, errors.Wrap(errors.Errorf("%q / %q / %q", jobName, className, testName), errDescNoMatch) |
| 78 | +} |
| 79 | + |
| 80 | +func loadFlakeConfigFile(fileName string) ([]*flakeDetectionPolicy, error) { |
| 81 | + ymlConfigFile, err := os.Open(fileName) |
| 82 | + if err != nil { |
| 83 | + return nil, errors.Wrap(err, fmt.Sprintf("open flake config file: %s", fileName)) |
| 84 | + } |
| 85 | + defer ymlConfigFile.Close() |
| 86 | + |
| 87 | + ymlConfigFileData, err := io.ReadAll(ymlConfigFile) |
| 88 | + if err != nil { |
| 89 | + return nil, errors.Wrap(err, fmt.Sprintf("read flake config file: %s", fileName)) |
| 90 | + } |
| 91 | + |
| 92 | + flakeConfigs := make([]flakeDetectionPolicyConfig, 0) |
| 93 | + err = yaml.Unmarshal(ymlConfigFileData, &flakeConfigs) |
| 94 | + if err != nil { |
| 95 | + return nil, errors.Wrap(err, fmt.Sprintf("parse flake config file: %s", fileName)) |
| 96 | + } |
| 97 | + |
| 98 | + detectionPolicies := make([]*flakeDetectionPolicy, 0, len(flakeConfigs)) |
| 99 | + for _, flakeConfig := range flakeConfigs { |
| 100 | + detectionPolicy, errNewPolicy := newFlakeDetectionPolicy(flakeConfig) |
| 101 | + if errNewPolicy != nil { |
| 102 | + return nil, errors.Wrap(err, fmt.Sprintf("create flake detection policy from config: %v", flakeConfig)) |
| 103 | + } |
| 104 | + |
| 105 | + detectionPolicies = append(detectionPolicies, detectionPolicy) |
| 106 | + } |
| 107 | + |
| 108 | + return detectionPolicies, nil |
| 109 | +} |
0 commit comments