From 43d2551042051c3e634fd0ba8214b66684d12b06 Mon Sep 17 00:00:00 2001 From: marco Date: Thu, 10 Apr 2025 12:36:50 +0200 Subject: [PATCH 1/3] CI: lint go code --- .github/workflows/lint.yml | 41 +++++++++++ waf-check/.golangci.yml | 145 +++++++++++++++++++++++++++++++++++++ waf-check/go.mod | 21 +----- waf-check/go.sum | 39 +--------- waf-check/main.go | 1 - waf-check/manager.go | 1 - waf-check/result.go | 2 +- waf-check/utils.go | 3 +- 8 files changed, 191 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 waf-check/.golangci.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..1a089d11cee --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,41 @@ +name: Static Analysis + +on: + push: + branches: [ master ] + paths: + - '.github/workflows/lint.yaml' + - 'waf-check/**' + pull_request: + branches: [ master ] + paths: + - '.github/workflows/lint.yaml' + - 'waf-check/**' + + +jobs: + build: + name: "Lint" + runs-on: ubuntu-latest + + permissions: + contents: read + + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: waf-check/go.mod + + - name: golangci-lint + uses: golangci/golangci-lint-action@v7 + with: + version: v2.0 + args: --issues-exit-code=1 --timeout 10m + only-new-issues: false + working-directory: waf-check diff --git a/waf-check/.golangci.yml b/waf-check/.golangci.yml new file mode 100644 index 00000000000..91d4f2aea0c --- /dev/null +++ b/waf-check/.golangci.yml @@ -0,0 +1,145 @@ +version: "2" + +linters: + default: all + disable: + - errcheck + - errorlint + - exhaustruct + - forbidigo + - ineffassign + - intrange + - musttag + - noctx + - perfsprint + - unparam + - usestdlibvars + - wastedassign + + - cyclop # revive + - funlen # revive + - gocognit # revive + - gocyclo # revive + - lll # revive + + - gosec # (gas): Inspects source code for security problems + - wrapcheck # Checks that errors returned from external packages are wrapped + - mnd # An analyzer to detect magic numbers. + - nlreturn # nlreturn checks for a new line before return and branch statements to increase code clarity + - whitespace # Whitespace is a linter that checks for unnecessary newlines at the start and end of functions, if, for, etc. + - wsl # add or remove empty lines + - err113 # Go linter to check the errors handling expressions + - tagliatelle # Checks the struct tags. + - varnamelen # checks that the length of a variable's name matches its scope + + settings: + + depguard: + rules: + yaml: + files: + - '!**/config.go' + deny: + - pkg: gopkg.in/yaml.v2 + desc: yaml.v2 is deprecated for new code in favor of yaml.v3 + + errcheck: + check-type-assertions: false + + gocritic: + enable-all: true + disabled-checks: + - httpNoBody + - hugeParam + - paramTypeCombine + - exitAfterDefer + + govet: + disable: + - fieldalignment + enable-all: true + + misspell: + locale: US + + nlreturn: + block-size: 5 + + nolintlint: + require-explanation: false # don't require an explanation for nolint directives + require-specific: false # don't require nolint directives to be specific about which linter is being skipped + allow-unused: false # report any unused nolint directives + + revive: + severity: error + enable-all-rules: true + rules: + - name: add-constant + disabled: true + - name: cognitive-complexity + arguments: + # lower this after refactoring + - 17 + - name: cyclomatic + arguments: + # lower this after refactoring + - 12 + - name: deep-exit + disabled: true + - name: empty-lines + disabled: true + - name: increment-decrement + disabled: true + - name: line-length-limit + arguments: + # lower this after refactoring + - 111 + - name: struct-tag + disabled: true + + staticcheck: + checks: + - all + - -SA4006 + + wsl: + # Allow blocks to end with comments + allow-trailing-comment: true + + exclusions: + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + rules: + + - linters: + - govet + text: 'shadow: declaration of "(err|ctx)" shadows declaration' + + paths: + - third_party$ + - builtin$ + - examples$ + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 + +formatters: + enable: + - gci + - gofumpt + + settings: + gci: + sections: + - standard + - default + + exclusions: + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/waf-check/go.mod b/waf-check/go.mod index 8d35c530c23..6543d9433c1 100644 --- a/waf-check/go.mod +++ b/waf-check/go.mod @@ -1,22 +1,5 @@ module waf-check -go 1.21.3 +go 1.24 -require ( - github.com/cheggaaa/pb/v3 v3.1.4 - github.com/google/uuid v1.5.0 - github.com/schollz/progressbar/v3 v3.14.1 - gopkg.in/yaml.v2 v2.4.0 -) - -require ( - github.com/VividCortex/ewma v1.2.0 // indirect - github.com/fatih/color v1.15.0 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect - github.com/rivo/uniseg v0.4.4 // indirect - golang.org/x/sys v0.14.0 // indirect - golang.org/x/term v0.14.0 // indirect -) +require gopkg.in/yaml.v2 v2.4.0 diff --git a/waf-check/go.sum b/waf-check/go.sum index 898cf93dd42..dd0bc19f1fe 100644 --- a/waf-check/go.sum +++ b/waf-check/go.sum @@ -1,41 +1,4 @@ -github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= -github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= -github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI= -github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/waf-check/main.go b/waf-check/main.go index d211b31c0f4..e02b1687376 100644 --- a/waf-check/main.go +++ b/waf-check/main.go @@ -64,5 +64,4 @@ func main() { } fmt.Printf("everything went well!\n") - } diff --git a/waf-check/manager.go b/waf-check/manager.go index d4135036ce4..4b63a325a23 100644 --- a/waf-check/manager.go +++ b/waf-check/manager.go @@ -95,7 +95,6 @@ func (m *Manager) processFile(file string) { } m.resultsChan <- result - } func NewManager(config Config, filesList []string) Manager { diff --git a/waf-check/result.go b/waf-check/result.go index c6d2ef154cc..9680153ff31 100644 --- a/waf-check/result.go +++ b/waf-check/result.go @@ -33,7 +33,7 @@ func GetResult(resultsChan chan Result, outputFolder string) error { if err != nil { log.Fatalf("unable to marshal: %+v", result) } - if err := os.WriteFile(failedTestsPath, jsonData, 0644); err != nil { + if err := os.WriteFile(failedTestsPath, jsonData, 0o644); err != nil { log.Fatalf("unable to write failed report '%s': %s", failedTestsPath, err) } fmt.Printf(" - failed tests report: '%s'\n", failedTestsPath) diff --git a/waf-check/utils.go b/waf-check/utils.go index b30d26f9030..1c3e4ae078e 100644 --- a/waf-check/utils.go +++ b/waf-check/utils.go @@ -22,7 +22,6 @@ func listJSONFiles(folderPath string) ([]string, error) { } return nil }) - if err != nil { return filesList, fmt.Errorf("error walking '%s': %s", folderPath, err) } @@ -71,7 +70,7 @@ func unzip(src, dest string) error { } defer r.Close() - os.MkdirAll(dest, 0755) + os.MkdirAll(dest, 0o755) for _, f := range r.File { fPath := filepath.Join(dest, f.Name) From abf1a17af1b2b9554576cfaa9d0d50138ed7d26e Mon Sep 17 00:00:00 2001 From: marco Date: Thu, 10 Apr 2025 13:36:03 +0200 Subject: [PATCH 2/3] update actions --- .github/workflows/waf-check.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/waf-check.yaml b/.github/workflows/waf-check.yaml index 4e415a4937b..4b315c445d8 100644 --- a/.github/workflows/waf-check.yaml +++ b/.github/workflows/waf-check.yaml @@ -31,9 +31,12 @@ jobs: steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 - - uses: actions/setup-go@v4 with: - go-version: '1.22' + fetch-depth: 0 + + - uses: actions/setup-go@v5 + with: + go-version-file: waf-check/go.mod - name: Split dateset run: | cd waf-check && go build -o waf-check @@ -101,4 +104,4 @@ jobs: if: always() with: token: ${{ secrets.GITHUB_TOKEN }} - name: waf-check-dataset \ No newline at end of file + name: waf-check-dataset From ac7275734b1b80fbb4619fb9b70be094c698783f Mon Sep 17 00:00:00 2001 From: marco Date: Fri, 11 Apr 2025 16:47:53 +0200 Subject: [PATCH 3/3] lint --- waf-check/.golangci.yml | 18 +++--------------- waf-check/config.go | 12 ++++++------ waf-check/dataset.go | 4 +--- waf-check/main.go | 3 +++ waf-check/manager.go | 5 ++--- waf-check/request.go | 6 +++--- waf-check/result.go | 4 ++-- waf-check/utils.go | 4 ++-- 8 files changed, 22 insertions(+), 34 deletions(-) diff --git a/waf-check/.golangci.yml b/waf-check/.golangci.yml index 91d4f2aea0c..a0d0079b356 100644 --- a/waf-check/.golangci.yml +++ b/waf-check/.golangci.yml @@ -4,17 +4,13 @@ linters: default: all disable: - errcheck - - errorlint - exhaustruct - forbidigo - - ineffassign - - intrange - musttag - noctx - perfsprint - unparam - - usestdlibvars - - wastedassign + - prealloc - cyclop # revive - funlen # revive @@ -49,7 +45,6 @@ linters: gocritic: enable-all: true disabled-checks: - - httpNoBody - hugeParam - paramTypeCombine - exitAfterDefer @@ -79,28 +74,21 @@ linters: - name: cognitive-complexity arguments: # lower this after refactoring - - 17 + - 19 - name: cyclomatic arguments: # lower this after refactoring - - 12 + - 13 - name: deep-exit disabled: true - - name: empty-lines - disabled: true - - name: increment-decrement - disabled: true - name: line-length-limit arguments: # lower this after refactoring - 111 - - name: struct-tag - disabled: true staticcheck: checks: - all - - -SA4006 wsl: # Allow blocks to end with comments diff --git a/waf-check/config.go b/waf-check/config.go index a1a35b6c019..d5f39058864 100644 --- a/waf-check/config.go +++ b/waf-check/config.go @@ -16,10 +16,10 @@ type Config struct { NbRoutines int `yaml:"nb_routines"` OutputFolder string `yaml:"output_folder"` DownloadDataset bool `yaml:"download_dataset"` - statusCode int `yaml:"-"` - path string `yaml:"-"` - batch bool `yaml:"-"` - dirCount int `yaml:"-"` + statusCode int + path string + batch bool + dirCount int } func LoadConfig() (Config, error) { @@ -38,12 +38,12 @@ func LoadConfig() (Config, error) { data, err := os.ReadFile(*configFile) if err != nil { - return config, fmt.Errorf("error reading YAML file: %s", err) + return config, fmt.Errorf("error reading YAML file: %w", err) } err = yaml.Unmarshal(data, &config) if err != nil { - return config, fmt.Errorf("error unmarshaling YAML: %s", err) + return config, fmt.Errorf("error unmarshaling YAML: %w", err) } if *outputFolder != "" { diff --git a/waf-check/dataset.go b/waf-check/dataset.go index 8c61c88d828..c82cb323b38 100644 --- a/waf-check/dataset.go +++ b/waf-check/dataset.go @@ -5,9 +5,7 @@ import ( "os" ) -const ( - datasetURL = "https://downloads.openappsec.io/waf-comparison-project/legitimate.zip" -) +const datasetURL = "https://downloads.openappsec.io/waf-comparison-project/legitimate.zip" func DownloadDataset(datasetFolder string) ([]string, error) { fmt.Printf("Downloading dataset to '%s'\n", datasetFolder) diff --git a/waf-check/main.go b/waf-check/main.go index e02b1687376..2feee584799 100644 --- a/waf-check/main.go +++ b/waf-check/main.go @@ -24,6 +24,9 @@ func main() { } } else { filesList, err = listJSONFiles(config.DatasetFolder) + if err != nil { + log.Fatalf("fail listing files: %s", err) + } } if len(filesList) == 0 { diff --git a/waf-check/manager.go b/waf-check/manager.go index 4b63a325a23..e1f508e7414 100644 --- a/waf-check/manager.go +++ b/waf-check/manager.go @@ -90,8 +90,7 @@ func (m *Manager) processFile(file string) { result.FailedTests = append(result.FailedTests, failedTest) } resp.Body.Close() - result.DoneTests += 1 - + result.DoneTests++ } m.resultsChan <- result @@ -110,7 +109,7 @@ func NewManager(config Config, filesList []string) Manager { } func (m *Manager) Run() error { - for i := 0; i < m.NbWorker; i++ { + for range m.NbWorker { go func() { for file := range m.filesChan { m.processFile(file) diff --git a/waf-check/request.go b/waf-check/request.go index 23abd1ceac7..7598291de30 100644 --- a/waf-check/request.go +++ b/waf-check/request.go @@ -33,10 +33,10 @@ func NewHTTPRequest(baseURL string, request *Request) (http.Request, error) { return http.Request{}, fmt.Errorf("error joining URL: %w", err) } - if request.Method == "POST" || request.Method == "PUT" { + if request.Method == http.MethodPost || request.Method == http.MethodPut { req, err = http.NewRequest(request.Method, request.FullURL, bytes.NewBufferString(request.Data)) } else { - req, err = http.NewRequest(request.Method, request.FullURL, nil) + req, err = http.NewRequest(request.Method, request.FullURL, http.NoBody) } if err != nil { @@ -58,7 +58,7 @@ func (r *Request) Curl() string { for key, value := range r.Headers { curlCmd.WriteString(fmt.Sprintf(" -H '%s: %s'", key, value)) } - if r.Method == "POST" || r.Method == "PUT" { + if r.Method == http.MethodPost || r.Method == http.MethodPut { curlCmd.WriteString(fmt.Sprintf(" -d '%s'", r.Data)) } curlCmd.WriteString(fmt.Sprintf(" '%s'", r.FullURL)) diff --git a/waf-check/result.go b/waf-check/result.go index 9680153ff31..26ee203b131 100644 --- a/waf-check/result.go +++ b/waf-check/result.go @@ -14,11 +14,11 @@ func GetResult(resultsChan chan Result, outputFolder string) error { totalFileRun := 0 totalTestFail := 0 for result := range resultsChan { - totalFileRun += 1 + totalFileRun++ totalTestRun += result.DoneTests totalTestFail += len(result.FailedTests) - resultCpt += 1 + resultCpt++ percentage := (100.0 * float32(len(result.FailedTests))) / float32(result.DoneTests) fmt.Printf("'%s' result:\n", result.Filename) diff --git a/waf-check/utils.go b/waf-check/utils.go index 1c3e4ae078e..c71b762579d 100644 --- a/waf-check/utils.go +++ b/waf-check/utils.go @@ -23,7 +23,7 @@ func listJSONFiles(folderPath string) ([]string, error) { return nil }) if err != nil { - return filesList, fmt.Errorf("error walking '%s': %s", folderPath, err) + return filesList, fmt.Errorf("error walking '%s': %w", folderPath, err) } return filesList, nil @@ -110,7 +110,7 @@ func splitIntoDirectories(files []string, dirCount int) [][]string { extraFiles := totalFiles % dirCount start := 0 - for i := 0; i < dirCount; i++ { + for i := range dirCount { end := start + minFilesPerDir if i < extraFiles { end++