feat: change 2ms version to v5 and other small fixes

.2ms.yml

@@ -3018,4 +3018,14 @@ ignore-result:
 - e6cb4b56f380f519250ef0bd150327710d49f220 # secret found in ruleids.go
 - edb22f489a7846d8a39074541c92d27e3a88c123 # unit test from engine_test.go
 - fe01911702da491dde97b22544ea2fd7027698a4 # unit test from engine_test.go
-- 5ad4e2c9342c26b419d89ce606af840b222ddc29 # FP
+- 5ad4e2c9342c26b419d89ce606af840b222ddc29 # FP
+- 30fe0105a11a47e0827f1e24abcb68ba5d226000 # unit test from engine_test.go
+- 5cef27dfd6ea6fab0b6aa0c3d6917c5bcdb54d8a # unit test from engine_test.go
+- 62be1ddd8021f334f3a61daf8bfcc8324c0d3c62 # unit test from engine_test.go
+- e8754bbeeb7c9b532bff5df09b18a44f4886e3e3 # unit test from engine_test.go
+- 9b801040827c63c43e1412150afd1cc8da46b6e5 # unit test from engine_test.go
+- 375f259080819eeaf745f3dc52fc27a6fe1321ff # unit test from engine_test.go
+- 5c764e8a91fe4cc22c0080c32dd3ccba4ad63e20 # unit test from engine_test.go
+- 71c523a79d530ca7b0aa5038c1d1425f26c9bde1 # unit test from engine_test.go
+- a439ca168c536084477f2ae6379b4b480e3c696d # unit test from engine_test.go
+- b3ba72354ad0d897ef3205ae6e3c51236da12489 # unit test from engine_test.go

README.md

@@ -354,14 +354,14 @@ Other fields are optional and can be seen in the example bellow of a file with a
 ```yaml
 - ruleId: 01ab7659-d25a-4a1c-9f98-dee9d0cf2e70 # REQUIRED: unique id, must match default rule id to override that default rule. Rule ids can be used as values in --rule and --ignore-rule flags
   ruleName: Custom-Api-Key # should be human-readable name. If left empty for new rule, ruleName will take the value of ruleId. If left empty for override, default rule name will be considered. Rule names can be used as values in --rule and --ignore-rule flags 
-  description: Custom rule 
-  regex: (?i)\b\w*secret\w*\b\s*:?=\s*["']?([A-Za-z0-9/_+=-]{8,150})["']? # REQUIRED: golang regular expression used to find secrets. If capture group is present in regex, it used to find the secret, otherwise whole regex is used. which group is considered the secret can be defined with secretGroup
+  description: Custom rule
+  regex: (?i)\b\w*secret\w*\b\s*:?=\s*["']?([A-Za-z0-9/_+=-]{8,150})["']? # REQUIRED: golang regular expression used to find secrets. For regexes, if enclosed in "", make sure to escape backslashes (\\, \\b, etc.). If capture group is present in regex, it's used to find the secret, otherwise whole regex is used. Which group is considered the secret can be defined with secretGroup
   keywords: # Keywords are used for pre-regex check filtering. Rules that contain keywords will perform a quick string compare check to make sure the keyword(s) are in the content being scanned.
     - access
     - api
-  entropy: 3.5 # shannon entropy, measures how random a string is. The value will be higher the more random a string is. Default rules that use entropy have values between 2.0 and 4.5. Leave empty to consider matches regardless of entropy
+  entropy: 3.5 # minimum shannon entropy, which measures how random a string is. The more unique characters a string has, the higher the entropy. The value of entropy will tend to become log2(unique chars), so long as all unique are equally present in the string ('abcd' string has entropy of log2(4)=2, but so does 'aabbccdd'). To test entropy values, use https://textcompare.io/shannon-entropy-calculator. Default rules that use entropy have values between 2.0 and 4.5, though these minimums can sometimes be 1-2 lower than the entropy of a true positive. Leave entropy empty to consider matches regardless of entropy
   secretGroup: 1 # defines which capture group of regex match is considered the secret. Is also used as the group that will have its entropy checked if `entropy` is set. Can be left empty, in which case the first capture group to match will be considered the secret
-  path: (?i)\.(?:tf|hcl)$ # regex to limit the rule to specific file paths. For example, only .tf and .hcl files
+  path: "(?i)\\.(?:tf|hcl)$" # regex to limit the rule to specific file paths, for example, only .tf and .hcl files. For regexes, if enclosed in "", make sure to escape backslashes (\\, \\b, etc.)
   severity: High # severity, can only be one of [Critical, High, Medium, Low, Info]
   tags: # identifiers for the rule, tags can be used as values of --rule and --ignore-rule flags
     - api-key
@@ -371,8 +371,8 @@ Other fields are optional and can be seen in the example bellow of a file with a
   deprecated: false # if true, the rule will not be used in the scan, regardless of --rule flag
   allowLists: # allowed values to ignore if matched
     - description: Allowlist for Custom Rule
-      matchCondition: OR # determines whether all criteria in the allowList must match. Can be AND or OR. Defaults to OR if not specified
-      regexTarget: match - # determines whether the regexes in allowList are tested against the rule.Regex match or the full line being scanned. Can be 'match' or 'line'. Defaults to 'match' if not specified
+      matchCondition: OR # Can be AND or OR. determines whether all criteria in the allowList must match. Defaults to OR if not specified
+      regexTarget: match - # Can be match or line. Determines whether the regexes in allowList are tested against the rule.Regex match or the full line being scanned. Defaults to "match" if not specified
       regexes: # allowed regex patterns
         - (?i)(?:access(?:ibility|or)|access[_.-]?id|random[_.-]?access|api[_.-]?(?:id|name|version)|rapid|capital|[a-z0-9-]*?api[a-z0-9-]*?:jar:|author|X-MS-Exchange-Organization-Auth|Authentication-Results|(?:credentials?[_.-]?id|withCredentials)|(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}|(?:bucket|foreign|hot|idx|natural|primary|pub(?:lic)?|schema|sequence)[_.-]?key|(?:turkey)|key[_.-]?(?:alias|board|code|frame|id|length|mesh|name|pair|press(?:ed)?|ring|selector|signature|size|stone|storetype|word|up|down|left|right)|KeyVault(?:[A-Za-z]*?(?:Administrator|Reader|Contributor|Owner|Operator|User|Officer))\s*[:=]\s*['"]?[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}['"]?|key[_.-]?vault[_.-]?(?:id|name)|keyVaultToStoreSecrets|key(?:store|tab)[_.-]?(?:file|path)|issuerkeyhash|(?-i:[DdMm]onkey|[DM]ONKEY)|keying|(?:secret)[_.-]?(?:length|name|size)|UserSecretsId|(?:csrf)[_.-]?token|(?:io\.jsonwebtoken[
           \t]?:[

benches/process_items_test.go

@@ -12,11 +12,11 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/checkmarx/2ms/v4/engine"
-	"github.com/checkmarx/2ms/v4/internal/workerpool"
-	"github.com/checkmarx/2ms/v4/lib/reporting"
-	"github.com/checkmarx/2ms/v4/lib/secrets"
-	"github.com/checkmarx/2ms/v4/plugins"
+	"github.com/checkmarx/2ms/v5/engine"
+	"github.com/checkmarx/2ms/v5/internal/workerpool"
+	"github.com/checkmarx/2ms/v5/lib/reporting"
+	"github.com/checkmarx/2ms/v5/lib/secrets"
+	"github.com/checkmarx/2ms/v5/plugins"
 	"github.com/rs/zerolog"
 )
 

cmd/config.go

@@ -8,8 +8,8 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/checkmarx/2ms/v4/engine/rules/ruledefine"
-	"github.com/checkmarx/2ms/v4/lib/utils"
+	"github.com/checkmarx/2ms/v5/engine/rules/ruledefine"
+	"github.com/checkmarx/2ms/v5/lib/utils"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"

cmd/config_test.go

@@ -6,7 +6,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/checkmarx/2ms/v4/engine/rules/ruledefine"
+	"github.com/checkmarx/2ms/v5/engine/rules/ruledefine"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"

cmd/main.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/checkmarx/2ms/v4/engine"
-	"github.com/checkmarx/2ms/v4/lib/config"
-	"github.com/checkmarx/2ms/v4/plugins"
+	"github.com/checkmarx/2ms/v5/engine"
+	"github.com/checkmarx/2ms/v5/lib/config"
+	"github.com/checkmarx/2ms/v5/plugins"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"

cmd/main_test.go

@@ -6,9 +6,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/checkmarx/2ms/v4/engine"
-	"github.com/checkmarx/2ms/v4/engine/rules/ruledefine"
-	"github.com/checkmarx/2ms/v4/internal/resources"
+	"github.com/checkmarx/2ms/v5/engine"
+	"github.com/checkmarx/2ms/v5/engine/rules/ruledefine"
+	"github.com/checkmarx/2ms/v5/internal/resources"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 )

docs/list-of-rules.md

@@ -115,7 +115,7 @@ Here is a complete list of all the rules that are currently implemented.
 | Intra42-Client-Secret | Found a Intra42 client secret, which could lead to unauthorized access to the 42School API and sensitive data. | client-secret |  |
 | Jfrog-Api-Key | Found a JFrog API Key, posing a risk of unauthorized access to software artifact repositories and build pipelines. | api-key |  |
 | Jfrog-Identity-Token | Discovered a JFrog Identity Token, potentially compromising access to JFrog services and sensitive software artifacts. | access-token |  |
-| jwt | Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data. | access-token |  |
+| Jwt | Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data. | access-token |  |
 | Jwt-Base64 | Detected a Base64-encoded JSON Web Token, posing a risk of exposing encoded authentication and data exchange information. | access-token |  |
 | Kraken-Access-Token | Identified a Kraken Access Token, potentially compromising cryptocurrency trading accounts and financial security. | access-token |  |
 | Kubernetes-Secret-Yaml | Possible Kubernetes Secret detected, posing a risk of leaking credentials/tokens from your deployments | secret-key |  |

engine/detect/utils.go

@@ -6,7 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/checkmarx/2ms/v4/engine/constants"
+	"github.com/checkmarx/2ms/v5/engine/constants"
 	"github.com/zricethezav/gitleaks/v8/cmd/scm"
 	"github.com/zricethezav/gitleaks/v8/logging"
 	"github.com/zricethezav/gitleaks/v8/report"
@@ -115,6 +115,7 @@ func createScmLink(remote *sources.RemoteInfo, finding *report.Finding) string {
 // Another way to think about what this is doing is calculating the number of bits
 // needed to on average encode the data. So, the higher the entropy, the more random the data, the
 // more bits needed to encode that data.
+// For a quick calculator, see https://textcompare.io/shannon-entropy-calculator
 func shannonEntropy(data string) (entropy float64) {
 	if data == "" {
 		return 0

engine/engine.go

@@ -18,20 +18,20 @@ import (
 	"sync/atomic"
 	"text/tabwriter"
 
-	"github.com/checkmarx/2ms/v4/engine/chunk"
-	"github.com/checkmarx/2ms/v4/engine/detect"
-	"github.com/checkmarx/2ms/v4/engine/extra"
-	"github.com/checkmarx/2ms/v4/engine/linecontent"
-	"github.com/checkmarx/2ms/v4/engine/rules"
-	"github.com/checkmarx/2ms/v4/engine/rules/ruledefine"
-	"github.com/checkmarx/2ms/v4/engine/score"
-	"github.com/checkmarx/2ms/v4/engine/semaphore"
-	"github.com/checkmarx/2ms/v4/engine/validation"
-	"github.com/checkmarx/2ms/v4/internal/resources"
-	"github.com/checkmarx/2ms/v4/internal/workerpool"
-	"github.com/checkmarx/2ms/v4/lib/reporting"
-	"github.com/checkmarx/2ms/v4/lib/secrets"
-	"github.com/checkmarx/2ms/v4/plugins"
+	"github.com/checkmarx/2ms/v5/engine/chunk"
+	"github.com/checkmarx/2ms/v5/engine/detect"
+	"github.com/checkmarx/2ms/v5/engine/extra"
+	"github.com/checkmarx/2ms/v5/engine/linecontent"
+	"github.com/checkmarx/2ms/v5/engine/rules"
+	"github.com/checkmarx/2ms/v5/engine/rules/ruledefine"
+	"github.com/checkmarx/2ms/v5/engine/score"
+	"github.com/checkmarx/2ms/v5/engine/semaphore"
+	"github.com/checkmarx/2ms/v5/engine/validation"
+	"github.com/checkmarx/2ms/v5/internal/resources"
+	"github.com/checkmarx/2ms/v5/internal/workerpool"
+	"github.com/checkmarx/2ms/v5/lib/reporting"
+	"github.com/checkmarx/2ms/v5/lib/secrets"
+	"github.com/checkmarx/2ms/v5/plugins"
 	"github.com/rs/zerolog/log"
 	"github.com/sourcegraph/conc"
 	"github.com/spf13/cobra"

engine/engine_test.go

@@ -1,6 +1,6 @@
 package engine
 
-//go:generate mockgen -destination=plugins_mock_test.go -package=${GOPACKAGE} github.com/checkmarx/2ms/v4/plugins ISourceItem
+//go:generate mockgen -destination=plugins_mock_test.go -package=${GOPACKAGE} github.com/checkmarx/2ms/v5/plugins ISourceItem
 
 import (
 	"bytes"
@@ -15,23 +15,23 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/checkmarx/2ms/v4/engine/rules/ruledefine"
+	"github.com/checkmarx/2ms/v5/engine/rules/ruledefine"
 	"go.uber.org/mock/gomock"
 
-	"github.com/checkmarx/2ms/v4/engine/chunk"
-	"github.com/checkmarx/2ms/v4/engine/rules"
-	"github.com/checkmarx/2ms/v4/engine/semaphore"
-	"github.com/checkmarx/2ms/v4/internal/resources"
-	"github.com/checkmarx/2ms/v4/lib/secrets"
-	"github.com/checkmarx/2ms/v4/plugins"
+	"github.com/checkmarx/2ms/v5/engine/chunk"
+	"github.com/checkmarx/2ms/v5/engine/rules"
+	"github.com/checkmarx/2ms/v5/engine/semaphore"
+	"github.com/checkmarx/2ms/v5/internal/resources"
+	"github.com/checkmarx/2ms/v5/lib/secrets"
+	"github.com/checkmarx/2ms/v5/plugins"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/zricethezav/gitleaks/v8/config"
 	"github.com/zricethezav/gitleaks/v8/report"
 
-	"github.com/checkmarx/2ms/v4/engine/detect"
+	"github.com/checkmarx/2ms/v5/engine/detect"
 )
 
 // Removed global fsPlugin to avoid test interference
@@ -1382,29 +1382,29 @@ func TestMaxSecretSizeFlag(t *testing.T) {
 	secret := "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"
 
 	testCases := []struct {
-		name        string
-		limit       uint64
-		shouldFind  bool
+		name       string
+		limit      uint64
+		shouldFind bool
 	}{
 		{
-			name:        "no limit - finds secret",
-			limit:       0,
-			shouldFind:  true,
+			name:       "no limit - finds secret",
+			limit:      0,
+			shouldFind: true,
 		},
 		{
-			name:        "limit larger than secret - finds secret",
-			limit:       200,
-			shouldFind:  true,
+			name:       "limit larger than secret - finds secret",
+			limit:      200,
+			shouldFind: true,
 		},
 		{
-			name:        "limit smaller than secret - ignores secret",
-			limit:       10,
-			shouldFind:  false,
+			name:       "limit smaller than secret - ignores secret",
+			limit:      10,
+			shouldFind: false,
 		},
 		{
-			name:        "limit exactly at secret size boundary - finds secret",
-			limit:       40,
-			shouldFind:  true,
+			name:       "limit exactly at secret size boundary - finds secret",
+			limit:      40,
+			shouldFind: true,
 		},
 	}
 
@@ -1527,25 +1527,25 @@ fifth_token: ghp_aB3cD4eF5gH6iJ7kL8mN9oP0qR1sT2uV3wX4
 `
 
 	testCases := []struct {
-		name               string
-		limit              uint64
-		fragments          []string
-		expectedCount      int
-		shouldLogWarning   bool
+		name             string
+		limit            uint64
+		fragments        []string
+		expectedCount    int
+		shouldLogWarning bool
 	}{
 		{
-			name:               "no limit - no warning",
-			limit:              0,
-			fragments:          []string{multipleSecrets},
-			expectedCount:      5,
-			shouldLogWarning:   false,
+			name:             "no limit - no warning",
+			limit:            0,
+			fragments:        []string{multipleSecrets},
+			expectedCount:    5,
+			shouldLogWarning: false,
 		},
 		{
-			name:               "limit of 3 - warning logged when limit reached",
-			limit:              3,
-			fragments:          []string{multipleSecrets},
-			expectedCount:      3,
-			shouldLogWarning:   true,
+			name:             "limit of 3 - warning logged when limit reached",
+			limit:            3,
+			fragments:        []string{multipleSecrets},
+			expectedCount:    3,
+			shouldLogWarning: true,
 		},
 		{
 			name:  "limit of 2 across multiple fragments - warning logged",
@@ -1555,15 +1555,15 @@ fifth_token: ghp_aB3cD4eF5gH6iJ7kL8mN9oP0qR1sT2uV3wX4
 				"ghp_1234567890abcdefghijklmnopqrstuvwxyz",
 				"ghp_abcdefghijklmnopqrstuvwxyz1234567890",
 			},
-			expectedCount:      2,
-			shouldLogWarning:   true,
+			expectedCount:    2,
+			shouldLogWarning: true,
 		},
 		{
-			name:               "limit of 1 - warning logged immediately",
-			limit:              1,
-			fragments:          []string{multipleSecrets},
-			expectedCount:      1,
-			shouldLogWarning:   true,
+			name:             "limit of 1 - warning logged immediately",
+			limit:            1,
+			fragments:        []string{multipleSecrets},
+			expectedCount:    1,
+			shouldLogWarning: true,
 		},
 		{
 			name:             "limit higher than findings - no warning",

engine/extra/extra.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/checkmarx/2ms/v4/engine/rules/ruledefine"
-	"github.com/checkmarx/2ms/v4/lib/secrets"
+	"github.com/checkmarx/2ms/v5/engine/rules/ruledefine"
+	"github.com/checkmarx/2ms/v5/lib/secrets"
 )
 
 type addExtraFunc = func(*secrets.Secret) interface{}

engine/extra/extra_test.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/checkmarx/2ms/v4/engine/rules/ruledefine"
-	"github.com/checkmarx/2ms/v4/lib/secrets"
+	"github.com/checkmarx/2ms/v5/engine/rules/ruledefine"
+	"github.com/checkmarx/2ms/v5/lib/secrets"
 	"github.com/stretchr/testify/assert"
 )
 

engine/rules/ruledefine/freemius_test.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/checkmarx/2ms/v4/engine/detect"
+	"github.com/checkmarx/2ms/v5/engine/detect"
 	"github.com/stretchr/testify/assert"
 )