From af96ececc64986342166316033f886788056237d Mon Sep 17 00:00:00 2001 From: SATYAsasini Date: Thu, 23 Jul 2026 14:32:55 +0530 Subject: [PATCH 1/5] feat: add mask sensitive data in rawJson util function --- common-lib/utils/PayloadMaskUtil.go | 97 +++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 common-lib/utils/PayloadMaskUtil.go diff --git a/common-lib/utils/PayloadMaskUtil.go b/common-lib/utils/PayloadMaskUtil.go new file mode 100644 index 000000000..3160983e7 --- /dev/null +++ b/common-lib/utils/PayloadMaskUtil.go @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "encoding/json" + "strings" +) + +// SecretDataMaskPlaceholder is written in place of any sensitive value that is masked out of a payload. +const SecretDataMaskPlaceholder = "********" + +// unparseablePayloadPlaceholder is logged when a payload cannot be parsed as JSON. We fail closed here +// (redact the whole body) instead of returning the raw bytes, so an unexpected content type can never +// leak a secret into logs. +const unparseablePayloadPlaceholder = "[REDACTED: unparseable payload]" + +// sensitivePayloadKeys are the (lower-cased) JSON field-name fragments whose values must never be logged. +// Matching is case-insensitive and substring based, so "password" also covers "confirmPassword", +// "sshPassword", etc., and "token" covers "accessToken", "dockerToken", and so on. +var sensitivePayloadKeys = []string{ + "password", + "passphrase", + "secret", + "token", + "credential", + "privatekey", + "apikey", + "accesskey", + "authkey", +} + +// isSensitivePayloadKey reports whether a JSON key names a sensitive value that must be masked. +func isSensitivePayloadKey(key string) bool { + lowerKey := strings.ToLower(key) + for _, sensitiveKey := range sensitivePayloadKeys { + if strings.Contains(lowerKey, sensitiveKey) { + return true + } + } + return false +} + +// MaskSensitiveDataInJsonPayload returns a JSON string equivalent to payload with every sensitive value +// (see sensitivePayloadKeys) replaced by SecretDataMaskPlaceholder. It is meant for the generic request +// logging / audit path, where only the raw body bytes are available and the concrete DTO type is unknown. +// +// It fails closed: an empty payload returns "", and a payload that cannot be parsed as JSON returns +// unparseablePayloadPlaceholder rather than the raw bytes, so a secret can never leak on the error path. +func MaskSensitiveDataInJsonPayload(payload []byte) string { + if len(payload) == 0 { + return "" + } + var parsed interface{} + if err := json.Unmarshal(payload, &parsed); err != nil { + return unparseablePayloadPlaceholder + } + maskSensitiveValues(parsed) + masked, err := json.Marshal(parsed) + if err != nil { + return unparseablePayloadPlaceholder + } + return string(masked) +} + +// maskSensitiveValues walks a decoded JSON value in place, masking any value whose key is sensitive and +// recursing into nested objects and arrays. +func maskSensitiveValues(value interface{}) { + switch typed := value.(type) { + case map[string]interface{}: + for key, val := range typed { + if isSensitivePayloadKey(key) { + typed[key] = SecretDataMaskPlaceholder + } else { + maskSensitiveValues(val) + } + } + case []interface{}: + for _, item := range typed { + maskSensitiveValues(item) + } + } +} From 8dec43093fc23a92d607b60a1585305360271ae4 Mon Sep 17 00:00:00 2001 From: SATYAsasini Date: Thu, 23 Jul 2026 17:14:43 +0530 Subject: [PATCH 2/5] feat: migrate secretScanner to oss common-lib --- common-lib/utils/secretScanner/rules.go | 670 ++++++++++++++++++++++ common-lib/utils/secretScanner/scanner.go | 121 ++++ 2 files changed, 791 insertions(+) create mode 100644 common-lib/utils/secretScanner/rules.go create mode 100644 common-lib/utils/secretScanner/scanner.go diff --git a/common-lib/utils/secretScanner/rules.go b/common-lib/utils/secretScanner/rules.go new file mode 100644 index 000000000..99985f0ba --- /dev/null +++ b/common-lib/utils/secretScanner/rules.go @@ -0,0 +1,670 @@ +package secretScanner + +import ( + "fmt" + "regexp" +) + +// Reusable regex patterns +const ( + quote = `["']?` + connect = `\s*(:|=>|=)?\s*` + startSecret = `(^|\s+)` + endSecret = `[.,]?(\s+|$)` + + aws = `aws_?` +) + +// create rule struct +type Rule struct { + ID string + Severity string + Title string + Regex *regexp.Regexp + SecretGroupName string + Keywords []string +} + +var BuiltinRules = []Rule{ + { + ID: "aws-access-key-id", + Severity: "CRITICAL", + Title: "AWS Access Key ID", + Regex: regexp.MustCompile(fmt.Sprintf(`%s(?P(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})%s%s`, quote, quote, endSecret)), + SecretGroupName: "secret", + Keywords: []string{"AKIA", "AGPA", "AIDA", "AROA", "AIPA", "ANPA", "ANVA", "ASIA"}, + }, + { + ID: "aws-secret-access-key", + Severity: "CRITICAL", + Title: "AWS Secret Access Key", + Regex: regexp.MustCompile(fmt.Sprintf(`(?i)%s%s%s(sec(ret)?)?_?(access)?_?key%s%s%s(?P[A-Za-z0-9\/\+=]{40})%s%s`, startSecret, quote, aws, quote, connect, quote, quote, endSecret)), + SecretGroupName: "secret", + Keywords: []string{"key"}, + }, + { + ID: "github-pat", + Title: "GitHub Personal Access Token", + Severity: "CRITICAL", + Regex: regexp.MustCompile(`ghp_[0-9a-zA-Z]{36}`), + Keywords: []string{"ghp_"}, + }, + { + ID: "github-oauth", + Title: "GitHub OAuth Access Token", + Severity: "CRITICAL", + Regex: regexp.MustCompile(`gho_[0-9a-zA-Z]{36}`), + Keywords: []string{"gho_"}, + }, + { + ID: "github-app-token", + Title: "GitHub App Token", + Severity: "CRITICAL", + Regex: regexp.MustCompile(`(ghu|ghs)_[0-9a-zA-Z]{36}`), + Keywords: []string{"ghu_", "ghs_"}, + }, + { + ID: "github-refresh-token", + Title: "GitHub Refresh Token", + Severity: "CRITICAL", + Regex: regexp.MustCompile(`ghr_[0-9a-zA-Z]{76}`), + Keywords: []string{"ghr_"}, + }, + { + ID: "github-fine-grained-pat", + Title: "GitHub Fine-grained personal access tokens", + Severity: "CRITICAL", + Regex: regexp.MustCompile(`github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}`), + Keywords: []string{"github_pat_"}, + }, + { + ID: "gitlab-pat", + Title: "GitLab Personal Access Token", + Severity: "CRITICAL", + Regex: regexp.MustCompile(`glpat-[0-9a-zA-Z\-\_]{20}`), + Keywords: []string{"glpat-"}, + }, + { + // cf. https://huggingface.co/docs/hub/en/security-tokens + ID: "hugging-face-access-token", + Severity: "CRITICAL", + Title: "Hugging Face Access Token", + Regex: regexp.MustCompile(`hf_[A-Za-z0-9]{39}`), + Keywords: []string{"hf_"}, + }, + { + ID: "private-key", + Title: "Asymmetric Private Key", + Severity: "HIGH", + Regex: regexp.MustCompile(`(?i)-----\s*?BEGIN[ A-Z0-9_-]*?PRIVATE KEY( BLOCK)?\s*?-----[\s]*?(?P[\sA-Za-z0-9=+/\\\r\n]+)[\s]*?-----\s*?END[ A-Z0-9_-]*? PRIVATE KEY( BLOCK)?\s*?-----`), + SecretGroupName: "secret", + Keywords: []string{"-----"}, + }, + { + ID: "shopify-token", + Title: "Shopify token", + Severity: "HIGH", + Regex: regexp.MustCompile(`shp(ss|at|ca|pa)_[a-fA-F0-9]{32}`), + Keywords: []string{"shpss_", "shpat_", "shpca_", "shppa_"}, + }, + { + ID: "slack-access-token", + Title: "Slack token", + Severity: "HIGH", + Regex: regexp.MustCompile(`xox[baprs]-([0-9a-zA-Z]{10,48})`), + Keywords: []string{"xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"}, + }, + { + ID: "stripe-publishable-token", + Title: "Stripe Publishable Key", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)pk_(test|live)_[0-9a-z]{10,32}`), + Keywords: []string{"pk_test_", "pk_live_"}, + }, + { + ID: "stripe-secret-token", + Title: "Stripe Secret Key", + Severity: "CRITICAL", + Regex: regexp.MustCompile(`(?i)sk_(test|live)_[0-9a-z]{10,32}`), + Keywords: []string{"sk_test_", "sk_live_"}, + }, + { + ID: "pypi-upload-token", + Title: "PyPI upload token", + Severity: "HIGH", + Regex: regexp.MustCompile(`pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}`), + Keywords: []string{"pypi-AgEIcHlwaS5vcmc"}, + }, + { + ID: "gcp-service-account", + Title: "Google (GCP) Service-account", + Severity: "CRITICAL", + Regex: regexp.MustCompile(`\"type\": \"service_account\"`), + Keywords: []string{"\"type\": \"service_account\""}, + }, + { + ID: "heroku-api-key", + Title: "Heroku API Key", + Severity: "HIGH", + Regex: regexp.MustCompile(` (?i)(?Pheroku[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"heroku"}, + }, + { + ID: "slack-web-hook", + Title: "Slack Webhook", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9+\/]{44,48}`), + Keywords: []string{"hooks.slack.com"}, + }, + { + ID: "twilio-api-key", + Title: "Twilio API Key", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`SK[0-9a-fA-F]{32}`), + Keywords: []string{"SK"}, + }, + { + ID: "age-secret-key", + Title: "Age secret key", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}`), + Keywords: []string{"AGE-SECRET-KEY-1"}, + }, + { + ID: "facebook-token", + Title: "Facebook token", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Pfacebook[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-f0-9]{32})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"facebook"}, + }, + { + ID: "twitter-token", + Title: "Twitter token", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Ptwitter[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-f0-9]{35,44})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"twitter"}, + }, + { + ID: "adobe-client-id", + Title: "Adobe Client ID (Oauth Web)", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Padobe[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-f0-9]{32})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"adobe"}, + }, + { + ID: "adobe-client-secret", + Title: "Adobe Client Secret", + Severity: "LOW", + Regex: regexp.MustCompile(`(p8e-)(?i)[a-z0-9]{32}`), + Keywords: []string{"p8e-"}, + }, + { + ID: "alibaba-access-key-id", + Title: "Alibaba AccessKey ID", + Severity: "HIGH", + Regex: regexp.MustCompile(`([^0-9A-Za-z]|^)(?P(LTAI)(?i)[a-z0-9]{20})([^0-9A-Za-z]|$)`), + SecretGroupName: "secret", + Keywords: []string{"LTAI"}, + }, + { + ID: "alibaba-secret-key", + Title: "Alibaba Secret Key", + Severity: "HIGH", + Regex: regexp.MustCompile(`(?i)(?Palibaba[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9]{30})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"alibaba"}, + }, + { + ID: "asana-client-id", + Title: "Asana Client ID", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pasana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[0-9]{16})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"asana"}, + }, + { + ID: "asana-client-secret", + Title: "Asana Client Secret", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pasana[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9]{32})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"asana"}, + }, + { + ID: "atlassian-api-token", + Title: "Atlassian API token", + Severity: "HIGH", + Regex: regexp.MustCompile(`(?i)(?Patlassian[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9]{24})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"atlassian"}, + }, + { + ID: "bitbucket-client-id", + Title: "Bitbucket client ID", + Severity: "HIGH", + Regex: regexp.MustCompile(`(?i)(?Pbitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9]{32})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"bitbucket"}, + }, + { + ID: "bitbucket-client-secret", + Title: "Bitbucket client secret", + Severity: "HIGH", + Regex: regexp.MustCompile(`(?i)(?Pbitbucket[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9_\-]{64})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"bitbucket"}, + }, + { + ID: "beamer-api-token", + Title: "Beamer API token", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Pbeamer[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?Pb_[a-z0-9=_\-]{44})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"beamer"}, + }, + { + ID: "clojars-api-token", + Title: "Clojars API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(CLOJARS_)(?i)[a-z0-9]{60}`), + Keywords: []string{"CLOJARS_"}, + }, + { + ID: "contentful-delivery-api-token", + Title: "Contentful delivery API token", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Pcontentful[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9\-=_]{43})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"contentful"}, + }, + { + ID: "databricks-api-token", + Title: "Databricks API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`dapi[a-h0-9]{32}`), + Keywords: []string{"dapi"}, + }, + { + ID: "discord-api-token", + Title: "Discord API key", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pdiscord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-h0-9]{64})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"discord"}, + }, + { + ID: "discord-client-id", + Title: "Discord client ID", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pdiscord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[0-9]{18})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"discord"}, + }, + { + ID: "discord-client-secret", + Title: "Discord client secret", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pdiscord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9=_\-]{32})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"discord"}, + }, + { + ID: "doppler-api-token", + Title: "Doppler API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`['\"](dp\.pt\.)(?i)[a-z0-9]{43}['\"]`), + Keywords: []string{"dp.pt."}, + }, + { + ID: "dropbox-api-secret", + Title: "Dropbox API secret/key", + Severity: "HIGH", + Regex: regexp.MustCompile(`(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]`), + Keywords: []string{"dropbox"}, + }, + { + ID: "dropbox-short-lived-api-token", + Title: "Dropbox short lived API token", + Severity: "HIGH", + Regex: regexp.MustCompile(`(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]`), + Keywords: []string{"dropbox"}, + }, + { + ID: "dropbox-long-lived-api-token", + Title: "Dropbox long lived API token", + Severity: "HIGH", + Regex: regexp.MustCompile(`(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]`), + Keywords: []string{"dropbox"}, + }, + { + ID: "duffel-api-token", + Title: "Duffel API token", + Severity: "LOW", + Regex: regexp.MustCompile(`['\"]duffel_(test|live)_(?i)[a-z0-9_-]{43}['\"]`), + Keywords: []string{"duffel_test_", "duffel_live_"}, + }, + { + ID: "dynatrace-api-token", + Title: "Dynatrace API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`['\"]dt0c01\.(?i)[a-z0-9]{24}\.[a-z0-9]{64}['\"]`), + Keywords: []string{"dt0c01."}, + }, + { + ID: "easypost-api-token", + Title: "EasyPost API token", + Severity: "LOW", + Regex: regexp.MustCompile(`['\"]EZ[AT]K(?i)[a-z0-9]{54}['\"]`), + Keywords: []string{"EZAK", "EZAT"}, + }, + { + ID: "fastly-api-token", + Title: "Fastly API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pfastly[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9\-=_]{32})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"fastly"}, + }, + { + ID: "finicity-client-secret", + Title: "Finicity client secret", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pfinicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9]{20})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"finicity"}, + }, + { + ID: "finicity-api-token", + Title: "Finicity API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pfinicity[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-f0-9]{32})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"finicity"}, + }, + { + ID: "flutterwave-public-key", + Title: "Flutterwave public/secret key", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`FLW(PUB|SEC)K_TEST-(?i)[a-h0-9]{32}-X`), + Keywords: []string{"FLWSECK_TEST-", "FLWPUBK_TEST-"}, + }, + { + ID: "flutterwave-enc-key", + Title: "Flutterwave encrypted key", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`FLWSECK_TEST[a-h0-9]{12}`), + Keywords: []string{"FLWSECK_TEST"}, + }, + { + ID: "frameio-api-token", + Title: "Frame.io API token", + Severity: "LOW", + Regex: regexp.MustCompile(`fio-u-(?i)[a-z0-9\-_=]{64}`), + Keywords: []string{"fio-u-"}, + }, + { + ID: "gocardless-api-token", + Title: "GoCardless API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`['\"]live_(?i)[a-z0-9\-_=]{40}['\"]`), + Keywords: []string{"live_"}, + }, + { + ID: "grafana-api-token", + Title: "Grafana API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`['\"]eyJrIjoi(?i)[a-z0-9\-_=]{72,92}['\"]`), + Keywords: []string{"eyJrIjoi"}, + }, + { + ID: "hashicorp-tf-api-token", + Title: "HashiCorp Terraform user/org API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9\-_=]{60,70}['\"]`), + Keywords: []string{"atlasv1."}, + }, + { + ID: "hubspot-api-token", + Title: "HubSpot API token", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Phubspot[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"hubspot"}, + }, + { + ID: "intercom-api-token", + Title: "Intercom API token", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Pintercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9=_]{60})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"intercom"}, + }, + { + ID: "intercom-client-secret", + Title: "Intercom client secret/ID", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Pintercom[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"intercom"}, + }, + { + ID: "ionic-api-token", + Title: "Ionic API token", + Regex: regexp.MustCompile(`(?i)(ionic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](ion_[a-z0-9]{42})['\"]`), + Keywords: []string{"ionic"}, + }, + { + ID: "jwt-token", + Title: "JWT token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`ey[a-zA-Z0-9]{17,}\.ey[a-zA-Z0-9\/\\_-]{17,}\.(?:[a-zA-Z0-9\/\\_-]{10,}={0,2})?`), + Keywords: []string{"jwt"}, + }, + { + ID: "linear-api-token", + Title: "Linear API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`lin_api_(?i)[a-z0-9]{40}`), + Keywords: []string{"lin_api_"}, + }, + { + ID: "linear-client-secret", + Title: "Linear client secret/ID", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Plinear[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-f0-9]{32})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"linear"}, + }, + { + ID: "lob-api-key", + Title: "Lob API Key", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Plob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P(live|test)_[a-f0-9]{35})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"lob"}, + }, + { + ID: "lob-pub-api-key", + Title: "Lob Publishable API Key", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Plob[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P(test|live)_pub_[a-f0-9]{31})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"lob"}, + }, + { + ID: "mailchimp-api-key", + Title: "Mailchimp API key", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pmailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-f0-9]{32}-us20)['\"]`), + SecretGroupName: "secret", + Keywords: []string{"mailchimp"}, + }, + { + ID: "mailgun-token", + Title: "Mailgun private API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pmailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P(pub)?key-[a-f0-9]{32})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"mailgun"}, + }, + { + ID: "mailgun-signing-key", + Title: "Mailgun webhook signing key", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pmailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"mailgun"}, + }, + { + ID: "mapbox-api-token", + Title: "Mapbox API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(pk\.[a-z0-9]{60}\.[a-z0-9]{22})`), + Keywords: []string{"pk."}, + }, + { + ID: "messagebird-api-token", + Title: "MessageBird API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pmessagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9]{25})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"messagebird"}, + }, + { + ID: "messagebird-client-id", + Title: "MessageBird API client ID", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pmessagebird[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-h0-9]{8}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{4}-[a-h0-9]{12})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"messagebird"}, + }, + { + ID: "new-relic-user-api-key", + Title: "New Relic user API Key", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`['\"](NRAK-[A-Z0-9]{27})['\"]`), + Keywords: []string{"NRAK-"}, + }, + { + ID: "new-relic-user-api-id", + Title: "New Relic user API ID", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`(?i)(?Pnewrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[A-Z0-9]{64})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"newrelic"}, + }, + { + ID: "new-relic-browser-api-token", + Title: "New Relic ingest browser API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`['\"](NRJS-[a-f0-9]{19})['\"]`), + Keywords: []string{"NRJS-"}, + }, + { + ID: "npm-access-token", + Title: "npm access token", + Severity: "CRITICAL", + Regex: regexp.MustCompile(`['\"](npm_(?i)[a-z0-9]{36})['\"]`), + Keywords: []string{"npm_"}, + }, + { + ID: "planetscale-password", + Title: "PlanetScale password", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`pscale_pw_(?i)[a-z0-9\-_\.]{43}`), + Keywords: []string{"pscale_pw_"}, + }, + { + ID: "planetscale-api-token", + Title: "PlanetScale API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`pscale_tkn_(?i)[a-z0-9\-_\.]{43}`), + Keywords: []string{"pscale_tkn_"}, + }, + { + ID: "postman-api-token", + Title: "Postman API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`PMAK-(?i)[a-f0-9]{24}\-[a-f0-9]{34}`), + Keywords: []string{"PMAK-"}, + }, + { + ID: "pulumi-api-token", + Title: "Pulumi API token", + Severity: "HIGH", + Regex: regexp.MustCompile(`pul-[a-f0-9]{40}`), + Keywords: []string{"pul-"}, + }, + { + ID: "rubygems-api-token", + Title: "Rubygem API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`rubygems_[a-f0-9]{48}`), + Keywords: []string{"rubygems_"}, + }, + { + ID: "sendgrid-api-token", + Title: "SendGrid API token", + Severity: "MEDIUM", + Regex: regexp.MustCompile(`SG\.(?i)[a-z0-9_\-\.]{66}`), + Keywords: []string{"SG."}, + }, + { + ID: "sendinblue-api-token", + Title: "Sendinblue API token", + Severity: "LOW", + Regex: regexp.MustCompile(`xkeysib-[a-f0-9]{64}\-(?i)[a-z0-9]{16}`), + Keywords: []string{"xkeysib-"}, + }, + { + ID: "shippo-api-token", + Title: "Shippo API token", + Severity: "LOW", + Regex: regexp.MustCompile(`shippo_(live|test)_[a-f0-9]{40}`), + Keywords: []string{"shippo_live_", "shippo_test_"}, + }, + { + ID: "linkedin-client-secret", + Title: "LinkedIn Client secret", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Plinkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z]{16})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"linkedin"}, + }, + { + ID: "linkedin-client-id", + Title: "LinkedIn Client ID", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Plinkedin[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9]{14})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"linkedin"}, + }, + { + ID: "twitch-api-token", + Title: "Twitch API token", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Ptwitch[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](?P[a-z0-9]{30})['\"]`), + SecretGroupName: "secret", + Keywords: []string{"twitch"}, + }, + { + ID: "typeform-api-token", + Title: "Typeform API token", + Severity: "LOW", + Regex: regexp.MustCompile(`(?i)(?Ptypeform[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}(?Ptfp_[a-z0-9\-_\.=]{59})`), + SecretGroupName: "secret", + Keywords: []string{"typeform"}, + }, + { + ID: "dockerconfig-secret", + Title: "Dockerconfig secret exposed", + Severity: "HIGH", + Regex: regexp.MustCompile(`(?i)(\.(dockerconfigjson|dockercfg):\s*\|*\s*(?P(ey|ew)+[A-Za-z0-9\/\+=]+))`), + SecretGroupName: "secret", + Keywords: []string{"dockerc"}, + }, +} diff --git a/common-lib/utils/secretScanner/scanner.go b/common-lib/utils/secretScanner/scanner.go new file mode 100644 index 000000000..afd9aa7b0 --- /dev/null +++ b/common-lib/utils/secretScanner/scanner.go @@ -0,0 +1,121 @@ +package secretScanner + +import ( + "bufio" + "context" + "errors" + "io" + "log" +) + +const maxCapacity int = 256 * 1024 // 256KB +const DEVTRON = "DEVTRON" + +// MaskSecretsOnString takes an input string and masks any secrets found based on the provided rules +func MaskSecretsOnString(input string) string { + maskedInput := input + + for _, rule := range BuiltinRules { + maskedInput = rule.Regex.ReplaceAllString(maskedInput, "******") + + } + return maskedInput +} + +// MaskSecretsOnStream processes an input stream, masking secrets according to built-in rules. +func MaskSecretsOnStream(input io.Reader) (io.Reader, error) { + pr, pw := io.Pipe() + go func() { + defer pw.Close() + scanner := bufio.NewScanner(input) + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + processLines(scanner, input, pw, buf) + }() + return pr, nil +} + +func MaskSecretsOnStreamWithCtx(ctx context.Context, input io.Reader) (io.Reader, error) { + pr, pw := io.Pipe() + go func() { + defer pw.Close() + scanner := bufio.NewScanner(input) + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + processLinesWithCtx(ctx, scanner, input, pw, buf) + }() + return pr, nil +} + +// processLines handles the main scanning and processing of lines from the input. +func processLines(scanner *bufio.Scanner, input io.Reader, pw *io.PipeWriter, buf []byte) { + for scanner.Scan() { + line := scanner.Text() + writeMaskedLine(pw, line, true) + } + + if err := scanner.Err(); err != nil { + handleScanError(err, input, pw, buf) + } +} + +func processLinesWithCtx(ctx context.Context, scanner *bufio.Scanner, input io.Reader, pw *io.PipeWriter, buf []byte) { + for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + line := scanner.Text() + writeMaskedLine(pw, line, true) + } + } + + if err := scanner.Err(); err != nil { + handleScanError(err, input, pw, buf) + } +} + +// writeMaskedLine writes the masked version of a line to the pipe writer. +func writeMaskedLine(pw *io.PipeWriter, line string, lineChange bool) { + var err error + if len(line) == 0 { + _, err = pw.Write([]byte("\n")) + } else { + maskedString := MaskSecretsOnString(line) + if lineChange { + _, err = pw.Write([]byte(maskedString + "\n")) + } else { + _, err = pw.Write([]byte(maskedString)) + } + } + if err != nil { + log.Println(DEVTRON, "Error writing to pipe: %v\n", err) + } +} + +// handleScanError handles errors encountered during scanning. +func handleScanError(err error, input io.Reader, pw *io.PipeWriter, buf []byte) { + if errors.Is(err, bufio.ErrTooLong) { + processLargeInput(input, pw, buf) + } else { + log.Println(DEVTRON, "Scanner error: %v\n", err) + } +} + +// processLargeInput handles processing of large inputs that exceed the scanner buffer size. +func processLargeInput(input io.Reader, pw *io.PipeWriter, buf []byte) { + for { + n, err := input.Read(buf) + if err != nil { + if err == io.EOF { + break + } + log.Println(DEVTRON, "Error reading input: %v\n", err) + return + } + line := string(buf[:n]) + writeMaskedLine(pw, line, false) + } +} From fdc94f96de2f281491943262b8dd0ff79b63978b Mon Sep 17 00:00:00 2001 From: SATYAsasini Date: Thu, 23 Jul 2026 17:40:46 +0530 Subject: [PATCH 3/5] feat: remove stale secret scanner code --- common-lib/utils/PayloadMaskUtil.go | 97 ----------------------------- 1 file changed, 97 deletions(-) delete mode 100644 common-lib/utils/PayloadMaskUtil.go diff --git a/common-lib/utils/PayloadMaskUtil.go b/common-lib/utils/PayloadMaskUtil.go deleted file mode 100644 index 3160983e7..000000000 --- a/common-lib/utils/PayloadMaskUtil.go +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2024. Devtron Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package utils - -import ( - "encoding/json" - "strings" -) - -// SecretDataMaskPlaceholder is written in place of any sensitive value that is masked out of a payload. -const SecretDataMaskPlaceholder = "********" - -// unparseablePayloadPlaceholder is logged when a payload cannot be parsed as JSON. We fail closed here -// (redact the whole body) instead of returning the raw bytes, so an unexpected content type can never -// leak a secret into logs. -const unparseablePayloadPlaceholder = "[REDACTED: unparseable payload]" - -// sensitivePayloadKeys are the (lower-cased) JSON field-name fragments whose values must never be logged. -// Matching is case-insensitive and substring based, so "password" also covers "confirmPassword", -// "sshPassword", etc., and "token" covers "accessToken", "dockerToken", and so on. -var sensitivePayloadKeys = []string{ - "password", - "passphrase", - "secret", - "token", - "credential", - "privatekey", - "apikey", - "accesskey", - "authkey", -} - -// isSensitivePayloadKey reports whether a JSON key names a sensitive value that must be masked. -func isSensitivePayloadKey(key string) bool { - lowerKey := strings.ToLower(key) - for _, sensitiveKey := range sensitivePayloadKeys { - if strings.Contains(lowerKey, sensitiveKey) { - return true - } - } - return false -} - -// MaskSensitiveDataInJsonPayload returns a JSON string equivalent to payload with every sensitive value -// (see sensitivePayloadKeys) replaced by SecretDataMaskPlaceholder. It is meant for the generic request -// logging / audit path, where only the raw body bytes are available and the concrete DTO type is unknown. -// -// It fails closed: an empty payload returns "", and a payload that cannot be parsed as JSON returns -// unparseablePayloadPlaceholder rather than the raw bytes, so a secret can never leak on the error path. -func MaskSensitiveDataInJsonPayload(payload []byte) string { - if len(payload) == 0 { - return "" - } - var parsed interface{} - if err := json.Unmarshal(payload, &parsed); err != nil { - return unparseablePayloadPlaceholder - } - maskSensitiveValues(parsed) - masked, err := json.Marshal(parsed) - if err != nil { - return unparseablePayloadPlaceholder - } - return string(masked) -} - -// maskSensitiveValues walks a decoded JSON value in place, masking any value whose key is sensitive and -// recursing into nested objects and arrays. -func maskSensitiveValues(value interface{}) { - switch typed := value.(type) { - case map[string]interface{}: - for key, val := range typed { - if isSensitivePayloadKey(key) { - typed[key] = SecretDataMaskPlaceholder - } else { - maskSensitiveValues(val) - } - } - case []interface{}: - for _, item := range typed { - maskSensitiveValues(item) - } - } -} From f2bb66cc7c8eb02e0934514de064c5f4133532b9 Mon Sep 17 00:00:00 2001 From: SATYAsasini Date: Thu, 23 Jul 2026 19:25:18 +0530 Subject: [PATCH 4/5] feat: add rule to hide secret --- common-lib/utils/secretScanner/rules.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/common-lib/utils/secretScanner/rules.go b/common-lib/utils/secretScanner/rules.go index 99985f0ba..05ffb28f4 100644 --- a/common-lib/utils/secretScanner/rules.go +++ b/common-lib/utils/secretScanner/rules.go @@ -667,4 +667,12 @@ var BuiltinRules = []Rule{ SecretGroupName: "secret", Keywords: []string{"dockerc"}, }, + { + ID: "generic-credential-assignment", + Title: "Generic credential assignment", + Severity: "HIGH", + Regex: regexp.MustCompile(`(?i)(password|passwd|pwd|secret|credential|token|api[_-]?key)["']?\s*(:|=>|=)\s*["']?(?P[^"',}\s]{3,})`), + SecretGroupName: "secret", + Keywords: []string{"password", "passwd", "pwd", "secret", "credential", "token", "key"}, + }, } From 7ca23b6edb39ef3a9a99a46489b621e44467fe6d Mon Sep 17 00:00:00 2001 From: SATYAsasini Date: Fri, 24 Jul 2026 00:15:45 +0530 Subject: [PATCH 5/5] feat: add new function key value masking --- common-lib/utils/secretScanner/rules.go | 12 ++++-------- common-lib/utils/secretScanner/scanner.go | 6 ++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/common-lib/utils/secretScanner/rules.go b/common-lib/utils/secretScanner/rules.go index 05ffb28f4..33280daba 100644 --- a/common-lib/utils/secretScanner/rules.go +++ b/common-lib/utils/secretScanner/rules.go @@ -15,6 +15,10 @@ const ( aws = `aws_?` ) +// credentialAssignmentRegex matches a value assigned to a credential-named key, capturing the +// key+separator+quote as `pre` so it can be kept while only the value is masked. +var credentialAssignmentRegex = regexp.MustCompile(`(?i)(?P
(password|passwd|pwd|secret|credential|token|api[_-]?key)["']?\s*(:|=>|=)\s*["']?)[^"',}\s]{3,}`)
+
 // create rule struct
 type Rule struct {
 	ID              string
@@ -667,12 +671,4 @@ var BuiltinRules = []Rule{
 		SecretGroupName: "secret",
 		Keywords:        []string{"dockerc"},
 	},
-	{
-		ID:              "generic-credential-assignment",
-		Title:           "Generic credential assignment",
-		Severity:        "HIGH",
-		Regex:           regexp.MustCompile(`(?i)(password|passwd|pwd|secret|credential|token|api[_-]?key)["']?\s*(:|=>|=)\s*["']?(?P[^"',}\s]{3,})`),
-		SecretGroupName: "secret",
-		Keywords:        []string{"password", "passwd", "pwd", "secret", "credential", "token", "key"},
-	},
 }
diff --git a/common-lib/utils/secretScanner/scanner.go b/common-lib/utils/secretScanner/scanner.go
index afd9aa7b0..623be0c3e 100644
--- a/common-lib/utils/secretScanner/scanner.go
+++ b/common-lib/utils/secretScanner/scanner.go
@@ -22,6 +22,12 @@ func MaskSecretsOnString(input string) string {
 	return maskedInput
 }
 
+// MaskCredentialKeyValues masks values assigned to credential-named keys, keeping the surrounding
+// structure intact so the result stays parseable (e.g. valid JSON).
+func MaskCredentialKeyValues(input string) string {
+	return credentialAssignmentRegex.ReplaceAllString(input, "${pre}******")
+}
+
 // MaskSecretsOnStream processes an input stream, masking secrets according to built-in rules.
 func MaskSecretsOnStream(input io.Reader) (io.Reader, error) {
 	pr, pw := io.Pipe()