diff --git a/go.mod b/go.mod index d363725..88226ea 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,13 @@ module github.com/checkmarxDev/gpt-wrapper go 1.17 -require github.com/google/uuid v1.3.0 +require ( + github.com/google/uuid v1.3.0 + github.com/rs/zerolog v1.31.0 +) + +require ( + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + golang.org/x/sys v0.12.0 // indirect +) diff --git a/go.sum b/go.sum index 3dfe1c9..750c504 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,17 @@ +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/secrets/maskSecrets.go b/internal/secrets/maskSecrets.go index ee69b27..57f9bc7 100644 --- a/internal/secrets/maskSecrets.go +++ b/internal/secrets/maskSecrets.go @@ -3,7 +3,6 @@ package secrets import ( _ "embed" "encoding/json" - "fmt" "math" "regexp" "strings" @@ -37,11 +36,11 @@ type AllowRule struct { type SecretRule struct { ID string `json:"id"` Name string `json:"name"` + Multiline bool `json:"multiline"` Regex string `json:"regex"` Entropies []Entropy `json:"entropies"` - Multiline Multiline `json:"multiline"` AllowRules []AllowRule `json:"allowRules"` - SpecialMask string `json:"specialMask"` + GroupToMask int `json:"groupMask"` } type SecretRules struct { @@ -52,10 +51,11 @@ type SecretRules struct { type SecretRegex struct { QueryName string Regex *regexp.Regexp - Multiline Multiline + RegexStr string + Multiline bool + GroupToMask int Entropies []Entropy AllowRules []*regexp.Regexp - SpecialMask *regexp.Regexp } type Result struct { @@ -78,7 +78,6 @@ func LoadRegexps() ([]SecretRegex, []*regexp.Regexp, error) { var regexes []SecretRegex for _, regexStruct := range secretRules.Rules { regex := regexStruct.Regex - specialMask := regexStruct.SpecialMask var allowRules []*regexp.Regexp for _, rule := range regexStruct.AllowRules { @@ -88,12 +87,6 @@ func LoadRegexps() ([]SecretRegex, []*regexp.Regexp, error) { } } - var specialMaskCompiled *regexp.Regexp - if specialMask == "" { - specialMaskCompiled = nil - } else { - specialMaskCompiled, _ = regexp.Compile(specialMask) - } regexCompiled, _ := regexp.Compile(regex) secretRegex := &SecretRegex{ @@ -102,7 +95,8 @@ func LoadRegexps() ([]SecretRegex, []*regexp.Regexp, error) { AllowRules: allowRules, Multiline: regexStruct.Multiline, Entropies: regexStruct.Entropies, - SpecialMask: specialMaskCompiled, + GroupToMask: regexStruct.GroupToMask, + RegexStr: regexStruct.Regex, } regexes = append(regexes, *secretRegex) } @@ -118,31 +112,17 @@ func LoadRegexps() ([]SecretRegex, []*regexp.Regexp, error) { } // getLineNumber calculates the line number based on the match index -func getLineNumber(str string, index int) int { +func getLineNumber(text, portion string) int { + index := strings.Index(text, portion) + 1 lineNumber := 1 for i := 0; i < index; i++ { - if str[i] == '\n' && i != index-1 { + if text[i] == '\n' && i != index-1 { lineNumber++ } } return lineNumber } -func getLines(str string, firstLine int, lastLine int) string { - lineNumber := 1 - var returnLines []byte - for i := 0; i < len(str) && lineNumber <= lastLine; i++ { - if lineNumber >= firstLine && (str[i] != '\n' || lineNumber < lastLine) { - returnLines = append(returnLines, str[i]) - } - if str[i] == '\n' { - lineNumber++ - } - - } - return string(returnLines) -} - // CheckEntropyInterval - verifies if a given token's entropy is within expected bounds func CheckEntropyInterval(entropy Entropy, token string) (isEntropyInInterval bool, entropyLevel float64) { base64Entropy := calculateEntropy(token, Base64Chars) @@ -179,6 +159,54 @@ func calculateEntropy(token, charSet string) float64 { return math.Log2(length) - freq/length } +// maskRegexByMatchGroup masks a content using a regex and the group to be masked +func maskRegexByMatchGroup(groupToMask int, matchContent string, query *SecretRegex) string { + query.Regex = regexp.MustCompile(".*" + query.RegexStr) // add .* to match the last appearance + groups := query.Regex.FindAllStringSubmatch(matchContent, -1) + lastMatch := groups[len(groups)-1] + if len(lastMatch) < groupToMask { + return matchContent + } + return strings.Replace(matchContent, lastMatch[groupToMask], "", 1) +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// IsAllowRule check if string matches any of the allow rules for the secret queries +func IsAllowRule(s string, query *SecretRegex, allowRules []*regexp.Regexp) bool { + query.Regex = regexp.MustCompile(query.RegexStr) + regexMatch := query.Regex.FindStringIndex(s) + if regexMatch != nil { + allowRuleMatches := AllowRuleMatches(s, append(query.AllowRules, allowRules...)) + + for _, allowMatch := range allowRuleMatches { + allowStart, allowEnd := allowMatch[0], allowMatch[1] + regexStart, regexEnd := regexMatch[0], regexMatch[1] + + if (allowStart <= regexEnd && allowStart >= regexStart) || (regexStart <= allowEnd && regexStart >= allowStart) { + return true + } + } + } + + return false +} + +// AllowRuleMatches return all the allow rules matches for the secret queries +func AllowRuleMatches(s string, allowRules []*regexp.Regexp) [][]int { + allowRuleMatches := [][]int{} + for i := range allowRules { + res := allowRules[i].FindAllStringIndex(s, -1) + allowRuleMatches = append(allowRuleMatches, res...) + } + return allowRuleMatches +} + // ReplaceMatches If matches between the regex and the file content, then replace the match with the string "" func ReplaceMatches(fileName string, result string, regexs []SecretRegex, allowRegexes []*regexp.Regexp) (string, []Result, []maskedSecret.MaskedSecret) { var results []Result @@ -188,20 +216,17 @@ func ReplaceMatches(fileName string, result string, regexs []SecretRegex, allowR lines := strings.Split(strings.ReplaceAll(result, "\r\n", "\n"), "\n") // Replace matches for _, re := range regexs { - if re.Multiline.DetectLineGroup != 0 { + if re.Multiline { multilineRegexes = append(multilineRegexes, re) continue } for index, line := range lines { - originalLine := lines[index] lines[index] = re.Regex.ReplaceAllStringFunc(line, func(match string) string { - for _, allowRule := range append(re.AllowRules, allowRegexes...) { - if allowRule.FindString(line) != "" { - return match - } + if IsAllowRule(line, &re, append(re.AllowRules, allowRegexes...)) { + return match } - + re.Regex = regexp.MustCompile(re.RegexStr) groups := re.Regex.FindAllStringSubmatch(result, -1) for _, entropy := range re.Entropies { if len(groups) < entropy.Group { @@ -211,11 +236,7 @@ func ReplaceMatches(fileName string, result string, regexs []SecretRegex, allowR } } - startOfMatch := "" - if re.SpecialMask != nil { - startOfMatch = re.SpecialMask.FindString(line) - } - maskedSecret := fmt.Sprintf("%s", startOfMatch) + maskedSecret := maskRegexByMatchGroup(re.GroupToMask, match, &re) results = append(results, Result{QueryName: "Passwords And Secrets - " + re.QueryName, Line: index + 1, FileName: fileName, Severity: "HIGH"}) return maskedSecret }) @@ -232,14 +253,13 @@ func ReplaceMatches(fileName string, result string, regexs []SecretRegex, allowR result = strings.Join(lines[:], "\n") for _, re := range multilineRegexes { // Find all matches of the regular expression in the string - groups := re.Regex.FindStringSubmatchIndex(result) + re.Regex = regexp.MustCompile(re.RegexStr) + groups := re.Regex.FindStringSubmatch(result) // Iterate over each match for groups != nil { maskedSecretElement := maskedSecret.MaskedSecret{} - firstLine := getLineNumber(result, groups[0]) - lastLine := getLineNumber(result, groups[1]) - fullContext := getLines(result, firstLine, lastLine) + fullContext := groups[0] allowed := false for _, allowRule := range append(re.AllowRules, allowRegexes...) { @@ -254,40 +274,23 @@ func ReplaceMatches(fileName string, result string, regexs []SecretRegex, allowR } // Extract the matched substring - matchString := result[groups[0]:groups[1]] - - if len(groups) <= re.Multiline.DetectLineGroup*2 { - groups = nil - continue - } + matchString := groups[0] - stringToMask := result[groups[re.Multiline.DetectLineGroup*2]:groups[re.Multiline.DetectLineGroup*2+1]] - lineOfSecret := getLineNumber(result, groups[re.Multiline.DetectLineGroup*2]) - - startOfMatch := "" - if re.SpecialMask != nil { - partOfMatches := re.SpecialMask.FindAllStringIndex(stringToMask, -1) - if len(partOfMatches) != 0 { - partOfMatch := partOfMatches[len(partOfMatches)-1] - startOfMatch = stringToMask[0:partOfMatch[1]] - } - } - maskedSecret := fmt.Sprintf("%s", startOfMatch) + lineOfSecret := getLineNumber(result, groups[re.GroupToMask]) + maskedMatchString := maskRegexByMatchGroup(re.GroupToMask, matchString, &re) results = append(results, Result{QueryName: "Passwords And Secrets - " + re.QueryName, Line: lineOfSecret, FileName: fileName, Severity: "HIGH"}) - maskedMatchString := strings.Replace(matchString, stringToMask, maskedSecret, 1) - // Add the masked string to return maskedSecretElement.Masked = maskedMatchString maskedSecretElement.Secret = matchString - maskedSecretElement.Line = firstLine + maskedSecretElement.Line = lineOfSecret maskedSecrets = append(maskedSecrets, maskedSecretElement) result = strings.Replace(result, matchString, maskedMatchString, 1) - groups = re.Regex.FindStringSubmatchIndex(result) + groups = re.Regex.FindStringSubmatch(result) } } return result, results, maskedSecrets diff --git a/internal/secrets/maskSecrets_test.go b/internal/secrets/maskSecrets_test.go index e0e4089..e0aa7a7 100644 --- a/internal/secrets/maskSecrets_test.go +++ b/internal/secrets/maskSecrets_test.go @@ -5,12 +5,14 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + + "github.com/rs/zerolog/log" ) func TestSecretsDetection(t *testing.T) { - expectedResultsPath := "test/positive_expected_result.json" expectedResults, err := os.ReadFile(expectedResultsPath) if err != nil { @@ -47,7 +49,7 @@ func TestSecretsDetection(t *testing.T) { } for _, ree := range rs { - if ree.Multiline.DetectLineGroup != 0 { + if ree.Multiline { multiLineQueries = append(multiLineQueries, ree.QueryName) } } @@ -58,7 +60,6 @@ func TestSecretsDetection(t *testing.T) { } func processFile(t *testing.T, path string, rs []SecretRegex, allowrs []*regexp.Regexp) []Result { - fileContent, err := os.ReadFile(path) if err != nil { t.Fatal(err) @@ -70,8 +71,29 @@ func processFile(t *testing.T, path string, rs []SecretRegex, allowrs []*regexp. } +func diffActualExpectedVulnerabilities(actual, expected []Result) []string { + m := make(map[string]bool) + diff := make([]string, 0) + for i := range expected { + m[expected[i].QueryName+":"+filepath.Base(expected[i].FileName)+":"+strconv.Itoa(expected[i].Line)] = true + } + for i := range actual { + if _, ok := m[actual[i].QueryName+":"+filepath.Base(actual[i].FileName)+":"+strconv.Itoa(actual[i].Line)]; !ok { + diff = append(diff, actual[i].FileName+":"+strconv.Itoa(actual[i].Line)) + } + } + + return diff +} + func compareExpectedWithActual(expected, actual []Result, multiLineQueries []string) bool { if len(expected) != len(actual) { + log.Error().Msgf( + "Count of actual issues and expected vulnerabilities doesn't match\n -- \n"+ + "not present in expected and present in actual: %v\n"+ + "not present in actual and present in expected: %v\n", + diffActualExpectedVulnerabilities(actual, expected), + diffActualExpectedVulnerabilities(expected, actual)) return false } diff --git a/internal/secrets/regex_rules.json b/internal/secrets/regex_rules.json index f0cef4a..890cb7a 100644 --- a/internal/secrets/regex_rules.json +++ b/internal/secrets/regex_rules.json @@ -5,17 +5,21 @@ "name": "Generic Password", "regex": "(?i)['\"]?password['\"]?\\s*[:=]\\s*['\"]?([A-Za-z0-9/~^_!@&%()=?*+-. ]{4,})['\"]?", "allowRules": [ + { + "description": "Avoiding TF resource access", + "regex": "(?i)['\"]?password['\"]?\\s*=\\s*(([a-zA-z_]+(.))?[a-zA-z_]+\\s*(.)\\s*[a-zA-z_]+(.)[a-zA-z_]+)?(\\s*:\\s*null|null)$" + }, { "description": "Avoiding CF AllowUsersToChangePassword", "regex": "['\"]?AllowUsersToChangePassword['\"]?\\s*[:=]\\s*['\"]?([A-Za-z0-9/~^_!@&%()=?*+-.]{4,})['\"]?" } ], - "specialMask": "(?i)['\"]?password['\"]?\\s*[:=]\\s*" + "groupMask": 1 }, { "id": "3e2d3b2f-c22a-4df1-9cc6-a7a0aebb0c99", "name": "Generic Secret", - "regex": "(?i)['\"]?secret[_]?(key)?['\"]?\\s*(:|=)\\s*['\"]?([A-Za-z0-9/~^_!@&%()=?*+-]{10,})['\"]?", + "regex": "(?i)['\"]?secret[_]?(key)?['\"]?\\s*(:|=)\\s*['\"]?([A-Za-z0-9/~^_!@#&%(){};=?*+-<>,:;[\\]%$]{10,})['\"]?", "entropies": [ { "group": 3, @@ -41,15 +45,13 @@ "regex": "['\"]?Description['\"]?\\s*[:=]\\s*['\"]?([A-Za-z0-9/~^_!@&%()=?*${}+-.'\"]+)['\"]?" } ], - "specialMask": "(?i)['\"]?secret[_]?(key)?['\"]?\\s*(:|=)\\s*" + "groupMask": 3 }, { "id": "51b5b840-cd0c-4556-98a7-fe5f4def80cf", "name": "Asymmetric private key", - "regex": "-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----(\\s*([A-Za-z0-9+,:\\-\\/=\\n\\r]+))+\\s*-----END ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----", - "multiline": { - "detectLineGroup": 5 - }, + "regex": "-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----\\s*(([A-Za-z0-9+,:\\-\\/=\\n\\r]+\\s*)+)\n-----END ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----", + "multiline": true, "entropies": [ { "group": 5, @@ -57,17 +59,19 @@ "max": 12 } ], - "specialMask": "all" + "groupMask": 4 }, { "id": "a007a85e-a2a7-4a81-803a-7a2ca0c65abb", - "name": "Putty Private Key", - "regex": "PuTTY-User-Key-File-2" + "name": "Putty User Key File Content", + "regex": "['\"]?PuTTY-User-Key-File-\\d: ([\\w\\d-:\\n\\s+/=]+Private-MAC: [\\d\\w\"]+)['\"]?", + "multiline": true, + "groupMask": 1 }, { "id": "c4d3b58a-e6d4-450f-9340-04f1e702eaae", "name": "Password in URL", - "regex": "[a-zA-Z]{3,10}://[^/\\s:@$]*?:[^/\\s:@$]*?@[^/\\s:@$]*" + "regex": "['\"]?[a-zA-Z]{3,10}://[^/\\s:@$]*?:[^/\\s:@$]*?@[^/\\s:@$]*['\"]?" }, { "id": "76c0bcde-903d-456e-ac13-e58c34987852", @@ -95,41 +99,39 @@ "max": 7 } ], - "specialMask": "(?i)AWS_SECRET(_ACCESS)?(_KEY)?\\s*[:=]\\s*" + "groupMask": 3 }, { "id": "4b2b5fd3-364d-4093-bac2-17391b2a5297", "name": "K8s Environment Variable Password", "regex": "apiVersion((.*)\\s*)*env:((.*)\\s*)*name:\\s*\\w+(?i)pass((?i)word)?\\w*\\s*(value):\\s*([\"|'].*[\"|'])", - "multiline": { - "detectLineGroup": 7 - }, - "specialMask": "\\s*(value):\\s*" + "multiline": true, + "groupMask": 7 }, { "id": "d651cca2-2156-4d17-8e76-423e68de5c8b", "name": "Google OAuth", - "regex": "[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com" + "regex": "['\"]?[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com['\"]?" }, { "id": "ccde326f-ebc7-4772-8ad5-de66e90a8cc3", "name": "Slack Webhook", - "regex": "https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}" + "regex": "['\"]?https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}['\"]?" }, { "id": "d6214dca-a31b-425f-bcf7-f4faa772a1c0", "name": "MSTeams Webhook", - "regex": "https://[a-zA-Z0-9_]{1,24}.webhook.office.com/webhook(b2)?/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}@[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/IncomingWebhook/[a-z0-9]+/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + "regex": "['\"]?https://[a-zA-Z0-9_]{1,24}.webhook.office.com/webhook(b2)?/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}@[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/IncomingWebhook/[a-z0-9]+/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}['\"]?" }, { "id": "7908a9e3-5cac-41b1-b514-5f6d82ce02d5", "name": "Slack Token", - "regex": "(xox[p|b|o|a]-[0-9]{12}-[0-9]{12}-[0-9]{12}-[a-z0-9]{32})" + "regex": "['\"]?(xox[p|b|o|a]-[0-9]{12}-[0-9]{12}-[0-9]{12}-[a-z0-9]{32})['\"]?" }, { "id": "6abcae17-b175-4698-a9a5-b07661974749", "name": "Stripe API Key", - "regex": "sk_live_[0-9a-zA-Z]{24}[^0-9a-zA-Z]" + "regex": "['\"]?sk_live_[0-9a-zA-Z]{24}[^0-9a-zA-Z]['\"]?" }, { "id": "0b1b2482-51e7-49d1-893d-522afa4a6bd0", @@ -144,20 +146,19 @@ { "id": "e9856348-4069-4ac0-bd91-415f6a7b84a4", "name": "Google API Key", - "regex": "AIza[0-9A-Za-z\\-_]{35}" + "regex": "['\"]?AIza[0-9A-Za-z\\-_]{35}['\"]?" }, { "id": "9a3650af-5b88-48cd-ab89-cd77fd0b633f", "name": "Heroku API Key", - "regex": "(?i)heroku((.|\\n)*)\\b([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})\\b", - "multiline": { - "detectLineGroup": 3 - } + "regex": "['\"]?(?i)heroku((.|\\n)*)\\b([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})\\b['\"]?", + "multiline": true, + "groupMask": 3 }, { "id": "bb51eb1e-0357-44a2-86d7-dd5350cffd43", "name": "Square OAuth Secret", - "regex": "sq0csp-[0-9A-Za-z\\-_]{43}" + "regex": "['\"]?sq0csp-[0-9A-Za-z\\-_]{43}['\"]?" }, { "id": "ac8c8075-6ec0-4367-9e26-30ec8161d258", @@ -177,12 +178,12 @@ { "id": "54274b18-bfac-47ce-afd1-0f05bc3e3b59", "name": "Stripe Restricted API Key", - "regex": "rk_live_[0-9a-zA-Z]{24}" + "regex": "['\"]?rk_live_[0-9a-zA-Z]{24}['\"]?" }, { "id": "5176e805-0cda-44fa-ac96-c092c646180a", "name": "Facebook Access Token", - "regex": "EAACEdEose0cBA[0-9A-Za-z]+" + "regex": "['\"]?EAACEdEose0cBA[0-9A-Za-z]+['\"]?" }, { "id": "74736dd1-dd11-4139-beb6-41cd43a50317", @@ -194,12 +195,12 @@ "regex": "(?i)['\"]?api[_]?key['\"]?\\s*[:=]\\s*['\"]?(SK[0-9a-fA-F]{32})['\"]?" } ], - "specialMask": "(?i)['\"]?api[_]?key['\"]?\\s*[:=]\\s*" + "groupMask": 1 }, { "id": "62d0025d-9575-4eff-b60b-d3b4fcec0d04", "name": "Mailgun API Key", - "regex": "key-[0-9a-zA-Z]{32}" + "regex": "['\"]?key-[0-9a-zA-Z]{32}['\"]?" }, { "id": "50cc5f03-e686-4183-97e9-12f9b55d0f97", @@ -214,19 +215,25 @@ { "id": "7f370dd5-eea3-4e5f-8354-3cb2506f9f13", "name": "Generic Access Key", - "regex": "(?i)^\\s*['\"]?(access)[_]?key['\"]?\\s*[:=]\\s*['\"]?([[A-Za-z0-9\/~^_!@&%()=?*+-]+)['\"]?", - "specialMask": "(?i)['\"]?access[_]?key['\"]?\\s*[:=]\\s*" + "regex": "(?i)^\\s*['\"]?(access)[_]?key['\"]?\\s*[:=]\\s*['\"]?([[A-Za-z0-9/~^_!@&%()=?*+-]+)['\"]?", + "groupMask": 2 }, { "id": "2f665079-c383-4b33-896e-88268c1fa258", "name": "Generic Private Key", "regex": "(?i)['\"]?private[_]?key['\"]?\\s*[:=]\\s*['\"]?([[A-Za-z0-9/~^_!@&%()=?*+-]+)['\"]?", - "specialMask": "(?i)['\"]?private[_]?key['\"]?\\s*[:=]\\s*" + "allowRules": [ + { + "description": "Avoiding bash variables", + "regex": "(?i)['\"]?\\$\\s*\\{[^\\s\\}]+\\}['\"]?" + } + ], + "groupMask": 1 }, { "id": "baee238e-1921-4801-9c3f-79ae1d7b2cbc", "name": "Generic Token", - "regex": "(?i)['\"]?token(_)?(key)?['\"]?\\s*[:=]\\s*['\"]?([[A-Za-z0-9/~^_!@&%()=?*+-]+)['\"]?", + "regex": "(?i)['\"]?token(_)?(key)?['\"]?\\s*[:=]\\s*['\"]?([[A-Za-z0-9/~^_!@&%()=?*+-:]+)['\"]?", "allowRules": [ { "description": "Avoiding Amazon MWS Auth Token", @@ -285,14 +292,13 @@ "regex": "(?i)['\"]?next(_)?token['\"]?\\s*[:=]\\s*['\"]?([[A-Za-z0-9/~^_!@&%()=?*+-]+)['\"]?" } ], - "specialMask": "(?i)['\"]?token(_)?(key)?['\"]?\\s*[:=]\\s*" - + "groupMask": 3 }, { - "id": "e0f01838-b1c2-4669-b84b-981949ebe5ed", + "id": "36b8f84d-df4e-4d49-b662-bcde71a8764f", "name": "CloudFormation Secret Template", "regex": "(?i)['\"]?SecretStringTemplate['\"]?\\s*:\\s*['\"]?{([\\\":A-Za-z0-9/~^_!@&%()=?*+-]{10,})}", - "specialMask": "(?i)['\"]?SecretStringTemplate['\"]?\\s*:\\s*" + "groupMask": 1 }, { "id": "9fb1cd65-7a07-4531-9bcf-47589d0f82d6", @@ -308,7 +314,7 @@ "regex": "['\"]?EncryptionKey['\"]?\\s*[:=]\\s*['\"]?([A-Za-z0-9/~^_!@&%()=?*+-.]+)['\"]?" } ], - "specialMask": "(?i)['\"]?encryption[_]?key['\"]?\\s*[:=]\\s*" + "groupMask": 1 }, { "id": "8a879bc7-6f82-40fd-bb48-74d25d557fe8", @@ -319,7 +325,7 @@ "allowRules": [ { "description": "Avoiding TF variables", - "regex": "(?i)['\"]?[a-zA-Z_]+['\"]?\\s*=\\s*['\"]?(var.)['\"]?" + "regex": "(?i)['\"]?[a-zA-Z_]+['\"]?\\s*=\\s*['\"]?(var\\.)['\"]?" }, { "description": "!Ref is a cloudFormation reference", @@ -358,4 +364,4 @@ "regex": "(?i)['\"]?[a-zA-Z_]+['\"]?\\s*[=:]\\s*['\"]?(\\*[0-9A-F]{40})['\"]?" } ] -} +} \ No newline at end of file diff --git a/internal/secrets/test/negative39.yaml b/internal/secrets/test/negative39.yaml index d467c15..c4297b9 100644 --- a/internal/secrets/test/negative39.yaml +++ b/internal/secrets/test/negative39.yaml @@ -1,14 +1,51 @@ -Resources: - ElastiCacheReplicationGroup: - Type: AWS::ElastiCache::ReplicationGroup - Properties: - AuthToken: '{{resolve:secretsmanager:/elasticache/replicationgroup/authtoken:SecretString:password}}' - CacheNodeType: cache.m5.large - CacheSubnetGroupName: subnet-foobar - Engine: redis - EngineVersion: '5.0.0' - NumCacheClusters: 2 - ReplicationGroupDescription: foobar - SecurityGroupIds: - - sg-foobar - TransitEncryptionEnabled: True +name: Example Workflow + +on: workflow_call + +jobs: + build-deploy: + permissions: + contents: read + pages: write + id-token: write + + runs-on: ubuntu + + steps: + - uses: actions/checkout@v3 + +--- + +name: Example Workflow + +on: workflow_call + +jobs: + build-deploy: + permissions: + contents: read + pages: write + id-token: read + + runs-on: ubuntu + + steps: + - uses: actions/checkout@v3 + +--- + +name: Example Workflow + +on: workflow_call + +jobs: + build-deploy: + permissions: + contents: read + pages: write + id-token: none + + runs-on: ubuntu + + steps: + - uses: actions/checkout@v3 \ No newline at end of file diff --git a/internal/secrets/test/negative50.yaml b/internal/secrets/test/negative50.yaml index 233a7e9..17b6a0c 100644 --- a/internal/secrets/test/negative50.yaml +++ b/internal/secrets/test/negative50.yaml @@ -1,18 +1,25 @@ -Transform: 'AWS::Serverless-2016-10-31' -Metadata: - 'AWS::ServerlessRepo::Application': - Name: AthenaJdbcConnector - Description: 'This connector enables Amazon Athena to communicate with your Database instance(s) using JDBC driver.' - Author: 'default author' - SpdxLicenseId: Apache-2.0 - LicenseUrl: LICENSE.txt - ReadmeUrl: README.md - Labels: - - athena-federation - HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' - SemanticVersion: 2021.41.1 - SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' -Parameters: - SecretNamePrefix: - Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederation*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' - Type: String +Type: AWS::Glue::Connection +Properties: + CatalogId: "1111111111111" + ConnectionInput: + ConnectionProperties: + CONNECTION_URL: + Fn::Join: + - "" + - - "mongodb://{{resolve:secretsmanager:arn:" + - Ref: AWS::Partition + - :secretsmanager:*:1111111111111:secret:/test/resources/docdb-test:SecretString:endpoint::}}/test + USERNAME: + Fn::Join: + - "" + - - "{{resolve:secretsmanager:arn:" + - Ref: AWS::Partition + - :secretsmanager:eu-west-1:*:secret:/test/resources/docdb-test:SecretString:username::}} + PASSWORD: + Fn::Join: + - "" + - - "{{resolve:secretsmanager:arn:" + - Ref: AWS::Partition + - :secretsmanager:us-east-?:*:secret:tiny::}} + JDBC_ENFORCE_SSL: true + ConnectionType: MONGODB diff --git a/internal/secrets/test/negative52.yaml b/internal/secrets/test/negative52.yaml deleted file mode 100644 index c4297b9..0000000 --- a/internal/secrets/test/negative52.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: Example Workflow - -on: workflow_call - -jobs: - build-deploy: - permissions: - contents: read - pages: write - id-token: write - - runs-on: ubuntu - - steps: - - uses: actions/checkout@v3 - ---- - -name: Example Workflow - -on: workflow_call - -jobs: - build-deploy: - permissions: - contents: read - pages: write - id-token: read - - runs-on: ubuntu - - steps: - - uses: actions/checkout@v3 - ---- - -name: Example Workflow - -on: workflow_call - -jobs: - build-deploy: - permissions: - contents: read - pages: write - id-token: none - - runs-on: ubuntu - - steps: - - uses: actions/checkout@v3 \ No newline at end of file diff --git a/internal/secrets/test/negative53.yaml b/internal/secrets/test/negative53.yaml deleted file mode 100644 index 17b6a0c..0000000 --- a/internal/secrets/test/negative53.yaml +++ /dev/null @@ -1,25 +0,0 @@ -Type: AWS::Glue::Connection -Properties: - CatalogId: "1111111111111" - ConnectionInput: - ConnectionProperties: - CONNECTION_URL: - Fn::Join: - - "" - - - "mongodb://{{resolve:secretsmanager:arn:" - - Ref: AWS::Partition - - :secretsmanager:*:1111111111111:secret:/test/resources/docdb-test:SecretString:endpoint::}}/test - USERNAME: - Fn::Join: - - "" - - - "{{resolve:secretsmanager:arn:" - - Ref: AWS::Partition - - :secretsmanager:eu-west-1:*:secret:/test/resources/docdb-test:SecretString:username::}} - PASSWORD: - Fn::Join: - - "" - - - "{{resolve:secretsmanager:arn:" - - Ref: AWS::Partition - - :secretsmanager:us-east-?:*:secret:tiny::}} - JDBC_ENFORCE_SSL: true - ConnectionType: MONGODB diff --git a/internal/secrets/test/negative55.tf b/internal/secrets/test/negative55.tf new file mode 100644 index 0000000..0188abc --- /dev/null +++ b/internal/secrets/test/negative55.tf @@ -0,0 +1,20 @@ +#this is a problematic code where the query should report a result(s) +resource "google_container_cluster" "primary1" { + name = "marcellus-wallace" + location = "us-central1-a" + initial_node_count = 3 + + master_auth { + username = "" + password = local.rds_postgres_is_primary ? var.rds_postgres_password : null + + client_certificate_config { + issue_client_certificate = true + } + } + + timeouts { + create = "30m" + update = "40m" + } +} diff --git a/internal/secrets/test/negative56.yml b/internal/secrets/test/negative56.yml new file mode 100644 index 0000000..7d72319 --- /dev/null +++ b/internal/secrets/test/negative56.yml @@ -0,0 +1,14 @@ +stages: + - build + +variables: + GIT_PRIVATE_KEY: $GIT_PRIVATE_KEY + +job_build: + stage: build + script: + - if [[ -z "${GIT_PRIVATE_KEY:-}" ]]; then + echo "Missing GIT_PRIVATE_KEY variable!" + exit 1 + fi + - echo "Private key is set." diff --git a/internal/secrets/test/positive15.tf b/internal/secrets/test/positive15.tf index f40a318..a3679b4 100644 --- a/internal/secrets/test/positive15.tf +++ b/internal/secrets/test/positive15.tf @@ -19,8 +19,8 @@ EOF tags = merge({ Name = "${local.resource_prefix.value}-ec2" }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) @@ -34,8 +34,8 @@ resource "aws_ebs_volume" "web_host_storage" { tags = merge({ Name = "${local.resource_prefix.value}-ebs" }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) @@ -48,8 +48,8 @@ resource "aws_ebs_snapshot" "example_snapshot" { tags = merge({ Name = "${local.resource_prefix.value}-ebs-snapshot" }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) @@ -90,8 +90,8 @@ resource "aws_security_group" "web-node" { } depends_on = [aws_vpc.web_vpc] tags = { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" } @@ -104,8 +104,8 @@ resource "aws_vpc" "web_vpc" { tags = merge({ Name = "${local.resource_prefix.value}-vpc" }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) @@ -120,8 +120,8 @@ resource "aws_subnet" "web_subnet" { tags = merge({ Name = "${local.resource_prefix.value}-subnet" }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) @@ -136,8 +136,8 @@ resource "aws_subnet" "web_subnet2" { tags = merge({ Name = "${local.resource_prefix.value}-subnet2" }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) @@ -150,8 +150,8 @@ resource "aws_internet_gateway" "web_igw" { tags = merge({ Name = "${local.resource_prefix.value}-igw" }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) @@ -163,8 +163,8 @@ resource "aws_route_table" "web_rtb" { tags = merge({ Name = "${local.resource_prefix.value}-rtb" }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) @@ -198,8 +198,8 @@ resource "aws_network_interface" "web-eni" { tags = merge({ Name = "${local.resource_prefix.value}-primary_network_interface" }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) @@ -216,8 +216,8 @@ resource "aws_flow_log" "vpcflowlogs" { Name = "${local.resource_prefix.value}-flowlogs" Environment = local.resource_prefix.value }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) @@ -231,8 +231,8 @@ resource "aws_s3_bucket" "flowbucket" { Name = "${local.resource_prefix.value}-flowlogs" Environment = local.resource_prefix.value }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) diff --git a/internal/secrets/test/positive40.tf b/internal/secrets/test/positive40.tf index 4b43c68..ecbac65 100644 --- a/internal/secrets/test/positive40.tf +++ b/internal/secrets/test/positive40.tf @@ -19,8 +19,8 @@ EOF tags = merge({ Name = "${local.resource_prefix.value}-ec2" }, { - git_last_modified_by = "felipe.avelar@checkmarx.com" - git_modifiers = "felipe.avelar" + git_last_modified_by = "email@email.com" + git_modifiers = "foo.bar" git_org = "checkmarx" git_repo = "kics" }) diff --git a/internal/secrets/test/positive43.yaml b/internal/secrets/test/positive43.yaml new file mode 100644 index 0000000..d467c15 --- /dev/null +++ b/internal/secrets/test/positive43.yaml @@ -0,0 +1,14 @@ +Resources: + ElastiCacheReplicationGroup: + Type: AWS::ElastiCache::ReplicationGroup + Properties: + AuthToken: '{{resolve:secretsmanager:/elasticache/replicationgroup/authtoken:SecretString:password}}' + CacheNodeType: cache.m5.large + CacheSubnetGroupName: subnet-foobar + Engine: redis + EngineVersion: '5.0.0' + NumCacheClusters: 2 + ReplicationGroupDescription: foobar + SecurityGroupIds: + - sg-foobar + TransitEncryptionEnabled: True diff --git a/internal/secrets/test/positive44.yaml b/internal/secrets/test/positive44.yaml new file mode 100644 index 0000000..507beda --- /dev/null +++ b/internal/secrets/test/positive44.yaml @@ -0,0 +1,18 @@ +Transform: 'AWS::Serverless-2016-10-31' +Metadata: + 'AWS::ServerlessRepo::Application': + Name: AthenaJdbcConnector + Description: 'This connector enables Amazon Athena to communicate with your Database instance(s) using JDBC driver.' + Author: 'default author' + SpdxLicenseId: Apache-2.0 + LicenseUrl: LICENSE.txt + ReadmeUrl: README.md + Labels: + - athena-federation + HomePageUrl: 'https://github.com/awslabs/aws-athena-query-federation' + SemanticVersion: 2021.41.1 + SourceCodeUrl: 'https://github.com/awslabs/aws-athena-query-federation' +Parameters: + SecretNamePrefix: + Description: 'Used to create resource-based authorization policy for "secretsmanager:GetSecretValue" action. E.g. All Athena JDBC Federation secret names can be prefixed with "AthenaJdbcFederation" and authorization policy will allow "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:AthenaJdbcFederatione*". Parameter value in this case should be "AthenaJdbcFederation". If you do not have a prefix, you can manually update the IAM policy to add allow any secret names.' + Type: String diff --git a/internal/secrets/test/positive45.tf b/internal/secrets/test/positive45.tf new file mode 100644 index 0000000..e668c2a --- /dev/null +++ b/internal/secrets/test/positive45.tf @@ -0,0 +1,20 @@ +#this is a problematic code where the query should report a result(s) +resource "google_container_cluster" "primary1" { + name = "marcellus-wallace" + location = "us-central1-a" + initial_node_count = 3 + + master_auth { + username = "" + password = local.rds_postgres_is_primary ? var.rds_postgres_password : "null" + + client_certificate_config { + issue_client_certificate = true + } + } + + timeouts { + create = "30m" + update = "40m" + } +} diff --git a/internal/secrets/test/positive46.yaml b/internal/secrets/test/positive46.yaml new file mode 100644 index 0000000..2f20427 --- /dev/null +++ b/internal/secrets/test/positive46.yaml @@ -0,0 +1,20 @@ +version: '3.9' +services: + vulnerable_node: + restart: always + build: . + depends_on: + - postgres_db + ports: + - "3000:3000" + depends_on: + - postgres_db + + postgres_db: + restart: always + build: ./services/postgresql + ports: + - "5432:5432" + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres \ No newline at end of file diff --git a/internal/secrets/test/positive47.tf b/internal/secrets/test/positive47.tf new file mode 100644 index 0000000..3532ed7 --- /dev/null +++ b/internal/secrets/test/positive47.tf @@ -0,0 +1,11 @@ +resource "auth0_connection" "google_oauth2" { + name = "Google-OAuth2-Connection" + strategy = "google-oauth2" + options { + client_id = "53221331-2323wasdfa343rwhthfaf33feaf2fa7f.apps.googleusercontent.com" + client_secret = "F-oS9Su%}<>[];#" + allowed_audiences = [ "example.com", "api.example.com" ] + scopes = [ "email", "profile", "gmail", "youtube" ] + set_user_root_attributes = "on_each_login" + } +} diff --git a/internal/secrets/test/positive48.tf b/internal/secrets/test/positive48.tf new file mode 100644 index 0000000..3076177 --- /dev/null +++ b/internal/secrets/test/positive48.tf @@ -0,0 +1,20 @@ +resource "google_container_cluster" "primary1" { + name = "marcellus-wallace" + location = "us-central1-a" + initial_node_count = 3 + + master_auth { + username = "" + password = "varexample" + + client_certificate_config { + issue_client_certificate = true + password = var.example + } + } + + timeouts { + create = "30m" + update = "40m" + } +} diff --git a/internal/secrets/test/positive49.yml b/internal/secrets/test/positive49.yml new file mode 100644 index 0000000..f071956 --- /dev/null +++ b/internal/secrets/test/positive49.yml @@ -0,0 +1,14 @@ +stages: + - build + +variables: + GIT_PRIVATE_KEY: "heythisisaprivatekey!" + +job_build: + stage: build + script: + - if [[ -z "${GIT_PRIVATE_KEY:-}" ]]; then + echo "Missing GIT_PRIVATE_KEY variable!" + exit 1 + fi + - echo "Private key is set." diff --git a/internal/secrets/test/positive_expected_result.json b/internal/secrets/test/positive_expected_result.json index 7980643..52f9288 100644 --- a/internal/secrets/test/positive_expected_result.json +++ b/internal/secrets/test/positive_expected_result.json @@ -312,7 +312,7 @@ "fileName": "positive35.yaml" }, { - "queryName": "Passwords And Secrets - Putty Private Key", + "queryName": "Passwords And Secrets - Putty User Key File Content", "severity": "HIGH", "line": 5, "fileName": "positive36.tf" @@ -350,7 +350,7 @@ { "queryName": "Passwords And Secrets - Asymmetric private key", "severity": "HIGH", - "line": 7, + "line": 6, "fileName": "positive41.tf" }, { @@ -358,5 +358,53 @@ "severity": "HIGH", "line": 7, "fileName": "positive42.tf" + }, + { + "queryName": "Passwords And Secrets - Generic Token", + "severity": "HIGH", + "line": 5, + "fileName": "positive43.yaml" + }, + { + "queryName": "Passwords And Secrets - Generic Secret", + "severity": "HIGH", + "line": 17, + "fileName": "positive44.yaml" + }, + { + "queryName": "Passwords And Secrets - Generic Password", + "severity": "HIGH", + "line": 9, + "fileName": "positive45.tf" + }, + { + "queryName": "Passwords And Secrets - Generic Password", + "severity": "HIGH", + "line": 20, + "fileName": "positive46.yaml" + }, + { + "queryName": "Passwords And Secrets - Google OAuth", + "severity": "HIGH", + "line": 5, + "fileName": "positive47.tf" + }, + { + "queryName": "Passwords And Secrets - Generic Secret", + "severity": "HIGH", + "line": 6, + "fileName": "positive47.tf" + }, + { + "queryName": "Passwords And Secrets - Generic Password", + "severity": "HIGH", + "line": 8, + "fileName": "positive48.tf" + }, + { + "queryName": "Passwords And Secrets - Generic Private Key", + "severity": "HIGH", + "line": 5, + "fileName": "positive49.yml" } ]