feat: catch username password inside urls (#169)

Closes #165 

After researching the Gitleaks docs, I concluded that adding a toml file
won't extend the current rules. This is due to the way this project
extends Gitleaks.

**Proposed Changes**
- added a secrets/custom_rules.go that includes custom rules
configuration.
- a test to validate custom rules were added to the all rules array.
- added a function to append the custom rules to all rules array.

I submit this contribution under the Apache-2.0 license.

---------

Co-authored-by: Baruch Odem (Rothkoff) <[email protected]>
Co-authored-by: Baruch Odem <[email protected]>

secrets/rules/authenticated_url.go

@@ -0,0 +1,31 @@
+package rules
+
+import (
+	"regexp"
+
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func AuthenticatedURL() *config.Rule {
+	regex, _ := regexp.Compile(`:\/\/(.+:.+)?@`)
+	rule := config.Rule{
+		Description: "Identify username:password inside URLS",
+		RuleID:      "authenticated-url",
+		Regex:       regex,
+		Keywords:    []string{},
+		SecretGroup: 1,
+	}
+
+	tPositives := []string{
+		"mongodb+srv://radar:[email protected]/?retryWrites=true&w=majority",
+		"--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs",
+		"https://abc:[email protected]",
+	}
+
+	fPositives := []string{
+		"https://google.com",
+		"https://google.com?user=abc&password=123",
+	}
+
+	return validate(rule, tPositives, fPositives)
+}

secrets/rules/rule.go

@@ -0,0 +1,37 @@
+package rules
+
+import (
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/zricethezav/gitleaks/v8/config"
+	"github.com/zricethezav/gitleaks/v8/detect"
+)
+
+// Copied from https://github.com/gitleaks/gitleaks/blob/463d24618fa42fc7629dc30c9744ebe36c5df1ab/cmd/generate/config/rules/rule.go
+func validate(r config.Rule, truePositives []string, falsePositives []string) *config.Rule {
+	// normalize keywords like in the config package
+	var keywords []string
+	for _, k := range r.Keywords {
+		keywords = append(keywords, strings.ToLower(k))
+	}
+	r.Keywords = keywords
+
+	rules := make(map[string]config.Rule)
+	rules[r.RuleID] = r
+	d := detect.NewDetector(config.Config{
+		Rules:    rules,
+		Keywords: keywords,
+	})
+	for _, tp := range truePositives {
+		if len(d.DetectString(tp)) != 1 {
+			log.Fatal().Msgf("Failed to validate. For rule ID [%s], true positive [%s] was not detected by regexp [%s]", r.RuleID, tp, r.Regex)
+		}
+	}
+	for _, fp := range falsePositives {
+		if len(d.DetectString(fp)) != 0 {
+			log.Fatal().Msgf("Failed to validate. For rule ID [%s], false positive [%s] was detected by regexp [%s]", r.RuleID, fp, r.Regex)
+		}
+	}
+	return &r
+}

secrets/rules/rule_test.go

@@ -0,0 +1,28 @@
+package rules_test
+
+import (
+	"testing"
+
+	"github.com/checkmarx/2ms/secrets/rules"
+	"github.com/zricethezav/gitleaks/v8/config"
+)
+
+func Test2msRules(t *testing.T) {
+	t.Parallel()
+
+	testsRules := []struct {
+		name     string
+		validate func() *config.Rule
+	}{
+		{name: "AuthenticatedURL", validate: rules.AuthenticatedURL},
+	}
+
+	for _, tRule := range testsRules {
+		testRule := tRule // fix for loop variable being captured by func literal
+		t.Run(testRule.name, func(t *testing.T) {
+			t.Parallel()
+
+			testRule.validate()
+		})
+	}
+}

secrets/secrets.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/checkmarx/2ms/plugins"
 	"github.com/checkmarx/2ms/reporting"
+	internalRules "github.com/checkmarx/2ms/secrets/rules"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules"
@@ -377,6 +378,7 @@ func loadAllRules() ([]Rule, error) {
 	allRules = append(allRules, Rule{Rule: *rules.YandexAWSAccessToken(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.YandexAccessToken(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}})
+	allRules = append(allRules, Rule{Rule: *internalRules.AuthenticatedURL(), Tags: []string{TagSensitiveUrl}})
 
 	return allRules, nil
 }

secrets/secrets_test.go

@@ -364,6 +364,31 @@ func TestSecrets(t *testing.T) {
 		Name       string
 		ShouldFind bool
 	}{
+		{
+			Content:    "",
+			Name:       "empty",
+			ShouldFind: false,
+		},
+		{
+			Content:    "mongodb+srv://radar:[email protected]/?retryWrites=true&w=majority",
+			Name:       "Authenticated URL",
+			ShouldFind: true,
+		},
+		{
+			Content:    "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs",
+			Name:       "Authenticated URL",
+			ShouldFind: true,
+		},
+		{
+			Content:    "https://abc:[email protected]",
+			Name:       "Basic Authenticated URL",
+			ShouldFind: true,
+		},
+		{
+			Content:    "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",
+			Name:       "GitHub Personal Access Token",
+			ShouldFind: true,
+		},
 		{
 			Content: "AKCp8jRRiQSAbghbuZmHKZcaKGEqbAASGH2SAb3rxXJQsSq9dGga8gFXe6aHpcRmzuHxN6oaT",
 			Name:    "JFROG Secret without keyword",