feat: upgrade gitleaks to v8.18.0 with new rules (#183)

Added rules:

- Authress Service Client Access Key
- Defined Networking API token
- OpenAI API Key
- Snyk API token

Added script to compare our rules with the rules in the latest
_gitleaks_ version.

---------

Co-authored-by: Baruch Odem <[email protected]>

.ci/check_new_rules.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	regexGitleaksRules = regexp.MustCompile(`configRules\s*=\s*append\(configRules,\s*rules\.([a-zA-Z0-9_]+)\(`)
+	regexGitleaksRules = regexp.MustCompile(`^[^/\n\r]configRules\s*=\s*append\(configRules,\s*rules\.([a-zA-Z0-9_]+)\(`)
 	regex2msRules      = regexp.MustCompile(`allRules\s*=\s*append\(allRules,\s*Rule{Rule:\s*\*rules\.([a-zA-Z0-9_]+)\(\),`)
 )
 
@@ -61,7 +61,7 @@ func main() {
 
 		os.Exit(1)
 	} else {
-		fmt.Printf("No differences found.")
+		fmt.Println("No differences found.")
 		os.Exit(0)
 	}
 }

go.mod

@@ -4,14 +4,14 @@ go 1.20
 
 require (
 	github.com/bwmarrin/discordgo v0.27.1
-	github.com/gitleaks/go-gitdiff v0.8.0
+	github.com/gitleaks/go-gitdiff v0.9.0
 	github.com/rs/zerolog v1.29.0
 	github.com/slack-go/slack v0.12.2
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.8.1
-	github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9
+	github.com/zricethezav/gitleaks/v8 v8.18.0
 	golang.org/x/time v0.1.0
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -68,8 +68,8 @@ github.com/fatih/semgroup v1.2.0/go.mod h1:1KAD4iIYfXjE4U13B48VM4z9QUwV5Tt8O4rS8
 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
-github.com/gitleaks/go-gitdiff v0.8.0 h1:7aExTZm+K/M/EQKOyYcub8rIAdWK6ONxPGuRzxmWW+0=
-github.com/gitleaks/go-gitdiff v0.8.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
+github.com/gitleaks/go-gitdiff v0.9.0 h1:SHAU2l0ZBEo8g82EeFewhVy81sb7JCxW76oSPtR/Nqg=
+github.com/gitleaks/go-gitdiff v0.9.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -227,8 +227,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9 h1:gw0iPgtVuWBW1XQoZed9Y0rWaZ9la1qOooa6aRHsEFo=
-github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9/go.mod h1:/0z7cslO7d0y29YRvHgYefeTu7UIqOmx95A4wMhcQtE=
+github.com/zricethezav/gitleaks/v8 v8.18.0 h1:+zXcDpHATT9E/eA9UZqcKNW/O1mg882NLmO/6z4CFK0=
+github.com/zricethezav/gitleaks/v8 v8.18.0/go.mod h1:JulwKdEMpiOxVFQxZFFixY51QzDZPn1xJ1/p7YqX4hQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

lib/channels.go

@@ -0,0 +1,12 @@
+package lib
+
+import "sync"
+
+func BindChannels[T any](source <-chan T, dest chan<- T, wg *sync.WaitGroup) {
+	if wg != nil {
+		defer wg.Done()
+	}
+	for item := range source {
+		dest <- item
+	}
+}

plugins/git.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/checkmarx/2ms/lib"
 	"github.com/gitleaks/go-gitdiff/gitdiff"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
@@ -68,13 +69,10 @@ func (p *GitPlugin) buildScanOptions() string {
 }
 
 func (p *GitPlugin) scanGit(path string, scanOptions string, itemsChan chan Item, errChan chan error) {
-	fileChan, err := git.GitLog(path, scanOptions)
-	if err != nil {
-		errChan <- fmt.Errorf("error while scanning git repository: %w", err)
-	}
-	log.Debug().Msgf("scanned git repository: %s", path)
+	diffs, close := p.readGitLog(path, scanOptions, errChan)
+	defer close()
 
-	for file := range fileChan {
+	for file := range diffs {
 		log.Debug().Msgf("file: %s; Commit: %s", file.NewName, file.PatchHeader.Title)
 		if file.IsBinary || file.IsDelete {
 			continue
@@ -97,6 +95,25 @@ func (p *GitPlugin) scanGit(path string, scanOptions string, itemsChan chan Item
 	}
 }
 
+func (p *GitPlugin) readGitLog(path string, scanOptions string, errChan chan error) (<-chan *gitdiff.File, func()) {
+	gitLog, err := git.NewGitLogCmd(path, scanOptions)
+	if err != nil {
+		errChan <- fmt.Errorf("error while scanning git repository: %w", err)
+	}
+	wait := func() {
+		err := gitLog.Wait()
+		if err != nil {
+			errChan <- fmt.Errorf("error while waiting for git log to finish: %w", err)
+		}
+	}
+	log.Debug().Msgf("scanning git repository: %s", path)
+
+	p.WaitGroup.Add(1)
+	go lib.BindChannels[error](gitLog.ErrCh(), errChan, p.WaitGroup)
+
+	return gitLog.DiffFilesCh(), wait
+}
+
 func validGitRepoArgs(cmd *cobra.Command, args []string) error {
 	stat, err := os.Stat(args[0])
 	if err != nil {

secrets/rules/rules.go

@@ -41,6 +41,7 @@ func getDefaultRules() *[]Rule {
 	allRules = append(allRules, Rule{Rule: *rules.AsanaClientID(), Tags: []string{TagClientId}})
 	allRules = append(allRules, Rule{Rule: *rules.AsanaClientSecret(), Tags: []string{TagClientSecret}})
 	allRules = append(allRules, Rule{Rule: *rules.Atlassian(), Tags: []string{TagApiToken}})
+	allRules = append(allRules, Rule{Rule: *rules.Authress(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.AWS(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.BitBucketClientID(), Tags: []string{TagClientId}})
 	allRules = append(allRules, Rule{Rule: *rules.BitBucketClientSecret(), Tags: []string{TagClientSecret}})
@@ -55,6 +56,7 @@ func getDefaultRules() *[]Rule {
 	allRules = append(allRules, Rule{Rule: *rules.Contentful(), Tags: []string{TagApiToken}})
 	allRules = append(allRules, Rule{Rule: *rules.Databricks(), Tags: []string{TagApiToken}})
 	allRules = append(allRules, Rule{Rule: *rules.DatadogtokenAccessToken(), Tags: []string{TagAccessToken, TagClientId}})
+	allRules = append(allRules, Rule{Rule: *rules.DefinedNetworkingAPIToken(), Tags: []string{TagApiToken}})
 	allRules = append(allRules, Rule{Rule: *rules.DigitalOceanPAT(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.DigitalOceanOAuthToken(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.DigitalOceanRefreshToken(), Tags: []string{TagRefreshToken}})
@@ -129,6 +131,7 @@ func getDefaultRules() *[]Rule {
 	allRules = append(allRules, Rule{Rule: *rules.NPM(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.NytimesAccessToken(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.OktaAccessToken(), Tags: []string{TagAccessToken}})
+	allRules = append(allRules, Rule{Rule: *rules.OpenAI(), Tags: []string{TagApiKey}})
 	allRules = append(allRules, Rule{Rule: *rules.PlaidAccessID(), Tags: []string{TagClientId}})
 	allRules = append(allRules, Rule{Rule: *rules.PlaidSecretKey(), Tags: []string{TagSecretKey}})
 	allRules = append(allRules, Rule{Rule: *rules.PlaidAccessToken(), Tags: []string{TagApiToken}})
@@ -169,6 +172,7 @@ func getDefaultRules() *[]Rule {
 	allRules = append(allRules, Rule{Rule: *rules.SquareSpaceAccessToken(), Tags: []string{TagAccessToken}})
 	allRules = append(allRules, Rule{Rule: *rules.SumoLogicAccessID(), Tags: []string{TagAccessId}})
 	allRules = append(allRules, Rule{Rule: *rules.SumoLogicAccessToken(), Tags: []string{TagAccessToken}})
+	allRules = append(allRules, Rule{Rule: *rules.Snyk(), Tags: []string{TagApiKey}})
 	allRules = append(allRules, Rule{Rule: *rules.TeamsWebhook(), Tags: []string{TagWebhook}})
 	allRules = append(allRules, Rule{Rule: *rules.TelegramBotToken(), Tags: []string{TagApiToken}})
 	allRules = append(allRules, Rule{Rule: *rules.TravisCIAccessToken(), Tags: []string{TagAccessToken}})