From a50ba8a2e83147a6b5e9cfcd2eddcc57ac5bd4b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Cunha=20Martins?= Date: Fri, 20 Oct 2023 21:10:35 +0100 Subject: [PATCH 1/3] Fix the regex to the detection serverless + improve detection of files Ansible With this Pr: - improve the regex to detection of serverless files -improve logic to detection of Ansible files --- pkg/analyzer/analyzer.go | 20 ++++++++++++++++---- pkg/analyzer/analyzer_test.go | 31 +++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/pkg/analyzer/analyzer.go b/pkg/analyzer/analyzer.go index 7830cd414bf..53624544c52 100644 --- a/pkg/analyzer/analyzer.go +++ b/pkg/analyzer/analyzer.go @@ -60,7 +60,7 @@ var ( pulumiRuntimeRegex = regexp.MustCompile(`runtime\s*:`) pulumiResourcesRegex = regexp.MustCompile(`resources\s*:`) serverlessServiceRegex = regexp.MustCompile(`service\s*:`) - serverlessProviderRegex = regexp.MustCompile(`provider\s*:`) + serverlessProviderRegex = regexp.MustCompile(`(^|\n)provider\s*:`) cicdOnRegex = regexp.MustCompile(`\s*on:\s*`) cicdJobsRegex = regexp.MustCompile(`\s*jobs:\s*`) cicdStepsRegex = regexp.MustCompile(`\s*steps:\s*`) @@ -100,6 +100,9 @@ var ( "pulumi": {"pulumi"}, "serverlessfw": {"serverlessfw"}, } + listKeywordsAnsible = []string{"playbooks", "name", "gather_facts", + "hosts", "tasks", "become", "with_items", "with_dict", + "when", "become_pass", "become_exe", "become_flags"} ) const ( @@ -543,9 +546,18 @@ func checkYamlPlatform(content []byte, path string) string { } } - // Since Ansible has no defining property - // and no other type matched for YAML file extension, assume the file type is Ansible - return ansible + // check if the file contains some keywords related with Ansiable + isAnsible := false + for _, keyword := range listKeywordsAnsible { + if _, ok := yamlContent[keyword]; ok { + isAnsible = true + } + } + + if isAnsible { + return ansible + } + return "" } // computeValues computes expected Lines of Code to be scanned from locCount channel diff --git a/pkg/analyzer/analyzer_test.go b/pkg/analyzer/analyzer_test.go index 8bc7389069d..843e4140d1d 100644 --- a/pkg/analyzer/analyzer_test.go +++ b/pkg/analyzer/analyzer_test.go @@ -28,10 +28,11 @@ func TestAnalyzer_Analyze(t *testing.T) { wantExclude: []string{ filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json"), filepath.FromSlash("../../test/fixtures/analyzer_test/pnpm-lock.yaml"), - filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink")}, + filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink"), + filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml")}, typesFromFlag: []string{""}, excludeTypesFromFlag: []string{""}, - wantLOC: 835, + wantLOC: 834, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -193,11 +194,11 @@ func TestAnalyzer_Analyze(t *testing.T) { paths: []string{ filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml"), }, - wantTypes: []string{"ansible"}, - wantExclude: []string{}, + wantTypes: []string{}, + wantExclude: []string{filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml")}, typesFromFlag: []string{""}, excludeTypesFromFlag: []string{""}, - wantLOC: 1, + wantLOC: 0, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -208,17 +209,25 @@ func TestAnalyzer_Analyze(t *testing.T) { wantTypes: []string{"ansible", "pulumi"}, wantExclude: []string{ filepath.FromSlash("../../test/fixtures/analyzer_test/azureResourceManager.json"), + filepath.FromSlash("../../test/fixtures/analyzer_test/cloudformation.yaml"), + filepath.FromSlash("../../test/fixtures/analyzer_test/crossplane.yaml"), + filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink"), + filepath.FromSlash("../../test/fixtures/analyzer_test/docker-compose.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/gdm.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/helm/Chart.yaml"), + filepath.FromSlash("../../test/fixtures/analyzer_test/helm/templates/service.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/helm/values.yaml"), + filepath.FromSlash("../../test/fixtures/analyzer_test/k8s.yaml"), + filepath.FromSlash("../../test/fixtures/analyzer_test/knative.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json"), filepath.FromSlash("../../test/fixtures/analyzer_test/openAPI.json"), filepath.FromSlash("../../test/fixtures/analyzer_test/openAPI_test/openAPI.json"), + filepath.FromSlash("../../test/fixtures/analyzer_test/openAPI_test/openAPI.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/pnpm-lock.yaml"), - filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink")}, + filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml")}, typesFromFlag: []string{"ansible", "pulumi"}, excludeTypesFromFlag: []string{""}, - wantLOC: 533, + wantLOC: 416, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -247,10 +256,11 @@ func TestAnalyzer_Analyze(t *testing.T) { wantExclude: []string{ filepath.FromSlash("../../test/fixtures/analyzer_test/pnpm-lock.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json"), - filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink")}, + filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink"), + filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml")}, typesFromFlag: []string{""}, excludeTypesFromFlag: []string{""}, - wantLOC: 835, + wantLOC: 834, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, @@ -263,10 +273,11 @@ func TestAnalyzer_Analyze(t *testing.T) { filepath.FromSlash("../../test/fixtures/analyzer_test/pnpm-lock.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/not_openapi.json"), filepath.FromSlash("../../test/fixtures/analyzer_test/dead_symlink"), + filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml"), }, typesFromFlag: []string{""}, excludeTypesFromFlag: []string{""}, - wantLOC: 835, + wantLOC: 834, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, From 731f7ff70eb3bcceec2b667830a4cb3b59c55838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Cunha=20Martins?= Date: Mon, 23 Oct 2023 12:30:12 +0100 Subject: [PATCH 2/3] improve logic before search for keywords make a previous search for playbooks --- pkg/analyzer/analyzer.go | 31 ++++++++++++++++++++++--------- pkg/analyzer/analyzer_test.go | 5 +++-- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/pkg/analyzer/analyzer.go b/pkg/analyzer/analyzer.go index 53624544c52..8c89bef1f61 100644 --- a/pkg/analyzer/analyzer.go +++ b/pkg/analyzer/analyzer.go @@ -100,9 +100,10 @@ var ( "pulumi": {"pulumi"}, "serverlessfw": {"serverlessfw"}, } - listKeywordsAnsible = []string{"playbooks", "name", "gather_facts", + listKeywordsAnsible = []string{"name", "gather_facts", "hosts", "tasks", "become", "with_items", "with_dict", "when", "become_pass", "become_exe", "become_flags"} + playBooks = "playbooks" ) const ( @@ -547,19 +548,31 @@ func checkYamlPlatform(content []byte, path string) string { } // check if the file contains some keywords related with Ansiable - isAnsible := false - for _, keyword := range listKeywordsAnsible { - if _, ok := yamlContent[keyword]; ok { - isAnsible = true - } - } - - if isAnsible { + if checkForAnsible(yamlContent) { return ansible } return "" } +func checkForAnsible(yamlContent model.Document) bool { + isAnsible := false + if play := yamlContent[playBooks]; play != nil { + if listOfPlayBooks, ok := play.([]interface{}); ok { + for _, value := range listOfPlayBooks { + castingValue, ok := value.(map[string]interface{}) + if ok { + for _, keyword := range listKeywordsAnsible { + if _, ok := castingValue[keyword]; ok { + isAnsible = true + } + } + } + } + } + } + return isAnsible +} + // computeValues computes expected Lines of Code to be scanned from locCount channel // and creates the types and unwanted slices from the channels removing any duplicates func computeValues(types, unwanted chan string, locCount chan int, done chan bool) (typesS, unwantedS []string, locTotal int) { diff --git a/pkg/analyzer/analyzer_test.go b/pkg/analyzer/analyzer_test.go index 843e4140d1d..62ad13a89bf 100644 --- a/pkg/analyzer/analyzer_test.go +++ b/pkg/analyzer/analyzer_test.go @@ -224,10 +224,11 @@ func TestAnalyzer_Analyze(t *testing.T) { filepath.FromSlash("../../test/fixtures/analyzer_test/openAPI_test/openAPI.json"), filepath.FromSlash("../../test/fixtures/analyzer_test/openAPI_test/openAPI.yaml"), filepath.FromSlash("../../test/fixtures/analyzer_test/pnpm-lock.yaml"), - filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml")}, + filepath.FromSlash("../../test/fixtures/analyzer_test/undetected.yaml"), + filepath.FromSlash("../../test/fixtures/analyzer_test/github.yaml")}, typesFromFlag: []string{"ansible", "pulumi"}, excludeTypesFromFlag: []string{""}, - wantLOC: 416, + wantLOC: 374, wantErr: false, gitIgnoreFileName: "", excludeGitIgnore: false, From 9d12e0c4d68d4d99de1e80916d62eb3c4a6852f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Cunha=20Martins?= <75611928+JoaoCxMartins@users.noreply.github.com> Date: Tue, 31 Oct 2023 09:30:19 +0000 Subject: [PATCH 3/3] Update pkg/analyzer/analyzer.go Co-authored-by: Gabriel --- pkg/analyzer/analyzer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/analyzer/analyzer.go b/pkg/analyzer/analyzer.go index 8c89bef1f61..82a1fc95b81 100644 --- a/pkg/analyzer/analyzer.go +++ b/pkg/analyzer/analyzer.go @@ -547,7 +547,7 @@ func checkYamlPlatform(content []byte, path string) string { } } - // check if the file contains some keywords related with Ansiable + // check if the file contains some keywords related with Ansible if checkForAnsible(yamlContent) { return ansible }