From f45589fdc3df6afc68613ef580b56612a431f32d Mon Sep 17 00:00:00 2001 From: attiasas Date: Sun, 15 Oct 2023 14:54:33 +0300 Subject: [PATCH 01/27] Start implement recursive scan --- xray/commands/audit/scarunner.go | 65 ++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index ae4a25b1a..ef9cbeb3d 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -24,6 +24,71 @@ import ( "time" ) +type ScaScanResult struct { + workingDir string + Te + err error +} + +func scaScan(params *AuditParams, results *Results) (err error) { + currentWorkingDir, err := os.Getwd() + if errorutils.CheckError(err) != nil { + return + } + defer func() { + // Make sure to return to the original working directory, each scan may change it. + err = errors.Join(err, os.Chdir(currentWorkingDir)) + }() + for _, wd := range getDirectoriesToScan(currentWorkingDir, params) { + log.Info("Running SCA scan for vulnerable dependencies scan in", wd, "directory...") + wdScanErr := runScaScanInWorkingDir(params, results, wd) + if wdScanErr != nil { + err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s\n", wd, wdScanErr.Error())) + continue + } + } + return +} + +func getDirectoriesToScan(currentWorkingDir string, params *AuditParams) []string { + workingDirs := datastructures.MakeSet[string]() + if len(params.workingDirs) == 0 { + workingDirs.Add(currentWorkingDir) + } + for _, wd := range params.workingDirs { + workingDirs.Add(wd) + } + return workingDirs.ToSlice() +} + +func runScaScanInWorkingDir(params *AuditParams, results *Results, workingDir string) (err error) { + // Change to the working directory. + err = os.Chdir(workingDir) + if err != nil { + return + } + // Prepare the working directory information for the scan. + technologies := getWorkingDirTechnologies(workingDir, params) + serverDetails, err := params.ServerDetails() + if err != nil { + return + } +} + +func getWorkingDirTechnologies(workingDir string, params *AuditParams) (technologies []string) { + //coreutils.Technology + requestedTechnologies := params.Technologies() + if len(requestedTechnologies) != 0 { + technologies = requestedTechnologies + } else { + technologies = coreutils.DetectedTechnologiesList() + } + if len(technologies) == 0 { + log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") + return + } +} + func runScaScan(params *AuditParams, results *Results) (err error) { rootDir, err := os.Getwd() if errorutils.CheckError(err) != nil { From aa27d55cefb3c864cbeda212afc04b0c80d26f25 Mon Sep 17 00:00:00 2001 From: attiasas Date: Mon, 16 Oct 2023 16:57:31 +0300 Subject: [PATCH 02/27] continue to implement recursive scan --- utils/coreutils/techutils.go | 4 + xray/commands/audit/audit.go | 30 ++++-- xray/commands/audit/scarunner.go | 178 +++++++++++++++++++++++++------ xray/utils/analyzermanager.go | 29 +++-- xray/utils/results.go | 46 ++++++++ 5 files changed, 229 insertions(+), 58 deletions(-) create mode 100644 xray/utils/results.go diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 3880da6ad..64da5ccc3 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -191,6 +191,10 @@ func DetectedTechnologiesList() (technologies []string) { return techStringsList } +func DetectTechnologiesDescriptors(path string, requestedTechnologies []Technology) (t map[Technology][]string) { + return +} + // DetectTechnologies tries to detect all technologies types according to the files in the given path. // 'isCiSetup' will limit the search of possible techs to Maven, Gradle, and npm. // 'recursive' will determine if the search will be limited to files in the root path or not. diff --git a/xray/commands/audit/audit.go b/xray/commands/audit/audit.go index 1347e5357..1b798a203 100644 --- a/xray/commands/audit/audit.go +++ b/xray/commands/audit/audit.go @@ -136,23 +136,31 @@ func (auditCmd *AuditCommand) CommandName() string { return "generic_audit" } -type Results struct { - IsMultipleRootProject bool - ScaError error - JasError error - ExtendedScanResults *xrayutils.ExtendedScanResults -} +// type ScaScanResult struct { +// Technology coreutils.Technology +// XrayResults []services.ScanResponse +// Descriptors []string +// } -func NewAuditResults() *Results { - return &Results{ExtendedScanResults: &xrayutils.ExtendedScanResults{}} -} +// type Results struct { +// ScaResults []ScaScanResult +// IsMultipleRootProject bool +// ScaError error + +// ExtendedScanResults *xrayutils.ExtendedScanResults +// JasError error +// } + +// func NewAuditResults() *Results { +// return &Results{ExtendedScanResults: &xrayutils.ExtendedScanResults{}} +// } // Runs an audit scan based on the provided auditParams. // Returns an audit Results object containing all the scan results. // If the current server is entitled for JAS, the advanced security results will be included in the scan results. -func RunAudit(auditParams *AuditParams) (results *Results, err error) { +func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) { // Initialize Results struct - results = NewAuditResults() + results = xrayutils.NewAuditResults() serverDetails, err := auditParams.ServerDetails() if err != nil { diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index ef9cbeb3d..b4a6e5026 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -4,8 +4,12 @@ import ( "encoding/json" "errors" "fmt" + "os" + "time" + "github.com/jfrog/build-info-go/utils/pythonutils" "github.com/jfrog/gofrog/datastructures" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca" _go "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca/go" @@ -19,77 +23,187 @@ import ( clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" + "github.com/jfrog/jfrog-client-go/xray/services" xrayCmdUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" - "os" - "time" ) -type ScaScanResult struct { - workingDir string - Te - err error -} - -func scaScan(params *AuditParams, results *Results) (err error) { +func scaScan(params *AuditParams, results *xrayutils.Results) (err error) { + // Prepare currentWorkingDir, err := os.Getwd() if errorutils.CheckError(err) != nil { return } + serverDetails, err := params.ServerDetails() + if err != nil { + return + } + scans := getScaScansToPreform(currentWorkingDir, params) + if len(scans) == 0 { + log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") + return + } + // Make sure to return to the original working directory, executeScaScan may change it. defer func() { - // Make sure to return to the original working directory, each scan may change it. err = errors.Join(err, os.Chdir(currentWorkingDir)) }() - for _, wd := range getDirectoriesToScan(currentWorkingDir, params) { - log.Info("Running SCA scan for vulnerable dependencies scan in", wd, "directory...") - wdScanErr := runScaScanInWorkingDir(params, results, wd) - if wdScanErr != nil { - err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s\n", wd, wdScanErr.Error())) + for _, scan := range scans { + // Run the scan. + log.Info("Running SCA scan for vulnerable dependencies in", scan.WorkingDirectory, "directory...") + if wdScanErr := executeScaScan(serverDetails, params, &scan); wdScanErr != nil { + err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s\n", scan.WorkingDirectory, wdScanErr.Error())) continue } + if len(scan.XrayResults) > 0 { + // Add the scan results to the results. + results.ScaResults = append(results.ScaResults, scan) + } } return } +func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansToPreform []xrayutils.ScaScanResult) { + directories := getDirectoriesToScan(currentWorkingDir, params) + for _, wd := range directories { + technologiesToDescriptors := coreutils.DetectTechnologiesDescriptors(wd, getTechnologiesToDetect(params)) + + scansToPreform = append(scansToPreform, xrayutils.ScaScanResult{WorkingDirectory: wd}) + } +} + func getDirectoriesToScan(currentWorkingDir string, params *AuditParams) []string { workingDirs := datastructures.MakeSet[string]() - if len(params.workingDirs) == 0 { - workingDirs.Add(currentWorkingDir) - } for _, wd := range params.workingDirs { workingDirs.Add(wd) } + if workingDirs.Size() == 0 { + workingDirs.Add(currentWorkingDir) + } return workingDirs.ToSlice() } -func runScaScanInWorkingDir(params *AuditParams, results *Results, workingDir string) (err error) { - // Change to the working directory. - err = os.Chdir(workingDir) +// Preform the SCA scan for the given scan information. +// This method may change the working directory to the scan's working directory. +func executeScaScan(serverDetails *config.ServerDetails, params *AuditParams, scan *xrayutils.ScaScanResult) (err error) { + if scan.Technology == coreutils.Dotnet { + return + } + // Get the dependency tree for the technology in the working directory. + if err = os.Chdir(scan.WorkingDirectory); err != nil { + return + } + flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology) + if techErr != nil { + return fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", scan.Technology, techErr.Error()) + } + if flattenTree == nil || len(flattenTree.Nodes) == 0 { + return errors.New("no dependencies were found. Please try to build your project and re-run the audit command") + } + // Scan the dependency tree. + scanResults, xrayErr := runScaWithTech(scan.Technology, params, serverDetails, flattenTree, fullDependencyTrees) + if xrayErr != nil { + err = errors.Join(err, ) + return fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s\n", scan.Technology, xrayErr.Error()) + } + addThirdPartyDependenciesToParams(params, scan.Technology, flattenTree, fullDependencyTrees) + scan.XrayResults = append(scan.XrayResults, scanResults...) + return +} + +func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []services.ScanResponse, err error) { + scanGraphParams := scangraph.NewScanGraphParams(). + SetServerDetails(serverDetails). + SetXrayGraphScanParams(params.xrayGraphScanParams). + SetXrayVersion(params.xrayVersion). + SetFixableOnly(params.fixableOnly). + SetSeverityLevel(params.minSeverityFilter) + techResults, err = sca.RunXrayDependenciesTreeScanGraph(flatTree, params.Progress(), tech, scanGraphParams) if err != nil { return } + techResults = sca.BuildImpactPathsForScanResponse(techResults, fullDependencyTrees) + return +} + +func runScaScanInWorkingDir(params *AuditParams, results *xrayutils.Results, workingDir string) (err error) { // Prepare the working directory information for the scan. - technologies := getWorkingDirTechnologies(workingDir, params) + technologies := coreutils.DetectTechnologiesDescriptors(workingDir, getTechnologiesToDetect(params)) + if len(technologies) == 0 { + log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") + return + } serverDetails, err := params.ServerDetails() if err != nil { return } + // Run the scan for each technology. + for tech, detectedDescriptors := range technologies { + if tech == coreutils.Dotnet { + continue + } + // Get the dependency tree for the technology. + flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, tech) + if techErr != nil { + err = errors.Join(err, fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", tech, techErr.Error())) + continue + } + if flattenTree == nil || len(flattenTree.Nodes) == 0 { + err = errors.Join(err, errors.New("no dependencies were found. Please try to build your project and re-run the audit command")) + continue + } + // Scan the dependency tree. + scanResults, xrayErr := runScaOnTech(tech, params, serverDetails, flattenTree, fullDependencyTrees, results) + if xrayErr != nil { + err = errors.Join(err, fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s\n", tech, xrayErr.Error())) + continue + } + addThirdPartyDependenciesToParams(params, tech, flattenTree, fullDependencyTrees) + results.ScaResults = append(results.ScaResults, xrayutils.ScaScanResult{Technology: tech, XrayResults: scanResults, Descriptors: detectedDescriptors}) + } + return } -func getWorkingDirTechnologies(workingDir string, params *AuditParams) (technologies []string) { - //coreutils.Technology - requestedTechnologies := params.Technologies() - if len(requestedTechnologies) != 0 { - technologies = requestedTechnologies +func getTechnologiesToDetect(params *AuditParams) (technologies []coreutils.Technology) { + if len(params.Technologies()) != 0 { + technologies = coreutils.ToTechnologies(params.Technologies()) } else { - technologies = coreutils.DetectedTechnologiesList() + technologies = coreutils.GetAllTechnologiesList() } - if len(technologies) == 0 { - log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") + return +} + +func addThirdPartyDependenciesToParams(params *AuditParams, tech coreutils.Technology, flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) { + var dependenciesForApplicabilityScan []string + if shouldUseAllDependencies(params.thirdPartyApplicabilityScan, tech) { + dependenciesForApplicabilityScan = getDirectDependenciesFromTree([]*xrayCmdUtils.GraphNode{flatTree}) + } else { + dependenciesForApplicabilityScan = getDirectDependenciesFromTree(fullDependencyTrees) + } + params.AppendDependenciesForApplicabilityScan(dependenciesForApplicabilityScan) +} + +func runScaOnTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, results *xrayutils.Results) (techResults []services.ScanResponse, err error) { + scanGraphParams := scangraph.NewScanGraphParams(). + SetServerDetails(serverDetails). + SetXrayGraphScanParams(params.xrayGraphScanParams). + SetXrayVersion(params.xrayVersion). + SetFixableOnly(params.fixableOnly). + SetSeverityLevel(params.minSeverityFilter) + techResults, err = sca.RunXrayDependenciesTreeScanGraph(flatTree, params.Progress(), tech, scanGraphParams) + if err != nil { return } + techResults = sca.BuildImpactPathsForScanResponse(techResults, fullDependencyTrees) + + // results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...) + // if !results.IsMultipleRootProject { + // results.IsMultipleRootProject = len(fullDependencyTrees) > 1 + // } + + // results.ExtendedScanResults.ScannedTechnologies = append(results.ExtendedScanResults.ScannedTechnologies, tech) + return } -func runScaScan(params *AuditParams, results *Results) (err error) { +func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { rootDir, err := os.Getwd() if errorutils.CheckError(err) != nil { return @@ -110,7 +224,7 @@ func runScaScan(params *AuditParams, results *Results) (err error) { } // Audits the project found in the current directory using Xray. -func runScaScanOnWorkingDir(params *AuditParams, results *Results, workingDir, rootDir string) (err error) { +func runScaScanOnWorkingDir(params *AuditParams, results *xrayutils.Results, workingDir, rootDir string) (err error) { err = os.Chdir(workingDir) if err != nil { return diff --git a/xray/utils/analyzermanager.go b/xray/utils/analyzermanager.go index cf894d660..7f3ec48b3 100644 --- a/xray/utils/analyzermanager.go +++ b/xray/utils/analyzermanager.go @@ -14,8 +14,6 @@ import ( "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" - "github.com/jfrog/jfrog-client-go/xray/services" - "github.com/owenrumney/go-sarif/v2/sarif" ) const ( @@ -78,21 +76,22 @@ var exitCodeErrorsMap = map[int]string{ unsupportedOsExitCode: "got unsupported operating system error from analyzer manager", } -type ExtendedScanResults struct { - XrayResults []services.ScanResponse - XrayVersion string - ScannedTechnologies []coreutils.Technology +// type ExtendedScanResults struct { +// XrayResults []services.ScanResponse +// XrayVersion string +// ScannedTechnologies []coreutils.Technology - ApplicabilityScanResults []*sarif.Run - SecretsScanResults []*sarif.Run - IacScanResults []*sarif.Run - SastScanResults []*sarif.Run - EntitledForJas bool -} +// // DependenciesResults [] +// ApplicabilityScanResults []*sarif.Run +// SecretsScanResults []*sarif.Run +// IacScanResults []*sarif.Run +// SastScanResults []*sarif.Run +// EntitledForJas bool +// } -func (e *ExtendedScanResults) getXrayScanResults() []services.ScanResponse { - return e.XrayResults -} +// func (e *ExtendedScanResults) getXrayScanResults() []services.ScanResponse { +// return e.XrayResults +// } type AnalyzerManager struct { AnalyzerManagerFullPath string diff --git a/xray/utils/results.go b/xray/utils/results.go new file mode 100644 index 000000000..38a11701d --- /dev/null +++ b/xray/utils/results.go @@ -0,0 +1,46 @@ +package utils + +import ( + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-client-go/xray/services" + "github.com/owenrumney/go-sarif/v2/sarif" +) + +type Results struct { + ScaResults []ScaScanResult + XrayVersion string + ScaError error + + IsMultipleRootProject bool // TODO: remove this + ExtendedScanResults *ExtendedScanResults + JasError error +} + +func NewAuditResults() *Results { + return &Results{ExtendedScanResults: &ExtendedScanResults{}} +} + +type ScaScanResult struct { + Technology coreutils.Technology + WorkingDirectory string + Descriptors []string + XrayResults []services.ScanResponse + + // IsMultipleRootProject bool +} + +type ExtendedScanResults struct { + XrayResults []services.ScanResponse // TODO: remove this + XrayVersion string // TODO: remove this + ScannedTechnologies []coreutils.Technology // TODO: remove this + + ApplicabilityScanResults []*sarif.Run + SecretsScanResults []*sarif.Run + IacScanResults []*sarif.Run + SastScanResults []*sarif.Run + EntitledForJas bool +} + +func (e *ExtendedScanResults) getXrayScanResults() []services.ScanResponse { + return e.XrayResults +} \ No newline at end of file From c7405abd2c6ca0099921cbe745ec7fd0934ce75b Mon Sep 17 00:00:00 2001 From: attiasas Date: Wed, 18 Oct 2023 16:01:50 +0300 Subject: [PATCH 03/27] fix --- xray/commands/audit/scarunner.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index b4a6e5026..ac3f38b28 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -64,10 +64,11 @@ func scaScan(params *AuditParams, results *xrayutils.Results) (err error) { func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansToPreform []xrayutils.ScaScanResult) { directories := getDirectoriesToScan(currentWorkingDir, params) for _, wd := range directories { - technologiesToDescriptors := coreutils.DetectTechnologiesDescriptors(wd, getTechnologiesToDetect(params)) + // technologiesToDescriptors := coreutils.DetectTechnologiesDescriptors(wd, getTechnologiesToDetect(params)) scansToPreform = append(scansToPreform, xrayutils.ScaScanResult{WorkingDirectory: wd}) } + return } func getDirectoriesToScan(currentWorkingDir string, params *AuditParams) []string { From eed6fd20595f18aa4bf23074179ee0b7a40269ac Mon Sep 17 00:00:00 2001 From: attiasas Date: Thu, 19 Oct 2023 17:54:25 +0300 Subject: [PATCH 04/27] implement recursive and descriptors --- utils/coreutils/techutils.go | 135 +++++++++++++++++++++++- xray/commands/audit/audit.go | 14 +-- xray/commands/audit/jasrunner.go | 14 ++- xray/commands/audit/jasrunner_test.go | 10 +- xray/commands/audit/scarunner.go | 143 +++++++++++++++----------- xray/commands/scan/buildscan.go | 4 +- xray/commands/scan/scan.go | 5 +- xray/utils/results.go | 90 +++++++++++++--- xray/utils/resultstable.go | 34 +++--- xray/utils/resultstable_test.go | 2 +- xray/utils/resultwriter.go | 70 +++++++------ 11 files changed, 372 insertions(+), 149 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 64da5ccc3..1a1edf79a 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -2,11 +2,15 @@ package coreutils import ( "fmt" - "github.com/jfrog/jfrog-client-go/utils/errorutils" - "github.com/jfrog/jfrog-client-go/utils/log" "os" + "path/filepath" "strings" + "github.com/jfrog/jfrog-client-go/artifactory/services/fspatterns" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/log" + + "golang.org/x/exp/maps" "golang.org/x/text/cases" "golang.org/x/text/language" @@ -110,6 +114,7 @@ var technologiesData = map[Technology]TechData{ Poetry: { packageType: Pypi, indicators: []string{"pyproject.toml", "poetry.lock"}, + packageDescriptors: []string{"pyproject.toml"}, packageInstallationCommand: "add", packageVersionOperator: "==", applicabilityScannable: true, @@ -179,7 +184,11 @@ func DetectedTechnologiesList() (technologies []string) { if errorutils.CheckError(err) != nil { return } - detectedTechnologies, err := DetectTechnologies(wd, false, false) + return DetectedTechnologiesListInPath(wd, false) +} + +func DetectedTechnologiesListInPath(path string, recursive bool) (technologies []string) { + detectedTechnologies, err := DetectTechnologies(path, false, recursive) if err != nil { return } @@ -191,10 +200,109 @@ func DetectedTechnologiesList() (technologies []string) { return techStringsList } -func DetectTechnologiesDescriptors(path string, requestedTechnologies []Technology) (t map[Technology][]string) { +// If recursive is true, the search will not be limited to files in the root path. +// If requestedTechs is empty, all technologies will be checked. +// If excludePathPattern is not empty, files/directories that match the wildcard pattern will be excluded from the search. +func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs []string, excludePathPattern string) (technologiesDetected map[Technology]map[string][]string) { + filesList, err := fspatterns.ListFiles(path, recursive, false, true, excludePathPattern) + if err != nil { + return + } + technologiesDetected = mapIndicatorsToTechnologies(mapWorkingDirectoriesToIndicators(filesList), ToTechnologies(requestedTechs)) + log.Debug(fmt.Sprintf("Detected %d technologies at %s: %s.", len(technologiesDetected), path, maps.Keys(technologiesDetected))) return } +func mapWorkingDirectoriesToIndicators(files []string) (workingDirectoryToIndicators map[string][]string) { + workingDirectoryToIndicators = make(map[string][]string) + for _, path := range files { + directory := filepath.Dir(path) + for _, techData := range technologiesData { + for _, indicator := range techData.indicators { + if strings.HasSuffix(path, indicator) { + workingDirectoryToIndicators[directory] = append(workingDirectoryToIndicators[directory], path) + } + } + if isDescriptor(path, techData) { + workingDirectoryToIndicators[directory] = append(workingDirectoryToIndicators[directory], path) + } + } + } + log.Debug(fmt.Sprintf("mapped indicators:\n%s", workingDirectoryToIndicators)) + return +} + +func isDescriptor(path string, techData TechData) bool { + for _, descriptor := range techData.packageDescriptors { + if strings.HasSuffix(path, descriptor) { + return true + } + } + return false +} + +func mapIndicatorsToTechnologies(workingDirectoryToIndicators map[string][]string, requestedTechs []Technology) (technologiesDetected map[Technology]map[string][]string) { + + technologies := requestedTechs + if len(technologies) == 0 { + technologies = GetAllTechnologiesList() + } + technologiesDetected = make(map[Technology]map[string][]string) + + for _, tech := range technologies { + techWorkingDirs := make(map[string][]string) + for wd, indicators := range workingDirectoryToIndicators { + + + // What to do about no descriptors found and only indicators? + // Implement exclude + // Collect only descriptors + + // Map tech for indicators/descriptors to tech + for _, indicator := range indicators { + // Don't allow working directory if sub directory already exists as key + if isDescriptor(indicator, technologiesData[tech]) && !hasSubDirKey(wd, techWorkingDirs) { + techWorkingDirs[wd] = append(techWorkingDirs[wd], indicator) + } + } + } + if len(techWorkingDirs) > 0 { + // Found indicators/descriptors for technology, add to detected. + technologiesDetected[tech] = techWorkingDirs + } + } + + for _, tech := range requestedTechs { + if _, exist := technologiesDetected[tech]; !exist { + // Requested (forced with flag) technology and not found any indicators/descriptors in detection, add as detected. + technologiesDetected[tech] = map[string][]string{} + } + } + return +} + +func hasSubDirKey(path string, workingDirectoryToIndicators map[string][]string) bool { + for wd := range workingDirectoryToIndicators { + if strings.HasPrefix(path, wd) { + return true + } + } + return false +} + +// func detectTechnologiesDescriptorsByFilePaths(paths []string) (technologiesToDescriptors map[Technology][]string) { +// detected := make(map[Technology][]string) +// for _, path := range paths { +// for techName, techData := range technologiesData { +// for _, descriptor := range techData.packageDescriptors { +// if strings.HasSuffix(path, descriptor) { +// detected[techName] = append(detected[techName], path) +// } +// } +// } +// } +// } + // DetectTechnologies tries to detect all technologies types according to the files in the given path. // 'isCiSetup' will limit the search of possible techs to Maven, Gradle, and npm. // 'recursive' will determine if the search will be limited to files in the root path or not. @@ -209,6 +317,7 @@ func DetectTechnologies(path string, isCiSetup, recursive bool) (map[Technology] if err != nil { return nil, err } + log.Info(fmt.Sprintf("Scanning %d file(s):%s", len(filesList), filesList)) detectedTechnologies := detectTechnologiesByFilePaths(filesList, isCiSetup) return detectedTechnologies, nil } @@ -245,6 +354,24 @@ func detectTechnologiesByFilePaths(paths []string, isCiSetup bool) (detected map return detected } +// func t(path string, techData TechData) (exclude bool, detected bool) { +// // If the project contains a file/directory with a name that ends with an excluded suffix, then this technology is excluded. +// for _, excludeFile := range techData.exclude { +// if strings.HasSuffix(path, excludeFile) { +// return true, false +// } +// } +// // If this technology was already excluded, there's no need to look for indicator files/directories. +// if _, exist := exclude[techName]; !exist { +// // If the project contains a file/directory with a name that ends with the indicator suffix, then the project probably uses this technology. +// for _, indicator := range techData.indicators { +// if strings.HasSuffix(path, indicator) { +// return false, true +// } +// } +// } +// } + // DetectedTechnologiesToSlice returns a string slice that includes all the names of the detected technologies. func DetectedTechnologiesToSlice(detected map[Technology]bool) []string { keys := make([]string, 0, len(detected)) diff --git a/xray/commands/audit/audit.go b/xray/commands/audit/audit.go index 1b798a203..6513c8544 100644 --- a/xray/commands/audit/audit.go +++ b/xray/commands/audit/audit.go @@ -107,10 +107,10 @@ func (auditCmd *AuditCommand) Run() (err error) { messages = []string{coreutils.PrintTitle("The ‘jf audit’ command also supports JFrog Advanced Security features, such as 'Contextual Analysis', 'Secret Detection', 'IaC Scan' and ‘SAST’.\nThis feature isn't enabled on your system. Read more - ") + coreutils.PrintLink("https://jfrog.com/xray/")} } // Print Scan results on all cases except if errors accrued on SCA scan and no security/license issues found. - printScanResults := !(auditResults.ScaError != nil && xrayutils.IsEmptyScanResponse(auditResults.ExtendedScanResults.XrayResults)) + printScanResults := !(auditResults.ScaError != nil && !auditResults.IsScaIssuesFound()) if printScanResults { - if err = xrayutils.NewResultsWriter(auditResults.ExtendedScanResults). - SetIsMultipleRootProject(auditResults.IsMultipleRootProject). + if err = xrayutils.NewResultsWriter(auditResults). + SetIsMultipleRootProject(auditResults.IsMultipleProject()). SetIncludeVulnerabilities(auditCmd.IncludeVulnerabilities). SetIncludeLicenses(auditCmd.IncludeLicenses). SetOutputFormat(auditCmd.OutputFormat()). @@ -126,7 +126,7 @@ func (auditCmd *AuditCommand) Run() (err error) { } // Only in case Xray's context was given (!auditCmd.IncludeVulnerabilities), and the user asked to fail the build accordingly, do so. - if auditCmd.Fail && !auditCmd.IncludeVulnerabilities && xrayutils.CheckIfFailBuild(auditResults.ExtendedScanResults.XrayResults) { + if auditCmd.Fail && !auditCmd.IncludeVulnerabilities && xrayutils.CheckIfFailBuild(auditResults.GetScaScansXrayResults()) { err = xrayutils.NewFailBuildError() } return @@ -173,7 +173,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) if err = clientutils.ValidateMinimumVersion(clientutils.Xray, auditParams.xrayVersion, scangraph.GraphScanMinXrayVersion); err != nil { return } - results.ExtendedScanResults.XrayVersion = auditParams.xrayVersion + results.XrayVersion = auditParams.xrayVersion results.ExtendedScanResults.EntitledForJas, err = isEntitledForJas(xrayManager, auditParams.xrayVersion) if err != nil { return @@ -186,7 +186,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. - results.ScaError = runScaScan(auditParams, results) + results.ScaError = scaScan(auditParams, results) // runScaScan(auditParams, results) // Wait for the Download of the AnalyzerManager to complete. if err = errGroup.Wait(); err != nil { @@ -195,7 +195,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) // Run scanners only if the user is entitled for Advanced Security if results.ExtendedScanResults.EntitledForJas { - results.JasError = runJasScannersAndSetResults(results.ExtendedScanResults, auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.Progress(), auditParams.xrayGraphScanParams.MultiScanId, auditParams.thirdPartyApplicabilityScan) + results.JasError = runJasScannersAndSetResults(results, auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.Progress(), auditParams.xrayGraphScanParams.MultiScanId, auditParams.thirdPartyApplicabilityScan) } return } diff --git a/xray/commands/audit/jasrunner.go b/xray/commands/audit/jasrunner.go index c528fdadc..07a25da6d 100644 --- a/xray/commands/audit/jasrunner.go +++ b/xray/commands/audit/jasrunner.go @@ -13,7 +13,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" ) -func runJasScannersAndSetResults(scanResults *utils.ExtendedScanResults, directDependencies []string, +func runJasScannersAndSetResults(scanResults *utils.Results, directDependencies []string, serverDetails *config.ServerDetails, workingDirs []string, progress io.ProgressMgr, multiScanId string, thirdPartyApplicabilityScan bool) (err error) { if serverDetails == nil || len(serverDetails.Url) == 0 { log.Warn("To include 'Advanced Security' scan as part of the audit output, please run the 'jf c add' command before running this command.") @@ -30,7 +30,11 @@ func runJasScannersAndSetResults(scanResults *utils.ExtendedScanResults, directD if progress != nil { progress.SetHeadlineMsg("Running applicability scanning") } - scanResults.ApplicabilityScanResults, err = applicability.RunApplicabilityScan(scanResults.XrayResults, directDependencies, scanResults.ScannedTechnologies, scanner, thirdPartyApplicabilityScan) + // for _, scaResult := range scanResults.ScaResults { + // // Do ny working dir, collect techs to save scanning + // scanResults.ExtendedScanResults.ApplicabilityScanResults, err = applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan) + // } + scanResults.ExtendedScanResults.ApplicabilityScanResults, err = applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan) if err != nil { return } @@ -41,20 +45,20 @@ func runJasScannersAndSetResults(scanResults *utils.ExtendedScanResults, directD if progress != nil { progress.SetHeadlineMsg("Running secrets scanning") } - scanResults.SecretsScanResults, err = secrets.RunSecretsScan(scanner) + scanResults.ExtendedScanResults.SecretsScanResults, err = secrets.RunSecretsScan(scanner) if err != nil { return } if progress != nil { progress.SetHeadlineMsg("Running IaC scanning") } - scanResults.IacScanResults, err = iac.RunIacScan(scanner) + scanResults.ExtendedScanResults.IacScanResults, err = iac.RunIacScan(scanner) if err != nil { return } if progress != nil { progress.SetHeadlineMsg("Running SAST scanning") } - scanResults.SastScanResults, err = sast.RunSastScan(scanner) + scanResults.ExtendedScanResults.SastScanResults, err = sast.RunSastScan(scanner) return } diff --git a/xray/commands/audit/jasrunner_test.go b/xray/commands/audit/jasrunner_test.go index 0d92f1b18..d4a8ebda2 100644 --- a/xray/commands/audit/jasrunner_test.go +++ b/xray/commands/audit/jasrunner_test.go @@ -21,14 +21,16 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { defer func() { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() - scanResults := &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Yarn}} + // scanResults := &utils.Results{ScaResults: }// &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Yarn}} + scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}} err = runJasScannersAndSetResults(scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, nil, "", false) // Expect error: assert.Error(t, err) } func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { - scanResults := &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Pip}} + // scanResults := &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Pip}} + scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Pip, XrayResults: jas.FakeBasicXrayResults}}} err := runJasScannersAndSetResults(scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, nil, "", false) assert.NoError(t, err) } @@ -36,7 +38,9 @@ func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { mockDirectDependencies := []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"} assert.NoError(t, rtutils.DownloadAnalyzerManagerIfNeeded()) - scanResults := &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Yarn}} + + // scanResults := &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Yarn}} + scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}} err := runJasScannersAndSetResults(scanResults, mockDirectDependencies, &jas.FakeServerDetails, nil, nil, "", false) // Expect error: diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index ac3f38b28..5f6f337da 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -42,36 +42,58 @@ func scaScan(params *AuditParams, results *xrayutils.Results) (err error) { log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") return } - // Make sure to return to the original working directory, executeScaScan may change it. defer func() { + // Make sure to return to the original working directory, executeScaScan may change it err = errors.Join(err, os.Chdir(currentWorkingDir)) }() + + printScansInformation(scans) for _, scan := range scans { - // Run the scan. - log.Info("Running SCA scan for vulnerable dependencies in", scan.WorkingDirectory, "directory...") + // Run the scan + log.Info("Running SCA scan for", scan.Technology ,"vulnerable dependencies in", scan.WorkingDirectory, "directory...") if wdScanErr := executeScaScan(serverDetails, params, &scan); wdScanErr != nil { - err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s\n", scan.WorkingDirectory, wdScanErr.Error())) + err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, wdScanErr.Error())) continue } - if len(scan.XrayResults) > 0 { - // Add the scan results to the results. - results.ScaResults = append(results.ScaResults, scan) - } + // Add the scan to the results + results.ScaResults = append(results.ScaResults, scan) } return } func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansToPreform []xrayutils.ScaScanResult) { - directories := getDirectoriesToScan(currentWorkingDir, params) - for _, wd := range directories { - // technologiesToDescriptors := coreutils.DetectTechnologiesDescriptors(wd, getTechnologiesToDetect(params)) - - scansToPreform = append(scansToPreform, xrayutils.ScaScanResult{WorkingDirectory: wd}) + isRecursive := true + excludePattern := "" + + for _, requestedDirectory := range getRequestedDirectoriesToScan(currentWorkingDir, params) { + // Detect descriptors from files + techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, isRecursive, params.Technologies(), excludePattern) + // Create scans to preform + for tech, workingDirs := range techToWorkingDirs { + if tech == coreutils.Dotnet { + // We detect Dotnet and Nuget the same way, if one detected so does the other. + // We don't need to scan for both and get duplicate results. + continue + } + if len(workingDirs) == 0 { + // Requested technology (from params) descriptors was not found, scan only requested directory for this technology. + scansToPreform = append(scansToPreform, xrayutils.ScaScanResult{WorkingDirectory: requestedDirectory, Technology: tech}) + } + for workingDir, descriptors := range workingDirs { + // Add scan for each detected working directory. + scansToPreform = append(scansToPreform, xrayutils.ScaScanResult{WorkingDirectory: workingDir, Technology: tech, Descriptors: descriptors}) + } + } } return } -func getDirectoriesToScan(currentWorkingDir string, params *AuditParams) []string { +func printScansInformation(scans []xrayutils.ScaScanResult) { + scansJson, _ := json.MarshalIndent(scans, "", " ") + log.Info(fmt.Sprintf("Preforming %d SCA scans:\n%s", len(scans), string(scansJson))) +} + +func getRequestedDirectoriesToScan(currentWorkingDir string, params *AuditParams) []string { workingDirs := datastructures.MakeSet[string]() for _, wd := range params.workingDirs { workingDirs.Add(wd) @@ -85,16 +107,13 @@ func getDirectoriesToScan(currentWorkingDir string, params *AuditParams) []strin // Preform the SCA scan for the given scan information. // This method may change the working directory to the scan's working directory. func executeScaScan(serverDetails *config.ServerDetails, params *AuditParams, scan *xrayutils.ScaScanResult) (err error) { - if scan.Technology == coreutils.Dotnet { - return - } // Get the dependency tree for the technology in the working directory. if err = os.Chdir(scan.WorkingDirectory); err != nil { return } flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology) if techErr != nil { - return fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", scan.Technology, techErr.Error()) + return fmt.Errorf("failed while building '%s' dependency tree:\n%s", scan.Technology, techErr.Error()) } if flattenTree == nil || len(flattenTree.Nodes) == 0 { return errors.New("no dependencies were found. Please try to build your project and re-run the audit command") @@ -102,9 +121,9 @@ func executeScaScan(serverDetails *config.ServerDetails, params *AuditParams, sc // Scan the dependency tree. scanResults, xrayErr := runScaWithTech(scan.Technology, params, serverDetails, flattenTree, fullDependencyTrees) if xrayErr != nil { - err = errors.Join(err, ) - return fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s\n", scan.Technology, xrayErr.Error()) + return fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s", scan.Technology, xrayErr.Error()) } + scan.IsMultipleRootProject = clientutils.Pointer(len(fullDependencyTrees) > 1) addThirdPartyDependenciesToParams(params, scan.Technology, flattenTree, fullDependencyTrees) scan.XrayResults = append(scan.XrayResults, scanResults...) return @@ -125,43 +144,43 @@ func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetail return } -func runScaScanInWorkingDir(params *AuditParams, results *xrayutils.Results, workingDir string) (err error) { - // Prepare the working directory information for the scan. - technologies := coreutils.DetectTechnologiesDescriptors(workingDir, getTechnologiesToDetect(params)) - if len(technologies) == 0 { - log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") - return - } - serverDetails, err := params.ServerDetails() - if err != nil { - return - } - // Run the scan for each technology. - for tech, detectedDescriptors := range technologies { - if tech == coreutils.Dotnet { - continue - } - // Get the dependency tree for the technology. - flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, tech) - if techErr != nil { - err = errors.Join(err, fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", tech, techErr.Error())) - continue - } - if flattenTree == nil || len(flattenTree.Nodes) == 0 { - err = errors.Join(err, errors.New("no dependencies were found. Please try to build your project and re-run the audit command")) - continue - } - // Scan the dependency tree. - scanResults, xrayErr := runScaOnTech(tech, params, serverDetails, flattenTree, fullDependencyTrees, results) - if xrayErr != nil { - err = errors.Join(err, fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s\n", tech, xrayErr.Error())) - continue - } - addThirdPartyDependenciesToParams(params, tech, flattenTree, fullDependencyTrees) - results.ScaResults = append(results.ScaResults, xrayutils.ScaScanResult{Technology: tech, XrayResults: scanResults, Descriptors: detectedDescriptors}) - } - return -} +// func runScaScanInWorkingDir(params *AuditParams, results *xrayutils.Results, workingDir string) (err error) { +// // Prepare the working directory information for the scan. +// technologies := coreutils.DetectTechnologiesDescriptors(workingDir, getTechnologiesToDetect(params), true) +// if len(technologies) == 0 { +// log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") +// return +// } +// serverDetails, err := params.ServerDetails() +// if err != nil { +// return +// } +// // Run the scan for each technology. +// for tech, detectedDescriptors := range technologies { +// if tech == coreutils.Dotnet { +// continue +// } +// // Get the dependency tree for the technology. +// flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, tech) +// if techErr != nil { +// err = errors.Join(err, fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", tech, techErr.Error())) +// continue +// } +// if flattenTree == nil || len(flattenTree.Nodes) == 0 { +// err = errors.Join(err, errors.New("no dependencies were found. Please try to build your project and re-run the audit command")) +// continue +// } +// // Scan the dependency tree. +// scanResults, xrayErr := runScaOnTech(tech, params, serverDetails, flattenTree, fullDependencyTrees, results) +// if xrayErr != nil { +// err = errors.Join(err, fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s\n", tech, xrayErr.Error())) +// continue +// } +// addThirdPartyDependenciesToParams(params, tech, flattenTree, fullDependencyTrees) +// results.ScaResults = append(results.ScaResults, xrayutils.ScaScanResult{Technology: tech, XrayResults: scanResults, Descriptors: detectedDescriptors}) +// } +// return +// } func getTechnologiesToDetect(params *AuditParams) (technologies []coreutils.Technology) { if len(params.Technologies()) != 0 { @@ -285,11 +304,11 @@ func runScaScanOnWorkingDir(params *AuditParams, results *xrayutils.Results, wor } params.AppendDependenciesForApplicabilityScan(dependenciesForApplicabilityScan) - results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...) - if !results.IsMultipleRootProject { - results.IsMultipleRootProject = len(fullDependencyTrees) > 1 - } - results.ExtendedScanResults.ScannedTechnologies = append(results.ExtendedScanResults.ScannedTechnologies, tech) + // results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...) + // if !results.IsMultipleRootProject { + // results.IsMultipleRootProject = len(fullDependencyTrees) > 1 + // } + // results.ExtendedScanResults.ScannedTechnologies = append(results.ExtendedScanResults.ScannedTechnologies, tech) } return } diff --git a/xray/commands/scan/buildscan.go b/xray/commands/scan/buildscan.go index d44a9dcb2..f1b9bd24a 100644 --- a/xray/commands/scan/buildscan.go +++ b/xray/commands/scan/buildscan.go @@ -126,9 +126,9 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS XrayDataUrl: buildScanResults.MoreDetailsUrl, }} - extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: scanResponse, XrayVersion: xrayVersion} + scanResults := &xrutils.Results{ScaResults: []xrutils.ScaScanResult{{XrayResults: scanResponse}}, XrayVersion: xrayVersion} - resultsPrinter := xrutils.NewResultsWriter(extendedScanResults). + resultsPrinter := xrutils.NewResultsWriter(scanResults). SetOutputFormat(bsc.outputFormat). SetIncludeVulnerabilities(bsc.includeVulnerabilities). SetIncludeLicenses(false). diff --git a/xray/commands/scan/scan.go b/xray/commands/scan/scan.go index 7a291a21c..f9ae9a181 100644 --- a/xray/commands/scan/scan.go +++ b/xray/commands/scan/scan.go @@ -241,9 +241,10 @@ func (scanCmd *ScanCommand) Run() (err error) { } scanErrors = appendErrorSlice(scanErrors, fileProducerErrors) scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors) - extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: flatResults, XrayVersion: xrayVersion} - if err = xrutils.NewResultsWriter(extendedScanResults). + scanResults := &xrutils.Results{ScaResults: []xrutils.ScaScanResult{{XrayResults: flatResults}}, XrayVersion: xrayVersion} + + if err = xrutils.NewResultsWriter(scanResults). SetOutputFormat(scanCmd.outputFormat). SetIncludeVulnerabilities(scanCmd.includeVulnerabilities). SetIncludeLicenses(scanCmd.includeLicenses). diff --git a/xray/utils/results.go b/xray/utils/results.go index 38a11701d..1b03236e4 100644 --- a/xray/utils/results.go +++ b/xray/utils/results.go @@ -1,6 +1,7 @@ package utils import ( + "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-client-go/xray/services" "github.com/owenrumney/go-sarif/v2/sarif" @@ -11,7 +12,7 @@ type Results struct { XrayVersion string ScaError error - IsMultipleRootProject bool // TODO: remove this + // IsMultipleRootProject bool // TODO: remove this ExtendedScanResults *ExtendedScanResults JasError error } @@ -20,19 +21,77 @@ func NewAuditResults() *Results { return &Results{ExtendedScanResults: &ExtendedScanResults{}} } +func (r *Results) GetScaScansXrayResults() (results []services.ScanResponse) { + for _, scaResult := range r.ScaResults { + results = append(results, scaResult.XrayResults...) + } + return +} + +func (r *Results) GetScaScannedTechnologies() []coreutils.Technology { + technologies := datastructures.MakeSet[coreutils.Technology]() + for _, scaResult := range r.ScaResults { + technologies.Add(scaResult.Technology) + } + return technologies.ToSlice() +} + +func (r *Results) IsMultipleProject() bool { + // Maybe we should check by working directory as well? + if len(r.ScaResults) == 0 { + return false + } + if len(r.ScaResults) == 1 { + return *r.ScaResults[0].IsMultipleRootProject + } + return true +} + +func (r *Results) IsScaIssuesFound() bool { + for _, scan := range r.ScaResults { + if scan.HasInformation() { + return true + } + } + return false +} + +func (r *Results) IsIssuesFound() bool { + if r.IsScaIssuesFound() { + return true + } + if r.ExtendedScanResults.IsIssuesFound() { + return true + } + return false +} + type ScaScanResult struct { - Technology coreutils.Technology - WorkingDirectory string - Descriptors []string - XrayResults []services.ScanResponse + Technology coreutils.Technology `json:"Technology"` + WorkingDirectory string `json:"WorkingDirectory"` + Descriptors []string `json:"Descriptors"` + XrayResults []services.ScanResponse `json:"XrayResults,omitempty"` - // IsMultipleRootProject bool + IsMultipleRootProject *bool `json:"IsMultipleRootProject,omitempty"` +} + +// func (s ScaScanResult) IsMultipleRootProject() bool { +// return len(s.XrayResults) > 1 +// } + +func (s ScaScanResult) HasInformation() bool { + for _, scan := range s.XrayResults { + if len(scan.Vulnerabilities) > 0 || len(scan.Violations) > 0 || len(scan.Licenses) > 0 { + return true + } + } + return false } type ExtendedScanResults struct { - XrayResults []services.ScanResponse // TODO: remove this - XrayVersion string // TODO: remove this - ScannedTechnologies []coreutils.Technology // TODO: remove this + // XrayResults []services.ScanResponse // TODO: remove this + // XrayVersion string // TODO: remove this + // ScannedTechnologies []coreutils.Technology // TODO: remove this ApplicabilityScanResults []*sarif.Run SecretsScanResults []*sarif.Run @@ -41,6 +100,13 @@ type ExtendedScanResults struct { EntitledForJas bool } -func (e *ExtendedScanResults) getXrayScanResults() []services.ScanResponse { - return e.XrayResults -} \ No newline at end of file +func (e *ExtendedScanResults) IsIssuesFound() bool { + return GetResultsLocationCount(e.ApplicabilityScanResults...) > 0 || + GetResultsLocationCount(e.SecretsScanResults...) > 0 || + GetResultsLocationCount(e.IacScanResults...) > 0 || + GetResultsLocationCount(e.SastScanResults...) > 0 +} + +// func (e *ExtendedScanResults) getXrayScanResults() []services.ScanResponse { +// return e.XrayResults +// } \ No newline at end of file diff --git a/xray/utils/resultstable.go b/xray/utils/resultstable.go index 0b308b9ff..0586bf680 100644 --- a/xray/utils/resultstable.go +++ b/xray/utils/resultstable.go @@ -37,8 +37,8 @@ const ( // In case one (or more) of the violations contains the field FailBuild set to true, CliError with exit code 3 will be returned. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintViolationsTable(violations []services.Violation, extendedResults *ExtendedScanResults, multipleRoots, printExtended bool, scanType services.ScanType) error { - securityViolationsRows, licenseViolationsRows, operationalRiskViolationsRows, err := prepareViolations(violations, extendedResults, multipleRoots, true, true) +func PrintViolationsTable(violations []services.Violation, results *Results, multipleRoots, printExtended bool, scanType services.ScanType) error { + securityViolationsRows, licenseViolationsRows, operationalRiskViolationsRows, err := prepareViolations(violations, results, multipleRoots, true, true) if err != nil { return err } @@ -72,11 +72,11 @@ func PrintViolationsTable(violations []services.Violation, extendedResults *Exte } // Prepare violations for all non-table formats (without style or emoji) -func PrepareViolations(violations []services.Violation, extendedResults *ExtendedScanResults, multipleRoots, simplifiedOutput bool) ([]formats.VulnerabilityOrViolationRow, []formats.LicenseRow, []formats.OperationalRiskViolationRow, error) { - return prepareViolations(violations, extendedResults, multipleRoots, false, simplifiedOutput) +func PrepareViolations(violations []services.Violation, results *Results, multipleRoots, simplifiedOutput bool) ([]formats.VulnerabilityOrViolationRow, []formats.LicenseRow, []formats.OperationalRiskViolationRow, error) { + return prepareViolations(violations, results, multipleRoots, false, simplifiedOutput) } -func prepareViolations(violations []services.Violation, extendedResults *ExtendedScanResults, multipleRoots, isTable, simplifiedOutput bool) ([]formats.VulnerabilityOrViolationRow, []formats.LicenseRow, []formats.OperationalRiskViolationRow, error) { +func prepareViolations(violations []services.Violation, results *Results, multipleRoots, isTable, simplifiedOutput bool) ([]formats.VulnerabilityOrViolationRow, []formats.LicenseRow, []formats.OperationalRiskViolationRow, error) { if simplifiedOutput { violations = simplifyViolations(violations, multipleRoots) } @@ -91,12 +91,12 @@ func prepareViolations(violations []services.Violation, extendedResults *Extende switch violation.ViolationType { case "security": cves := convertCves(violation.Cves) - if extendedResults.EntitledForJas { + if results.ExtendedScanResults.EntitledForJas { for i := range cves { - cves[i].Applicability = getCveApplicabilityField(cves[i], extendedResults.ApplicabilityScanResults, violation.Components) + cves[i].Applicability = getCveApplicabilityField(cves[i], results.ExtendedScanResults.ApplicabilityScanResults, violation.Components) } } - applicabilityStatus := getApplicableCveStatus(extendedResults.EntitledForJas, extendedResults.ApplicabilityScanResults, cves) + applicabilityStatus := getApplicableCveStatus(results.ExtendedScanResults.EntitledForJas, results.ExtendedScanResults.ApplicabilityScanResults, cves) currSeverity := GetSeverity(violation.Severity, applicabilityStatus) jfrogResearchInfo := convertJfrogResearchInformation(violation.ExtendedInformation) for compIndex := 0; compIndex < len(impactedPackagesNames); compIndex++ { @@ -182,8 +182,8 @@ func prepareViolations(violations []services.Violation, extendedResults *Extende // In case multipleRoots is true, the field Component will show the root of each impact path, otherwise it will show the root's child. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintVulnerabilitiesTable(vulnerabilities []services.Vulnerability, extendedResults *ExtendedScanResults, multipleRoots, printExtended bool, scanType services.ScanType) error { - vulnerabilitiesRows, err := prepareVulnerabilities(vulnerabilities, extendedResults, multipleRoots, true, true) +func PrintVulnerabilitiesTable(vulnerabilities []services.Vulnerability, results *Results, multipleRoots, printExtended bool, scanType services.ScanType) error { + vulnerabilitiesRows, err := prepareVulnerabilities(vulnerabilities, results, multipleRoots, true, true) if err != nil { return err } @@ -192,7 +192,7 @@ func PrintVulnerabilitiesTable(vulnerabilities []services.Vulnerability, extende return coreutils.PrintTable(formats.ConvertToVulnerabilityScanTableRow(vulnerabilitiesRows), "Vulnerable Components", "✨ No vulnerable components were found ✨", printExtended) } var emptyTableMessage string - if len(extendedResults.ScannedTechnologies) > 0 { + if len(results.ScaResults) > 0 { emptyTableMessage = "✨ No vulnerable dependencies were found ✨" } else { emptyTableMessage = coreutils.PrintYellow("🔧 Couldn't determine a package manager or build tool used by this project 🔧") @@ -201,11 +201,11 @@ func PrintVulnerabilitiesTable(vulnerabilities []services.Vulnerability, extende } // Prepare vulnerabilities for all non-table formats (without style or emoji) -func PrepareVulnerabilities(vulnerabilities []services.Vulnerability, extendedResults *ExtendedScanResults, multipleRoots, simplifiedOutput bool) ([]formats.VulnerabilityOrViolationRow, error) { - return prepareVulnerabilities(vulnerabilities, extendedResults, multipleRoots, false, simplifiedOutput) +func PrepareVulnerabilities(vulnerabilities []services.Vulnerability, results *Results, multipleRoots, simplifiedOutput bool) ([]formats.VulnerabilityOrViolationRow, error) { + return prepareVulnerabilities(vulnerabilities, results, multipleRoots, false, simplifiedOutput) } -func prepareVulnerabilities(vulnerabilities []services.Vulnerability, extendedResults *ExtendedScanResults, multipleRoots, isTable, simplifiedOutput bool) ([]formats.VulnerabilityOrViolationRow, error) { +func prepareVulnerabilities(vulnerabilities []services.Vulnerability, results *Results, multipleRoots, isTable, simplifiedOutput bool) ([]formats.VulnerabilityOrViolationRow, error) { if simplifiedOutput { vulnerabilities = simplifyVulnerabilities(vulnerabilities, multipleRoots) } @@ -216,12 +216,12 @@ func prepareVulnerabilities(vulnerabilities []services.Vulnerability, extendedRe return nil, err } cves := convertCves(vulnerability.Cves) - if extendedResults.EntitledForJas { + if results.ExtendedScanResults.EntitledForJas { for i := range cves { - cves[i].Applicability = getCveApplicabilityField(cves[i], extendedResults.ApplicabilityScanResults, vulnerability.Components) + cves[i].Applicability = getCveApplicabilityField(cves[i], results.ExtendedScanResults.ApplicabilityScanResults, vulnerability.Components) } } - applicabilityStatus := getApplicableCveStatus(extendedResults.EntitledForJas, extendedResults.ApplicabilityScanResults, cves) + applicabilityStatus := getApplicableCveStatus(results.ExtendedScanResults.EntitledForJas, results.ExtendedScanResults.ApplicabilityScanResults, cves) currSeverity := GetSeverity(vulnerability.Severity, applicabilityStatus) jfrogResearchInfo := convertJfrogResearchInformation(vulnerability.ExtendedInformation) for compIndex := 0; compIndex < len(impactedPackagesNames); compIndex++ { diff --git a/xray/utils/resultstable_test.go b/xray/utils/resultstable_test.go index d0589fe24..14d315b00 100644 --- a/xray/utils/resultstable_test.go +++ b/xray/utils/resultstable_test.go @@ -25,7 +25,7 @@ func TestPrintViolationsTable(t *testing.T) { } for _, test := range tests { - err := PrintViolationsTable(test.violations, &ExtendedScanResults{}, false, true, services.Binary) + err := PrintViolationsTable(test.violations, NewAuditResults(), false, true, services.Binary) assert.NoError(t, err) if CheckIfFailBuild([]services.ScanResponse{{Violations: test.violations}}) { err = NewFailBuildError() diff --git a/xray/utils/resultwriter.go b/xray/utils/resultwriter.go index 35a8fafd4..5d7702841 100644 --- a/xray/utils/resultwriter.go +++ b/xray/utils/resultwriter.go @@ -38,7 +38,7 @@ var CurationOutputFormats = []string{string(Table), string(Json)} type ResultsWriter struct { // The scan results. - results *ExtendedScanResults + results *Results // SimpleJsonError Errors to be added to output of the SimpleJson format. simpleJsonError []formats.SimpleJsonError // Format The output format. @@ -57,8 +57,8 @@ type ResultsWriter struct { messages []string } -func NewResultsWriter(extendedScanResults *ExtendedScanResults) *ResultsWriter { - return &ResultsWriter{results: extendedScanResults} +func NewResultsWriter(scanResults *Results) *ResultsWriter { + return &ResultsWriter{results: scanResults} } func (rw *ResultsWriter) SetOutputFormat(format OutputFormat) *ResultsWriter { @@ -115,7 +115,7 @@ func (rw *ResultsWriter) PrintScanResults() error { } return PrintJson(jsonTable) case Json: - return PrintJson(rw.results.getXrayScanResults()) + return PrintJson(rw.results.GetScaScansXrayResults()) case Sarif: sarifReport, err := GenereateSarifReportFromResults(rw.results, rw.isMultipleRoots, rw.includeLicenses) if err != nil { @@ -131,8 +131,8 @@ func (rw *ResultsWriter) PrintScanResults() error { } func (rw *ResultsWriter) printScanResultsTables() (err error) { printMessages(rw.messages) - violations, vulnerabilities, licenses := SplitScanResults(rw.results.getXrayScanResults()) - if len(rw.results.getXrayScanResults()) > 0 { + violations, vulnerabilities, licenses := SplitScanResults(rw.results.ScaResults) + if rw.results.IsIssuesFound() { var resultsPath string if resultsPath, err = writeJsonResults(rw.results); err != nil { return @@ -153,13 +153,13 @@ func (rw *ResultsWriter) printScanResultsTables() (err error) { return } } - if err = PrintSecretsTable(rw.results.SecretsScanResults, rw.results.EntitledForJas); err != nil { + if err = PrintSecretsTable(rw.results.ExtendedScanResults.SecretsScanResults, rw.results.ExtendedScanResults.EntitledForJas); err != nil { return } - if err = PrintIacTable(rw.results.IacScanResults, rw.results.EntitledForJas); err != nil { + if err = PrintIacTable(rw.results.ExtendedScanResults.IacScanResults, rw.results.ExtendedScanResults.EntitledForJas); err != nil { return } - return PrintSastTable(rw.results.SastScanResults, rw.results.EntitledForJas) + return PrintSastTable(rw.results.ExtendedScanResults.SastScanResults, rw.results.ExtendedScanResults.EntitledForJas) } func printMessages(messages []string) { @@ -175,21 +175,21 @@ func printMessage(message string) { log.Output("💬" + message) } -func GenereateSarifReportFromResults(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses bool) (report *sarif.Report, err error) { +func GenereateSarifReportFromResults(results *Results, isMultipleRoots, includeLicenses bool) (report *sarif.Report, err error) { report, err = NewReport() if err != nil { return } - xrayRun, err := convertXrayResponsesToSarifRun(extendedResults, isMultipleRoots, includeLicenses) + xrayRun, err := convertXrayResponsesToSarifRun(results, isMultipleRoots, includeLicenses) if err != nil { return } report.Runs = append(report.Runs, xrayRun) - report.Runs = append(report.Runs, extendedResults.ApplicabilityScanResults...) - report.Runs = append(report.Runs, extendedResults.IacScanResults...) - report.Runs = append(report.Runs, extendedResults.SecretsScanResults...) - report.Runs = append(report.Runs, extendedResults.SastScanResults...) + report.Runs = append(report.Runs, results.ExtendedScanResults.ApplicabilityScanResults...) + report.Runs = append(report.Runs, results.ExtendedScanResults.IacScanResults...) + report.Runs = append(report.Runs, results.ExtendedScanResults.SecretsScanResults...) + report.Runs = append(report.Runs, results.ExtendedScanResults.SastScanResults...) return } @@ -202,13 +202,13 @@ func ConvertSarifReportToString(report *sarif.Report) (sarifStr string, err erro return clientUtils.IndentJson(out), nil } -func convertXrayResponsesToSarifRun(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses bool) (run *sarif.Run, err error) { - xrayJson, err := convertXrayScanToSimpleJson(extendedResults, isMultipleRoots, includeLicenses, true) +func convertXrayResponsesToSarifRun(results *Results, isMultipleRoots, includeLicenses bool) (run *sarif.Run, err error) { + xrayJson, err := convertXrayScanToSimpleJson(results, isMultipleRoots, includeLicenses, true) if err != nil { return } xrayRun := sarif.NewRunWithInformationURI("JFrog Xray SCA", BaseDocumentationURL+"sca") - xrayRun.Tool.Driver.Version = &extendedResults.XrayVersion + xrayRun.Tool.Driver.Version = &results.XrayVersion if len(xrayJson.Vulnerabilities) > 0 || len(xrayJson.SecurityViolations) > 0 { if err = extractXrayIssuesToSarifRun(xrayRun, xrayJson); err != nil { return @@ -351,18 +351,18 @@ func addXrayRule(ruleId, ruleDescription, maxCveScore, summary, markdownDescript }) } -func convertXrayScanToSimpleJson(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, simplifiedOutput bool) (formats.SimpleJsonResults, error) { - violations, vulnerabilities, licenses := SplitScanResults(extendedResults.XrayResults) +func convertXrayScanToSimpleJson(results *Results, isMultipleRoots, includeLicenses, simplifiedOutput bool) (formats.SimpleJsonResults, error) { + violations, vulnerabilities, licenses := SplitScanResults(results.ScaResults) jsonTable := formats.SimpleJsonResults{} if len(vulnerabilities) > 0 { - vulJsonTable, err := PrepareVulnerabilities(vulnerabilities, extendedResults, isMultipleRoots, simplifiedOutput) + vulJsonTable, err := PrepareVulnerabilities(vulnerabilities, results, isMultipleRoots, simplifiedOutput) if err != nil { return formats.SimpleJsonResults{}, err } jsonTable.Vulnerabilities = vulJsonTable } if len(violations) > 0 { - secViolationsJsonTable, licViolationsJsonTable, opRiskViolationsJsonTable, err := PrepareViolations(violations, extendedResults, isMultipleRoots, simplifiedOutput) + secViolationsJsonTable, licViolationsJsonTable, opRiskViolationsJsonTable, err := PrepareViolations(violations, results, isMultipleRoots, simplifiedOutput) if err != nil { return formats.SimpleJsonResults{}, err } @@ -386,14 +386,14 @@ func (rw *ResultsWriter) convertScanToSimpleJson() (formats.SimpleJsonResults, e if err != nil { return formats.SimpleJsonResults{}, err } - if len(rw.results.SecretsScanResults) > 0 { - jsonTable.Secrets = PrepareSecrets(rw.results.SecretsScanResults) + if len(rw.results.ExtendedScanResults.SecretsScanResults) > 0 { + jsonTable.Secrets = PrepareSecrets(rw.results.ExtendedScanResults.SecretsScanResults) } - if len(rw.results.IacScanResults) > 0 { - jsonTable.Iacs = PrepareIacs(rw.results.IacScanResults) + if len(rw.results.ExtendedScanResults.IacScanResults) > 0 { + jsonTable.Iacs = PrepareIacs(rw.results.ExtendedScanResults.IacScanResults) } - if len(rw.results.SastScanResults) > 0 { - jsonTable.Sast = PrepareSast(rw.results.SastScanResults) + if len(rw.results.ExtendedScanResults.SastScanResults) > 0 { + jsonTable.Sast = PrepareSast(rw.results.ExtendedScanResults.SastScanResults) } jsonTable.Errors = rw.simpleJsonError @@ -483,19 +483,21 @@ func findMaxCVEScore(cves []formats.CveRow) (string, error) { } // Splits scan responses into aggregated lists of violations, vulnerabilities and licenses. -func SplitScanResults(results []services.ScanResponse) ([]services.Violation, []services.Vulnerability, []services.License) { +func SplitScanResults(results []ScaScanResult) ([]services.Violation, []services.Vulnerability, []services.License) { var violations []services.Violation var vulnerabilities []services.Vulnerability var licenses []services.License - for _, result := range results { - violations = append(violations, result.Violations...) - vulnerabilities = append(vulnerabilities, result.Vulnerabilities...) - licenses = append(licenses, result.Licenses...) + for _, scan := range results { + for _, result := range scan.XrayResults { + violations = append(violations, result.Violations...) + vulnerabilities = append(vulnerabilities, result.Vulnerabilities...) + licenses = append(licenses, result.Licenses...) + } } return violations, vulnerabilities, licenses } -func writeJsonResults(results *ExtendedScanResults) (resultsPath string, err error) { +func writeJsonResults(results *Results) (resultsPath string, err error) { out, err := fileutils.CreateTempFile() if errorutils.CheckError(err) != nil { return From ccb5f11d78693fd70821d08bcb5897e3e6d02f7f Mon Sep 17 00:00:00 2001 From: attiasas Date: Thu, 19 Oct 2023 18:34:40 +0300 Subject: [PATCH 05/27] continue to implement --- utils/coreutils/techutils.go | 47 ++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 1a1edf79a..b1b3416f4 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -218,13 +218,10 @@ func mapWorkingDirectoriesToIndicators(files []string) (workingDirectoryToIndica for _, path := range files { directory := filepath.Dir(path) for _, techData := range technologiesData { - for _, indicator := range techData.indicators { - if strings.HasSuffix(path, indicator) { - workingDirectoryToIndicators[directory] = append(workingDirectoryToIndicators[directory], path) - } - } if isDescriptor(path, techData) { workingDirectoryToIndicators[directory] = append(workingDirectoryToIndicators[directory], path) + } else if isIndicator(path, techData) { + workingDirectoryToIndicators[directory] = append(workingDirectoryToIndicators[directory], path) } } } @@ -232,6 +229,15 @@ func mapWorkingDirectoriesToIndicators(files []string) (workingDirectoryToIndica return } +func isIndicator(path string, techData TechData) bool { + for _, indicator := range techData.indicators { + if strings.HasSuffix(path, indicator) { + return true + } + } + return false +} + func isDescriptor(path string, techData TechData) bool { for _, descriptor := range techData.packageDescriptors { if strings.HasSuffix(path, descriptor) { @@ -251,22 +257,27 @@ func mapIndicatorsToTechnologies(workingDirectoryToIndicators map[string][]strin for _, tech := range technologies { techWorkingDirs := make(map[string][]string) + foundIndicator := false for wd, indicators := range workingDirectoryToIndicators { // What to do about no descriptors found and only indicators? // Implement exclude - // Collect only descriptors + // Collect only descriptors, indicators adds techWorkingDirs // Map tech for indicators/descriptors to tech - for _, indicator := range indicators { - // Don't allow working directory if sub directory already exists as key - if isDescriptor(indicator, technologiesData[tech]) && !hasSubDirKey(wd, techWorkingDirs) { - techWorkingDirs[wd] = append(techWorkingDirs[wd], indicator) + for _, path := range indicators { + if isDescriptor(path, technologiesData[tech]) { + techWorkingDirs[wd] = append(techWorkingDirs[wd], path) } + // else if isIndicator(path, technologiesData[tech]) { + // foundIndicator = true + // } } } - if len(techWorkingDirs) > 0 { + // Don't allow working directory if sub directory already exists as key for the same technology + techWorkingDirs = cleanSubDirectories(techWorkingDirs) + if foundIndicator || len(techWorkingDirs) > 0 { // Found indicators/descriptors for technology, add to detected. technologiesDetected[tech] = techWorkingDirs } @@ -281,9 +292,19 @@ func mapIndicatorsToTechnologies(workingDirectoryToIndicators map[string][]strin return } -func hasSubDirKey(path string, workingDirectoryToIndicators map[string][]string) bool { +func cleanSubDirectories(workingDirectoryToIndicators map[string][]string) (result map[string][]string) { + result = make(map[string][]string) + for wd, indicators := range workingDirectoryToIndicators { + if !hasSubDirKey(wd, workingDirectoryToIndicators) { + result[wd] = indicators + } + } + return +} + +func hasSubDirKey(dir string, workingDirectoryToIndicators map[string][]string) bool { for wd := range workingDirectoryToIndicators { - if strings.HasPrefix(path, wd) { + if dir != wd && strings.HasPrefix(dir, wd) { return true } } From 9d4a8a216247ca05fbde7f52a636d55ff1670e5b Mon Sep 17 00:00:00 2001 From: attiasas Date: Sun, 22 Oct 2023 10:41:36 +0300 Subject: [PATCH 06/27] done implementation --- utils/coreutils/techutils.go | 112 +++++++++----- xray/commands/audit/scarunner.go | 249 ++++++++++++++++--------------- xray/utils/results.go | 32 ++-- 3 files changed, 223 insertions(+), 170 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index b1b3416f4..ab0a17570 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -1,6 +1,7 @@ package coreutils import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -114,7 +115,7 @@ var technologiesData = map[Technology]TechData{ Poetry: { packageType: Pypi, indicators: []string{"pyproject.toml", "poetry.lock"}, - packageDescriptors: []string{"pyproject.toml"}, + packageDescriptors: []string{"pyproject.toml"}, packageInstallationCommand: "add", packageVersionOperator: "==", applicabilityScannable: true, @@ -203,30 +204,56 @@ func DetectedTechnologiesListInPath(path string, recursive bool) (technologies [ // If recursive is true, the search will not be limited to files in the root path. // If requestedTechs is empty, all technologies will be checked. // If excludePathPattern is not empty, files/directories that match the wildcard pattern will be excluded from the search. -func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs []string, excludePathPattern string) (technologiesDetected map[Technology]map[string][]string) { +func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs []string, requestedDescriptors map[Technology][]string, excludePathPattern string) (technologiesDetected map[Technology]map[string][]string) { filesList, err := fspatterns.ListFiles(path, recursive, false, true, excludePathPattern) if err != nil { return } - technologiesDetected = mapIndicatorsToTechnologies(mapWorkingDirectoriesToIndicators(filesList), ToTechnologies(requestedTechs)) + workingDirectoryToIndicators, excludedTechAtWorkingDir := mapFilesToRelevantWorkingDirectories(filesList, requestedDescriptors) + technologiesDetected = mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators, excludedTechAtWorkingDir, ToTechnologies(requestedTechs)) log.Debug(fmt.Sprintf("Detected %d technologies at %s: %s.", len(technologiesDetected), path, maps.Keys(technologiesDetected))) - return + return } -func mapWorkingDirectoriesToIndicators(files []string) (workingDirectoryToIndicators map[string][]string) { +func mapFilesToRelevantWorkingDirectories(files []string, requestedDescriptors map[Technology][]string) (workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology) { workingDirectoryToIndicators = make(map[string][]string) + excludedTechAtWorkingDir = make(map[string][]Technology) for _, path := range files { directory := filepath.Dir(path) - for _, techData := range technologiesData { - if isDescriptor(path, techData) { + for tech, techData := range technologiesData { + // Check if the working directory contains indicators/descriptors for the technology + if isDescriptor(path, techData) || isRequestedDescriptor(path, requestedDescriptors[tech]) { workingDirectoryToIndicators[directory] = append(workingDirectoryToIndicators[directory], path) } else if isIndicator(path, techData) { workingDirectoryToIndicators[directory] = append(workingDirectoryToIndicators[directory], path) } + // Check if the working directory contains a file/directory with a name that ends with an excluded suffix + if isExclude(path, techData) { + excludedTechAtWorkingDir[directory] = append(excludedTechAtWorkingDir[directory], tech) + } + } + } + strJson, _ := json.MarshalIndent(workingDirectoryToIndicators, "", " ") + log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson)) + return +} + +func isDescriptor(path string, techData TechData) bool { + for _, descriptor := range techData.packageDescriptors { + if strings.HasSuffix(path, descriptor) { + return true + } + } + return false +} + +func isRequestedDescriptor(path string, requestedDescriptors []string) bool { + for _, requestedDescriptor := range requestedDescriptors { + if strings.HasSuffix(path, requestedDescriptor) { + return true } } - log.Debug(fmt.Sprintf("mapped indicators:\n%s", workingDirectoryToIndicators)) - return + return false } func isIndicator(path string, techData TechData) bool { @@ -238,41 +265,42 @@ func isIndicator(path string, techData TechData) bool { return false } -func isDescriptor(path string, techData TechData) bool { - for _, descriptor := range techData.packageDescriptors { - if strings.HasSuffix(path, descriptor) { +func isExclude(path string, techData TechData) bool { + for _, exclude := range techData.exclude { + if strings.HasSuffix(path, exclude) { return true } } return false } -func mapIndicatorsToTechnologies(workingDirectoryToIndicators map[string][]string, requestedTechs []Technology) (technologiesDetected map[Technology]map[string][]string) { - +func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedTechs []Technology) (technologiesDetected map[Technology]map[string][]string) { + // Get the relevant technologies to check technologies := requestedTechs if len(technologies) == 0 { technologies = GetAllTechnologiesList() } technologiesDetected = make(map[Technology]map[string][]string) - + // Map working directories to technologies for _, tech := range technologies { techWorkingDirs := make(map[string][]string) foundIndicator := false for wd, indicators := range workingDirectoryToIndicators { - - - // What to do about no descriptors found and only indicators? - // Implement exclude - // Collect only descriptors, indicators adds techWorkingDirs - - // Map tech for indicators/descriptors to tech + if excludedTechs, exist := excludedTechAtWorkingDir[wd]; exist { + for _, excludedTech := range excludedTechs { + if excludedTech == tech { + // Exclude this technology from this working directory + continue + } + } + } + // Check if the working directory contains indicators/descriptors for the technology for _, path := range indicators { if isDescriptor(path, technologiesData[tech]) { techWorkingDirs[wd] = append(techWorkingDirs[wd], path) + } else if isIndicator(path, technologiesData[tech]) { + foundIndicator = true } - // else if isIndicator(path, technologiesData[tech]) { - // foundIndicator = true - // } } } // Don't allow working directory if sub directory already exists as key for the same technology @@ -286,29 +314,45 @@ func mapIndicatorsToTechnologies(workingDirectoryToIndicators map[string][]strin for _, tech := range requestedTechs { if _, exist := technologiesDetected[tech]; !exist { // Requested (forced with flag) technology and not found any indicators/descriptors in detection, add as detected. + log.Warn(fmt.Sprintf("Requested technology %s but not found any indicators/descriptors in detection.", tech)) technologiesDetected[tech] = map[string][]string{} } } return } -func cleanSubDirectories(workingDirectoryToIndicators map[string][]string) (result map[string][]string) { +func cleanSubDirectories(workingDirectoryToFiles map[string][]string) (result map[string][]string) { result = make(map[string][]string) - for wd, indicators := range workingDirectoryToIndicators { - if !hasSubDirKey(wd, workingDirectoryToIndicators) { - result[wd] = indicators - } + for wd, files := range workingDirectoryToFiles { + root := getExistingRootDir(wd, workingDirectoryToFiles) + result[root] = append(result[root], files...) + // if root == wd { + // // Current working directory is the root + // result[wd] = files + // } else { + // // add descriptors from sub projects to the root + // result[root] = append(result[root], files...) + // } } return } -func hasSubDirKey(dir string, workingDirectoryToIndicators map[string][]string) bool { +func getExistingRootDir(path string, workingDirectoryToIndicators map[string][]string) (rootDir string) { + rootDir = path for wd := range workingDirectoryToIndicators { - if dir != wd && strings.HasPrefix(dir, wd) { - return true + if strings.HasPrefix(rootDir, wd) { + rootDir = wd } } - return false + return + + // // TODO: make sure to get the top most root! + // for wd := range workingDirectoryToIndicators { + // if path != wd && strings.HasPrefix(path, wd) { + // return wd + // } + // } + // return "" } // func detectTechnologiesDescriptorsByFilePaths(paths []string) (technologiesToDescriptors map[Technology][]string) { diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index 5f6f337da..ccad27e24 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -50,24 +50,23 @@ func scaScan(params *AuditParams, results *xrayutils.Results) (err error) { printScansInformation(scans) for _, scan := range scans { // Run the scan - log.Info("Running SCA scan for", scan.Technology ,"vulnerable dependencies in", scan.WorkingDirectory, "directory...") - if wdScanErr := executeScaScan(serverDetails, params, &scan); wdScanErr != nil { + log.Info("Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") + if wdScanErr := executeScaScan(serverDetails, params, scan); wdScanErr != nil { err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, wdScanErr.Error())) continue } // Add the scan to the results - results.ScaResults = append(results.ScaResults, scan) + results.ScaResults = append(results.ScaResults, *scan) } return } -func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansToPreform []xrayutils.ScaScanResult) { +func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansToPreform []*xrayutils.ScaScanResult) { isRecursive := true excludePattern := "" - for _, requestedDirectory := range getRequestedDirectoriesToScan(currentWorkingDir, params) { // Detect descriptors from files - techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, isRecursive, params.Technologies(), excludePattern) + techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, isRecursive, params.Technologies(), getRequestedDescriptors(params), excludePattern) // Create scans to preform for tech, workingDirs := range techToWorkingDirs { if tech == coreutils.Dotnet { @@ -77,20 +76,30 @@ func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansT } if len(workingDirs) == 0 { // Requested technology (from params) descriptors was not found, scan only requested directory for this technology. - scansToPreform = append(scansToPreform, xrayutils.ScaScanResult{WorkingDirectory: requestedDirectory, Technology: tech}) + scansToPreform = append(scansToPreform, &xrayutils.ScaScanResult{WorkingDirectory: requestedDirectory, Technology: tech}) } for workingDir, descriptors := range workingDirs { // Add scan for each detected working directory. - scansToPreform = append(scansToPreform, xrayutils.ScaScanResult{WorkingDirectory: workingDir, Technology: tech, Descriptors: descriptors}) + scansToPreform = append(scansToPreform, &xrayutils.ScaScanResult{WorkingDirectory: workingDir, Technology: tech, Descriptors: descriptors}) } } } return } -func printScansInformation(scans []xrayutils.ScaScanResult) { - scansJson, _ := json.MarshalIndent(scans, "", " ") - log.Info(fmt.Sprintf("Preforming %d SCA scans:\n%s", len(scans), string(scansJson))) +func getRequestedDescriptors(params *AuditParams) map[coreutils.Technology][]string { + requestedDescriptors := map[coreutils.Technology][]string{} + if params.PipRequirementsFile() != "" { + requestedDescriptors[coreutils.Pip] = []string{params.PipRequirementsFile()} + } + return requestedDescriptors +} + +func printScansInformation(scans []*xrayutils.ScaScanResult) { + scansJson, err := json.MarshalIndent(scans, "", " ") + if err == nil { + log.Info(fmt.Sprintf("Preforming %d SCA scans:\n%s", len(scans), string(scansJson))) + } } func getRequestedDirectoriesToScan(currentWorkingDir string, params *AuditParams) []string { @@ -182,14 +191,14 @@ func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetail // return // } -func getTechnologiesToDetect(params *AuditParams) (technologies []coreutils.Technology) { - if len(params.Technologies()) != 0 { - technologies = coreutils.ToTechnologies(params.Technologies()) - } else { - technologies = coreutils.GetAllTechnologiesList() - } - return -} +// func getTechnologiesToDetect(params *AuditParams) (technologies []coreutils.Technology) { +// if len(params.Technologies()) != 0 { +// technologies = coreutils.ToTechnologies(params.Technologies()) +// } else { +// technologies = coreutils.GetAllTechnologiesList() +// } +// return +// } func addThirdPartyDependenciesToParams(params *AuditParams, tech coreutils.Technology, flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) { var dependenciesForApplicabilityScan []string @@ -201,117 +210,117 @@ func addThirdPartyDependenciesToParams(params *AuditParams, tech coreutils.Techn params.AppendDependenciesForApplicabilityScan(dependenciesForApplicabilityScan) } -func runScaOnTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, results *xrayutils.Results) (techResults []services.ScanResponse, err error) { - scanGraphParams := scangraph.NewScanGraphParams(). - SetServerDetails(serverDetails). - SetXrayGraphScanParams(params.xrayGraphScanParams). - SetXrayVersion(params.xrayVersion). - SetFixableOnly(params.fixableOnly). - SetSeverityLevel(params.minSeverityFilter) - techResults, err = sca.RunXrayDependenciesTreeScanGraph(flatTree, params.Progress(), tech, scanGraphParams) - if err != nil { - return - } - techResults = sca.BuildImpactPathsForScanResponse(techResults, fullDependencyTrees) +// func runScaOnTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, results *xrayutils.Results) (techResults []services.ScanResponse, err error) { +// scanGraphParams := scangraph.NewScanGraphParams(). +// SetServerDetails(serverDetails). +// SetXrayGraphScanParams(params.xrayGraphScanParams). +// SetXrayVersion(params.xrayVersion). +// SetFixableOnly(params.fixableOnly). +// SetSeverityLevel(params.minSeverityFilter) +// techResults, err = sca.RunXrayDependenciesTreeScanGraph(flatTree, params.Progress(), tech, scanGraphParams) +// if err != nil { +// return +// } +// techResults = sca.BuildImpactPathsForScanResponse(techResults, fullDependencyTrees) - // results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...) - // if !results.IsMultipleRootProject { - // results.IsMultipleRootProject = len(fullDependencyTrees) > 1 - // } +// // results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...) +// // if !results.IsMultipleRootProject { +// // results.IsMultipleRootProject = len(fullDependencyTrees) > 1 +// // } - // results.ExtendedScanResults.ScannedTechnologies = append(results.ExtendedScanResults.ScannedTechnologies, tech) - return -} +// // results.ExtendedScanResults.ScannedTechnologies = append(results.ExtendedScanResults.ScannedTechnologies, tech) +// return +// } -func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { - rootDir, err := os.Getwd() - if errorutils.CheckError(err) != nil { - return - } - for _, wd := range params.workingDirs { - if len(params.workingDirs) > 1 { - log.Info("Running SCA scan for vulnerable dependencies scan in", wd, "directory...") - } else { - log.Info("Running SCA scan for vulnerable dependencies...") - } - wdScanErr := runScaScanOnWorkingDir(params, results, wd, rootDir) - if wdScanErr != nil { - err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s\n", wd, wdScanErr.Error())) - continue - } - } - return -} +// func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { +// rootDir, err := os.Getwd() +// if errorutils.CheckError(err) != nil { +// return +// } +// for _, wd := range params.workingDirs { +// if len(params.workingDirs) > 1 { +// log.Info("Running SCA scan for vulnerable dependencies scan in", wd, "directory...") +// } else { +// log.Info("Running SCA scan for vulnerable dependencies...") +// } +// wdScanErr := runScaScanOnWorkingDir(params, results, wd, rootDir) +// if wdScanErr != nil { +// err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s\n", wd, wdScanErr.Error())) +// continue +// } +// } +// return +// } // Audits the project found in the current directory using Xray. -func runScaScanOnWorkingDir(params *AuditParams, results *xrayutils.Results, workingDir, rootDir string) (err error) { - err = os.Chdir(workingDir) - if err != nil { - return - } - defer func() { - err = errors.Join(err, os.Chdir(rootDir)) - }() +// func runScaScanOnWorkingDir(params *AuditParams, results *xrayutils.Results, workingDir, rootDir string) (err error) { +// err = os.Chdir(workingDir) +// if err != nil { +// return +// } +// defer func() { +// err = errors.Join(err, os.Chdir(rootDir)) +// }() - var technologies []string - requestedTechnologies := params.Technologies() - if len(requestedTechnologies) != 0 { - technologies = requestedTechnologies - } else { - technologies = coreutils.DetectedTechnologiesList() - } - if len(technologies) == 0 { - log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") - return - } - serverDetails, err := params.ServerDetails() - if err != nil { - return - } +// var technologies []string +// requestedTechnologies := params.Technologies() +// if len(requestedTechnologies) != 0 { +// technologies = requestedTechnologies +// } else { +// technologies = coreutils.DetectedTechnologiesList() +// } +// if len(technologies) == 0 { +// log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") +// return +// } +// serverDetails, err := params.ServerDetails() +// if err != nil { +// return +// } - for _, tech := range coreutils.ToTechnologies(technologies) { - if tech == coreutils.Dotnet { - continue - } - flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, tech) - if techErr != nil { - err = errors.Join(err, fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", tech, techErr.Error())) - continue - } - if flattenTree == nil || len(flattenTree.Nodes) == 0 { - err = errors.Join(err, errors.New("no dependencies were found. Please try to build your project and re-run the audit command")) - continue - } +// for _, tech := range coreutils.ToTechnologies(technologies) { +// if tech == coreutils.Dotnet { +// continue +// } +// flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, tech) +// if techErr != nil { +// err = errors.Join(err, fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", tech, techErr.Error())) +// continue +// } +// if flattenTree == nil || len(flattenTree.Nodes) == 0 { +// err = errors.Join(err, errors.New("no dependencies were found. Please try to build your project and re-run the audit command")) +// continue +// } - scanGraphParams := scangraph.NewScanGraphParams(). - SetServerDetails(serverDetails). - SetXrayGraphScanParams(params.xrayGraphScanParams). - SetXrayVersion(params.xrayVersion). - SetFixableOnly(params.fixableOnly). - SetSeverityLevel(params.minSeverityFilter) - techResults, techErr := sca.RunXrayDependenciesTreeScanGraph(flattenTree, params.Progress(), tech, scanGraphParams) - if techErr != nil { - err = errors.Join(err, fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s\n", tech, techErr.Error())) - continue - } - techResults = sca.BuildImpactPathsForScanResponse(techResults, fullDependencyTrees) +// scanGraphParams := scangraph.NewScanGraphParams(). +// SetServerDetails(serverDetails). +// SetXrayGraphScanParams(params.xrayGraphScanParams). +// SetXrayVersion(params.xrayVersion). +// SetFixableOnly(params.fixableOnly). +// SetSeverityLevel(params.minSeverityFilter) +// techResults, techErr := sca.RunXrayDependenciesTreeScanGraph(flattenTree, params.Progress(), tech, scanGraphParams) +// if techErr != nil { +// err = errors.Join(err, fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s\n", tech, techErr.Error())) +// continue +// } +// techResults = sca.BuildImpactPathsForScanResponse(techResults, fullDependencyTrees) - var dependenciesForApplicabilityScan []string - if shouldUseAllDependencies(params.thirdPartyApplicabilityScan, tech) { - dependenciesForApplicabilityScan = getDirectDependenciesFromTree([]*xrayCmdUtils.GraphNode{flattenTree}) - } else { - dependenciesForApplicabilityScan = getDirectDependenciesFromTree(fullDependencyTrees) - } - params.AppendDependenciesForApplicabilityScan(dependenciesForApplicabilityScan) +// var dependenciesForApplicabilityScan []string +// if shouldUseAllDependencies(params.thirdPartyApplicabilityScan, tech) { +// dependenciesForApplicabilityScan = getDirectDependenciesFromTree([]*xrayCmdUtils.GraphNode{flattenTree}) +// } else { +// dependenciesForApplicabilityScan = getDirectDependenciesFromTree(fullDependencyTrees) +// } +// params.AppendDependenciesForApplicabilityScan(dependenciesForApplicabilityScan) - // results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...) - // if !results.IsMultipleRootProject { - // results.IsMultipleRootProject = len(fullDependencyTrees) > 1 - // } - // results.ExtendedScanResults.ScannedTechnologies = append(results.ExtendedScanResults.ScannedTechnologies, tech) - } - return -} +// // results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...) +// // if !results.IsMultipleRootProject { +// // results.IsMultipleRootProject = len(fullDependencyTrees) > 1 +// // } +// // results.ExtendedScanResults.ScannedTechnologies = append(results.ExtendedScanResults.ScannedTechnologies, tech) +// } +// return +// } // When building pip dependency tree using pipdeptree, some of the direct dependencies are recognized as transitive and missed by the CA scanner. // Our solution for this case is to send all dependencies to the CA scanner. diff --git a/xray/utils/results.go b/xray/utils/results.go index 1b03236e4..c86a3c0d3 100644 --- a/xray/utils/results.go +++ b/xray/utils/results.go @@ -8,13 +8,13 @@ import ( ) type Results struct { - ScaResults []ScaScanResult - XrayVersion string - ScaError error - + ScaResults []ScaScanResult + XrayVersion string + ScaError error + // IsMultipleRootProject bool // TODO: remove this - ExtendedScanResults *ExtendedScanResults - JasError error + ExtendedScanResults *ExtendedScanResults + JasError error } func NewAuditResults() *Results { @@ -67,12 +67,12 @@ func (r *Results) IsIssuesFound() bool { } type ScaScanResult struct { - Technology coreutils.Technology `json:"Technology"` - WorkingDirectory string `json:"WorkingDirectory"` - Descriptors []string `json:"Descriptors"` - XrayResults []services.ScanResponse `json:"XrayResults,omitempty"` + Technology coreutils.Technology `json:"Technology"` + WorkingDirectory string `json:"WorkingDirectory"` + Descriptors []string `json:"Descriptors,omitempty"` + XrayResults []services.ScanResponse `json:"XrayResults,omitempty"` - IsMultipleRootProject *bool `json:"IsMultipleRootProject,omitempty"` + IsMultipleRootProject *bool `json:"IsMultipleRootProject,omitempty"` } // func (s ScaScanResult) IsMultipleRootProject() bool { @@ -101,12 +101,12 @@ type ExtendedScanResults struct { } func (e *ExtendedScanResults) IsIssuesFound() bool { - return GetResultsLocationCount(e.ApplicabilityScanResults...) > 0 || - GetResultsLocationCount(e.SecretsScanResults...) > 0 || - GetResultsLocationCount(e.IacScanResults...) > 0 || - GetResultsLocationCount(e.SastScanResults...) > 0 + return GetResultsLocationCount(e.ApplicabilityScanResults...) > 0 || + GetResultsLocationCount(e.SecretsScanResults...) > 0 || + GetResultsLocationCount(e.IacScanResults...) > 0 || + GetResultsLocationCount(e.SastScanResults...) > 0 } // func (e *ExtendedScanResults) getXrayScanResults() []services.ScanResponse { // return e.XrayResults -// } \ No newline at end of file +// } From d463fa14dd62c33aa0d2139c79bad8e2978ceb19 Mon Sep 17 00:00:00 2001 From: attiasas Date: Sun, 22 Oct 2023 10:50:27 +0300 Subject: [PATCH 07/27] fix static --- utils/coreutils/techutils.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index ab0a17570..ef9570cd9 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -233,8 +233,10 @@ func mapFilesToRelevantWorkingDirectories(files []string, requestedDescriptors m } } } - strJson, _ := json.MarshalIndent(workingDirectoryToIndicators, "", " ") - log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson)) + strJson, err := json.MarshalIndent(workingDirectoryToIndicators, "", " ") + if err == nil { + log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson)) + } return } From d284a1547ca7a4f948f97074ee5d0ab26d0b056b Mon Sep 17 00:00:00 2001 From: attiasas Date: Sun, 22 Oct 2023 11:59:15 +0300 Subject: [PATCH 08/27] fix test --- xray/commands/audit/jasrunner_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xray/commands/audit/jasrunner_test.go b/xray/commands/audit/jasrunner_test.go index d4a8ebda2..8154874d6 100644 --- a/xray/commands/audit/jasrunner_test.go +++ b/xray/commands/audit/jasrunner_test.go @@ -22,7 +22,7 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() // scanResults := &utils.Results{ScaResults: }// &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Yarn}} - scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}} + scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} err = runJasScannersAndSetResults(scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, nil, "", false) // Expect error: assert.Error(t, err) @@ -30,7 +30,7 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { // scanResults := &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Pip}} - scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Pip, XrayResults: jas.FakeBasicXrayResults}}} + scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} err := runJasScannersAndSetResults(scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, nil, "", false) assert.NoError(t, err) } @@ -40,7 +40,7 @@ func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { assert.NoError(t, rtutils.DownloadAnalyzerManagerIfNeeded()) // scanResults := &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Yarn}} - scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}} + scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} err := runJasScannersAndSetResults(scanResults, mockDirectDependencies, &jas.FakeServerDetails, nil, nil, "", false) // Expect error: From c13dadf4617d1f186f1bcaf648e64610468a0ccd Mon Sep 17 00:00:00 2001 From: attiasas Date: Sun, 22 Oct 2023 13:03:24 +0300 Subject: [PATCH 09/27] clean up --- utils/coreutils/techutils.go | 57 +++------ xray/commands/audit/audit.go | 24 +--- xray/commands/audit/jasrunner.go | 4 - xray/commands/audit/jasrunner_test.go | 6 +- xray/commands/audit/scarunner.go | 172 ++------------------------ xray/utils/analyzermanager.go | 17 --- xray/utils/results.go | 14 --- 7 files changed, 26 insertions(+), 268 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index ef9570cd9..f01e36c18 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -63,66 +63,63 @@ type TechData struct { var technologiesData = map[Technology]TechData{ Maven: { - indicators: []string{"pom.xml"}, - ciSetupSupport: true, packageDescriptors: []string{"pom.xml"}, + ciSetupSupport: true, execCommand: "mvn", applicabilityScannable: true, }, Gradle: { + packageDescriptors: []string{"build.gradle", "build.gradle.kts"}, indicators: []string{".gradle", ".gradle.kts"}, ciSetupSupport: true, - packageDescriptors: []string{"build.gradle", "build.gradle.kts"}, applicabilityScannable: true, }, Npm: { - indicators: []string{"package.json", "package-lock.json", "npm-shrinkwrap.json"}, + packageDescriptors: []string{"package.json"}, + indicators: []string{"package-lock.json", "npm-shrinkwrap.json"}, exclude: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, ciSetupSupport: true, - packageDescriptors: []string{"package.json"}, formal: string(Npm), packageVersionOperator: "@", packageInstallationCommand: "install", applicabilityScannable: true, }, Yarn: { - indicators: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, packageDescriptors: []string{"package.json"}, + indicators: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, packageVersionOperator: "@", applicabilityScannable: true, }, Go: { - indicators: []string{"go.mod"}, packageDescriptors: []string{"go.mod"}, packageVersionOperator: "@v", packageInstallationCommand: "get", }, Pip: { packageType: Pypi, - indicators: []string{"setup.py", "requirements.txt"}, packageDescriptors: []string{"setup.py", "requirements.txt"}, exclude: []string{"Pipfile", "Pipfile.lock", "pyproject.toml", "poetry.lock"}, applicabilityScannable: true, }, Pipenv: { packageType: Pypi, - indicators: []string{"Pipfile", "Pipfile.lock"}, packageDescriptors: []string{"Pipfile"}, + indicators: []string{"Pipfile.lock"}, packageVersionOperator: "==", packageInstallationCommand: "install", applicabilityScannable: true, }, Poetry: { packageType: Pypi, - indicators: []string{"pyproject.toml", "poetry.lock"}, packageDescriptors: []string{"pyproject.toml"}, + indicators: []string{"poetry.lock"}, packageInstallationCommand: "add", packageVersionOperator: "==", applicabilityScannable: true, }, Nuget: { - indicators: []string{".sln", ".csproj"}, - formal: "NuGet", + packageDescriptors: []string{".sln", ".csproj"}, + formal: "NuGet", // .NET CLI is used for NuGet projects execCommand: "dotnet", packageInstallationCommand: "add", @@ -130,8 +127,8 @@ var technologiesData = map[Technology]TechData{ packageVersionOperator: " -v ", }, Dotnet: { - indicators: []string{".sln", ".csproj"}, - formal: ".NET", + packageDescriptors: []string{".sln", ".csproj"}, + formal: ".NET", }, } @@ -215,6 +212,7 @@ func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs [ return } +// Map files to relevant working directories according to the technologies' indicators/descriptors and requested descriptors. func mapFilesToRelevantWorkingDirectories(files []string, requestedDescriptors map[Technology][]string) (workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology) { workingDirectoryToIndicators = make(map[string][]string) excludedTechAtWorkingDir = make(map[string][]Technology) @@ -276,6 +274,7 @@ func isExclude(path string, techData TechData) bool { return false } +// Map working directories to technologies according to the given workingDirectoryToIndicators map files. func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedTechs []Technology) (technologiesDetected map[Technology]map[string][]string) { // Get the relevant technologies to check technologies := requestedTechs @@ -323,22 +322,17 @@ func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string return } +// Remove sub directories from the given workingDirectoryToFiles map. func cleanSubDirectories(workingDirectoryToFiles map[string][]string) (result map[string][]string) { result = make(map[string][]string) for wd, files := range workingDirectoryToFiles { root := getExistingRootDir(wd, workingDirectoryToFiles) result[root] = append(result[root], files...) - // if root == wd { - // // Current working directory is the root - // result[wd] = files - // } else { - // // add descriptors from sub projects to the root - // result[root] = append(result[root], files...) - // } } return } +// Get the root directory of the given path according to the given workingDirectoryToIndicators map. func getExistingRootDir(path string, workingDirectoryToIndicators map[string][]string) (rootDir string) { rootDir = path for wd := range workingDirectoryToIndicators { @@ -347,29 +341,8 @@ func getExistingRootDir(path string, workingDirectoryToIndicators map[string][]s } } return - - // // TODO: make sure to get the top most root! - // for wd := range workingDirectoryToIndicators { - // if path != wd && strings.HasPrefix(path, wd) { - // return wd - // } - // } - // return "" } -// func detectTechnologiesDescriptorsByFilePaths(paths []string) (technologiesToDescriptors map[Technology][]string) { -// detected := make(map[Technology][]string) -// for _, path := range paths { -// for techName, techData := range technologiesData { -// for _, descriptor := range techData.packageDescriptors { -// if strings.HasSuffix(path, descriptor) { -// detected[techName] = append(detected[techName], path) -// } -// } -// } -// } -// } - // DetectTechnologies tries to detect all technologies types according to the files in the given path. // 'isCiSetup' will limit the search of possible techs to Maven, Gradle, and npm. // 'recursive' will determine if the search will be limited to files in the root path or not. diff --git a/xray/commands/audit/audit.go b/xray/commands/audit/audit.go index 6513c8544..26d57ed5d 100644 --- a/xray/commands/audit/audit.go +++ b/xray/commands/audit/audit.go @@ -2,6 +2,8 @@ package audit import ( "errors" + "os" + rtutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-core/v2/xray/scangraph" @@ -10,7 +12,6 @@ import ( "github.com/jfrog/jfrog-client-go/xray" "github.com/jfrog/jfrog-client-go/xray/services" "golang.org/x/sync/errgroup" - "os" xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils" ) @@ -136,25 +137,6 @@ func (auditCmd *AuditCommand) CommandName() string { return "generic_audit" } -// type ScaScanResult struct { -// Technology coreutils.Technology -// XrayResults []services.ScanResponse -// Descriptors []string -// } - -// type Results struct { -// ScaResults []ScaScanResult -// IsMultipleRootProject bool -// ScaError error - -// ExtendedScanResults *xrayutils.ExtendedScanResults -// JasError error -// } - -// func NewAuditResults() *Results { -// return &Results{ExtendedScanResults: &xrayutils.ExtendedScanResults{}} -// } - // Runs an audit scan based on the provided auditParams. // Returns an audit Results object containing all the scan results. // If the current server is entitled for JAS, the advanced security results will be included in the scan results. @@ -186,7 +168,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) } // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. - results.ScaError = scaScan(auditParams, results) // runScaScan(auditParams, results) + results.ScaError = runScaScan(auditParams, results) // runScaScan(auditParams, results) // Wait for the Download of the AnalyzerManager to complete. if err = errGroup.Wait(); err != nil { diff --git a/xray/commands/audit/jasrunner.go b/xray/commands/audit/jasrunner.go index 07a25da6d..b9dc0c154 100644 --- a/xray/commands/audit/jasrunner.go +++ b/xray/commands/audit/jasrunner.go @@ -30,10 +30,6 @@ func runJasScannersAndSetResults(scanResults *utils.Results, directDependencies if progress != nil { progress.SetHeadlineMsg("Running applicability scanning") } - // for _, scaResult := range scanResults.ScaResults { - // // Do ny working dir, collect techs to save scanning - // scanResults.ExtendedScanResults.ApplicabilityScanResults, err = applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan) - // } scanResults.ExtendedScanResults.ApplicabilityScanResults, err = applicability.RunApplicabilityScan(scanResults.GetScaScansXrayResults(), directDependencies, scanResults.GetScaScannedTechnologies(), scanner, thirdPartyApplicabilityScan) if err != nil { return diff --git a/xray/commands/audit/jasrunner_test.go b/xray/commands/audit/jasrunner_test.go index 8154874d6..2c6445b66 100644 --- a/xray/commands/audit/jasrunner_test.go +++ b/xray/commands/audit/jasrunner_test.go @@ -21,7 +21,6 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { defer func() { assert.NoError(t, os.Unsetenv(coreutils.HomeDir)) }() - // scanResults := &utils.Results{ScaResults: }// &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Yarn}} scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} err = runJasScannersAndSetResults(scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, &jas.FakeServerDetails, nil, nil, "", false) // Expect error: @@ -29,19 +28,16 @@ func TestGetExtendedScanResults_AnalyzerManagerDoesntExist(t *testing.T) { } func TestGetExtendedScanResults_ServerNotValid(t *testing.T) { - // scanResults := &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Pip}} scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Pip, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} err := runJasScannersAndSetResults(scanResults, []string{"issueId_1_direct_dependency", "issueId_2_direct_dependency"}, nil, nil, nil, "", false) assert.NoError(t, err) } func TestGetExtendedScanResults_AnalyzerManagerReturnsError(t *testing.T) { - mockDirectDependencies := []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"} assert.NoError(t, rtutils.DownloadAnalyzerManagerIfNeeded()) - // scanResults := &utils.ExtendedScanResults{XrayResults: jas.FakeBasicXrayResults, ScannedTechnologies: []coreutils.Technology{coreutils.Yarn}} scanResults := &utils.Results{ScaResults: []utils.ScaScanResult{{Technology: coreutils.Yarn, XrayResults: jas.FakeBasicXrayResults}}, ExtendedScanResults: &utils.ExtendedScanResults{}} - err := runJasScannersAndSetResults(scanResults, mockDirectDependencies, &jas.FakeServerDetails, nil, nil, "", false) + err := runJasScannersAndSetResults(scanResults, []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"}, &jas.FakeServerDetails, nil, nil, "", false) // Expect error: assert.ErrorContains(t, err, "failed to run Applicability scan") diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index ccad27e24..c2938a1db 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -27,7 +27,7 @@ import ( xrayCmdUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" ) -func scaScan(params *AuditParams, results *xrayutils.Results) (err error) { +func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { // Prepare currentWorkingDir, err := os.Getwd() if errorutils.CheckError(err) != nil { @@ -42,12 +42,12 @@ func scaScan(params *AuditParams, results *xrayutils.Results) (err error) { log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") return } + printScansInformation(scans) + defer func() { // Make sure to return to the original working directory, executeScaScan may change it err = errors.Join(err, os.Chdir(currentWorkingDir)) }() - - printScansInformation(scans) for _, scan := range scans { // Run the scan log.Info("Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") @@ -61,11 +61,12 @@ func scaScan(params *AuditParams, results *xrayutils.Results) (err error) { return } +// Calculate the scans to preform func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansToPreform []*xrayutils.ScaScanResult) { isRecursive := true excludePattern := "" for _, requestedDirectory := range getRequestedDirectoriesToScan(currentWorkingDir, params) { - // Detect descriptors from files + // Detect descriptors and technologies in the requested directory. techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, isRecursive, params.Technologies(), getRequestedDescriptors(params), excludePattern) // Create scans to preform for tech, workingDirs := range techToWorkingDirs { @@ -75,7 +76,7 @@ func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansT continue } if len(workingDirs) == 0 { - // Requested technology (from params) descriptors was not found, scan only requested directory for this technology. + // Requested technology (from params) descriptors/indicators was not found, scan only requested directory for this technology. scansToPreform = append(scansToPreform, &xrayutils.ScaScanResult{WorkingDirectory: requestedDirectory, Technology: tech}) } for workingDir, descriptors := range workingDirs { @@ -114,7 +115,7 @@ func getRequestedDirectoriesToScan(currentWorkingDir string, params *AuditParams } // Preform the SCA scan for the given scan information. -// This method may change the working directory to the scan's working directory. +// This method will change the working directory to the scan's working directory. func executeScaScan(serverDetails *config.ServerDetails, params *AuditParams, scan *xrayutils.ScaScanResult) (err error) { // Get the dependency tree for the technology in the working directory. if err = os.Chdir(scan.WorkingDirectory); err != nil { @@ -153,53 +154,6 @@ func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetail return } -// func runScaScanInWorkingDir(params *AuditParams, results *xrayutils.Results, workingDir string) (err error) { -// // Prepare the working directory information for the scan. -// technologies := coreutils.DetectTechnologiesDescriptors(workingDir, getTechnologiesToDetect(params), true) -// if len(technologies) == 0 { -// log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") -// return -// } -// serverDetails, err := params.ServerDetails() -// if err != nil { -// return -// } -// // Run the scan for each technology. -// for tech, detectedDescriptors := range technologies { -// if tech == coreutils.Dotnet { -// continue -// } -// // Get the dependency tree for the technology. -// flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, tech) -// if techErr != nil { -// err = errors.Join(err, fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", tech, techErr.Error())) -// continue -// } -// if flattenTree == nil || len(flattenTree.Nodes) == 0 { -// err = errors.Join(err, errors.New("no dependencies were found. Please try to build your project and re-run the audit command")) -// continue -// } -// // Scan the dependency tree. -// scanResults, xrayErr := runScaOnTech(tech, params, serverDetails, flattenTree, fullDependencyTrees, results) -// if xrayErr != nil { -// err = errors.Join(err, fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s\n", tech, xrayErr.Error())) -// continue -// } -// addThirdPartyDependenciesToParams(params, tech, flattenTree, fullDependencyTrees) -// results.ScaResults = append(results.ScaResults, xrayutils.ScaScanResult{Technology: tech, XrayResults: scanResults, Descriptors: detectedDescriptors}) -// } -// return -// } - -// func getTechnologiesToDetect(params *AuditParams) (technologies []coreutils.Technology) { -// if len(params.Technologies()) != 0 { -// technologies = coreutils.ToTechnologies(params.Technologies()) -// } else { -// technologies = coreutils.GetAllTechnologiesList() -// } -// return -// } - func addThirdPartyDependenciesToParams(params *AuditParams, tech coreutils.Technology, flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) { var dependenciesForApplicabilityScan []string if shouldUseAllDependencies(params.thirdPartyApplicabilityScan, tech) { @@ -210,118 +164,6 @@ func addThirdPartyDependenciesToParams(params *AuditParams, tech coreutils.Techn params.AppendDependenciesForApplicabilityScan(dependenciesForApplicabilityScan) } -// func runScaOnTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, results *xrayutils.Results) (techResults []services.ScanResponse, err error) { -// scanGraphParams := scangraph.NewScanGraphParams(). -// SetServerDetails(serverDetails). -// SetXrayGraphScanParams(params.xrayGraphScanParams). -// SetXrayVersion(params.xrayVersion). -// SetFixableOnly(params.fixableOnly). -// SetSeverityLevel(params.minSeverityFilter) -// techResults, err = sca.RunXrayDependenciesTreeScanGraph(flatTree, params.Progress(), tech, scanGraphParams) -// if err != nil { -// return -// } -// techResults = sca.BuildImpactPathsForScanResponse(techResults, fullDependencyTrees) - -// // results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...) -// // if !results.IsMultipleRootProject { -// // results.IsMultipleRootProject = len(fullDependencyTrees) > 1 -// // } - -// // results.ExtendedScanResults.ScannedTechnologies = append(results.ExtendedScanResults.ScannedTechnologies, tech) -// return -// } - -// func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { -// rootDir, err := os.Getwd() -// if errorutils.CheckError(err) != nil { -// return -// } -// for _, wd := range params.workingDirs { -// if len(params.workingDirs) > 1 { -// log.Info("Running SCA scan for vulnerable dependencies scan in", wd, "directory...") -// } else { -// log.Info("Running SCA scan for vulnerable dependencies...") -// } -// wdScanErr := runScaScanOnWorkingDir(params, results, wd, rootDir) -// if wdScanErr != nil { -// err = errors.Join(err, fmt.Errorf("audit command in '%s' failed:\n%s\n", wd, wdScanErr.Error())) -// continue -// } -// } -// return -// } - -// Audits the project found in the current directory using Xray. -// func runScaScanOnWorkingDir(params *AuditParams, results *xrayutils.Results, workingDir, rootDir string) (err error) { -// err = os.Chdir(workingDir) -// if err != nil { -// return -// } -// defer func() { -// err = errors.Join(err, os.Chdir(rootDir)) -// }() - -// var technologies []string -// requestedTechnologies := params.Technologies() -// if len(requestedTechnologies) != 0 { -// technologies = requestedTechnologies -// } else { -// technologies = coreutils.DetectedTechnologiesList() -// } -// if len(technologies) == 0 { -// log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") -// return -// } -// serverDetails, err := params.ServerDetails() -// if err != nil { -// return -// } - -// for _, tech := range coreutils.ToTechnologies(technologies) { -// if tech == coreutils.Dotnet { -// continue -// } -// flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, tech) -// if techErr != nil { -// err = errors.Join(err, fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", tech, techErr.Error())) -// continue -// } -// if flattenTree == nil || len(flattenTree.Nodes) == 0 { -// err = errors.Join(err, errors.New("no dependencies were found. Please try to build your project and re-run the audit command")) -// continue -// } - -// scanGraphParams := scangraph.NewScanGraphParams(). -// SetServerDetails(serverDetails). -// SetXrayGraphScanParams(params.xrayGraphScanParams). -// SetXrayVersion(params.xrayVersion). -// SetFixableOnly(params.fixableOnly). -// SetSeverityLevel(params.minSeverityFilter) -// techResults, techErr := sca.RunXrayDependenciesTreeScanGraph(flattenTree, params.Progress(), tech, scanGraphParams) -// if techErr != nil { -// err = errors.Join(err, fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s\n", tech, techErr.Error())) -// continue -// } -// techResults = sca.BuildImpactPathsForScanResponse(techResults, fullDependencyTrees) - -// var dependenciesForApplicabilityScan []string -// if shouldUseAllDependencies(params.thirdPartyApplicabilityScan, tech) { -// dependenciesForApplicabilityScan = getDirectDependenciesFromTree([]*xrayCmdUtils.GraphNode{flattenTree}) -// } else { -// dependenciesForApplicabilityScan = getDirectDependenciesFromTree(fullDependencyTrees) -// } -// params.AppendDependenciesForApplicabilityScan(dependenciesForApplicabilityScan) - -// // results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...) -// // if !results.IsMultipleRootProject { -// // results.IsMultipleRootProject = len(fullDependencyTrees) > 1 -// // } -// // results.ExtendedScanResults.ScannedTechnologies = append(results.ExtendedScanResults.ScannedTechnologies, tech) -// } -// return -// } - // When building pip dependency tree using pipdeptree, some of the direct dependencies are recognized as transitive and missed by the CA scanner. // Our solution for this case is to send all dependencies to the CA scanner. // When thirdPartyApplicabilityScan is true, use flatten graph to include all the dependencies in applicability scanning. diff --git a/xray/utils/analyzermanager.go b/xray/utils/analyzermanager.go index 7f3ec48b3..bec032c7e 100644 --- a/xray/utils/analyzermanager.go +++ b/xray/utils/analyzermanager.go @@ -76,23 +76,6 @@ var exitCodeErrorsMap = map[int]string{ unsupportedOsExitCode: "got unsupported operating system error from analyzer manager", } -// type ExtendedScanResults struct { -// XrayResults []services.ScanResponse -// XrayVersion string -// ScannedTechnologies []coreutils.Technology - -// // DependenciesResults [] -// ApplicabilityScanResults []*sarif.Run -// SecretsScanResults []*sarif.Run -// IacScanResults []*sarif.Run -// SastScanResults []*sarif.Run -// EntitledForJas bool -// } - -// func (e *ExtendedScanResults) getXrayScanResults() []services.ScanResponse { -// return e.XrayResults -// } - type AnalyzerManager struct { AnalyzerManagerFullPath string MultiScanId string diff --git a/xray/utils/results.go b/xray/utils/results.go index c86a3c0d3..89ba3f275 100644 --- a/xray/utils/results.go +++ b/xray/utils/results.go @@ -12,7 +12,6 @@ type Results struct { XrayVersion string ScaError error - // IsMultipleRootProject bool // TODO: remove this ExtendedScanResults *ExtendedScanResults JasError error } @@ -37,7 +36,6 @@ func (r *Results) GetScaScannedTechnologies() []coreutils.Technology { } func (r *Results) IsMultipleProject() bool { - // Maybe we should check by working directory as well? if len(r.ScaResults) == 0 { return false } @@ -75,10 +73,6 @@ type ScaScanResult struct { IsMultipleRootProject *bool `json:"IsMultipleRootProject,omitempty"` } -// func (s ScaScanResult) IsMultipleRootProject() bool { -// return len(s.XrayResults) > 1 -// } - func (s ScaScanResult) HasInformation() bool { for _, scan := range s.XrayResults { if len(scan.Vulnerabilities) > 0 || len(scan.Violations) > 0 || len(scan.Licenses) > 0 { @@ -89,10 +83,6 @@ func (s ScaScanResult) HasInformation() bool { } type ExtendedScanResults struct { - // XrayResults []services.ScanResponse // TODO: remove this - // XrayVersion string // TODO: remove this - // ScannedTechnologies []coreutils.Technology // TODO: remove this - ApplicabilityScanResults []*sarif.Run SecretsScanResults []*sarif.Run IacScanResults []*sarif.Run @@ -106,7 +96,3 @@ func (e *ExtendedScanResults) IsIssuesFound() bool { GetResultsLocationCount(e.IacScanResults...) > 0 || GetResultsLocationCount(e.SastScanResults...) > 0 } - -// func (e *ExtendedScanResults) getXrayScanResults() []services.ScanResponse { -// return e.XrayResults -// } From d97299eacb172298b35f9558c2de230553b2aaeb Mon Sep 17 00:00:00 2001 From: attiasas Date: Sun, 22 Oct 2023 13:25:47 +0300 Subject: [PATCH 10/27] add parameters --- utils/coreutils/techutils.go | 18 ------------------ xray/commands/audit/auditparams.go | 21 +++++++++++++++++++++ xray/commands/audit/scarunner.go | 4 +--- xray/utils/results.go | 11 +++++------ 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index f01e36c18..fe43da4e2 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -394,24 +394,6 @@ func detectTechnologiesByFilePaths(paths []string, isCiSetup bool) (detected map return detected } -// func t(path string, techData TechData) (exclude bool, detected bool) { -// // If the project contains a file/directory with a name that ends with an excluded suffix, then this technology is excluded. -// for _, excludeFile := range techData.exclude { -// if strings.HasSuffix(path, excludeFile) { -// return true, false -// } -// } -// // If this technology was already excluded, there's no need to look for indicator files/directories. -// if _, exist := exclude[techName]; !exist { -// // If the project contains a file/directory with a name that ends with the indicator suffix, then the project probably uses this technology. -// for _, indicator := range techData.indicators { -// if strings.HasSuffix(path, indicator) { -// return false, true -// } -// } -// } -// } - // DetectedTechnologiesToSlice returns a string slice that includes all the names of the detected technologies. func DetectedTechnologiesToSlice(detected map[Technology]bool) []string { keys := make([]string, 0, len(detected)) diff --git a/xray/commands/audit/auditparams.go b/xray/commands/audit/auditparams.go index 9dc42e3c2..ac78a70ee 100644 --- a/xray/commands/audit/auditparams.go +++ b/xray/commands/audit/auditparams.go @@ -8,6 +8,8 @@ import ( type AuditParams struct { xrayGraphScanParams *services.XrayGraphScanParams workingDirs []string + excludePattern string + recursively bool installFunc func(tech string) error fixableOnly bool minSeverityFilter string @@ -21,6 +23,7 @@ func NewAuditParams() *AuditParams { return &AuditParams{ xrayGraphScanParams: &services.XrayGraphScanParams{}, AuditBasicParams: &xrayutils.AuditBasicParams{}, + recursively: true, } } @@ -40,6 +43,24 @@ func (params *AuditParams) XrayVersion() string { return params.xrayVersion } +func (params *AuditParams) ExcludePattern() string { + return params.excludePattern +} + +func (params *AuditParams) Recursively() bool { + return params.recursively +} + +func (params *AuditParams) SetRecursively(recursively bool) *AuditParams { + params.recursively = recursively + return params +} + +func (params *AuditParams) SetExcludePattern(excludePattern string) *AuditParams { + params.excludePattern = excludePattern + return params +} + func (params *AuditParams) SetXrayGraphScanParams(xrayGraphScanParams *services.XrayGraphScanParams) *AuditParams { params.xrayGraphScanParams = xrayGraphScanParams return params diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index c2938a1db..99a9553cc 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -63,11 +63,9 @@ func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { // Calculate the scans to preform func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansToPreform []*xrayutils.ScaScanResult) { - isRecursive := true - excludePattern := "" for _, requestedDirectory := range getRequestedDirectoriesToScan(currentWorkingDir, params) { // Detect descriptors and technologies in the requested directory. - techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, isRecursive, params.Technologies(), getRequestedDescriptors(params), excludePattern) + techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, params.Recursively(), params.Technologies(), getRequestedDescriptors(params), params.ExcludePattern()) // Create scans to preform for tech, workingDirs := range techToWorkingDirs { if tech == coreutils.Dotnet { diff --git a/xray/utils/results.go b/xray/utils/results.go index 89ba3f275..8d1eb507f 100644 --- a/xray/utils/results.go +++ b/xray/utils/results.go @@ -65,12 +65,11 @@ func (r *Results) IsIssuesFound() bool { } type ScaScanResult struct { - Technology coreutils.Technology `json:"Technology"` - WorkingDirectory string `json:"WorkingDirectory"` - Descriptors []string `json:"Descriptors,omitempty"` - XrayResults []services.ScanResponse `json:"XrayResults,omitempty"` - - IsMultipleRootProject *bool `json:"IsMultipleRootProject,omitempty"` + Technology coreutils.Technology `json:"Technology"` + WorkingDirectory string `json:"WorkingDirectory"` + XrayResults []services.ScanResponse `json:"XrayResults,omitempty"` + Descriptors []string `json:"Descriptors,omitempty"` + IsMultipleRootProject *bool `json:"IsMultipleRootProject,omitempty"` } func (s ScaScanResult) HasInformation() bool { From 4d71df5c58d60ee8fb0025a0ca48f57b89a930cf Mon Sep 17 00:00:00 2001 From: attiasas Date: Sun, 22 Oct 2023 17:42:52 +0300 Subject: [PATCH 11/27] add new params --- utils/coreutils/techutils.go | 59 +++++++++++++++++++----------- xray/commands/audit/audit.go | 4 +- xray/commands/audit/auditparams.go | 21 +++++------ xray/commands/audit/scarunner.go | 11 +++++- 4 files changed, 60 insertions(+), 35 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index fe43da4e2..33dc095af 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-client-go/artifactory/services/fspatterns" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -63,34 +64,36 @@ type TechData struct { var technologiesData = map[Technology]TechData{ Maven: { - packageDescriptors: []string{"pom.xml"}, + indicators: []string{"pom.xml"}, ciSetupSupport: true, + packageDescriptors: []string{"pom.xml"}, execCommand: "mvn", applicabilityScannable: true, }, Gradle: { - packageDescriptors: []string{"build.gradle", "build.gradle.kts"}, indicators: []string{".gradle", ".gradle.kts"}, ciSetupSupport: true, + packageDescriptors: []string{"build.gradle", "build.gradle.kts"}, applicabilityScannable: true, }, Npm: { - packageDescriptors: []string{"package.json"}, - indicators: []string{"package-lock.json", "npm-shrinkwrap.json"}, + indicators: []string{"package.json", "package-lock.json", "npm-shrinkwrap.json"}, exclude: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, ciSetupSupport: true, + packageDescriptors: []string{"package.json"}, formal: string(Npm), packageVersionOperator: "@", packageInstallationCommand: "install", applicabilityScannable: true, }, Yarn: { - packageDescriptors: []string{"package.json"}, indicators: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, + packageDescriptors: []string{"package.json"}, packageVersionOperator: "@", applicabilityScannable: true, }, Go: { + indicators: []string{"go.mod"}, packageDescriptors: []string{"go.mod"}, packageVersionOperator: "@v", packageInstallationCommand: "get", @@ -98,26 +101,28 @@ var technologiesData = map[Technology]TechData{ Pip: { packageType: Pypi, packageDescriptors: []string{"setup.py", "requirements.txt"}, + indicators: []string{"setup.py", "requirements.txt"}, exclude: []string{"Pipfile", "Pipfile.lock", "pyproject.toml", "poetry.lock"}, applicabilityScannable: true, }, Pipenv: { packageType: Pypi, + indicators: []string{"Pipfile", "Pipfile.lock"}, packageDescriptors: []string{"Pipfile"}, - indicators: []string{"Pipfile.lock"}, packageVersionOperator: "==", packageInstallationCommand: "install", applicabilityScannable: true, }, Poetry: { packageType: Pypi, + indicators: []string{"pyproject.toml", "poetry.lock"}, packageDescriptors: []string{"pyproject.toml"}, - indicators: []string{"poetry.lock"}, packageInstallationCommand: "add", packageVersionOperator: "==", applicabilityScannable: true, }, Nuget: { + indicators: []string{".sln", ".csproj"}, packageDescriptors: []string{".sln", ".csproj"}, formal: "NuGet", // .NET CLI is used for NuGet projects @@ -127,6 +132,7 @@ var technologiesData = map[Technology]TechData{ packageVersionOperator: " -v ", }, Dotnet: { + indicators: []string{".sln", ".csproj"}, packageDescriptors: []string{".sln", ".csproj"}, formal: ".NET", }, @@ -207,23 +213,31 @@ func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs [ return } workingDirectoryToIndicators, excludedTechAtWorkingDir := mapFilesToRelevantWorkingDirectories(filesList, requestedDescriptors) - technologiesDetected = mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators, excludedTechAtWorkingDir, ToTechnologies(requestedTechs)) - log.Debug(fmt.Sprintf("Detected %d technologies at %s: %s.", len(technologiesDetected), path, maps.Keys(technologiesDetected))) + strJson, err := json.MarshalIndent(workingDirectoryToIndicators, "", " ") + if err == nil && len(workingDirectoryToIndicators) > 0 { + log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson)) + } + technologiesDetected = mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators, excludedTechAtWorkingDir, ToTechnologies(requestedTechs), requestedDescriptors) + if len(technologiesDetected) > 0 { + log.Debug(fmt.Sprintf("Detected %d technologies at %s: %s.", len(technologiesDetected), path, maps.Keys(technologiesDetected))) + } return } // Map files to relevant working directories according to the technologies' indicators/descriptors and requested descriptors. func mapFilesToRelevantWorkingDirectories(files []string, requestedDescriptors map[Technology][]string) (workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology) { - workingDirectoryToIndicators = make(map[string][]string) + workingDirectoryToIndicatorsSet := make(map[string]*datastructures.Set[string]) excludedTechAtWorkingDir = make(map[string][]Technology) for _, path := range files { directory := filepath.Dir(path) for tech, techData := range technologiesData { // Check if the working directory contains indicators/descriptors for the technology - if isDescriptor(path, techData) || isRequestedDescriptor(path, requestedDescriptors[tech]) { - workingDirectoryToIndicators[directory] = append(workingDirectoryToIndicators[directory], path) - } else if isIndicator(path, techData) { - workingDirectoryToIndicators[directory] = append(workingDirectoryToIndicators[directory], path) + relevant := isIndicator(path, techData) || isDescriptor(path, techData) || isRequestedDescriptor(path, requestedDescriptors[tech]) + if relevant { + if _, exist := workingDirectoryToIndicatorsSet[directory]; !exist { + workingDirectoryToIndicatorsSet[directory] = datastructures.MakeSet[string]() + } + workingDirectoryToIndicatorsSet[directory].Add(path) } // Check if the working directory contains a file/directory with a name that ends with an excluded suffix if isExclude(path, techData) { @@ -231,9 +245,9 @@ func mapFilesToRelevantWorkingDirectories(files []string, requestedDescriptors m } } } - strJson, err := json.MarshalIndent(workingDirectoryToIndicators, "", " ") - if err == nil { - log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson)) + workingDirectoryToIndicators = make(map[string][]string) + for wd, indicators := range workingDirectoryToIndicatorsSet { + workingDirectoryToIndicators[wd] = indicators.ToSlice() } return } @@ -275,7 +289,7 @@ func isExclude(path string, techData TechData) bool { } // Map working directories to technologies according to the given workingDirectoryToIndicators map files. -func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedTechs []Technology) (technologiesDetected map[Technology]map[string][]string) { +func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedTechs []Technology, requestedDescriptors map[Technology][]string) (technologiesDetected map[Technology]map[string][]string) { // Get the relevant technologies to check technologies := requestedTechs if len(technologies) == 0 { @@ -297,17 +311,18 @@ func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string } // Check if the working directory contains indicators/descriptors for the technology for _, path := range indicators { - if isDescriptor(path, technologiesData[tech]) { + if isDescriptor(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { techWorkingDirs[wd] = append(techWorkingDirs[wd], path) - } else if isIndicator(path, technologiesData[tech]) { + } + if isIndicator(path, technologiesData[tech]) { foundIndicator = true } } } // Don't allow working directory if sub directory already exists as key for the same technology techWorkingDirs = cleanSubDirectories(techWorkingDirs) - if foundIndicator || len(techWorkingDirs) > 0 { - // Found indicators/descriptors for technology, add to detected. + if foundIndicator { + // Found indicators of the technology, add to detected. technologiesDetected[tech] = techWorkingDirs } } diff --git a/xray/commands/audit/audit.go b/xray/commands/audit/audit.go index 26d57ed5d..d6575f2d6 100644 --- a/xray/commands/audit/audit.go +++ b/xray/commands/audit/audit.go @@ -93,7 +93,9 @@ func (auditCmd *AuditCommand) Run() (err error) { SetMinSeverityFilter(auditCmd.minSeverityFilter). SetFixableOnly(auditCmd.fixableOnly). SetGraphBasicParams(auditCmd.AuditBasicParams). - SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan) + SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan). + SetRecursive(auditCmd.recursive). + SetExclusions(auditCmd.exclusions) auditResults, err := RunAudit(auditParams) if err != nil { return diff --git a/xray/commands/audit/auditparams.go b/xray/commands/audit/auditparams.go index ac78a70ee..beaf0b166 100644 --- a/xray/commands/audit/auditparams.go +++ b/xray/commands/audit/auditparams.go @@ -8,8 +8,8 @@ import ( type AuditParams struct { xrayGraphScanParams *services.XrayGraphScanParams workingDirs []string - excludePattern string - recursively bool + exclusions []string + recursive bool installFunc func(tech string) error fixableOnly bool minSeverityFilter string @@ -23,7 +23,6 @@ func NewAuditParams() *AuditParams { return &AuditParams{ xrayGraphScanParams: &services.XrayGraphScanParams{}, AuditBasicParams: &xrayutils.AuditBasicParams{}, - recursively: true, } } @@ -43,21 +42,21 @@ func (params *AuditParams) XrayVersion() string { return params.xrayVersion } -func (params *AuditParams) ExcludePattern() string { - return params.excludePattern +func (params *AuditParams) Exclusions() []string { + return params.exclusions } -func (params *AuditParams) Recursively() bool { - return params.recursively +func (params *AuditParams) Recursive() bool { + return params.recursive } -func (params *AuditParams) SetRecursively(recursively bool) *AuditParams { - params.recursively = recursively +func (params *AuditParams) SetRecursive(recursively bool) *AuditParams { + params.recursive = recursively return params } -func (params *AuditParams) SetExcludePattern(excludePattern string) *AuditParams { - params.excludePattern = excludePattern +func (params *AuditParams) SetExclusions(exclusions []string) *AuditParams { + params.exclusions = exclusions return params } diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index 99a9553cc..e35a719d4 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -20,6 +20,7 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca/yarn" "github.com/jfrog/jfrog-cli-core/v2/xray/scangraph" xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils" + "github.com/jfrog/jfrog-client-go/artifactory/services/fspatterns" clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -27,6 +28,8 @@ import ( xrayCmdUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" ) +var defaultExcludePatterns = []string{"*node_modules*", "*target*", "*venv*", "*test*"} + func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { // Prepare currentWorkingDir, err := os.Getwd() @@ -65,7 +68,7 @@ func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansToPreform []*xrayutils.ScaScanResult) { for _, requestedDirectory := range getRequestedDirectoriesToScan(currentWorkingDir, params) { // Detect descriptors and technologies in the requested directory. - techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, params.Recursively(), params.Technologies(), getRequestedDescriptors(params), params.ExcludePattern()) + techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, params.Recursive(), params.Technologies(), getRequestedDescriptors(params), getExcludePattern(params)) // Create scans to preform for tech, workingDirs := range techToWorkingDirs { if tech == coreutils.Dotnet { @@ -94,6 +97,12 @@ func getRequestedDescriptors(params *AuditParams) map[coreutils.Technology][]str return requestedDescriptors } +func getExcludePattern(params *AuditParams) string { + exclusions := params.Exclusions() + exclusions = append(exclusions, defaultExcludePatterns...) + return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, params.Recursive()) +} + func printScansInformation(scans []*xrayutils.ScaScanResult) { scansJson, err := json.MarshalIndent(scans, "", " ") if err == nil { From 9c665994d17bff43e52dc117011add2492747eb9 Mon Sep 17 00:00:00 2001 From: attiasas Date: Sun, 22 Oct 2023 17:44:29 +0300 Subject: [PATCH 12/27] format --- utils/coreutils/techutils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 33dc095af..8d245741a 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -100,8 +100,8 @@ var technologiesData = map[Technology]TechData{ }, Pip: { packageType: Pypi, - packageDescriptors: []string{"setup.py", "requirements.txt"}, indicators: []string{"setup.py", "requirements.txt"}, + packageDescriptors: []string{"setup.py", "requirements.txt"}, exclude: []string{"Pipfile", "Pipfile.lock", "pyproject.toml", "poetry.lock"}, applicabilityScannable: true, }, From ee1bd2a8d50cfc022826be7c5238deb7b7feb22b Mon Sep 17 00:00:00 2001 From: attiasas Date: Sun, 22 Oct 2023 17:57:03 +0300 Subject: [PATCH 13/27] fix requested descriptor indicator --- utils/coreutils/techutils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 8d245741a..1bd8bb0ca 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -314,7 +314,7 @@ func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string if isDescriptor(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { techWorkingDirs[wd] = append(techWorkingDirs[wd], path) } - if isIndicator(path, technologiesData[tech]) { + if isIndicator(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { foundIndicator = true } } From 15992e3193f73da2919f5795a7f37ff8c6826b13 Mon Sep 17 00:00:00 2001 From: attiasas Date: Tue, 24 Oct 2023 10:33:43 +0300 Subject: [PATCH 14/27] add tests --- utils/coreutils/techutils.go | 83 +++++++----- utils/coreutils/techutils_test.go | 214 +++++++++++++++++++++++++++++- 2 files changed, 266 insertions(+), 31 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 1bd8bb0ca..4742b3c26 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -230,6 +230,7 @@ func mapFilesToRelevantWorkingDirectories(files []string, requestedDescriptors m excludedTechAtWorkingDir = make(map[string][]Technology) for _, path := range files { directory := filepath.Dir(path) + for tech, techData := range technologiesData { // Check if the working directory contains indicators/descriptors for the technology relevant := isIndicator(path, techData) || isDescriptor(path, techData) || isRequestedDescriptor(path, requestedDescriptors[tech]) @@ -298,35 +299,12 @@ func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string technologiesDetected = make(map[Technology]map[string][]string) // Map working directories to technologies for _, tech := range technologies { - techWorkingDirs := make(map[string][]string) - foundIndicator := false - for wd, indicators := range workingDirectoryToIndicators { - if excludedTechs, exist := excludedTechAtWorkingDir[wd]; exist { - for _, excludedTech := range excludedTechs { - if excludedTech == tech { - // Exclude this technology from this working directory - continue - } - } - } - // Check if the working directory contains indicators/descriptors for the technology - for _, path := range indicators { - if isDescriptor(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { - techWorkingDirs[wd] = append(techWorkingDirs[wd], path) - } - if isIndicator(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { - foundIndicator = true - } - } - } - // Don't allow working directory if sub directory already exists as key for the same technology - techWorkingDirs = cleanSubDirectories(techWorkingDirs) - if foundIndicator { + techWorkingDirs := getTechInformationFromWorkingDir(tech, workingDirectoryToIndicators, excludedTechAtWorkingDir, requestedDescriptors) + if len(techWorkingDirs) > 0 { // Found indicators of the technology, add to detected. technologiesDetected[tech] = techWorkingDirs } } - for _, tech := range requestedTechs { if _, exist := technologiesDetected[tech]; !exist { // Requested (forced with flag) technology and not found any indicators/descriptors in detection, add as detected. @@ -337,7 +315,50 @@ func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string return } -// Remove sub directories from the given workingDirectoryToFiles map. +func getTechInformationFromWorkingDir(tech Technology, workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedDescriptors map[Technology][]string) (techWorkingDirs map[string][]string) { + techWorkingDirs = make(map[string][]string) + for wd, indicators := range workingDirectoryToIndicators { + descriptorsAtWd := []string{} + foundIndicator := false + if isTechExcludedInWorkingDir(tech, wd, excludedTechAtWorkingDir) { + // Exclude this technology from this working directory + continue + } + // Check if the working directory contains indicators/descriptors for the technology + for _, path := range indicators { + if isDescriptor(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { + descriptorsAtWd = append(descriptorsAtWd, path) + } + if isIndicator(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { + foundIndicator = true + } + } + if foundIndicator { + // Found indicators of the technology in the current working directory, add to detected. + techWorkingDirs[wd] = descriptorsAtWd + } + } + // Don't allow working directory if sub directory already exists as key for the same technology + techWorkingDirs = cleanSubDirectories(techWorkingDirs) + return +} + +func isTechExcludedInWorkingDir(tech Technology, wd string, excludedTechAtWorkingDir map[string][]Technology) bool { + if excludedTechs, exist := excludedTechAtWorkingDir[wd]; exist { + for _, excludedTech := range excludedTechs { + if excludedTech == tech { + return true + } + } + } + return false +} + +// Remove sub directories keys from the given workingDirectoryToFiles map. +// Keys: [dir/dir, dir/directory] -> [dir/dir, dir/directory] +// Keys: [dir, directory] -> [dir, directory] +// Keys: [dir/dir2, dir/dir2/dir3, dir/dir2/dir3/dir4] -> [dir/dir2] +// Values of removed sub directories will be added to the root directory. func cleanSubDirectories(workingDirectoryToFiles map[string][]string) (result map[string][]string) { result = make(map[string][]string) for wd, files := range workingDirectoryToFiles { @@ -348,11 +369,13 @@ func cleanSubDirectories(workingDirectoryToFiles map[string][]string) (result ma } // Get the root directory of the given path according to the given workingDirectoryToIndicators map. -func getExistingRootDir(path string, workingDirectoryToIndicators map[string][]string) (rootDir string) { - rootDir = path +func getExistingRootDir(path string, workingDirectoryToIndicators map[string][]string) (root string) { + root = path for wd := range workingDirectoryToIndicators { - if strings.HasPrefix(rootDir, wd) { - rootDir = wd + parentWd := filepath.Dir(wd) + parentRoot := filepath.Dir(root) + if parentRoot != parentWd && strings.HasPrefix(root, wd) { + root = wd } } return diff --git a/utils/coreutils/techutils_test.go b/utils/coreutils/techutils_test.go index 943813177..b342cbd92 100644 --- a/utils/coreutils/techutils_test.go +++ b/utils/coreutils/techutils_test.go @@ -1,9 +1,12 @@ package coreutils import ( - "github.com/stretchr/testify/assert" + "path/filepath" "reflect" "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/exp/maps" ) func TestDetectTechnologiesByFilePaths(t *testing.T) { @@ -31,6 +34,215 @@ func TestDetectTechnologiesByFilePaths(t *testing.T) { } } +func TestMapFilesToRelevantWorkingDirectories(t *testing.T) { + noRequest := map[Technology][]string{} + noExclude := map[string][]Technology{} + + tests := []struct { + name string + paths []string + requestedDescriptors map[Technology][]string + expectedWorkingDir map[string][]string + expectedExcluded map[string][]Technology + }{ + { + name: "noTechTest", + paths: []string{"pomxml", filepath.Join("sub1", "file"), filepath.Join("sub", "sub", "file")}, + requestedDescriptors: noRequest, + expectedWorkingDir: map[string][]string{}, + expectedExcluded: noExclude, + }, + { + name: "mavenTest", + paths: []string{"pom.xml", filepath.Join("sub1", "pom.xml"), filepath.Join("sub2", "pom.xml")}, + requestedDescriptors: noRequest, + expectedWorkingDir: map[string][]string{ + ".": {"pom.xml"}, + "sub1": {filepath.Join("sub1", "pom.xml")}, + "sub2": {filepath.Join("sub2", "pom.xml")}, + }, + expectedExcluded: noExclude, + }, + { + name: "npmTest", + paths: []string{filepath.Join("dir", "package.json"), filepath.Join("dir", "package-lock.json"), filepath.Join("dir2", "npm-shrinkwrap.json")}, + requestedDescriptors: noRequest, + expectedWorkingDir: map[string][]string{ + "dir": {filepath.Join("dir", "package.json"), filepath.Join("dir", "package-lock.json")}, + "dir2": {filepath.Join("dir2", "npm-shrinkwrap.json")}, + }, + expectedExcluded: noExclude, + }, + { + name: "yarnTest", + paths: []string{filepath.Join("dir", "package.json"), filepath.Join("dir", ".yarn")}, + requestedDescriptors: noRequest, + expectedWorkingDir: map[string][]string{"dir": {filepath.Join("dir", "package.json"), filepath.Join("dir", ".yarn")}}, + expectedExcluded: map[string][]Technology{"dir": {Npm}}, + }, + { + name: "golangTest", + paths: []string{filepath.Join("dir", "dir2", "go.mod")}, + requestedDescriptors: noRequest, + expectedWorkingDir: map[string][]string{filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}}, + expectedExcluded: noExclude, + }, + { + name: "pipTest", + paths: []string{ + filepath.Join("users_dir", "test", "package", "setup.py"), + filepath.Join("users_dir", "test", "package", "blabla.txt"), + filepath.Join("users_dir", "test", "package2", "requirements.txt"), + }, + requestedDescriptors: noRequest, + expectedWorkingDir: map[string][]string{ + filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, + filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}}, + expectedExcluded: noExclude, + }, + { + name: "pipRequestedDescriptorTest", + paths: []string{filepath.Join("dir", "blabla.txt"), filepath.Join("dir", "somefile")}, + requestedDescriptors: map[Technology][]string{Pip: {"blabla.txt"}}, + expectedWorkingDir: map[string][]string{"dir": {filepath.Join("dir", "blabla.txt")}}, + expectedExcluded: noExclude, + }, + { + name: "pipenvTest", + paths: []string{filepath.Join("users", "test", "package", "Pipfile")}, + requestedDescriptors: noRequest, + expectedWorkingDir: map[string][]string{filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile")}}, + expectedExcluded: map[string][]Technology{filepath.Join("users", "test", "package"): {Pip}}, + }, + { + name: "gradleTest", + paths: []string{filepath.Join("users", "test", "package", "build.gradle"), filepath.Join("dir", "build.gradle.kts"), filepath.Join("dir", "file")}, + requestedDescriptors: noRequest, + expectedWorkingDir: map[string][]string{ + filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "build.gradle")}, + "dir": {filepath.Join("dir", "build.gradle.kts")}, + }, + expectedExcluded: noExclude, + }, + { + name: "nugetTest", + paths: []string{filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj"), filepath.Join("dir", "file")}, + requestedDescriptors: noRequest, + expectedWorkingDir: map[string][]string{ + "dir": {filepath.Join("dir", "project.sln")}, + filepath.Join("dir", "sub1"): {filepath.Join("dir", "sub1", "project.csproj")}, + }, + expectedExcluded: noExclude, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + detectedWd, detectedExcluded := mapFilesToRelevantWorkingDirectories(test.paths, test.requestedDescriptors) + expectedKeys := maps.Keys(test.expectedWorkingDir) + actualKeys := maps.Keys(detectedWd) + assert.ElementsMatch(t, expectedKeys, actualKeys, "expected: %s, actual: %s", expectedKeys, actualKeys) + for key, value := range test.expectedWorkingDir { + assert.ElementsMatch(t, value, detectedWd[key], "expected: %s, actual: %s", value, detectedWd[key]) + } + assert.True(t, reflect.DeepEqual(test.expectedExcluded, detectedExcluded), "expected: %s, actual: %s", test.expectedExcluded, detectedExcluded) + }) + } +} + +func TestMapWorkingDirectoriesToTechnologies(t *testing.T) { + noRequestSpecialDescriptors := map[Technology][]string{} + noRequestTech := []Technology{} + // noExclude := map[string][]Technology{} + tests := []struct { + name string + workingDirectoryToIndicators map[string][]string + excludedTechAtWorkingDir map[string][]Technology + requestedTechs []Technology + requestedDescriptors map[Technology][]string + + expected map[Technology]map[string][]string + }{ + // { + // name: "noTechTest", + // workingDirectoryToIndicators: map[string][]string{}, + // excludedTechAtWorkingDir: noExclude, + // requestedTechs: noRequestTech, + // requestedDescriptors: noRequestSpecialDescriptors, + // expected: map[Technology]map[string][]string{}, + // }, + { + name: "all techs test", + workingDirectoryToIndicators: map[string][]string{ + "folder": {filepath.Join("folder", "pom.xml")}, + filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")}, + filepath.Join("folder", "sub2"): {filepath.Join("folder", "sub2", "pom.xml")}, + "dir": {filepath.Join("dir", "package.json"), filepath.Join("dir", "package-lock.json"), filepath.Join("dir", "build.gradle.kts"), filepath.Join("dir", "project.sln")}, + "directory": {filepath.Join("directory", "npm-shrinkwrap.json")}, + "dir3": {filepath.Join("dir3", "package.json"), filepath.Join("dir3", ".yarn")}, + filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}, + filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, + filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}, + filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile"), filepath.Join("users", "test", "package", "build.gradle")}, + filepath.Join("dir", "sub1"): {filepath.Join("dir", "sub1", "project.csproj")}, + }, + excludedTechAtWorkingDir: map[string][]Technology{ + filepath.Join("users", "test", "package"): {Pip}, + "dir3": {Npm}, + }, + requestedTechs: noRequestTech, + requestedDescriptors: noRequestSpecialDescriptors, + expected: map[Technology]map[string][]string{ + Maven: {"folder": {filepath.Join("folder", "pom.xml"), filepath.Join("folder", "sub1", "pom.xml"), filepath.Join("folder", "sub2", "pom.xml")}}, + Npm: { + "dir": {filepath.Join("dir", "package.json")}, + "directory": {}, + }, + Yarn: {"dir3": {filepath.Join("dir3", "package.json")}}, + Go: {filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}}, + Pip: { + filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, + filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}, + }, + Pipenv: {filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile")}}, + Gradle: { + "dir": {filepath.Join("dir", "build.gradle.kts")}, + filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "build.gradle")}, + }, + Nuget: {"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}}, + Dotnet: {"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}}, + }, + }, + // {"noTechTest", map[string][]string{}, map[string][]Technology{}, []Technology{}, map[Technology][]string{}, map[Technology]map[string][]string{}}, + // {"mavenTest", map[string][]string{".": {"dir/pom.xml"}, "sub1": {"dir/sub1/pom.xml"}, "sub2": {"dir/sub2/pom.xml"}}, map[string][]Technology{}, []Technology{Maven}, map[Technology][]string{}, map[Technology]map[string][]string{Maven: {"dir": {"dir/pom.xml","dir/sub1/pom.xml","dir/sub2/pom.xml"}}}}, + // {"npmTest", map[string][]string{"..": {"../package.json"}}, map[string][]Technology{}, []Technology{Npm}, map[Technology][]string{}, map[Technology]map[string][]string{Npm: {"..": {"../package.json"}}}}, + // {"yarnTest", map[string][]string{".": {"./package.json", "./.yarn"}}, map[string][]Technology{".": {Npm}}, []Technology{Yarn}, map[Technology][]string{}, map[Technology]map[string][]string{Yarn: {".": {"./package.json", "./.yarn"}}}}, + // {"golangTest", map[string][]string{"/Users/eco/dev/jfrog-cli-core": {"/Users/eco/dev/jfrog-cli-core/go.mod"}}, map[string][]Technology{}, []Technology{Go}, map[Technology][]string{}, map[Technology]map[string][]string{Go: {"/Users/eco/dev/jfrog-cli-core": {"/Users/eco/dev/jfrog-cli-core/go.mod"}}}}, + // {"pipRequestedDescriptorTest", map[string][]string{"dir": {"dir/blabla.txt"}}, map[string][]Technology{}, []Technology{Pip}, map[Technology][]string{Pip: {"blabla.txt"}}, map[Technology]map[string][]string{Pip: {"dir": {"dir/blabla.txt"}}}}, + // {"pipenvTest", map[string][]string{"c:\\users\\test\\package": {"c:\\users\\test\\package\\Pipfile"}}, map[string][]Technology{}, []Technology{Pipenv}, map[Technology][]string{}, map[Technology]map[string][]string{Pipenv: {"c:\\users\\test\\package": {"c:\\users\\test\\package\\Pipfile"}}}}, + // {"gradleTest", map[string][]string{"c:\\users\\test\\package": {"c:\\users\\test\\package\\build.gradle"}}, map[string][]Technology{}, []Technology{Gradle}, map[Technology][]string{}, map[Technology]map[string][]string{Gradle: {"c:\\users\\test\\package": {"c:\\users\\test\\package\\build.gradle"}}}}, + // {"nugetTest", map[string][]string{"c:\\users\\test\\package": {"c:\\users\\test\\package\\project.sln"}}, map[string][]Technology{}, []Technology{Nuget}, map[Technology][]string{}, map[Technology]map[string][]string{Nuget: {"c:\\users\\test\\package": {"c:\\users\\test\\package\\project.sln"}}}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + detectedTech := mapWorkingDirectoriesToTechnologies(test.workingDirectoryToIndicators, test.excludedTechAtWorkingDir, test.requestedTechs, test.requestedDescriptors) + expectedKeys := maps.Keys(test.expected) + detectedKeys := maps.Keys(detectedTech) + assert.ElementsMatch(t, expectedKeys, detectedKeys, "expected: %s, actual: %s", expectedKeys, detectedKeys) + for key, value := range test.expected { + actualKeys := maps.Keys(detectedTech[key]) + expectedKeys := maps.Keys(value) + assert.ElementsMatch(t, expectedKeys, actualKeys, "for tech %s, expected: %s, actual: %s", key, expectedKeys, actualKeys) + // assert.ElementsMatch(t, value, actual[key], "expected: %s, actual: %s", value, actual[key]) + for innerKey, innerValue := range value { + assert.ElementsMatch(t, innerValue, detectedTech[key][innerKey], "expected: %s, actual: %s", innerValue, detectedTech[key][innerKey]) + } + } + // assert.True(t, reflect.DeepEqual(test.expected, detectedTech), "expected: %s, actual: %s", test.expected, detectedTech) + }) + } +} + func TestContainsApplicabilityScannableTech(t *testing.T) { tests := []struct { name string From 09b8e91231ab904e2c33b2965705c5342adf59f4 Mon Sep 17 00:00:00 2001 From: attiasas Date: Tue, 24 Oct 2023 10:35:01 +0300 Subject: [PATCH 15/27] format --- utils/coreutils/techutils_test.go | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/utils/coreutils/techutils_test.go b/utils/coreutils/techutils_test.go index b342cbd92..51a35343c 100644 --- a/utils/coreutils/techutils_test.go +++ b/utils/coreutils/techutils_test.go @@ -152,7 +152,6 @@ func TestMapFilesToRelevantWorkingDirectories(t *testing.T) { func TestMapWorkingDirectoriesToTechnologies(t *testing.T) { noRequestSpecialDescriptors := map[Technology][]string{} noRequestTech := []Technology{} - // noExclude := map[string][]Technology{} tests := []struct { name string workingDirectoryToIndicators map[string][]string @@ -162,14 +161,14 @@ func TestMapWorkingDirectoriesToTechnologies(t *testing.T) { expected map[Technology]map[string][]string }{ - // { - // name: "noTechTest", - // workingDirectoryToIndicators: map[string][]string{}, - // excludedTechAtWorkingDir: noExclude, - // requestedTechs: noRequestTech, - // requestedDescriptors: noRequestSpecialDescriptors, - // expected: map[Technology]map[string][]string{}, - // }, + { + name: "noTechTest", + workingDirectoryToIndicators: map[string][]string{}, + excludedTechAtWorkingDir: map[string][]Technology{}, + requestedTechs: noRequestTech, + requestedDescriptors: noRequestSpecialDescriptors, + expected: map[Technology]map[string][]string{}, + }, { name: "all techs test", workingDirectoryToIndicators: map[string][]string{ @@ -212,15 +211,6 @@ func TestMapWorkingDirectoriesToTechnologies(t *testing.T) { Dotnet: {"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}}, }, }, - // {"noTechTest", map[string][]string{}, map[string][]Technology{}, []Technology{}, map[Technology][]string{}, map[Technology]map[string][]string{}}, - // {"mavenTest", map[string][]string{".": {"dir/pom.xml"}, "sub1": {"dir/sub1/pom.xml"}, "sub2": {"dir/sub2/pom.xml"}}, map[string][]Technology{}, []Technology{Maven}, map[Technology][]string{}, map[Technology]map[string][]string{Maven: {"dir": {"dir/pom.xml","dir/sub1/pom.xml","dir/sub2/pom.xml"}}}}, - // {"npmTest", map[string][]string{"..": {"../package.json"}}, map[string][]Technology{}, []Technology{Npm}, map[Technology][]string{}, map[Technology]map[string][]string{Npm: {"..": {"../package.json"}}}}, - // {"yarnTest", map[string][]string{".": {"./package.json", "./.yarn"}}, map[string][]Technology{".": {Npm}}, []Technology{Yarn}, map[Technology][]string{}, map[Technology]map[string][]string{Yarn: {".": {"./package.json", "./.yarn"}}}}, - // {"golangTest", map[string][]string{"/Users/eco/dev/jfrog-cli-core": {"/Users/eco/dev/jfrog-cli-core/go.mod"}}, map[string][]Technology{}, []Technology{Go}, map[Technology][]string{}, map[Technology]map[string][]string{Go: {"/Users/eco/dev/jfrog-cli-core": {"/Users/eco/dev/jfrog-cli-core/go.mod"}}}}, - // {"pipRequestedDescriptorTest", map[string][]string{"dir": {"dir/blabla.txt"}}, map[string][]Technology{}, []Technology{Pip}, map[Technology][]string{Pip: {"blabla.txt"}}, map[Technology]map[string][]string{Pip: {"dir": {"dir/blabla.txt"}}}}, - // {"pipenvTest", map[string][]string{"c:\\users\\test\\package": {"c:\\users\\test\\package\\Pipfile"}}, map[string][]Technology{}, []Technology{Pipenv}, map[Technology][]string{}, map[Technology]map[string][]string{Pipenv: {"c:\\users\\test\\package": {"c:\\users\\test\\package\\Pipfile"}}}}, - // {"gradleTest", map[string][]string{"c:\\users\\test\\package": {"c:\\users\\test\\package\\build.gradle"}}, map[string][]Technology{}, []Technology{Gradle}, map[Technology][]string{}, map[Technology]map[string][]string{Gradle: {"c:\\users\\test\\package": {"c:\\users\\test\\package\\build.gradle"}}}}, - // {"nugetTest", map[string][]string{"c:\\users\\test\\package": {"c:\\users\\test\\package\\project.sln"}}, map[string][]Technology{}, []Technology{Nuget}, map[Technology][]string{}, map[Technology]map[string][]string{Nuget: {"c:\\users\\test\\package": {"c:\\users\\test\\package\\project.sln"}}}}, } for _, test := range tests { @@ -233,12 +223,10 @@ func TestMapWorkingDirectoriesToTechnologies(t *testing.T) { actualKeys := maps.Keys(detectedTech[key]) expectedKeys := maps.Keys(value) assert.ElementsMatch(t, expectedKeys, actualKeys, "for tech %s, expected: %s, actual: %s", key, expectedKeys, actualKeys) - // assert.ElementsMatch(t, value, actual[key], "expected: %s, actual: %s", value, actual[key]) for innerKey, innerValue := range value { assert.ElementsMatch(t, innerValue, detectedTech[key][innerKey], "expected: %s, actual: %s", innerValue, detectedTech[key][innerKey]) } } - // assert.True(t, reflect.DeepEqual(test.expected, detectedTech), "expected: %s, actual: %s", test.expected, detectedTech) }) } } From 8ca6ad7a9903bb1aeec7d0c255ca3214d8ce7783 Mon Sep 17 00:00:00 2001 From: attiasas Date: Tue, 24 Oct 2023 14:12:16 +0300 Subject: [PATCH 16/27] use new on other scan results --- xray/commands/scan/buildscan.go | 4 +++- xray/commands/scan/scan.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/xray/commands/scan/buildscan.go b/xray/commands/scan/buildscan.go index f1b9bd24a..5980f6a84 100644 --- a/xray/commands/scan/buildscan.go +++ b/xray/commands/scan/buildscan.go @@ -126,7 +126,9 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS XrayDataUrl: buildScanResults.MoreDetailsUrl, }} - scanResults := &xrutils.Results{ScaResults: []xrutils.ScaScanResult{{XrayResults: scanResponse}}, XrayVersion: xrayVersion} + scanResults := xrutils.NewAuditResults() + scanResults.XrayVersion = xrayVersion + scanResults.ScaResults = []xrutils.ScaScanResult{{XrayResults: scanResponse}} resultsPrinter := xrutils.NewResultsWriter(scanResults). SetOutputFormat(bsc.outputFormat). diff --git a/xray/commands/scan/scan.go b/xray/commands/scan/scan.go index f9ae9a181..cf206809a 100644 --- a/xray/commands/scan/scan.go +++ b/xray/commands/scan/scan.go @@ -242,7 +242,9 @@ func (scanCmd *ScanCommand) Run() (err error) { scanErrors = appendErrorSlice(scanErrors, fileProducerErrors) scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors) - scanResults := &xrutils.Results{ScaResults: []xrutils.ScaScanResult{{XrayResults: flatResults}}, XrayVersion: xrayVersion} + scanResults := xrutils.NewAuditResults() + scanResults.XrayVersion = xrayVersion + scanResults.ScaResults = []xrutils.ScaScanResult{{XrayResults: flatResults}} if err = xrutils.NewResultsWriter(scanResults). SetOutputFormat(scanCmd.outputFormat). From 0c5fa7028da9ff26538ade04e1f3671d03f9b620 Mon Sep 17 00:00:00 2001 From: attiasas Date: Tue, 24 Oct 2023 14:20:18 +0300 Subject: [PATCH 17/27] remove recursive flag --- xray/commands/audit/audit.go | 1 - xray/commands/audit/auditparams.go | 10 ---------- xray/commands/audit/scarunner.go | 11 +++++++---- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/xray/commands/audit/audit.go b/xray/commands/audit/audit.go index d6575f2d6..ea9d68807 100644 --- a/xray/commands/audit/audit.go +++ b/xray/commands/audit/audit.go @@ -94,7 +94,6 @@ func (auditCmd *AuditCommand) Run() (err error) { SetFixableOnly(auditCmd.fixableOnly). SetGraphBasicParams(auditCmd.AuditBasicParams). SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan). - SetRecursive(auditCmd.recursive). SetExclusions(auditCmd.exclusions) auditResults, err := RunAudit(auditParams) if err != nil { diff --git a/xray/commands/audit/auditparams.go b/xray/commands/audit/auditparams.go index beaf0b166..97fa586c0 100644 --- a/xray/commands/audit/auditparams.go +++ b/xray/commands/audit/auditparams.go @@ -9,7 +9,6 @@ type AuditParams struct { xrayGraphScanParams *services.XrayGraphScanParams workingDirs []string exclusions []string - recursive bool installFunc func(tech string) error fixableOnly bool minSeverityFilter string @@ -46,15 +45,6 @@ func (params *AuditParams) Exclusions() []string { return params.exclusions } -func (params *AuditParams) Recursive() bool { - return params.recursive -} - -func (params *AuditParams) SetRecursive(recursively bool) *AuditParams { - params.recursive = recursively - return params -} - func (params *AuditParams) SetExclusions(exclusions []string) *AuditParams { params.exclusions = exclusions return params diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index e35a719d4..d6970d8d2 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -66,9 +66,10 @@ func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { // Calculate the scans to preform func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansToPreform []*xrayutils.ScaScanResult) { + recursive := len(currentWorkingDir) > 0 for _, requestedDirectory := range getRequestedDirectoriesToScan(currentWorkingDir, params) { // Detect descriptors and technologies in the requested directory. - techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, params.Recursive(), params.Technologies(), getRequestedDescriptors(params), getExcludePattern(params)) + techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, recursive, params.Technologies(), getRequestedDescriptors(params), getExcludePattern(params, recursive)) // Create scans to preform for tech, workingDirs := range techToWorkingDirs { if tech == coreutils.Dotnet { @@ -97,10 +98,12 @@ func getRequestedDescriptors(params *AuditParams) map[coreutils.Technology][]str return requestedDescriptors } -func getExcludePattern(params *AuditParams) string { +func getExcludePattern(params *AuditParams, recursive bool) string { exclusions := params.Exclusions() - exclusions = append(exclusions, defaultExcludePatterns...) - return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, params.Recursive()) + if len(exclusions) == 0 { + exclusions = append(exclusions, defaultExcludePatterns...) + } + return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, recursive) } func printScansInformation(scans []*xrayutils.ScaScanResult) { From 989332afd5d3d1dfac9a468637ce6decb0b74b2f Mon Sep 17 00:00:00 2001 From: attiasas Date: Tue, 24 Oct 2023 17:16:23 +0300 Subject: [PATCH 18/27] fix ismultipleProject --- xray/utils/results.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xray/utils/results.go b/xray/utils/results.go index 8d1eb507f..c298ab54f 100644 --- a/xray/utils/results.go +++ b/xray/utils/results.go @@ -40,6 +40,9 @@ func (r *Results) IsMultipleProject() bool { return false } if len(r.ScaResults) == 1 { + if r.ScaResults[0].IsMultipleRootProject == nil { + return false + } return *r.ScaResults[0].IsMultipleRootProject } return true From 1772407e9154157ab25e712521477a1be7788d43 Mon Sep 17 00:00:00 2001 From: attiasas Date: Wed, 25 Oct 2023 15:18:33 +0300 Subject: [PATCH 19/27] use relative filter --- artifactory/commands/buildinfo/adddependencies.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- utils/coreutils/techutils.go | 2 +- xray/commands/scan/scan.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/artifactory/commands/buildinfo/adddependencies.go b/artifactory/commands/buildinfo/adddependencies.go index 1754a546c..c4ce550a4 100644 --- a/artifactory/commands/buildinfo/adddependencies.go +++ b/artifactory/commands/buildinfo/adddependencies.go @@ -262,7 +262,7 @@ func collectPatternMatchingFiles(addDepsParams *specutils.CommonParams, rootPath return nil, err } - paths, err := fspatterns.ListFiles(rootPath, addDepsParams.IsRecursive(), addDepsParams.IsIncludeDirs(), true, excludePathPattern) + paths, err := fspatterns.ListFiles(rootPath, addDepsParams.IsRecursive(), addDepsParams.IsIncludeDirs(), false, true, excludePathPattern) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 7065e6f93..57103f170 100644 --- a/go.mod +++ b/go.mod @@ -99,6 +99,6 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231003120621-90e9d7ea05e9 +replace github.com/jfrog/jfrog-client-go => github.com/attiasas/jfrog-client-go v0.0.0-20231025121226-04e810f6f34a // replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231019085746-e1b192457664 diff --git a/go.sum b/go.sum index 369bfedd0..a71d0d087 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/attiasas/jfrog-client-go v0.0.0-20231025121226-04e810f6f34a h1:yto6FgxR7sT2twmbYbpTfBJDdaz305KksXscZbYSRTY= +github.com/attiasas/jfrog-client-go v0.0.0-20231025121226-04e810f6f34a/go.mod h1:fuxhYzWEkA16+ZV5cP/BJUGjA3SXVKbBoDmb8ZS6J4g= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -202,8 +204,6 @@ github.com/jfrog/gofrog v1.3.1 h1:QqAwQXCVReT724uga1AYqG/ZyrNQ6f+iTxmzkb+YFQk= github.com/jfrog/gofrog v1.3.1/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= -github.com/jfrog/jfrog-client-go v1.34.3 h1:kDfw3FUQQvOsTKFqonIgLlziez6CSX80xCYZIH9YYcg= -github.com/jfrog/jfrog-client-go v1.34.3/go.mod h1:fuxhYzWEkA16+ZV5cP/BJUGjA3SXVKbBoDmb8ZS6J4g= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 4742b3c26..e1290dea6 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -208,7 +208,7 @@ func DetectedTechnologiesListInPath(path string, recursive bool) (technologies [ // If requestedTechs is empty, all technologies will be checked. // If excludePathPattern is not empty, files/directories that match the wildcard pattern will be excluded from the search. func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs []string, requestedDescriptors map[Technology][]string, excludePathPattern string) (technologiesDetected map[Technology]map[string][]string) { - filesList, err := fspatterns.ListFiles(path, recursive, false, true, excludePathPattern) + filesList, err := fspatterns.ListFiles(path, recursive, false, true, true, excludePathPattern) if err != nil { return } diff --git a/xray/commands/scan/scan.go b/xray/commands/scan/scan.go index cf206809a..0747ca48a 100644 --- a/xray/commands/scan/scan.go +++ b/xray/commands/scan/scan.go @@ -413,7 +413,7 @@ func collectPatternMatchingFiles(fileData spec.File, rootPath string, dataHandle return err } - paths, err := fspatterns.ListFiles(rootPath, recursive, false, false, excludePathPattern) + paths, err := fspatterns.ListFiles(rootPath, recursive, false, false, false, excludePathPattern) if err != nil { return err } From 3a02b25f4e1e31697f7ed97c777e0dfb7303c902 Mon Sep 17 00:00:00 2001 From: attiasas Date: Wed, 1 Nov 2023 16:10:28 +0200 Subject: [PATCH 20/27] review changes --- utils/coreutils/techutils.go | 19 ++- utils/coreutils/techutils_test.go | 234 ++++++++++++++++++++++++++ xray/commands/audit/scarunner.go | 8 +- xray/commands/audit/scarunner_test.go | 179 +++++++++++++++++++- 4 files changed, 432 insertions(+), 8 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index e1290dea6..7cc94430c 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -188,10 +188,10 @@ func DetectedTechnologiesList() (technologies []string) { if errorutils.CheckError(err) != nil { return } - return DetectedTechnologiesListInPath(wd, false) + return detectedTechnologiesListInPath(wd, false) } -func DetectedTechnologiesListInPath(path string, recursive bool) (technologies []string) { +func detectedTechnologiesListInPath(path string, recursive bool) (technologies []string) { detectedTechnologies, err := DetectTechnologies(path, false, recursive) if err != nil { return @@ -214,7 +214,7 @@ func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs [ } workingDirectoryToIndicators, excludedTechAtWorkingDir := mapFilesToRelevantWorkingDirectories(filesList, requestedDescriptors) strJson, err := json.MarshalIndent(workingDirectoryToIndicators, "", " ") - if err == nil && len(workingDirectoryToIndicators) > 0 { + if errorutils.CheckError(err) == nil && len(workingDirectoryToIndicators) > 0 { log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson)) } technologiesDetected = mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators, excludedTechAtWorkingDir, ToTechnologies(requestedTechs), requestedDescriptors) @@ -225,6 +225,15 @@ func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs [ } // Map files to relevant working directories according to the technologies' indicators/descriptors and requested descriptors. +// files: The file paths to map. +// requestedDescriptors: Special requested descriptors (for example in Pip requirement.txt can have different path) for each technology. +// Returns: +// 1. workingDirectoryToIndicators: A map of working directories to the files that are relevant to the technologies. +// wd1: [wd1/indicator, wd1/descriptor] +// wd/wd2: [wd/wd2/indicator] +// 2. excludedTechAtWorkingDir: A map of working directories to the technologies that are excluded from the working directory. +// wd1: [tech1, tech2] +// wd/wd2: [tech1] func mapFilesToRelevantWorkingDirectories(files []string, requestedDescriptors map[Technology][]string) (workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology) { workingDirectoryToIndicatorsSet := make(map[string]*datastructures.Set[string]) excludedTechAtWorkingDir = make(map[string][]Technology) @@ -290,6 +299,10 @@ func isExclude(path string, techData TechData) bool { } // Map working directories to technologies according to the given workingDirectoryToIndicators map files. +// workingDirectoryToIndicators: A map of working directories to the files inside the directory that are relevant to the technologies. +// excludedTechAtWorkingDir: A map of working directories to the technologies that are excluded from the working directory. +// requestedTechs: The technologies to check, if empty all technologies will be checked. +// requestedDescriptors: Special requested descriptors (for example in Pip requirement.txt can have different path) for each technology to detect. func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedTechs []Technology, requestedDescriptors map[Technology][]string) (technologiesDetected map[Technology]map[string][]string) { // Get the relevant technologies to check technologies := requestedTechs diff --git a/utils/coreutils/techutils_test.go b/utils/coreutils/techutils_test.go index 51a35343c..7d170b9da 100644 --- a/utils/coreutils/techutils_test.go +++ b/utils/coreutils/techutils_test.go @@ -231,6 +231,240 @@ func TestMapWorkingDirectoriesToTechnologies(t *testing.T) { } } +func TestGetExistingRootDir(t *testing.T) { + tests := []struct { + name string + path string + workingDirectoryToIndicators map[string][]string + expected string + }{ + { + name: "empty", + path: "", + workingDirectoryToIndicators: map[string][]string{}, + expected: "", + }, + { + name: "no match", + path: "dir", + workingDirectoryToIndicators: map[string][]string{ + filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")}, + "dir2": {filepath.Join("dir2", "go.mod")}, + "dir3": {}, + filepath.Join("directory", "dir2"): {filepath.Join("directory", "dir2", "go.mod")}, + }, + expected: "dir", + }, + { + name: "match root", + path: filepath.Join("directory", "dir2"), + workingDirectoryToIndicators: map[string][]string{ + filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")}, + "dir2": {filepath.Join("dir2", "go.mod")}, + "dir3": {}, + filepath.Join("directory", "dir2"): {filepath.Join("directory", "dir2", "go.mod")}, + }, + expected: filepath.Join("directory", "dir2"), + }, + { + name: "match sub", + path: filepath.Join("directory", "dir2"), + workingDirectoryToIndicators: map[string][]string{ + filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")}, + "dir2": {filepath.Join("dir2", "go.mod")}, + "directory": {}, + filepath.Join("directory", "dir2"): {filepath.Join("directory", "dir2", "go.mod")}, + }, + expected: "directory", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expected, getExistingRootDir(test.path, test.workingDirectoryToIndicators)) + }) + } +} + +func TestCleanSubDirectories(t *testing.T) { + tests := []struct { + name string + workingDirectoryToFiles map[string][]string + expected map[string][]string + }{ + { + name: "empty", + workingDirectoryToFiles: map[string][]string{}, + expected: map[string][]string{}, + }, + { + name: "no sub directories", + workingDirectoryToFiles: map[string][]string{ + "directory": {filepath.Join("directory", "file")}, + filepath.Join("dir", "dir"): {filepath.Join("dir", "dir", "file")}, + filepath.Join("dir", "directory"): {filepath.Join("dir", "directory", "file")}, + }, + expected: map[string][]string{ + "directory": {filepath.Join("directory", "file")}, + filepath.Join("dir", "dir"): {filepath.Join("dir", "dir", "file")}, + filepath.Join("dir", "directory"): {filepath.Join("dir", "directory", "file")}, + }, + }, + { + name: "sub directories", + workingDirectoryToFiles: map[string][]string{ + filepath.Join("dir", "dir"): {filepath.Join("dir", "dir", "file")}, + filepath.Join("dir", "directory"): {filepath.Join("dir", "directory", "file")}, + "dir": {filepath.Join("dir", "file")}, + "directory": {filepath.Join("directory", "file")}, + filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "file")}, + filepath.Join("dir", "dir2", "dir3"): {filepath.Join("dir", "dir2", "dir3", "file")}, + filepath.Join("dir", "dir2", "dir3", "dir4"): {filepath.Join("dir", "dir2", "dir3", "dir4", "file")}, + }, + expected: map[string][]string{ + "directory": {filepath.Join("directory", "file")}, + "dir": { + filepath.Join("dir", "file"), + filepath.Join("dir", "dir", "file"), + filepath.Join("dir", "directory", "file"), + filepath.Join("dir", "dir2", "file"), + filepath.Join("dir", "dir2", "dir3", "file"), + filepath.Join("dir", "dir2", "dir3", "dir4", "file"), + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cleaned := cleanSubDirectories(test.workingDirectoryToFiles) + cleanedKeys := maps.Keys(cleaned) + expectedKeys := maps.Keys(test.expected) + assert.ElementsMatch(t, expectedKeys, cleanedKeys, "expected: %s, actual: %s", expectedKeys, cleanedKeys) + for key, value := range test.expected { + assert.ElementsMatch(t, value, cleaned[key], "expected: %s, actual: %s", value, cleaned[key]) + } + }) + } +} + +func TestGetTechInformationFromWorkingDir(t *testing.T) { + workingDirectoryToIndicators := map[string][]string{ + "folder": {filepath.Join("folder", "pom.xml")}, + filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")}, + filepath.Join("folder", "sub2"): {filepath.Join("folder", "sub2", "pom.xml")}, + "dir": {filepath.Join("dir", "package.json"), filepath.Join("dir", "package-lock.json"), filepath.Join("dir", "build.gradle.kts"), filepath.Join("dir", "project.sln"), filepath.Join("dir", "blabla.txt")}, + "directory": {filepath.Join("directory", "npm-shrinkwrap.json")}, + "dir3": {filepath.Join("dir3", "package.json"), filepath.Join("dir3", ".yarn")}, + filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}, + filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, + filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}, + filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile"), filepath.Join("users", "test", "package", "build.gradle")}, + filepath.Join("dir", "sub1"): {filepath.Join("dir", "sub1", "project.csproj")}, + } + excludedTechAtWorkingDir := map[string][]Technology{ + filepath.Join("users", "test", "package"): {Pip}, + "dir3": {Npm}, + } + + tests := []struct { + name string + tech Technology + requestedDescriptors map[Technology][]string + expected map[string][]string + }{ + { + name: "mavenTest", + tech: Maven, + requestedDescriptors: map[Technology][]string{}, + expected: map[string][]string{ + "folder": { + filepath.Join("folder", "pom.xml"), + filepath.Join("folder", "sub1", "pom.xml"), + filepath.Join("folder", "sub2", "pom.xml"), + }, + }, + }, + { + name: "npmTest", + tech: Npm, + requestedDescriptors: map[Technology][]string{}, + expected: map[string][]string{ + "dir": {filepath.Join("dir", "package.json")}, + "directory": {}, + }, + }, + { + name: "yarnTest", + tech: Yarn, + requestedDescriptors: map[Technology][]string{}, + expected: map[string][]string{"dir3": {filepath.Join("dir3", "package.json")}}, + }, + { + name: "golangTest", + tech: Go, + requestedDescriptors: map[Technology][]string{}, + expected: map[string][]string{filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}}, + }, + { + name: "pipTest", + tech: Pip, + requestedDescriptors: map[Technology][]string{}, + expected: map[string][]string{ + filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, + filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}, + }, + }, + { + name: "pipRequestedDescriptorTest", + tech: Pip, + requestedDescriptors: map[Technology][]string{Pip: {"blabla.txt"}}, + expected: map[string][]string{ + "dir": {filepath.Join("dir", "blabla.txt")}, + filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, + filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}, + }, + }, + { + name: "pipenvTest", + tech: Pipenv, + requestedDescriptors: map[Technology][]string{}, + expected: map[string][]string{filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile")}}, + }, + { + name: "gradleTest", + tech: Gradle, + requestedDescriptors: map[Technology][]string{}, + expected: map[string][]string{ + filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "build.gradle")}, + "dir": {filepath.Join("dir", "build.gradle.kts")}, + }, + }, + { + name: "nugetTest", + tech: Nuget, + requestedDescriptors: map[Technology][]string{}, + expected: map[string][]string{"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}}, + }, + { + name: "dotnetTest", + tech: Dotnet, + requestedDescriptors: map[Technology][]string{}, + expected: map[string][]string{"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + techInformation := getTechInformationFromWorkingDir(test.tech, workingDirectoryToIndicators, excludedTechAtWorkingDir, test.requestedDescriptors) + expectedKeys := maps.Keys(test.expected) + actualKeys := maps.Keys(techInformation) + assert.ElementsMatch(t, expectedKeys, actualKeys, "expected: %s, actual: %s", expectedKeys, actualKeys) + for key, value := range test.expected { + assert.ElementsMatch(t, value, techInformation[key], "expected: %s, actual: %s", value, techInformation[key]) + } + }) + } +} + func TestContainsApplicabilityScannableTech(t *testing.T) { tests := []struct { name string diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index d6970d8d2..b13d2f2f8 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -45,7 +45,7 @@ func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") return } - printScansInformation(scans) + logScanInfo(scans) defer func() { // Make sure to return to the original working directory, executeScaScan may change it @@ -106,7 +106,7 @@ func getExcludePattern(params *AuditParams, recursive bool) string { return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, recursive) } -func printScansInformation(scans []*xrayutils.ScaScanResult) { +func logScanInfo(scans []*xrayutils.ScaScanResult) { scansJson, err := json.MarshalIndent(scans, "", " ") if err == nil { log.Info(fmt.Sprintf("Preforming %d SCA scans:\n%s", len(scans), string(scansJson))) @@ -129,14 +129,14 @@ func getRequestedDirectoriesToScan(currentWorkingDir string, params *AuditParams func executeScaScan(serverDetails *config.ServerDetails, params *AuditParams, scan *xrayutils.ScaScanResult) (err error) { // Get the dependency tree for the technology in the working directory. if err = os.Chdir(scan.WorkingDirectory); err != nil { - return + return errorutils.CheckError(err) } flattenTree, fullDependencyTrees, techErr := GetTechDependencyTree(params.AuditBasicParams, scan.Technology) if techErr != nil { return fmt.Errorf("failed while building '%s' dependency tree:\n%s", scan.Technology, techErr.Error()) } if flattenTree == nil || len(flattenTree.Nodes) == 0 { - return errors.New("no dependencies were found. Please try to build your project and re-run the audit command") + return errorutils.CheckErrorf("no dependencies were found. Please try to build your project and re-run the audit command") } // Scan the dependency tree. scanResults, xrayErr := runScaWithTech(scan.Technology, params, serverDetails, flattenTree, fullDependencyTrees) diff --git a/xray/commands/audit/scarunner_test.go b/xray/commands/audit/scarunner_test.go index 3e3d198d7..c8dfe6376 100644 --- a/xray/commands/audit/scarunner_test.go +++ b/xray/commands/audit/scarunner_test.go @@ -1,9 +1,17 @@ package audit import ( + "os" + "path/filepath" + "sort" + "testing" + + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils" + "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" "github.com/stretchr/testify/assert" - "testing" ) func TestGetDirectDependenciesList(t *testing.T) { @@ -42,3 +50,172 @@ func TestGetDirectDependenciesList(t *testing.T) { assert.ElementsMatch(t, test.expectedResult, result) } } + +func createTestDir(t *testing.T) (directory string, cleanUp func()) { + tmpDir, err := fileutils.CreateTempDir() + assert.NoError(t, err) + + // Temp dir structure: + // tempDir + // ├── dir + // │ ├── maven + // │ │ ├── maven-sub + // │ │ └── maven-sub + // │ ├── npm + // │ └── go + // ├── yarn + // │ ├── Pip + // │ └── Pipenv + // └── Nuget + // ├── Nuget-sub + + dir := createEmptyDir(t, filepath.Join(tmpDir, "dir")) + // Maven + maven := createEmptyDir(t, filepath.Join(dir, "maven")) + createEmptyFile(t, filepath.Join(maven, "pom.xml")) + mavenSub := createEmptyDir(t, filepath.Join(maven, "maven-sub")) + createEmptyFile(t, filepath.Join(mavenSub, "pom.xml")) + mavenSub2 := createEmptyDir(t, filepath.Join(maven, "maven-sub2")) + createEmptyFile(t, filepath.Join(mavenSub2, "pom.xml")) + // Npm + npm := createEmptyDir(t, filepath.Join(dir, "npm")) + createEmptyFile(t, filepath.Join(npm, "package.json")) + createEmptyFile(t, filepath.Join(npm, "package-lock.json")) + // Go + goDir := createEmptyDir(t, filepath.Join(dir, "go")) + createEmptyFile(t, filepath.Join(goDir, "go.mod")) + // Yarn + yarn := createEmptyDir(t, filepath.Join(tmpDir, "yarn")) + createEmptyFile(t, filepath.Join(yarn, "package.json")) + createEmptyFile(t, filepath.Join(yarn, "yarn.lock")) + // Pip + pip := createEmptyDir(t, filepath.Join(yarn, "Pip")) + createEmptyFile(t, filepath.Join(pip, "requirements.txt")) + // Pipenv + pipenv := createEmptyDir(t, filepath.Join(yarn, "Pipenv")) + createEmptyFile(t, filepath.Join(pipenv, "Pipfile")) + createEmptyFile(t, filepath.Join(pipenv, "Pipfile.lock")) + // Nuget + nuget := createEmptyDir(t, filepath.Join(tmpDir, "Nuget")) + createEmptyFile(t, filepath.Join(nuget, "project.sln")) + nugetSub := createEmptyDir(t, filepath.Join(nuget, "Nuget-sub")) + createEmptyFile(t, filepath.Join(nugetSub, "project.csproj")) + + return tmpDir, func() { + assert.NoError(t, fileutils.RemoveTempDir(tmpDir), "Couldn't removeAll: "+tmpDir) + } +} + +func createEmptyDir(t *testing.T, path string) string { + assert.NoError(t, fileutils.CreateDirIfNotExist(path)) + return path +} + +func createEmptyFile(t *testing.T, path string) string { + file, err := os.Create(path) + assert.NoError(t, err) + file.Close() + return path +} + +func TestGetScaScansToPreform(t *testing.T) { + + dir, cleanUp := createTestDir(t) + + tests := []struct { + name string + wd string + params func() *AuditParams + expected []*xrayutils.ScaScanResult + }{ + { + name: "Test specific technologies", + wd: dir, + params: func() *AuditParams { + param := NewAuditParams() + param.SetTechnologies([]string{"maven", "npm", "go"}) + return param + }, + expected: []*xrayutils.ScaScanResult{ + { + Technology: coreutils.Maven, + WorkingDirectory: filepath.Join(dir, "dir", "maven"), + Descriptors: []string{ + filepath.Join(dir, "dir", "maven", "pom.xml"), + filepath.Join(dir, "dir", "maven", "maven-sub", "pom.xml"), + filepath.Join(dir, "dir", "maven", "maven-sub2", "pom.xml"), + }, + }, + { + Technology: coreutils.Npm, + WorkingDirectory: filepath.Join(dir, "dir", "npm"), + Descriptors: []string{filepath.Join(dir, "dir", "npm", "package.json")}, + }, + { + Technology: coreutils.Go, + WorkingDirectory: filepath.Join(dir, "dir", "go"), + Descriptors: []string{filepath.Join(dir, "dir", "go", "go.mod")}, + }, + }, + }, + { + name: "Test all", + wd: dir, + params: func() *AuditParams { return NewAuditParams() }, + expected: []*xrayutils.ScaScanResult{ + { + Technology: coreutils.Maven, + WorkingDirectory: filepath.Join(dir, "dir", "maven"), + Descriptors: []string{ + filepath.Join(dir, "dir", "maven", "pom.xml"), + filepath.Join(dir, "dir", "maven", "maven-sub", "pom.xml"), + filepath.Join(dir, "dir", "maven", "maven-sub2", "pom.xml"), + }, + }, + { + Technology: coreutils.Npm, + WorkingDirectory: filepath.Join(dir, "dir", "npm"), + Descriptors: []string{filepath.Join(dir, "dir", "npm", "package.json")}, + }, + { + Technology: coreutils.Go, + WorkingDirectory: filepath.Join(dir, "dir", "go"), + Descriptors: []string{filepath.Join(dir, "dir", "go", "go.mod")}, + }, + { + Technology: coreutils.Yarn, + WorkingDirectory: filepath.Join(dir, "yarn"), + Descriptors: []string{filepath.Join(dir, "yarn", "package.json")}, + }, + { + Technology: coreutils.Pip, + WorkingDirectory: filepath.Join(dir, "yarn", "Pip"), + Descriptors: []string{filepath.Join(dir, "yarn", "Pip", "requirements.txt")}, + }, + { + Technology: coreutils.Pipenv, + WorkingDirectory: filepath.Join(dir, "yarn", "Pipenv"), + Descriptors: []string{filepath.Join(dir, "yarn", "Pipenv", "Pipfile")}, + }, + { + Technology: coreutils.Nuget, + WorkingDirectory: filepath.Join(dir, "Nuget"), + Descriptors: []string{filepath.Join(dir, "Nuget", "project.sln"), filepath.Join(dir, "Nuget", "Nuget-sub", "project.csproj")}, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := getScaScansToPreform(test.wd, test.params()) + for i := range result { + sort.Strings(result[i].Descriptors) + sort.Strings(test.expected[i].Descriptors) + } + assert.ElementsMatch(t, test.expected, result) + }) + } + + cleanUp() +} From 242e1bb9ce8c89bf288570cd7d47e9a3ab6e7ce2 Mon Sep 17 00:00:00 2001 From: attiasas Date: Wed, 1 Nov 2023 16:18:07 +0200 Subject: [PATCH 21/27] fix static --- xray/commands/audit/scarunner.go | 2 +- xray/commands/audit/scarunner_test.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index 0d03ee6c8..69550a1f7 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -9,8 +9,8 @@ import ( "github.com/jfrog/build-info-go/utils/pythonutils" "github.com/jfrog/gofrog/datastructures" - "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca" _go "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca/go" diff --git a/xray/commands/audit/scarunner_test.go b/xray/commands/audit/scarunner_test.go index c8dfe6376..900624f21 100644 --- a/xray/commands/audit/scarunner_test.go +++ b/xray/commands/audit/scarunner_test.go @@ -111,11 +111,10 @@ func createEmptyDir(t *testing.T, path string) string { return path } -func createEmptyFile(t *testing.T, path string) string { +func createEmptyFile(t *testing.T, path string) { file, err := os.Create(path) assert.NoError(t, err) file.Close() - return path } func TestGetScaScansToPreform(t *testing.T) { @@ -161,7 +160,7 @@ func TestGetScaScansToPreform(t *testing.T) { { name: "Test all", wd: dir, - params: func() *AuditParams { return NewAuditParams() }, + params: NewAuditParams, expected: []*xrayutils.ScaScanResult{ { Technology: coreutils.Maven, From c7e983b8863702cb561fb01af3ae867dbc616355 Mon Sep 17 00:00:00 2001 From: attiasas Date: Wed, 1 Nov 2023 16:18:54 +0200 Subject: [PATCH 22/27] fix gosec --- xray/commands/audit/scarunner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xray/commands/audit/scarunner_test.go b/xray/commands/audit/scarunner_test.go index 900624f21..ac717e4c5 100644 --- a/xray/commands/audit/scarunner_test.go +++ b/xray/commands/audit/scarunner_test.go @@ -114,7 +114,7 @@ func createEmptyDir(t *testing.T, path string) string { func createEmptyFile(t *testing.T, path string) { file, err := os.Create(path) assert.NoError(t, err) - file.Close() + assert.NoError(t, file.Close()) } func TestGetScaScansToPreform(t *testing.T) { From 75b8bf2fcf95ace901c7599434792780ac64ebe8 Mon Sep 17 00:00:00 2001 From: attiasas Date: Wed, 1 Nov 2023 16:33:01 +0200 Subject: [PATCH 23/27] update client go to dev --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index aafb518d2..46fdf6d14 100644 --- a/go.mod +++ b/go.mod @@ -99,6 +99,6 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/attiasas/jfrog-client-go v0.0.0-20231025121226-04e810f6f34a +replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231101142932-422f20520a28 replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231031143744-13f94ab07bbc diff --git a/go.sum b/go.sum index 82b7ea08a..2dce7ebfc 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,6 @@ github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/attiasas/jfrog-client-go v0.0.0-20231025121226-04e810f6f34a h1:yto6FgxR7sT2twmbYbpTfBJDdaz305KksXscZbYSRTY= -github.com/attiasas/jfrog-client-go v0.0.0-20231025121226-04e810f6f34a/go.mod h1:fuxhYzWEkA16+ZV5cP/BJUGjA3SXVKbBoDmb8ZS6J4g= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -204,6 +202,8 @@ github.com/jfrog/gofrog v1.3.1 h1:QqAwQXCVReT724uga1AYqG/ZyrNQ6f+iTxmzkb+YFQk= github.com/jfrog/gofrog v1.3.1/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= +github.com/jfrog/jfrog-client-go v1.28.1-0.20231101142932-422f20520a28 h1:CeuORbXaa9E+jDTT/DX1Ozuo8HGzDO7B8PIs0O35MNo= +github.com/jfrog/jfrog-client-go v1.28.1-0.20231101142932-422f20520a28/go.mod h1:fuxhYzWEkA16+ZV5cP/BJUGjA3SXVKbBoDmb8ZS6J4g= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= From 273bbeea92ccd0bf9749ca7c2edd1c9a33493cff Mon Sep 17 00:00:00 2001 From: attiasas Date: Thu, 2 Nov 2023 12:39:22 +0200 Subject: [PATCH 24/27] review changes add tests --- xray/commands/audit/scarunner.go | 4 +- xray/commands/audit/scarunner_test.go | 86 +++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index 69550a1f7..4e321a268 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -29,7 +29,7 @@ import ( xrayCmdUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" ) -var defaultExcludePatterns = []string{"*node_modules*", "*target*", "*venv*", "*test*"} +var DefaultExcludePatterns = []string{"*node_modules*", "*target*", "*venv*", "*test*"} func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { // Prepare @@ -102,7 +102,7 @@ func getRequestedDescriptors(params *AuditParams) map[coreutils.Technology][]str func getExcludePattern(params *AuditParams, recursive bool) string { exclusions := params.Exclusions() if len(exclusions) == 0 { - exclusions = append(exclusions, defaultExcludePatterns...) + exclusions = append(exclusions, DefaultExcludePatterns...) } return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, recursive) } diff --git a/xray/commands/audit/scarunner_test.go b/xray/commands/audit/scarunner_test.go index ac717e4c5..ff831e082 100644 --- a/xray/commands/audit/scarunner_test.go +++ b/xray/commands/audit/scarunner_test.go @@ -117,6 +117,92 @@ func createEmptyFile(t *testing.T, path string) { assert.NoError(t, file.Close()) } +func TestGetExcludePattern(t *testing.T) { + tests := []struct { + name string + params func() *AuditParams + recursive bool + expected string + }{ + { + name: "Test exclude pattern recursive", + params: func() *AuditParams { + param := NewAuditParams() + param.SetExclusions([]string{"exclude1", "exclude2"}) + return param + }, + recursive: true, + expected: "(^exclude1$)|(^exclude2$)", + }, + { + name: "Test no exclude pattern recursive", + params: NewAuditParams, + recursive: true, + expected: "(^.*node_modules.*$)|(^.*target.*$)|(^.*venv.*$)|(^.*test.*$)", + }, + { + name: "Test exclude pattern not recursive", + params: func() *AuditParams { + param := NewAuditParams() + param.SetExclusions([]string{"exclude1", "exclude2"}) + return param + }, + recursive: false, + expected: "(^exclude1$)|(^exclude2$)", + }, + { + name: "Test no exclude pattern", + params: NewAuditParams, + recursive: false, + expected: "(^.*node_modules.*$)|(^.*target.*$)|(^.*venv.*$)|(^.*test.*$)", + }, + // name: "Test exclude pattern not recursive", + // params: func() *AuditParams { + // param := NewAuditParams() + // param.SetExclusions([]string{"exclude1", "exclude2"}) + // return param + // }, + // recursive: false, + // expected: "(^exclude1$)|(^exclude2$)", + + // }, + // { + // name: "Test no exclude pattern", + // params: NewAuditParams, + // expected: []string{}, + // }, + // { + + // { + // name: "Test no exclude pattern recursive", + // params: NewAuditParams, + // recursive: false, + // expected: []string{}, + // }, + // { + // name: "Test exclude pattern", + // params: func() *AuditParams { + // param := NewAuditParams() + // param.SetExclusions([]string{"exclude1", "exclude2"}) + // return param + // }, + // expected: []string{"exclude1", "exclude2"}, + // }, + // { + // name: "Test no exclude pattern", + // params: NewAuditParams, + // expected: []string{}, + // }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := getExcludePattern(test.params(), test.recursive) + assert.Equal(t, test.expected, result) + }) + } +} + func TestGetScaScansToPreform(t *testing.T) { dir, cleanUp := createTestDir(t) From 909090900e1ec6db94cdcc6684cd0e361c07e250 Mon Sep 17 00:00:00 2001 From: attiasas Date: Thu, 2 Nov 2023 12:41:21 +0200 Subject: [PATCH 25/27] cleanup --- xray/commands/audit/scarunner_test.go | 37 --------------------------- 1 file changed, 37 deletions(-) diff --git a/xray/commands/audit/scarunner_test.go b/xray/commands/audit/scarunner_test.go index ff831e082..758db9d3c 100644 --- a/xray/commands/audit/scarunner_test.go +++ b/xray/commands/audit/scarunner_test.go @@ -156,43 +156,6 @@ func TestGetExcludePattern(t *testing.T) { recursive: false, expected: "(^.*node_modules.*$)|(^.*target.*$)|(^.*venv.*$)|(^.*test.*$)", }, - // name: "Test exclude pattern not recursive", - // params: func() *AuditParams { - // param := NewAuditParams() - // param.SetExclusions([]string{"exclude1", "exclude2"}) - // return param - // }, - // recursive: false, - // expected: "(^exclude1$)|(^exclude2$)", - - // }, - // { - // name: "Test no exclude pattern", - // params: NewAuditParams, - // expected: []string{}, - // }, - // { - - // { - // name: "Test no exclude pattern recursive", - // params: NewAuditParams, - // recursive: false, - // expected: []string{}, - // }, - // { - // name: "Test exclude pattern", - // params: func() *AuditParams { - // param := NewAuditParams() - // param.SetExclusions([]string{"exclude1", "exclude2"}) - // return param - // }, - // expected: []string{"exclude1", "exclude2"}, - // }, - // { - // name: "Test no exclude pattern", - // params: NewAuditParams, - // expected: []string{}, - // }, } for _, test := range tests { From d1a100ad8402f095b5359e1b072605d103b84274 Mon Sep 17 00:00:00 2001 From: attiasas Date: Thu, 2 Nov 2023 15:19:10 +0200 Subject: [PATCH 26/27] review changes --- utils/coreutils/techutils.go | 9 +++++---- utils/coreutils/utils.go | 11 +++++++++++ xray/commands/audit/scarunner.go | 20 +++++++++++--------- xray/commands/audit/scarunner_test.go | 3 ++- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 7cc94430c..fa043f20e 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -1,7 +1,6 @@ package coreutils import ( - "encoding/json" "fmt" "os" "path/filepath" @@ -207,14 +206,16 @@ func detectedTechnologiesListInPath(path string, recursive bool) (technologies [ // If recursive is true, the search will not be limited to files in the root path. // If requestedTechs is empty, all technologies will be checked. // If excludePathPattern is not empty, files/directories that match the wildcard pattern will be excluded from the search. -func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs []string, requestedDescriptors map[Technology][]string, excludePathPattern string) (technologiesDetected map[Technology]map[string][]string) { +func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs []string, requestedDescriptors map[Technology][]string, excludePathPattern string) (technologiesDetected map[Technology]map[string][]string, err error) { filesList, err := fspatterns.ListFiles(path, recursive, false, true, true, excludePathPattern) if err != nil { return } workingDirectoryToIndicators, excludedTechAtWorkingDir := mapFilesToRelevantWorkingDirectories(filesList, requestedDescriptors) - strJson, err := json.MarshalIndent(workingDirectoryToIndicators, "", " ") - if errorutils.CheckError(err) == nil && len(workingDirectoryToIndicators) > 0 { + var strJson string + if strJson, err = GetJsonIndent(workingDirectoryToIndicators); err != nil { + return + } else if len(workingDirectoryToIndicators) > 0 { log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson)) } technologiesDetected = mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators, excludedTechAtWorkingDir, ToTechnologies(requestedTechs), requestedDescriptors) diff --git a/utils/coreutils/utils.go b/utils/coreutils/utils.go index 463a65d50..2269381d2 100644 --- a/utils/coreutils/utils.go +++ b/utils/coreutils/utils.go @@ -2,6 +2,7 @@ package coreutils import ( "bytes" + "encoding/json" "errors" "fmt" "io/fs" @@ -502,6 +503,16 @@ func parseYesNo(s string, def bool) (ans, valid bool) { return false, false } +func GetJsonIndent(o any) (strJson string, err error) { + byteJson, err := json.MarshalIndent(o, "", " ") + if err != nil { + err = errorutils.CheckError(err) + return + } + strJson = string(byteJson) + return +} + func GetCliUserAgent() string { if cliUserAgentVersion == "" { return cliUserAgentName diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index 4e321a268..17eaa1d9b 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -41,12 +41,17 @@ func runScaScan(params *AuditParams, results *xrayutils.Results) (err error) { if err != nil { return } + scans := getScaScansToPreform(currentWorkingDir, params) if len(scans) == 0 { log.Info("Couldn't determine a package manager or build tool used by this project. Skipping the SCA scan...") return } - logScanInfo(scans) + scanInfo, err := coreutils.GetJsonIndent(scans) + if err != nil { + return + } + log.Info(fmt.Sprintf("Preforming %d SCA scans:\n%s", len(scans), scanInfo)) defer func() { // Make sure to return to the original working directory, executeScaScan may change it @@ -70,7 +75,11 @@ func getScaScansToPreform(currentWorkingDir string, params *AuditParams) (scansT recursive := len(currentWorkingDir) > 0 for _, requestedDirectory := range getRequestedDirectoriesToScan(currentWorkingDir, params) { // Detect descriptors and technologies in the requested directory. - techToWorkingDirs := coreutils.DetectTechnologiesDescriptors(requestedDirectory, recursive, params.Technologies(), getRequestedDescriptors(params), getExcludePattern(params, recursive)) + techToWorkingDirs, err := coreutils.DetectTechnologiesDescriptors(requestedDirectory, recursive, params.Technologies(), getRequestedDescriptors(params), getExcludePattern(params, recursive)) + if err != nil { + log.Warn("Couldn't detect technologies in", requestedDirectory, "directory.", err.Error()) + continue + } // Create scans to preform for tech, workingDirs := range techToWorkingDirs { if tech == coreutils.Dotnet { @@ -107,13 +116,6 @@ func getExcludePattern(params *AuditParams, recursive bool) string { return fspatterns.PrepareExcludePathPattern(exclusions, clientutils.WildCardPattern, recursive) } -func logScanInfo(scans []*xrayutils.ScaScanResult) { - scansJson, err := json.MarshalIndent(scans, "", " ") - if err == nil { - log.Info(fmt.Sprintf("Preforming %d SCA scans:\n%s", len(scans), string(scansJson))) - } -} - func getRequestedDirectoriesToScan(currentWorkingDir string, params *AuditParams) []string { workingDirs := datastructures.MakeSet[string]() for _, wd := range params.workingDirs { diff --git a/xray/commands/audit/scarunner_test.go b/xray/commands/audit/scarunner_test.go index 758db9d3c..177d30e76 100644 --- a/xray/commands/audit/scarunner_test.go +++ b/xray/commands/audit/scarunner_test.go @@ -256,7 +256,8 @@ func TestGetScaScansToPreform(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result := getScaScansToPreform(test.wd, test.params()) + result, err := getScaScansToPreform(test.wd, test.params()) + assert.NoError(t, err) for i := range result { sort.Strings(result[i].Descriptors) sort.Strings(test.expected[i].Descriptors) From f1246d95c34de08e0aee49d1c60cbb1bfb2733b6 Mon Sep 17 00:00:00 2001 From: attiasas Date: Thu, 2 Nov 2023 15:29:37 +0200 Subject: [PATCH 27/27] fix tests --- xray/commands/audit/scarunner_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xray/commands/audit/scarunner_test.go b/xray/commands/audit/scarunner_test.go index 177d30e76..758db9d3c 100644 --- a/xray/commands/audit/scarunner_test.go +++ b/xray/commands/audit/scarunner_test.go @@ -256,8 +256,7 @@ func TestGetScaScansToPreform(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result, err := getScaScansToPreform(test.wd, test.params()) - assert.NoError(t, err) + result := getScaScansToPreform(test.wd, test.params()) for i := range result { sort.Strings(result[i].Descriptors) sort.Strings(test.expected[i].Descriptors)