Merge branch 'dev' into missing-context

cli/docs/flags.go

@@ -116,6 +116,7 @@ const (
 	ThirdPartyContextualAnalysis = "third-party-contextual-analysis"
 	RequirementsFile             = "requirements-file"
 	WorkingDirs                  = "working-dirs"
+	OutputDir                    = "output-dir"
 
 	// Unique curation flags
 	CurationOutput = "curation-format"
@@ -152,7 +153,7 @@ var commandFlags = map[string][]string{
 		url, user, password, accessToken, ServerId, InsecureTls, Project, Watches, RepoPath, Licenses, OutputFormat, ExcludeTestDeps,
 		useWrapperAudit, DepType, RequirementsFile, Fail, ExtendedTable, WorkingDirs, ExclusionsAudit, Mvn, Gradle, Npm,
 		Pnpm, Yarn, Go, Nuget, Pip, Pipenv, Poetry, MinSeverity, FixableOnly, ThirdPartyContextualAnalysis, Threads,
-		Sca, Iac, Sast, Secrets, WithoutCA, ScanVuln,
+		Sca, Iac, Sast, Secrets, WithoutCA, ScanVuln, OutputDir,
 	},
 	CurationAudit: {
 		CurationOutput, WorkingDirs, Threads, RequirementsFile,
@@ -228,6 +229,7 @@ var flagsMap = map[string]components.Flag{
 		components.WithBoolDefaultValue(true),
 	),
 	WorkingDirs: components.NewStringFlag(WorkingDirs, "A comma-separated list of relative working directories, to determine audit targets locations."),
+	OutputDir:   components.NewStringFlag(OutputDir, "Target directory to save partial results to.", components.SetHiddenStrFlag()),
 	ExclusionsAudit: components.NewStringFlag(
 		Exclusions,
 		"List of exclusions separated by semicolons, utilized to skip sub-projects from undergoing an audit. These exclusions may incorporate the * and ? wildcards.",

cli/scancommands.go

@@ -19,6 +19,7 @@ import (
 	"github.com/jfrog/jfrog-cli-security/commands/enrich"
 	"github.com/jfrog/jfrog-cli-security/utils/xray"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
+	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 	"github.com/urfave/cli"
 	"os"
@@ -455,6 +456,11 @@ func CreateAuditCmd(c *components.Context) (*audit.AuditCommand, error) {
 	if err != nil {
 		return nil, err
 	}
+	scansOutputDir, err := getAndValidateOutputDirExistsIfProvided(c)
+	if err != nil {
+		return nil, err
+	}
+
 	auditCmd.SetAnalyticsMetricsService(xsc.NewAnalyticsMetricsService(serverDetails))
 
 	auditCmd.SetTargetRepoPath(addTrailingSlashToRepoPathIfNeeded(c)).
@@ -465,7 +471,8 @@ func CreateAuditCmd(c *components.Context) (*audit.AuditCommand, error) {
 		SetPrintExtendedTable(c.GetBoolFlagValue(flags.ExtendedTable)).
 		SetMinSeverityFilter(minSeverity).
 		SetFixableOnly(c.GetBoolFlagValue(flags.FixableOnly)).
-		SetThirdPartyApplicabilityScan(c.GetBoolFlagValue(flags.ThirdPartyContextualAnalysis))
+		SetThirdPartyApplicabilityScan(c.GetBoolFlagValue(flags.ThirdPartyContextualAnalysis)).
+		SetScansResultsOutputDir(scansOutputDir)
 
 	if c.GetStringFlagValue(flags.Watches) != "" {
 		auditCmd.SetWatches(splitByCommaAndTrim(c.GetStringFlagValue(flags.Watches)))
@@ -629,6 +636,21 @@ func getCurationCommand(c *components.Context) (*curation.CurationAuditCommand,
 	return curationAuditCommand, nil
 }
 
+func getAndValidateOutputDirExistsIfProvided(c *components.Context) (string, error) {
+	scansOutputDir := c.GetStringFlagValue(flags.OutputDir)
+	if scansOutputDir == "" {
+		return "", nil
+	}
+	exists, err := fileutils.IsDirExists(scansOutputDir, false)
+	if err != nil {
+		return "", err
+	}
+	if !exists {
+		return "", fmt.Errorf("output directory path for saving scans results was provided, but the directory doesn't exist: '%s'", scansOutputDir)
+	}
+	return scansOutputDir, nil
+}
+
 func DockerScanMockCommand() components.Command {
 	// Mock how the CLI handles docker commands:
 	// https://github.com/jfrog/jfrog-cli/blob/v2/buildtools/cli.go#L691

commands/audit/audit.go

@@ -3,19 +3,17 @@ package audit
 import (
 	"errors"
 	"fmt"
-
 	jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
+	"github.com/jfrog/jfrog-cli-security/jas"
 	"github.com/jfrog/jfrog-cli-security/jas/applicability"
 	"github.com/jfrog/jfrog-cli-security/jas/runner"
 	"github.com/jfrog/jfrog-cli-security/jas/secrets"
+	"github.com/jfrog/jfrog-cli-security/utils"
 	"github.com/jfrog/jfrog-cli-security/utils/xray/scangraph"
 	"github.com/jfrog/jfrog-cli-security/utils/xsc"
 
-	"github.com/jfrog/jfrog-cli-security/jas"
-	"github.com/jfrog/jfrog-cli-security/utils"
-
 	xrayutils "github.com/jfrog/jfrog-cli-security/utils/xray"
 	clientutils "github.com/jfrog/jfrog-client-go/utils"
 	"github.com/jfrog/jfrog-client-go/xray"
@@ -115,7 +113,8 @@ func (auditCmd *AuditCommand) Run() (err error) {
 		SetGraphBasicParams(auditCmd.AuditBasicParams).
 		SetCommonGraphScanParams(auditCmd.CreateCommonGraphScanParams()).
 		SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan).
-		SetThreads(auditCmd.Threads)
+		SetThreads(auditCmd.Threads).
+		SetScansResultsOutputDir(auditCmd.scanResultsOutputDir)
 	auditParams.SetIsRecursiveScan(isRecursiveScan).SetExclusions(auditCmd.Exclusions())
 
 	auditResults, err := RunAudit(auditParams)
@@ -256,7 +255,7 @@ func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.SecurityPa
 	if err != nil {
 		return fmt.Errorf("failed to create jas scanner: %s", err.Error())
 	}
-	if err = runner.AddJasScannersTasks(auditParallelRunner, scanResults, auditParams.DirectDependencies(), serverDetails, auditParams.thirdPartyApplicabilityScan, scanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan, auditParams.ScansToPerform(), auditParams.configProfile); err != nil {
+	if err = runner.AddJasScannersTasks(auditParallelRunner, scanResults, auditParams.DirectDependencies(), serverDetails, auditParams.thirdPartyApplicabilityScan, scanner, applicability.ApplicabilityScannerType, secrets.SecretsScannerType, auditParallelRunner.AddErrorToChan, auditParams.ScansToPerform(), auditParams.configProfile, auditParams.scanResultsOutputDir); err != nil {
 		return fmt.Errorf("%s failed to run JAS scanners: %s", clientutils.GetLogMsgPrefix(threadId, false), err.Error())
 	}
 	return

commands/audit/audit_test.go

@@ -6,11 +6,12 @@ import (
 	coreTests "github.com/jfrog/jfrog-cli-core/v2/utils/tests"
 	"github.com/jfrog/jfrog-cli-security/utils"
 	"github.com/jfrog/jfrog-cli-security/utils/xray/scangraph"
-	clientTests "github.com/jfrog/jfrog-client-go/utils/tests"
+	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
+	scanservices "github.com/jfrog/jfrog-client-go/xray/services"
 	"github.com/jfrog/jfrog-client-go/xsc/services"
 	"github.com/stretchr/testify/assert"
-	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -95,36 +96,30 @@ func TestAuditWithConfigProfile(t *testing.T) {
 			mockServer, serverDetails := utils.XrayServer(t, utils.EntitlementsMinVersion)
 			defer mockServer.Close()
 
+			tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
+			defer createTempDirCallback()
+			testDirPath := filepath.Join("..", "..", "tests", "testdata", "projects", "jas", "jas")
+			assert.NoError(t, biutils.CopyDir(testDirPath, tempDirPath, true, nil))
+
 			auditBasicParams := (&utils.AuditBasicParams{}).
 				SetServerDetails(serverDetails).
 				SetOutputFormat(format.Table).
 				SetUseJas(true)
 
 			configProfile := testcase.configProfile
 			auditParams := NewAuditParams().
+				SetWorkingDirs([]string{tempDirPath}).
 				SetGraphBasicParams(auditBasicParams).
 				SetConfigProfile(&configProfile).
 				SetCommonGraphScanParams(&scangraph.CommonGraphScanParams{
 					RepoPath:               "",
-					ProjectKey:             "",
-					Watches:                nil,
-					ScanType:               "dependency",
+					ScanType:               scanservices.Dependency,
 					IncludeVulnerabilities: true,
 					XscVersion:             services.ConfigProfileMinXscVersion,
 					MultiScanId:            "random-msi",
 				})
 			auditParams.SetIsRecursiveScan(true)
 
-			tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
-			defer createTempDirCallback()
-			testDirPath := filepath.Join("..", "..", "tests", "testdata", "projects", "jas", "jas")
-			assert.NoError(t, biutils.CopyDir(testDirPath, tempDirPath, true, nil))
-
-			baseWd, err := os.Getwd()
-			assert.NoError(t, err)
-			chdirCallback := clientTests.ChangeDirWithCallback(t, baseWd, tempDirPath)
-			defer chdirCallback()
-
 			auditResults, err := RunAudit(auditParams)
 			assert.NoError(t, err)
 
@@ -149,3 +144,53 @@ func TestAuditWithConfigProfile(t *testing.T) {
 		})
 	}
 }
+
+// This test tests audit flow when providing --output-dir flag
+func TestAuditWithScansOutputDir(t *testing.T) {
+	mockServer, serverDetails := utils.XrayServer(t, utils.EntitlementsMinVersion)
+	defer mockServer.Close()
+
+	outputDirPath, removeOutputDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
+	defer removeOutputDirCallback()
+
+	tempDirPath, createTempDirCallback := coreTests.CreateTempDirWithCallbackAndAssert(t)
+	defer createTempDirCallback()
+	testDirPath := filepath.Join("..", "..", "tests", "testdata", "projects", "jas", "jas")
+	assert.NoError(t, biutils.CopyDir(testDirPath, tempDirPath, true, nil))
+
+	auditBasicParams := (&utils.AuditBasicParams{}).
+		SetServerDetails(serverDetails).
+		SetOutputFormat(format.Table).
+		SetUseJas(true)
+
+	auditParams := NewAuditParams().
+		SetWorkingDirs([]string{tempDirPath}).
+		SetGraphBasicParams(auditBasicParams).
+		SetCommonGraphScanParams(&scangraph.CommonGraphScanParams{
+			ScanType:               scanservices.Dependency,
+			IncludeVulnerabilities: true,
+			MultiScanId:            utils.TestScaScanId,
+		}).
+		SetScansResultsOutputDir(outputDirPath)
+	auditParams.SetIsRecursiveScan(true)
+
+	_, err := RunAudit(auditParams)
+	assert.NoError(t, err)
+
+	filesList, err := fileutils.ListFiles(outputDirPath, false)
+	assert.NoError(t, err)
+	assert.Len(t, filesList, 5)
+
+	var fileNamesWithoutSuffix []string
+	for _, fileName := range filesList {
+		// Removing <hash>.json suffix to so we can check by suffix all expected files exist
+		splitName := strings.Split(fileName, "_")
+		fileNamesWithoutSuffix = append(fileNamesWithoutSuffix, splitName[0])
+	}
+
+	assert.Contains(t, fileNamesWithoutSuffix, filepath.Join(outputDirPath, "sca"))
+	assert.Contains(t, fileNamesWithoutSuffix, filepath.Join(outputDirPath, "iac"))
+	assert.Contains(t, fileNamesWithoutSuffix, filepath.Join(outputDirPath, "sast"))
+	assert.Contains(t, fileNamesWithoutSuffix, filepath.Join(outputDirPath, "secrets"))
+	assert.Contains(t, fileNamesWithoutSuffix, filepath.Join(outputDirPath, "applicability"))
+}

commands/audit/auditparams.go

@@ -21,6 +21,7 @@ type AuditParams struct {
 	thirdPartyApplicabilityScan bool
 	threads                     int
 	configProfile               *clientservices.ConfigProfile
+	scanResultsOutputDir        string
 }
 
 func NewAuditParams() *AuditParams {
@@ -99,6 +100,11 @@ func (params *AuditParams) SetConfigProfile(configProfile *clientservices.Config
 	return params
 }
 
+func (params *AuditParams) SetScansResultsOutputDir(outputDir string) *AuditParams {
+	params.scanResultsOutputDir = outputDir
+	return params
+}
+
 func (params *AuditParams) createXrayGraphScanParams() *services.XrayGraphScanParams {
 	return &services.XrayGraphScanParams{
 		RepoPath:               params.commonGraphScanParams.RepoPath,

commands/audit/scarunner.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-
 	"github.com/jfrog/build-info-go/utils/pythonutils"
 	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	"golang.org/x/exp/slices"
@@ -153,6 +152,7 @@ func executeScaScanTask(auditParallelRunner *utils.SecurityParallelRunner, serve
 		auditParallelRunner.ResultsMu.Lock()
 		addThirdPartyDependenciesToParams(auditParams, scan.Technology, treeResult.FlatTree, treeResult.FullDepTrees)
 		scan.XrayResults = append(scan.XrayResults, scanResults...)
+		err = dumpScanResponseToFileIfNeeded(scanResults, auditParams.scanResultsOutputDir, utils.ScaScan)
 		auditParallelRunner.ResultsMu.Unlock()
 		return
 	}
@@ -384,3 +384,15 @@ func buildDependencyTree(scan *utils.ScaScanResult, params *AuditParams) (*Depen
 	}
 	return &treeResult, nil
 }
+
+// If an output dir was provided through --output-dir flag, we create in the provided path new file containing the scan results
+func dumpScanResponseToFileIfNeeded(results []services.ScanResponse, scanResultsOutputDir string, scanType utils.SubScanType) (err error) {
+	if scanResultsOutputDir == "" || results == nil {
+		return
+	}
+	fileContent, err := json.Marshal(results)
+	if err != nil {
+		return fmt.Errorf("failed to write %s scan results to file: %s", scanType, err.Error())
+	}
+	return utils.DumpContentToFile(fileContent, scanResultsOutputDir, scanType.String())
+}

commands/scan/buildscan.go

@@ -155,7 +155,7 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS
 
 	resultsPrinter := utils.NewResultsWriter(scanResults).
 		SetOutputFormat(bsc.outputFormat).
-		SetHasViolationContext(bsc.hasViolationContext()).
+		SetHasViolationContext(true).
 		SetIncludeVulnerabilities(bsc.includeVulnerabilities).
 		SetIncludeLicenses(false).
 		SetIsMultipleRootProject(true).
@@ -176,18 +176,11 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS
 				return false, err
 			}
 		}
-		if bsc.includeVulnerabilities {
-			resultsPrinter.SetIncludeVulnerabilities(true)
-			if err = resultsPrinter.PrintScanResults(); err != nil {
-				return false, err
-			}
-		}
 	}
 	err = utils.RecordSecurityCommandSummary(utils.NewBuildScanSummary(
 		scanResults,
 		bsc.serverDetails,
 		bsc.includeVulnerabilities,
-		bsc.hasViolationContext(),
 		params.BuildName, params.BuildNumber,
 	))
 	return
@@ -197,10 +190,6 @@ func (bsc *BuildScanCommand) CommandName() string {
 	return "xr_build_scan"
 }
 
-func (bsc *BuildScanCommand) hasViolationContext() bool {
-	return bsc.buildConfiguration.GetProject() != ""
-}
-
 // There are two cases. when serverDetails.Url is configured and when serverDetails.XrayUrl and serverDetails.ArtifactoryUrl are configured
 // The function will return the Url if configured and will trim xray if serverDetails.Url is not configured
 func getActualUrl(serverDetails config.ServerDetails) (string, error) {

commands/scan/scan.go

@@ -450,7 +450,7 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledFo
 						log.Error(fmt.Sprintf("failed to create jas scanner: %s", err.Error()))
 						indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})
 					}
-					err = runner.AddJasScannersTasks(jasFileProducerConsumer, &scanResults, &depsList, scanCmd.serverDetails, false, scanner, applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc, utils.GetAllSupportedScans(), nil)
+					err = runner.AddJasScannersTasks(jasFileProducerConsumer, &scanResults, &depsList, scanCmd.serverDetails, false, scanner, applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType, jasErrHandlerFunc, utils.GetAllSupportedScans(), nil, "")
 					if err != nil {
 						log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error()))
 						indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})