Feature - whitesourceExecuteScan - adding ability to scan multiple do…

…cker images (SAP#4755)

* added-multiple-images-scan-logic

* amended-description

* added-reference-to-common-pipeline-env

cmd/whitesourceExecuteScan.go

@@ -196,28 +196,25 @@ func runWhitesourceExecuteScan(ctx context.Context, config *ScanOptions, scan *w
 }
 
 func runWhitesourceScan(ctx context.Context, config *ScanOptions, scan *ws.Scan, utils whitesourceUtils, sys whitesource, commonPipelineEnvironment *whitesourceExecuteScanCommonPipelineEnvironment, influx *whitesourceExecuteScanInflux) error {
+
 	// Download Docker image for container scan
 	// ToDo: move it to improve testability
 	if config.BuildTool == "docker" {
-		saveImageOptions := containerSaveImageOptions{
-			ContainerImage:            config.ScanImage,
-			ContainerRegistryURL:      config.ScanImageRegistryURL,
-			ContainerRegistryUser:     config.ContainerRegistryUser,
-			ContainerRegistryPassword: config.ContainerRegistryPassword,
-			DockerConfigJSON:          config.DockerConfigJSON,
-			FilePath:                  config.ProjectName,
-			ImageFormat:               "legacy", // keep the image format legacy or whitesource is not able to read layers
-		}
-		dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", ImageFormat: "legacy"}
-		dClient := &piperDocker.Client{}
-		dClient.SetOptions(dClientOptions)
-		if _, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils); err != nil {
-			if strings.Contains(fmt.Sprint(err), "no image found") {
-				log.SetErrorCategory(log.ErrorConfiguration)
+		if len(config.ScanImages) != 0 {
+			for _, image := range config.ScanImages {
+				config.ScanImage = image
+				err := downloadDockerImageAsTar(config, utils)
+				if err != nil {
+					return errors.Wrapf(err, "failed to download docker image")
+				}
 			}
-			return errors.Wrapf(err, "failed to download Docker image %v", config.ScanImage)
-		}
 
+		} else {
+			err := downloadDockerImageAsTar(config, utils)
+			if err != nil {
+				return errors.Wrapf(err, "failed to download docker image")
+			}
+		}
 	}
 
 	// Start the scan
@@ -375,8 +372,11 @@ func resolveProjectIdentifiers(config *ScanOptions, scan *ws.Scan, utils whiteso
 	if err := resolveProductToken(config, sys); err != nil {
 		return errors.Wrap(err, "error resolving product token")
 	}
-	if err := resolveAggregateProjectToken(config, sys); err != nil {
-		return errors.Wrap(err, "error resolving aggregate project token")
+
+	if !config.SkipParentProjectResolution {
+		if err := resolveAggregateProjectToken(config, sys); err != nil {
+			return errors.Wrap(err, "error resolving aggregate project token")
+		}
 	}
 
 	scan.ProductToken = config.ProductToken
@@ -465,33 +465,34 @@ func validateProductVersion(version string) string {
 
 func wsScanOptions(config *ScanOptions) *ws.ScanOptions {
 	return &ws.ScanOptions{
-		BuildTool:                  config.BuildTool,
-		ScanType:                   "", // no longer provided via config
-		OrgToken:                   config.OrgToken,
-		UserToken:                  config.UserToken,
-		ProductName:                config.ProductName,
-		ProductToken:               config.ProductToken,
-		ProductVersion:             config.Version,
-		ProjectName:                config.ProjectName,
-		BuildDescriptorFile:        config.BuildDescriptorFile,
-		BuildDescriptorExcludeList: config.BuildDescriptorExcludeList,
-		PomPath:                    config.BuildDescriptorFile,
-		M2Path:                     config.M2Path,
-		GlobalSettingsFile:         config.GlobalSettingsFile,
-		ProjectSettingsFile:        config.ProjectSettingsFile,
-		InstallArtifacts:           config.InstallArtifacts,
-		DefaultNpmRegistry:         config.DefaultNpmRegistry,
-		AgentDownloadURL:           config.AgentDownloadURL,
-		AgentFileName:              config.AgentFileName,
-		ConfigFilePath:             config.ConfigFilePath,
-		Includes:                   config.Includes,
-		Excludes:                   config.Excludes,
-		JreDownloadURL:             config.JreDownloadURL,
-		AgentURL:                   config.AgentURL,
-		ServiceURL:                 config.ServiceURL,
-		ScanPath:                   config.ScanPath,
-		InstallCommand:             config.InstallCommand,
-		Verbose:                    GeneralConfig.Verbose,
+		BuildTool:                   config.BuildTool,
+		ScanType:                    "", // no longer provided via config
+		OrgToken:                    config.OrgToken,
+		UserToken:                   config.UserToken,
+		ProductName:                 config.ProductName,
+		ProductToken:                config.ProductToken,
+		ProductVersion:              config.Version,
+		ProjectName:                 config.ProjectName,
+		BuildDescriptorFile:         config.BuildDescriptorFile,
+		BuildDescriptorExcludeList:  config.BuildDescriptorExcludeList,
+		PomPath:                     config.BuildDescriptorFile,
+		M2Path:                      config.M2Path,
+		GlobalSettingsFile:          config.GlobalSettingsFile,
+		ProjectSettingsFile:         config.ProjectSettingsFile,
+		InstallArtifacts:            config.InstallArtifacts,
+		DefaultNpmRegistry:          config.DefaultNpmRegistry,
+		AgentDownloadURL:            config.AgentDownloadURL,
+		AgentFileName:               config.AgentFileName,
+		ConfigFilePath:              config.ConfigFilePath,
+		Includes:                    config.Includes,
+		Excludes:                    config.Excludes,
+		JreDownloadURL:              config.JreDownloadURL,
+		AgentURL:                    config.AgentURL,
+		ServiceURL:                  config.ServiceURL,
+		ScanPath:                    config.ScanPath,
+		InstallCommand:              config.InstallCommand,
+		Verbose:                     GeneralConfig.Verbose,
+		SkipParentProjectResolution: config.SkipParentProjectResolution,
 	}
 }
 
@@ -1086,3 +1087,27 @@ func createToolRecordWhitesource(utils whitesourceUtils, workspace string, confi
 	}
 	return record.GetFileName(), nil
 }
+
+func downloadDockerImageAsTar(config *ScanOptions, utils whitesourceUtils) error {
+
+	saveImageOptions := containerSaveImageOptions{
+		ContainerImage:            config.ScanImage,
+		ContainerRegistryURL:      config.ScanImageRegistryURL,
+		ContainerRegistryUser:     config.ContainerRegistryUser,
+		ContainerRegistryPassword: config.ContainerRegistryPassword,
+		DockerConfigJSON:          config.DockerConfigJSON,
+		FilePath:                  config.ScanPath + "/" + config.ScanImage, // previously was config.ProjectName
+		ImageFormat:               "legacy",                                 // keep the image format legacy or whitesource is not able to read layers
+	}
+	dClientOptions := piperDocker.ClientOptions{ImageName: saveImageOptions.ContainerImage, RegistryURL: saveImageOptions.ContainerRegistryURL, LocalPath: "", ImageFormat: "legacy"}
+	dClient := &piperDocker.Client{}
+	dClient.SetOptions(dClientOptions)
+	if _, err := runContainerSaveImage(&saveImageOptions, &telemetry.CustomData{}, "./cache", "", dClient, utils); err != nil {
+		if strings.Contains(fmt.Sprint(err), "no image found") {
+			log.SetErrorCategory(log.ErrorConfiguration)
+		}
+		return errors.Wrapf(err, "failed to download Docker image %v", config.ScanImage)
+	}
+
+	return nil
+}

pkg/whitesource/configHelper.go

@@ -46,7 +46,14 @@ func (s *ScanOptions) RewriteUAConfigurationFile(utils Utils, projectName string
 	newConfig := properties.LoadMap(newConfigMap)
 
 	now := time.Now().Format("20060102150405")
-	newConfigFilePath := fmt.Sprintf("%v.%v", s.ConfigFilePath, now)
+
+	var newConfigFilePath string
+
+	if s.ScanPath != "." {
+		newConfigFilePath = fmt.Sprintf("%v/%v.%v", s.ScanPath, s.ConfigFilePath, now)
+	} else {
+		newConfigFilePath = fmt.Sprintf("%v.%v", s.ConfigFilePath, now)
+	}
 
 	var configContent bytes.Buffer
 	_, err = newConfig.Write(&configContent, properties.UTF8)

pkg/whitesource/scanOptions.go

@@ -46,5 +46,7 @@ type ScanOptions struct {
 
 	InstallCommand string
 
+	SkipParentProjectResolution bool
+
 	Verbose bool
 }

pkg/whitesource/scanUA.go

@@ -98,8 +98,10 @@ func (s *Scan) ExecuteUAScanInPath(config *ScanOptions, utils Utils, scanPath st
 	// ToDo: Check if Download of Docker/container image should be done here instead of in cmd/whitesourceExecuteScan.go
 
 	// ToDo: check if this is required
-	if err := s.AppendScannedProject(s.AggregateProjectName); err != nil {
-		return err
+	if !config.SkipParentProjectResolution {
+		if err := s.AppendScannedProject(s.AggregateProjectName); err != nil {
+			return err
+		}
 	}
 
 	configPath, err := config.RewriteUAConfigurationFile(utils, s.AggregateProjectName)

resources/metadata/whitesourceExecuteScan.yaml

@@ -377,6 +377,25 @@ spec:
           - PARAMETERS
           - STAGES
           - STEPS
+      - name: scanImages
+        type: "[]string"
+        description: "For `buildTool: docker`: Allowing to scan multiple docker images. In case parent project will not contain any dependecies, use skipParentProjectResolution parameter"
+        resourceRef:
+          - name: commonPipelineEnvironment
+            param: container/imageNameTags
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+      - name: skipParentProjectResolution
+        type: bool
+        description: "Parameter for multi-module, multi-images projects to skip the parent project resolution for reporing purpose
+        Could be used if parent project is set as just a placeholder for scan and doesn't contain any dependencies."
+        scope:
+          - PARAMETERS
+          - STAGES
+          - STEPS
+        default: false
       - name: scanImageRegistryUrl
         type: string
         description: "For `buildTool: docker`: Defines the registry where the scanImage is located."