CLID-265,CLID-259: generates folders based on the filtered catalog digest

v2/internal/pkg/api/v2alpha1/type_image.go

@@ -54,6 +54,10 @@ func (it ImageType) IsOperator() bool {
 	return it == TypeOperatorBundle || it == TypeOperatorCatalog || it == TypeOperatorRelatedImage
 }
 
+func (it ImageType) IsOperatorCatalog() bool {
+	return it == TypeOperatorCatalog
+}
+
 func (it ImageType) IsAdditionalImage() bool {
 	return it == TypeGeneric
 }

v2/internal/pkg/api/v2alpha1/type_internal.go

@@ -191,7 +191,8 @@ type RelatedImage struct {
 	TargetTag string `json:"targetTag"`
 	// Used to keep specific naming info for the related image
 	// if set should be used when mirroring
-	TargetCatalog string `json:"targetCatalog"`
+	TargetCatalog    string `json:"targetCatalog"`
+	IsCatalogRebuilt bool   `json:"isCatalogRebuilt"`
 }
 
 type CollectorSchema struct {
@@ -219,7 +220,8 @@ type CopyImageSchema struct {
 	Origin string
 	// Type: metadata to explain why this image is being copied
 	// it doesn´t need to be persisted to json
-	Type ImageType `json:"-"`
+	Type             ImageType `json:"-"`
+	IsCatalogRebuilt bool      `json:"isCatalogRebuilt"`
 }
 
 // SignatureContentSchema

v2/internal/pkg/cli/const.go

@@ -16,6 +16,7 @@ const (
 	releaseImageExtractDir        string = "hold-release"
 	cincinnatiGraphDataDir        string = "cincinnati-graph-data"
 	operatorImageExtractDir       string = "hold-operator"
+	operatorCatalogsDir           string = "operator-catalogs"
 	signaturesDir                 string = "signatures"
 	registryLogFilename           string = "registry.log"
 	startMessage                  string = "starting local storage on localhost:%v"

v2/internal/pkg/cli/executor.go

@@ -689,6 +689,7 @@ func (o *ExecutorSchema) setupWorkingDir() error {
 		return err
 	}
 
+	//TODO ALEX REMOVE ME WHEN filtered_collector.go is the default for operators
 	// create operator cache dir
 	o.Log.Trace("creating operator cache directory %s ", o.Opts.Global.WorkingDir+"/"+operatorImageExtractDir)
 	err = o.MakeDir.makeDirAll(o.Opts.Global.WorkingDir+"/"+operatorImageExtractDir, 0755)
@@ -697,6 +698,13 @@ func (o *ExecutorSchema) setupWorkingDir() error {
 		return err
 	}
 
+	o.Log.Trace("creating operator cache directory %s ", filepath.Join(o.Opts.Global.WorkingDir, operatorCatalogsDir))
+	err = o.MakeDir.makeDirAll(filepath.Join(o.Opts.Global.WorkingDir, operatorCatalogsDir), 0755)
+	if err != nil {
+		o.Log.Error(" setupWorkingDir for operator cache %v ", err)
+		return err
+	}
+
 	// create cluster-resources directory and clean it
 	o.Log.Trace("creating cluster-resources directory %s ", o.Opts.Global.WorkingDir+"/"+clusterResourcesDir)
 	err = os.RemoveAll(o.Opts.Global.WorkingDir + "/" + clusterResourcesDir)
@@ -1031,13 +1039,12 @@ func (o *ExecutorSchema) CollectAll(ctx context.Context) (v2alpha1.CollectorSche
 
 	// CLID-230 rebuild-catalogs
 	oImgs := oCollector.AllImages
-	if o.alphaCtlgFilter && (o.Opts.IsMirrorToDisk() || o.Opts.IsMirrorToMirror()) {
-		results, delImgs, err := o.ImageBuilder.RebuildCatalogs(ctx, oCollector)
+	if o.alphaCtlgFilter && (o.Opts.IsMirrorToDisk() || o.Opts.IsMirrorToMirror()) { //TODO ALEX ADD A CHECK TO SEE IF THE CATALOG WAS ALREADY REBUILT BEFORE, IF YES, SKIP, IT WILL AVOID REBUILDING CATALOGS ALREADY REBUILT PREVIOUSLY
+		results, err := o.ImageBuilder.RebuildCatalogs(ctx, oCollector)
 		if err != nil {
 			o.closeAll()
 			return v2alpha1.CollectorSchema{}, err
 		}
-		oImgs = excludeImages(oImgs, delImgs)
 		if o.Opts.IsMirrorToMirror() {
 			oImgs = append(oImgs, results...)
 		}

v2/internal/pkg/imagebuilder/interface.go

@@ -12,5 +12,5 @@ type ImageBuilderInterface interface {
 	BuildAndPush(ctx context.Context, targetRef string, layoutPath layout.Path, cmd []string, layers ...v1.Layer) error
 	SaveImageLayoutToDir(ctx context.Context, imgRef string, layoutDir string) (layout.Path, error)
 	ProcessImageIndex(ctx context.Context, idx v1.ImageIndex, v2format *bool, cmd []string, targetRef string, layers ...v1.Layer) (v1.ImageIndex, error)
-	RebuildCatalogs(ctx context.Context, collectorSchema v2alpha1.CollectorSchema) ([]v2alpha1.CopyImageSchema, []v2alpha1.Image, error)
+	RebuildCatalogs(ctx context.Context, collectorSchema v2alpha1.CollectorSchema) ([]v2alpha1.CopyImageSchema, error)
 }

v2/internal/pkg/imagebuilder/rebuild_catalog.go

@@ -33,11 +33,10 @@ const (
 // RebuildCatalogs - uses buildah library that reads a containerfile and builds mult-arch manifestlist
 // NB - due to the unshare (reexec) for buildah no unit tests have been implemented
 // The final goal is to implement integration tests for this functionality
-func (o *ImageBuilder) RebuildCatalogs(ctx context.Context, catalogSchema v2alpha1.CollectorSchema) ([]v2alpha1.CopyImageSchema, []v2alpha1.Image, error) {
+func (o *ImageBuilder) RebuildCatalogs(ctx context.Context, catalogSchema v2alpha1.CollectorSchema) ([]v2alpha1.CopyImageSchema, error) {
 
 	// set variables
 	catalogs := catalogSchema.CatalogToFBCMap
-	excludeCatalogs := []v2alpha1.Image{}
 	result := []v2alpha1.CopyImageSchema{}
 	containerTemplate := `
 FROM {{ .Catalog }} AS builder
@@ -58,57 +57,42 @@ COPY --from=builder /tmp/cache /tmp/cache
 `
 
 	for _, v := range catalogs {
+		filteredDir := filepath.Dir(v.FilteredConfigPath)
+
 		o.Logger.Info("🔂 rebuilding catalog (pulling catalog image) %s", v.OperatorFilter.Catalog)
 		contents := bytes.NewBufferString("")
 		tmpl, err := template.New("Containerfile").Parse(containerTemplate)
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 		err = tmpl.Execute(contents, map[string]interface{}{
 			"Catalog": v.OperatorFilter.Catalog,
 		})
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 
 		// write the Containerfile content to a file
-		containerfilePath := filepath.Join("tmp", "Containerfile")
-		os.MkdirAll("tmp", 0755)
-		defer os.RemoveAll("tmp")
+		containerfilePath := filepath.Join(filteredDir, "Containerfile")
 
 		err = os.WriteFile(containerfilePath, contents.Bytes(), 0755)
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 
-		var localDest, remoteDest string
-
 		imgSpec, err := image.ParseRef(v.OperatorFilter.Catalog)
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 
-		switch {
-		case len(v.OperatorFilter.TargetCatalog) > 0 && len(v.OperatorFilter.TargetTag) > 0:
-			localDest = dockerProtocol + strings.Join([]string{o.LocalFQDN, v.OperatorFilter.TargetCatalog}, "/") + ":" + v.OperatorFilter.TargetTag
-			remoteDest = strings.Join([]string{o.Destination, v.OperatorFilter.TargetCatalog}, "/") + ":" + v.OperatorFilter.TargetTag
-		case len(v.OperatorFilter.TargetCatalog) > 0 && len(v.OperatorFilter.TargetTag) == 0:
-			localDest = dockerProtocol + strings.Join([]string{o.LocalFQDN, v.OperatorFilter.TargetCatalog}, "/") + ":" + imgSpec.Tag
-			remoteDest = strings.Join([]string{o.Destination, v.OperatorFilter.TargetCatalog}, "/") + ":" + imgSpec.Tag
-		case len(v.OperatorFilter.TargetCatalog) == 0 && len(v.OperatorFilter.TargetTag) > 0:
-			localDest = dockerProtocol + strings.Join([]string{o.LocalFQDN, imgSpec.PathComponent}, "/") + ":" + v.OperatorFilter.TargetTag
-			remoteDest = strings.Join([]string{o.Destination, imgSpec.PathComponent}, "/") + ":" + v.OperatorFilter.TargetTag
-		case len(v.OperatorFilter.TargetCatalog) == 0 && len(v.OperatorFilter.TargetTag) == 0:
-			localDest = dockerProtocol + strings.Join([]string{o.LocalFQDN, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag
-			remoteDest = strings.Join([]string{o.Destination, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag
-		}
+		srcCache := dockerProtocol + strings.Join([]string{o.LocalFQDN, imgSpec.PathComponent}, "/") + ":" + filepath.Base(filteredDir)
 
 		// this is safe as we know that there is a docker:// prefix
-		updatedDest := strings.Split(localDest, dockerProtocol)[1]
+		updatedDest := strings.Split(srcCache, dockerProtocol)[1]
 
 		buildOptions, err := getStandardBuildOptions(updatedDest, o.SrcTlsVerify)
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 
 		buildOptions.DefaultMountsFilePath = ""
@@ -117,12 +101,12 @@ COPY --from=builder /tmp/cache /tmp/cache
 
 		buildStoreOptions, err := storage.DefaultStoreOptions()
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 
 		buildStore, err := storage.GetStore(buildStoreOptions)
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 		defer buildStore.Shutdown(false)
 
@@ -137,7 +121,7 @@ COPY --from=builder /tmp/cache /tmp/cache
 			o.Logger.Debug("  image reference  : %s", ref.String())
 		}
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 
 		var retries *uint
@@ -157,51 +141,74 @@ COPY --from=builder /tmp/cache /tmp/cache
 			MaxRetries:             retries,
 		}
 
-		destImageRef, err := alltransports.ParseImageName(localDest)
+		destImageRef, err := alltransports.ParseImageName(srcCache)
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 
 		_, list, err := manifests.LoadFromImage(buildStore, id)
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 
-		o.Logger.Debug("local cache destination (rebuilt-catalog) %s", localDest)
+		o.Logger.Debug("local cache destination (rebuilt-catalog) %s", srcCache)
 		o.Logger.Debug("destination image reference %v", destImageRef)
 		o.Logger.Debug("pushing manifest list to remote registry")
 		// push the manifest list to local cache
 		_, digest, err := list.Push(ctx, destImageRef, manifestPushOptions)
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
+		}
+
+		digestOnly := digest.String()
+		if strings.Contains(digestOnly, ":") {
+			digestOnly = strings.Split(digest.String(), ":")[1]
+		}
+
+		err = os.WriteFile(filepath.Join(filteredDir, "digest"), []byte(digestOnly), 0755)
+		if err != nil {
+			return result, err
 		}
 
 		_, err = buildStore.DeleteImage(id, true)
 		if err != nil {
-			return result, excludeCatalogs, err
+			return result, err
 		}
 
 		o.Logger.Info("✅ successfully pushed catalog manifest list")
 		o.Logger.Debug("  digest           : %s", digest)
 
-		excludeImg := v2alpha1.Image{
-			Name: v.OperatorFilter.Catalog,
-		}
-		excludeCatalogs = append(excludeCatalogs, excludeImg)
-
 		if o.Mode == MirrorToMirror {
+
+			var dest string
+			for _, img := range catalogSchema.AllImages {
+				if img.Type.IsOperatorCatalog() && strings.Split(img.Origin, dockerProtocol)[1] == v.OperatorFilter.Catalog && !strings.Contains(img.Destination, o.LocalFQDN) {
+					if v.OperatorFilter.TargetTag != "" {
+						dest = img.Destination
+					} else {
+						//TODO ALEX so far we are not considering digests only, it is needed to cover the digest only as well
+						parts := strings.Split(img.Destination, ":")
+						parts[len(parts)-1] = filepath.Base(filteredDir)
+
+						dest = strings.Join(parts, ":")
+					}
+
+				}
+			}
+
 			copyImage := v2alpha1.CopyImageSchema{
-				Origin:      v.OperatorFilter.Catalog,
-				Source:      localDest,
-				Destination: remoteDest,
-				Type:        v2alpha1.TypeOperatorCatalog,
+				Origin:           v.OperatorFilter.Catalog,
+				Source:           srcCache,
+				Destination:      dest,
+				Type:             v2alpha1.TypeOperatorCatalog,
+				IsCatalogRebuilt: true,
 			}
-			result = append(result, copyImage)
+			result = append(result, copyImage) //TODO ALEX currently in mirrorToMirror both original and filtered are being mirrored, find a way to keep the original only in the cache for mirrorToMirror
 		}
 	}
 	o.Logger.Info("✅ completed rebuild catalog/s")
 	o.Logger.Debug("result %v", result)
-	return result, excludeCatalogs, nil
+	return result, nil
 }
 
 func newSystemContext(tlsVerify bool) *types.SystemContext {

v2/internal/pkg/operator/common.go

@@ -148,7 +148,7 @@ func (o OperatorCollector) prepareD2MCopyBatch(images map[string][]v2alpha1.Rela
 				o.Log.Debug("delete mode, catalog index %s : SKIPPED", img.Image)
 			} else {
 				if _, found := alreadyIncluded[img.Image]; !found {
-					result = append(result, v2alpha1.CopyImageSchema{Origin: imgSpec.ReferenceWithTransport, Source: src, Destination: dest, Type: img.Type})
+					result = append(result, v2alpha1.CopyImageSchema{Origin: imgSpec.ReferenceWithTransport, Source: src, Destination: dest, Type: img.Type, IsCatalogRebuilt: img.IsCatalogRebuilt})
 					alreadyIncluded[img.Image] = struct{}{}
 				}
 			}
@@ -202,12 +202,12 @@ func (o OperatorCollector) prepareM2DCopyBatch(images map[string][]v2alpha1.Rela
 			o.Log.Debug("destination %s", dest)
 
 			if _, found := alreadyIncluded[img.Image]; !found {
-				result = append(result, v2alpha1.CopyImageSchema{Source: src, Destination: dest, Origin: imgSpec.ReferenceWithTransport, Type: img.Type})
+				result = append(result, v2alpha1.CopyImageSchema{Source: src, Destination: dest, Origin: imgSpec.ReferenceWithTransport, Type: img.Type, IsCatalogRebuilt: img.IsCatalogRebuilt})
 				// OCPBUGS-37948 + CLID-196
 				// Keep a copy of the catalog image in local cache for delete workflow
 				if img.Type == v2alpha1.TypeOperatorCatalog && o.Opts.Mode == mirror.MirrorToMirror {
 					cacheDest := strings.Replace(dest, o.destinationRegistry(), o.LocalStorageFQDN, 1)
-					result = append(result, v2alpha1.CopyImageSchema{Source: src, Destination: cacheDest, Origin: imgSpec.ReferenceWithTransport, Type: img.Type})
+					result = append(result, v2alpha1.CopyImageSchema{Source: src, Destination: cacheDest, Origin: imgSpec.ReferenceWithTransport, Type: img.Type, IsCatalogRebuilt: img.IsCatalogRebuilt})
 
 				}
 				alreadyIncluded[img.Image] = struct{}{}

v2/internal/pkg/operator/const.go

@@ -1,16 +1,20 @@
 package operator
 
 const (
-	operatorImageExtractDir        = "hold-operator"
-	dockerProtocol                 = "docker://"
-	ociProtocol                    = "oci://"
-	ociProtocolTrimmed             = "oci:"
-	operatorImageDir               = "operator-images"
-	blobsDir                       = "blobs/sha256"
-	collectorPrefix                = "[OperatorImageCollector] "
-	errMsg                         = collectorPrefix + "%s"
-	logsFile                       = "operator.log"
-	errorSemver             string = " semver %v "
-	filteredCatalogDir             = "filtered-operator"
-	digestIncorrectMessage  string = "the digests seem to be incorrect for %s: %s "
+	operatorImageExtractDir           = "hold-operator" //TODO ALEX REMOVE ME when filtered_collector.go is the default
+	dockerProtocol                    = "docker://"
+	ociProtocol                       = "oci://"
+	ociProtocolTrimmed                = "oci:"
+	operatorImageDir                  = "operator-images" //TODO ALEX REMOVE ME when filtered_collector.go is the default
+	operatorCatalogsDir        string = "operator-catalogs"
+	operatorCatalogConfigDir   string = "catalog-config"
+	operatorCatalogImageDir    string = "catalog-image"
+	operatorCatalogFilteredDir string = "filtered-catalogs"
+	blobsDir                          = "blobs/sha256"
+	collectorPrefix                   = "[OperatorImageCollector] "
+	errMsg                            = collectorPrefix + "%s"
+	logsFile                          = "operator.log"
+	errorSemver                string = " semver %v "
+	filteredCatalogDir                = "filtered-operator"
+	digestIncorrectMessage     string = "the digests seem to be incorrect for %s: %s "
 )