From 3eed6e62054c67a6ba4ac17f300c703d17475d44 Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Tue, 5 Nov 2024 19:56:19 +0100 Subject: [PATCH] clid-265: generates folders based on the filtered catalog digest --- v2/internal/pkg/api/v2alpha1/type_image.go | 4 + v2/internal/pkg/api/v2alpha1/type_internal.go | 6 +- v2/internal/pkg/cli/const.go | 1 + v2/internal/pkg/cli/executor.go | 13 +- v2/internal/pkg/imagebuilder/interface.go | 2 +- .../pkg/imagebuilder/rebuild_catalog.go | 101 ++--- v2/internal/pkg/operator/common.go | 6 +- v2/internal/pkg/operator/const.go | 28 +- .../pkg/operator/filtered_collector.go | 355 ++++++++++++------ v2/internal/pkg/release/graph_test.go | 4 +- 10 files changed, 329 insertions(+), 191 deletions(-) diff --git a/v2/internal/pkg/api/v2alpha1/type_image.go b/v2/internal/pkg/api/v2alpha1/type_image.go index 019b23f91..34111bd84 100644 --- a/v2/internal/pkg/api/v2alpha1/type_image.go +++ b/v2/internal/pkg/api/v2alpha1/type_image.go @@ -54,6 +54,10 @@ func (it ImageType) IsOperator() bool { return it == TypeOperatorBundle || it == TypeOperatorCatalog || it == TypeOperatorRelatedImage } +func (it ImageType) IsOperatorCatalog() bool { + return it == TypeOperatorCatalog +} + func (it ImageType) IsAdditionalImage() bool { return it == TypeGeneric } diff --git a/v2/internal/pkg/api/v2alpha1/type_internal.go b/v2/internal/pkg/api/v2alpha1/type_internal.go index 13330e220..bdf07b879 100644 --- a/v2/internal/pkg/api/v2alpha1/type_internal.go +++ b/v2/internal/pkg/api/v2alpha1/type_internal.go @@ -191,7 +191,8 @@ type RelatedImage struct { TargetTag string `json:"targetTag"` // Used to keep specific naming info for the related image // if set should be used when mirroring - TargetCatalog string `json:"targetCatalog"` + TargetCatalog string `json:"targetCatalog"` + IsCatalogRebuilt bool `json:"isCatalogRebuilt"` } type CollectorSchema struct { @@ -219,7 +220,8 @@ type CopyImageSchema struct { Origin string // Type: metadata to explain why this image is being copied // it doesn´t need to be persisted to json - Type ImageType `json:"-"` + Type ImageType `json:"-"` + IsCatalogRebuilt bool `json:"isCatalogRebuilt"` } // SignatureContentSchema diff --git a/v2/internal/pkg/cli/const.go b/v2/internal/pkg/cli/const.go index 9c39fe18d..c3c9700e8 100644 --- a/v2/internal/pkg/cli/const.go +++ b/v2/internal/pkg/cli/const.go @@ -16,6 +16,7 @@ const ( releaseImageExtractDir string = "hold-release" cincinnatiGraphDataDir string = "cincinnati-graph-data" operatorImageExtractDir string = "hold-operator" + operatorCatalogsDir string = "operator-catalogs" signaturesDir string = "signatures" registryLogFilename string = "registry.log" startMessage string = "starting local storage on localhost:%v" diff --git a/v2/internal/pkg/cli/executor.go b/v2/internal/pkg/cli/executor.go index c27bdaafa..146dc3edd 100644 --- a/v2/internal/pkg/cli/executor.go +++ b/v2/internal/pkg/cli/executor.go @@ -689,6 +689,7 @@ func (o *ExecutorSchema) setupWorkingDir() error { return err } + //TODO ALEX REMOVE ME WHEN filtered_collector.go is the default for operators // create operator cache dir o.Log.Trace("creating operator cache directory %s ", o.Opts.Global.WorkingDir+"/"+operatorImageExtractDir) err = o.MakeDir.makeDirAll(o.Opts.Global.WorkingDir+"/"+operatorImageExtractDir, 0755) @@ -697,6 +698,13 @@ func (o *ExecutorSchema) setupWorkingDir() error { return err } + o.Log.Trace("creating operator cache directory %s ", filepath.Join(o.Opts.Global.WorkingDir, operatorCatalogsDir)) + err = o.MakeDir.makeDirAll(filepath.Join(o.Opts.Global.WorkingDir, operatorCatalogsDir), 0755) + if err != nil { + o.Log.Error(" setupWorkingDir for operator cache %v ", err) + return err + } + // create cluster-resources directory and clean it o.Log.Trace("creating cluster-resources directory %s ", o.Opts.Global.WorkingDir+"/"+clusterResourcesDir) err = os.RemoveAll(o.Opts.Global.WorkingDir + "/" + clusterResourcesDir) @@ -1031,13 +1039,12 @@ func (o *ExecutorSchema) CollectAll(ctx context.Context) (v2alpha1.CollectorSche // CLID-230 rebuild-catalogs oImgs := oCollector.AllImages - if o.alphaCtlgFilter && (o.Opts.IsMirrorToDisk() || o.Opts.IsMirrorToMirror()) { - results, delImgs, err := o.ImageBuilder.RebuildCatalogs(ctx, oCollector) + if o.alphaCtlgFilter && (o.Opts.IsMirrorToDisk() || o.Opts.IsMirrorToMirror()) { //TODO ALEX ADD A CHECK TO SEE IF THE CATALOG WAS ALREADY REBUILT BEFORE, IF YES, SKIP, IT WILL AVOID REBUILDING CATALOGS ALREADY REBUILT PREVIOUSLY + results, err := o.ImageBuilder.RebuildCatalogs(ctx, oCollector) if err != nil { o.closeAll() return v2alpha1.CollectorSchema{}, err } - oImgs = excludeImages(oImgs, delImgs) if o.Opts.IsMirrorToMirror() { oImgs = append(oImgs, results...) } diff --git a/v2/internal/pkg/imagebuilder/interface.go b/v2/internal/pkg/imagebuilder/interface.go index bce7d4382..1a810abd5 100644 --- a/v2/internal/pkg/imagebuilder/interface.go +++ b/v2/internal/pkg/imagebuilder/interface.go @@ -12,5 +12,5 @@ type ImageBuilderInterface interface { BuildAndPush(ctx context.Context, targetRef string, layoutPath layout.Path, cmd []string, layers ...v1.Layer) error SaveImageLayoutToDir(ctx context.Context, imgRef string, layoutDir string) (layout.Path, error) ProcessImageIndex(ctx context.Context, idx v1.ImageIndex, v2format *bool, cmd []string, targetRef string, layers ...v1.Layer) (v1.ImageIndex, error) - RebuildCatalogs(ctx context.Context, collectorSchema v2alpha1.CollectorSchema) ([]v2alpha1.CopyImageSchema, []v2alpha1.Image, error) + RebuildCatalogs(ctx context.Context, collectorSchema v2alpha1.CollectorSchema) ([]v2alpha1.CopyImageSchema, error) } diff --git a/v2/internal/pkg/imagebuilder/rebuild_catalog.go b/v2/internal/pkg/imagebuilder/rebuild_catalog.go index 93da203ca..a39960ad2 100644 --- a/v2/internal/pkg/imagebuilder/rebuild_catalog.go +++ b/v2/internal/pkg/imagebuilder/rebuild_catalog.go @@ -33,11 +33,10 @@ const ( // RebuildCatalogs - uses buildah library that reads a containerfile and builds mult-arch manifestlist // NB - due to the unshare (reexec) for buildah no unit tests have been implemented // The final goal is to implement integration tests for this functionality -func (o *ImageBuilder) RebuildCatalogs(ctx context.Context, catalogSchema v2alpha1.CollectorSchema) ([]v2alpha1.CopyImageSchema, []v2alpha1.Image, error) { +func (o *ImageBuilder) RebuildCatalogs(ctx context.Context, catalogSchema v2alpha1.CollectorSchema) ([]v2alpha1.CopyImageSchema, error) { // set variables catalogs := catalogSchema.CatalogToFBCMap - excludeCatalogs := []v2alpha1.Image{} result := []v2alpha1.CopyImageSchema{} containerTemplate := ` FROM {{ .Catalog }} AS builder @@ -58,57 +57,42 @@ COPY --from=builder /tmp/cache /tmp/cache ` for _, v := range catalogs { + filteredDir := filepath.Dir(v.FilteredConfigPath) + o.Logger.Info("🔂 rebuilding catalog (pulling catalog image) %s", v.OperatorFilter.Catalog) contents := bytes.NewBufferString("") tmpl, err := template.New("Containerfile").Parse(containerTemplate) if err != nil { - return result, excludeCatalogs, err + return result, err } err = tmpl.Execute(contents, map[string]interface{}{ "Catalog": v.OperatorFilter.Catalog, }) if err != nil { - return result, excludeCatalogs, err + return result, err } // write the Containerfile content to a file - containerfilePath := filepath.Join("tmp", "Containerfile") - os.MkdirAll("tmp", 0755) - defer os.RemoveAll("tmp") + containerfilePath := filepath.Join(filteredDir, "Containerfile") err = os.WriteFile(containerfilePath, contents.Bytes(), 0755) if err != nil { - return result, excludeCatalogs, err + return result, err } - var localDest, remoteDest string - imgSpec, err := image.ParseRef(v.OperatorFilter.Catalog) if err != nil { - return result, excludeCatalogs, err + return result, err } - switch { - case len(v.OperatorFilter.TargetCatalog) > 0 && len(v.OperatorFilter.TargetTag) > 0: - localDest = dockerProtocol + strings.Join([]string{o.LocalFQDN, v.OperatorFilter.TargetCatalog}, "/") + ":" + v.OperatorFilter.TargetTag - remoteDest = strings.Join([]string{o.Destination, v.OperatorFilter.TargetCatalog}, "/") + ":" + v.OperatorFilter.TargetTag - case len(v.OperatorFilter.TargetCatalog) > 0 && len(v.OperatorFilter.TargetTag) == 0: - localDest = dockerProtocol + strings.Join([]string{o.LocalFQDN, v.OperatorFilter.TargetCatalog}, "/") + ":" + imgSpec.Tag - remoteDest = strings.Join([]string{o.Destination, v.OperatorFilter.TargetCatalog}, "/") + ":" + imgSpec.Tag - case len(v.OperatorFilter.TargetCatalog) == 0 && len(v.OperatorFilter.TargetTag) > 0: - localDest = dockerProtocol + strings.Join([]string{o.LocalFQDN, imgSpec.PathComponent}, "/") + ":" + v.OperatorFilter.TargetTag - remoteDest = strings.Join([]string{o.Destination, imgSpec.PathComponent}, "/") + ":" + v.OperatorFilter.TargetTag - case len(v.OperatorFilter.TargetCatalog) == 0 && len(v.OperatorFilter.TargetTag) == 0: - localDest = dockerProtocol + strings.Join([]string{o.LocalFQDN, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag - remoteDest = strings.Join([]string{o.Destination, imgSpec.PathComponent}, "/") + ":" + imgSpec.Tag - } + srcCache := dockerProtocol + strings.Join([]string{o.LocalFQDN, imgSpec.PathComponent}, "/") + ":" + filepath.Base(filteredDir) // this is safe as we know that there is a docker:// prefix - updatedDest := strings.Split(localDest, dockerProtocol)[1] + updatedDest := strings.Split(srcCache, dockerProtocol)[1] buildOptions, err := getStandardBuildOptions(updatedDest, o.SrcTlsVerify) if err != nil { - return result, excludeCatalogs, err + return result, err } buildOptions.DefaultMountsFilePath = "" @@ -117,12 +101,12 @@ COPY --from=builder /tmp/cache /tmp/cache buildStoreOptions, err := storage.DefaultStoreOptions() if err != nil { - return result, excludeCatalogs, err + return result, err } buildStore, err := storage.GetStore(buildStoreOptions) if err != nil { - return result, excludeCatalogs, err + return result, err } defer buildStore.Shutdown(false) @@ -137,7 +121,7 @@ COPY --from=builder /tmp/cache /tmp/cache o.Logger.Debug(" image reference : %s", ref.String()) } if err != nil { - return result, excludeCatalogs, err + return result, err } var retries *uint @@ -157,51 +141,74 @@ COPY --from=builder /tmp/cache /tmp/cache MaxRetries: retries, } - destImageRef, err := alltransports.ParseImageName(localDest) + destImageRef, err := alltransports.ParseImageName(srcCache) if err != nil { - return result, excludeCatalogs, err + return result, err } _, list, err := manifests.LoadFromImage(buildStore, id) if err != nil { - return result, excludeCatalogs, err + return result, err } - o.Logger.Debug("local cache destination (rebuilt-catalog) %s", localDest) + o.Logger.Debug("local cache destination (rebuilt-catalog) %s", srcCache) o.Logger.Debug("destination image reference %v", destImageRef) o.Logger.Debug("pushing manifest list to remote registry") // push the manifest list to local cache _, digest, err := list.Push(ctx, destImageRef, manifestPushOptions) if err != nil { - return result, excludeCatalogs, err + return result, err + } + + digestOnly := digest.String() + if strings.Contains(digestOnly, ":") { + digestOnly = strings.Split(digest.String(), ":")[1] + } + + err = os.WriteFile(filepath.Join(filteredDir, "digest"), []byte(digestOnly), 0755) + if err != nil { + return result, err } _, err = buildStore.DeleteImage(id, true) if err != nil { - return result, excludeCatalogs, err + return result, err } o.Logger.Info("✅ successfully pushed catalog manifest list") o.Logger.Debug(" digest : %s", digest) - excludeImg := v2alpha1.Image{ - Name: v.OperatorFilter.Catalog, - } - excludeCatalogs = append(excludeCatalogs, excludeImg) - if o.Mode == MirrorToMirror { + + var dest string + for _, img := range catalogSchema.AllImages { + if img.Type.IsOperatorCatalog() && strings.Split(img.Origin, dockerProtocol)[1] == v.OperatorFilter.Catalog && !strings.Contains(img.Destination, o.LocalFQDN) { + if v.OperatorFilter.TargetTag != "" { + dest = img.Destination + } else { + //TODO ALEX so far we are not considering digests only, it is needed to cover the digest only as well + parts := strings.Split(img.Destination, ":") + parts[len(parts)-1] = filepath.Base(filteredDir) + + dest = strings.Join(parts, ":") + } + + } + } + copyImage := v2alpha1.CopyImageSchema{ - Origin: v.OperatorFilter.Catalog, - Source: localDest, - Destination: remoteDest, - Type: v2alpha1.TypeOperatorCatalog, + Origin: v.OperatorFilter.Catalog, + Source: srcCache, + Destination: dest, + Type: v2alpha1.TypeOperatorCatalog, + IsCatalogRebuilt: true, } - result = append(result, copyImage) + result = append(result, copyImage) //TODO ALEX currently in mirrorToMirror both original and filtered are being mirrored, find a way to keep the original only in the cache for mirrorToMirror } } o.Logger.Info("✅ completed rebuild catalog/s") o.Logger.Debug("result %v", result) - return result, excludeCatalogs, nil + return result, nil } func newSystemContext(tlsVerify bool) *types.SystemContext { diff --git a/v2/internal/pkg/operator/common.go b/v2/internal/pkg/operator/common.go index 83c649935..c62b6433d 100644 --- a/v2/internal/pkg/operator/common.go +++ b/v2/internal/pkg/operator/common.go @@ -148,7 +148,7 @@ func (o OperatorCollector) prepareD2MCopyBatch(images map[string][]v2alpha1.Rela o.Log.Debug("delete mode, catalog index %s : SKIPPED", img.Image) } else { if _, found := alreadyIncluded[img.Image]; !found { - result = append(result, v2alpha1.CopyImageSchema{Origin: imgSpec.ReferenceWithTransport, Source: src, Destination: dest, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{Origin: imgSpec.ReferenceWithTransport, Source: src, Destination: dest, Type: img.Type, IsCatalogRebuilt: img.IsCatalogRebuilt}) alreadyIncluded[img.Image] = struct{}{} } } @@ -202,12 +202,12 @@ func (o OperatorCollector) prepareM2DCopyBatch(images map[string][]v2alpha1.Rela o.Log.Debug("destination %s", dest) if _, found := alreadyIncluded[img.Image]; !found { - result = append(result, v2alpha1.CopyImageSchema{Source: src, Destination: dest, Origin: imgSpec.ReferenceWithTransport, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{Source: src, Destination: dest, Origin: imgSpec.ReferenceWithTransport, Type: img.Type, IsCatalogRebuilt: img.IsCatalogRebuilt}) // OCPBUGS-37948 + CLID-196 // Keep a copy of the catalog image in local cache for delete workflow if img.Type == v2alpha1.TypeOperatorCatalog && o.Opts.Mode == mirror.MirrorToMirror { cacheDest := strings.Replace(dest, o.destinationRegistry(), o.LocalStorageFQDN, 1) - result = append(result, v2alpha1.CopyImageSchema{Source: src, Destination: cacheDest, Origin: imgSpec.ReferenceWithTransport, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{Source: src, Destination: cacheDest, Origin: imgSpec.ReferenceWithTransport, Type: img.Type, IsCatalogRebuilt: img.IsCatalogRebuilt}) } alreadyIncluded[img.Image] = struct{}{} diff --git a/v2/internal/pkg/operator/const.go b/v2/internal/pkg/operator/const.go index 1d68bcde8..ea50ffb94 100644 --- a/v2/internal/pkg/operator/const.go +++ b/v2/internal/pkg/operator/const.go @@ -1,16 +1,20 @@ package operator const ( - operatorImageExtractDir = "hold-operator" - dockerProtocol = "docker://" - ociProtocol = "oci://" - ociProtocolTrimmed = "oci:" - operatorImageDir = "operator-images" - blobsDir = "blobs/sha256" - collectorPrefix = "[OperatorImageCollector] " - errMsg = collectorPrefix + "%s" - logsFile = "operator.log" - errorSemver string = " semver %v " - filteredCatalogDir = "filtered-operator" - digestIncorrectMessage string = "the digests seem to be incorrect for %s: %s " + operatorImageExtractDir = "hold-operator" //TODO ALEX REMOVE ME when filtered_collector.go is the default + dockerProtocol = "docker://" + ociProtocol = "oci://" + ociProtocolTrimmed = "oci:" + operatorImageDir = "operator-images" //TODO ALEX REMOVE ME when filtered_collector.go is the default + operatorCatalogsDir string = "operator-catalogs" + operatorCatalogConfigDir string = "catalog-config" + operatorCatalogImageDir string = "catalog-image" + operatorCatalogFilteredDir string = "filtered-catalogs" + blobsDir = "blobs/sha256" + collectorPrefix = "[OperatorImageCollector] " + errMsg = collectorPrefix + "%s" + logsFile = "operator.log" + errorSemver string = " semver %v " + filteredCatalogDir = "filtered-operator" + digestIncorrectMessage string = "the digests seem to be incorrect for %s: %s " ) diff --git a/v2/internal/pkg/operator/filtered_collector.go b/v2/internal/pkg/operator/filtered_collector.go index eafbe40cf..12de104cb 100644 --- a/v2/internal/pkg/operator/filtered_collector.go +++ b/v2/internal/pkg/operator/filtered_collector.go @@ -2,6 +2,8 @@ package operator import ( "context" + "crypto/md5" + "encoding/json" "errors" "fmt" "io" @@ -12,10 +14,12 @@ import ( "sort" "strings" + "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" "github.com/openshift/oc-mirror/v2/internal/pkg/api/v2alpha1" "github.com/openshift/oc-mirror/v2/internal/pkg/image" "github.com/openshift/oc-mirror/v2/internal/pkg/mirror" + "github.com/operator-framework/operator-registry/alpha/declcfg" "github.com/otiai10/copy" ) @@ -30,10 +34,10 @@ type FilterCollector struct { func (o *FilterCollector) OperatorImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { var ( - allImages []v2alpha1.CopyImageSchema - label string - dir string - catalogName string + allImages []v2alpha1.CopyImageSchema + label string + catalogImageDir string + catalogName string ) o.Log.Debug(collectorPrefix+"setting copy option o.Opts.MultiArch=%s when collecting operator images", o.Opts.MultiArch) @@ -42,6 +46,7 @@ func (o *FilterCollector) OperatorImageCollector(ctx context.Context) (v2alpha1. copyImageSchemaMap := &v2alpha1.CopyImageSchemaMap{OperatorsByImage: make(map[string]map[string]struct{}), BundlesByImage: make(map[string]map[string]string)} for _, op := range o.Config.Mirror.Operators { + var catalogImage string // download the operator index image o.Log.Debug(collectorPrefix+"copying operator image %s", op.Catalog) @@ -79,37 +84,72 @@ func (o *FilterCollector) OperatorImageCollector(ctx context.Context) (v2alpha1. catalogDigest = d } - imageIndexDir := filepath.Join(imgSpec.ComponentName(), catalogDigest) - cacheDir := filepath.Join(o.Opts.Global.WorkingDir, operatorImageExtractDir, imageIndexDir) - dir = filepath.Join(o.Opts.Global.WorkingDir, operatorImageDir, imageIndexDir) - - if imgSpec.Transport == ociProtocol { - if _, err := os.Stat(dir); errors.Is(err, os.ErrNotExist) { - // delete the existing directory and untarred cache contents - os.RemoveAll(dir) - os.RemoveAll(cacheDir) - // copy all contents to the working dir - err := copy.Copy(imgSpec.PathComponent, dir) - if err != nil { - o.Log.Error(errMsg, err.Error()) - return v2alpha1.CollectorSchema{}, err - } - } + imageIndex := filepath.Join(imgSpec.ComponentName(), catalogDigest) + imageIndexDir := filepath.Join(o.Opts.Global.WorkingDir, operatorCatalogsDir, imageIndex) + configsDir := filepath.Join(imageIndexDir, operatorCatalogConfigDir) + catalogImageDir = filepath.Join(imageIndexDir, operatorCatalogImageDir) + filteredCatalogsDir := filepath.Join(imageIndexDir, operatorCatalogFilteredDir) + + err = createFolders([]string{configsDir, catalogImageDir, filteredCatalogsDir}) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return v2alpha1.CollectorSchema{}, err + } + + var filteredDC *declcfg.DeclarativeConfig + var isAlreadyFiltered bool + + filterDigest, err := digestOfFilter(op) + if err != nil { + return v2alpha1.CollectorSchema{}, err + } + + var isCatalogRebuilt bool + var srcFilteredCatalog string + filterPath := filepath.Join(filteredCatalogsDir, filterDigest, "digest") + filteredImageDigest, err := os.ReadFile(filterPath) + if err == nil && len(filterDigest) > 0 { + srcFilteredCatalog = dockerProtocol + strings.Join([]string{o.LocalStorageFQDN, imgSpec.PathComponent}, "/") + ":" + filterDigest + isAlreadyFiltered = o.isAlreadyFiltered(ctx, srcFilteredCatalog, string(filteredImageDigest)) + } + if isAlreadyFiltered { + filterConfigDir := filepath.Join(filteredCatalogsDir, filterDigest, operatorCatalogConfigDir) + filteredDC, err = o.ctlgHandler.getDeclarativeConfig(filterConfigDir) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return v2alpha1.CollectorSchema{}, err + } if len(op.TargetCatalog) > 0 { catalogName = op.TargetCatalog } else { catalogName = path.Base(imgSpec.Reference) } + catalogImage = strings.Split(srcFilteredCatalog, dockerProtocol)[1] + catalogDigest = string(filteredImageDigest) + isCatalogRebuilt = true } else { - if _, err := os.Stat(cacheDir); errors.Is(err, os.ErrNotExist) { - err := os.MkdirAll(dir, 0755) - if err != nil { - o.Log.Error(errMsg, err.Error()) - return v2alpha1.CollectorSchema{}, err + if imgSpec.Transport == ociProtocol { + if _, err := os.Stat(catalogImageDir); errors.Is(err, os.ErrNotExist) { //TODO ALEX CHECK IF THIS IS CORRECT AND FIX + // delete the existing directory and untarred cache contents + os.RemoveAll(catalogImageDir) + os.RemoveAll(configsDir) + // copy all contents to the working dir + err := copy.Copy(imgSpec.PathComponent, catalogImageDir) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return v2alpha1.CollectorSchema{}, err + } + } + + if len(op.TargetCatalog) > 0 { + catalogName = op.TargetCatalog + } else { + catalogName = path.Base(imgSpec.Reference) } + } else { src := dockerProtocol + op.Catalog - dest := ociProtocolTrimmed + dir + dest := ociProtocolTrimmed + catalogImageDir optsCopy := o.Opts optsCopy.Stdout = io.Discard @@ -120,122 +160,144 @@ func (o *FilterCollector) OperatorImageCollector(ctx context.Context) (v2alpha1. o.Log.Error(errMsg, err.Error()) } } - } - // it's in oci format so we can go directly to the index.json file - oci, err := o.Manifest.GetImageIndex(dir) - if err != nil { - o.Log.Error(errMsg, err.Error()) - return v2alpha1.CollectorSchema{}, err - } - - var catalogImage string - if isMultiManifestIndex(*oci) && imgSpec.Transport == ociProtocol { - err = o.Manifest.ConvertIndexToSingleManifest(dir, oci) + // it's in oci format so we can go directly to the index.json file + oci, err := o.Manifest.GetImageIndex(catalogImageDir) if err != nil { o.Log.Error(errMsg, err.Error()) return v2alpha1.CollectorSchema{}, err } - oci, err = o.Manifest.GetImageIndex(dir) - if err != nil { - o.Log.Error(errMsg, err.Error()) - return v2alpha1.CollectorSchema{}, err + if isMultiManifestIndex(*oci) && imgSpec.Transport == ociProtocol { + err = o.Manifest.ConvertIndexToSingleManifest(catalogImageDir, oci) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return v2alpha1.CollectorSchema{}, err + } + + oci, err = o.Manifest.GetImageIndex(catalogImageDir) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return v2alpha1.CollectorSchema{}, err + } + + catalogImage = ociProtocol + catalogImageDir + } else { + catalogImage = op.Catalog } - catalogImage = ociProtocol + dir - } else { - catalogImage = op.Catalog - } + if len(oci.Manifests) == 0 { + o.Log.Error(collectorPrefix+"no manifests found for %s ", op.Catalog) + return v2alpha1.CollectorSchema{}, fmt.Errorf(collectorPrefix+"no manifests found for %s ", op.Catalog) + } - if len(oci.Manifests) == 0 { - o.Log.Error(collectorPrefix+"no manifests found for %s ", op.Catalog) - return v2alpha1.CollectorSchema{}, fmt.Errorf(collectorPrefix+"no manifests found for %s ", op.Catalog) - } + validDigest, err := digest.Parse(oci.Manifests[0].Digest) + if err != nil { + o.Log.Error(collectorPrefix+digestIncorrectMessage, op.Catalog, err.Error()) + return v2alpha1.CollectorSchema{}, fmt.Errorf(collectorPrefix+"the digests seem to be incorrect for %s: %s ", op.Catalog, err.Error()) + } - validDigest, err := digest.Parse(oci.Manifests[0].Digest) - if err != nil { - o.Log.Error(collectorPrefix+digestIncorrectMessage, op.Catalog, err.Error()) - return v2alpha1.CollectorSchema{}, fmt.Errorf(collectorPrefix+"the digests seem to be incorrect for %s: %s ", op.Catalog, err.Error()) - } + manifest := validDigest.Encoded() + o.Log.Debug(collectorPrefix+"manifest %s", manifest) + // read the operator image manifest + manifestDir := filepath.Join(catalogImageDir, blobsDir, manifest) + oci, err = o.Manifest.GetImageManifest(manifestDir) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return v2alpha1.CollectorSchema{}, err + } - manifest := validDigest.Encoded() - o.Log.Debug(collectorPrefix+"manifest %s", manifest) - // read the operator image manifest - manifestDir := filepath.Join(dir, blobsDir, manifest) - oci, err = o.Manifest.GetImageManifest(manifestDir) - if err != nil { - o.Log.Error(errMsg, err.Error()) - return v2alpha1.CollectorSchema{}, err - } + // we need to check if oci returns multi manifests + // (from manifest list) also oci.Config will be nil + // we are only interested in the first manifest as all + // architecture "configs" will be exactly the same + if len(oci.Manifests) > 1 && oci.Config.Size == 0 { + subDigest, err := digest.Parse(oci.Manifests[0].Digest) + if err != nil { + o.Log.Error(collectorPrefix+digestIncorrectMessage, op.Catalog, err.Error()) + return v2alpha1.CollectorSchema{}, fmt.Errorf(collectorPrefix+"the digests seem to be incorrect for %s: %s ", op.Catalog, err.Error()) + } + manifestDir := filepath.Join(catalogImageDir, blobsDir, subDigest.Encoded()) + oci, err = o.Manifest.GetImageManifest(manifestDir) + if err != nil { + o.Log.Error(collectorPrefix+"manifest %s: %s ", op.Catalog, err.Error()) + return v2alpha1.CollectorSchema{}, fmt.Errorf(collectorPrefix+"manifest %s: %s ", op.Catalog, err.Error()) + } + } - // we need to check if oci returns multi manifests - // (from manifest list) also oci.Config will be nil - // we are only interested in the first manifest as all - // architecture "configs" will be exactly the same - if len(oci.Manifests) > 1 && oci.Config.Size == 0 { - subDigest, err := digest.Parse(oci.Manifests[0].Digest) + // read the config digest to get the detailed manifest + // looking for the lable to search for a specific folder + configDigest, err := digest.Parse(oci.Config.Digest) if err != nil { o.Log.Error(collectorPrefix+digestIncorrectMessage, op.Catalog, err.Error()) return v2alpha1.CollectorSchema{}, fmt.Errorf(collectorPrefix+"the digests seem to be incorrect for %s: %s ", op.Catalog, err.Error()) } - manifestDir := filepath.Join(dir, blobsDir, subDigest.Encoded()) - oci, err = o.Manifest.GetImageManifest(manifestDir) + catalogDir := filepath.Join(catalogImageDir, blobsDir, configDigest.Encoded()) + ocs, err := o.Manifest.GetOperatorConfig(catalogDir) if err != nil { - o.Log.Error(collectorPrefix+"manifest %s: %s ", op.Catalog, err.Error()) - return v2alpha1.CollectorSchema{}, fmt.Errorf(collectorPrefix+"manifest %s: %s ", op.Catalog, err.Error()) + o.Log.Error(errMsg, err.Error()) + return v2alpha1.CollectorSchema{}, err } - } - // read the config digest to get the detailed manifest - // looking for the lable to search for a specific folder - configDigest, err := digest.Parse(oci.Config.Digest) - if err != nil { - o.Log.Error(collectorPrefix+digestIncorrectMessage, op.Catalog, err.Error()) - return v2alpha1.CollectorSchema{}, fmt.Errorf(collectorPrefix+"the digests seem to be incorrect for %s: %s ", op.Catalog, err.Error()) - } - catalogDir := filepath.Join(dir, blobsDir, configDigest.Encoded()) - ocs, err := o.Manifest.GetOperatorConfig(catalogDir) - if err != nil { - o.Log.Error(errMsg, err.Error()) - return v2alpha1.CollectorSchema{}, err - } + label = ocs.Config.Labels.OperatorsOperatorframeworkIoIndexConfigsV1 + o.Log.Debug(collectorPrefix+"label %s", label) - label = ocs.Config.Labels.OperatorsOperatorframeworkIoIndexConfigsV1 - o.Log.Debug(collectorPrefix+"label %s", label) + // untar all the blobs for the operator + // if the layer with "label (from previous step) is found to a specific folder" + fromDir := strings.Join([]string{catalogImageDir, blobsDir}, "/") + err = o.Manifest.ExtractLayersOCI(fromDir, configsDir, label, oci) + if err != nil { + return v2alpha1.CollectorSchema{}, err + } - // untar all the blobs for the operator - // if the layer with "label (from previous step) is found to a specific folder" - fromDir := strings.Join([]string{dir, blobsDir}, "/") - err = o.Manifest.ExtractLayersOCI(fromDir, cacheDir, label, oci) - if err != nil { - return v2alpha1.CollectorSchema{}, err - } + originalDC, err := o.ctlgHandler.getDeclarativeConfig(filepath.Join(configsDir, label)) + if err != nil { + return v2alpha1.CollectorSchema{}, err + } - dc, err := o.ctlgHandler.getDeclarativeConfig(filepath.Join(cacheDir, label)) - if err != nil { - return v2alpha1.CollectorSchema{}, err - } + if !isFullCatalog(op) { + var filteredDigestPath string + var filterDigest string - filteredDC, err := filterCatalog(ctx, *dc, op) - if err != nil { - return v2alpha1.CollectorSchema{}, err - } + filteredDC, err = filterCatalog(ctx, *originalDC, op) + if err != nil { + return v2alpha1.CollectorSchema{}, err + } - filteredDCSavingDir := filepath.Join(o.Opts.Global.WorkingDir, filteredCatalogDir, imageIndexDir) - err = saveDeclarativeConfig(*filteredDC, filteredDCSavingDir) - if err != nil { - return v2alpha1.CollectorSchema{}, err - } + filterDigest, err = digestOfFilter(op) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return v2alpha1.CollectorSchema{}, err + } - if collectorSchema.CatalogToFBCMap == nil { - collectorSchema.CatalogToFBCMap = make(map[string]v2alpha1.CatalogFilterResult) - } - result := v2alpha1.CatalogFilterResult{ - OperatorFilter: op, - FilteredConfigPath: filteredDCSavingDir, + if filterDigest != "" { + filteredDigestPath = filepath.Join(filteredCatalogsDir, filterDigest, operatorCatalogConfigDir) + + err = createFolders([]string{filteredDigestPath}) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return v2alpha1.CollectorSchema{}, err + } + } + + err = saveDeclarativeConfig(*filteredDC, filteredDigestPath) + if err != nil { + return v2alpha1.CollectorSchema{}, err + } + + if collectorSchema.CatalogToFBCMap == nil { + collectorSchema.CatalogToFBCMap = make(map[string]v2alpha1.CatalogFilterResult) + } + result := v2alpha1.CatalogFilterResult{ + OperatorFilter: op, + FilteredConfigPath: filteredDigestPath, + } + collectorSchema.CatalogToFBCMap[op.Catalog] = result + + } else { + filteredDC = originalDC + } } - collectorSchema.CatalogToFBCMap[op.Catalog] = result ri, err := o.ctlgHandler.getRelatedImagesFromCatalog(filteredDC, copyImageSchemaMap) if err != nil { @@ -266,11 +328,12 @@ func (o *FilterCollector) OperatorImageCollector(ctx context.Context) (v2alpha1. relatedImages[componentName] = []v2alpha1.RelatedImage{ { - Name: catalogName, - Image: catalogImage, - Type: v2alpha1.TypeOperatorCatalog, - TargetTag: targetTag, - TargetCatalog: targetCatalog, + Name: catalogName, + Image: catalogImage, + Type: v2alpha1.TypeOperatorCatalog, + TargetTag: targetTag, + TargetCatalog: targetCatalog, + IsCatalogRebuilt: isCatalogRebuilt, }, } } @@ -308,3 +371,53 @@ func (o *FilterCollector) OperatorImageCollector(ctx context.Context) (v2alpha1. return collectorSchema, nil } + +func isFullCatalog(catalog v2alpha1.Operator) bool { + return len(catalog.IncludeConfig.Packages) == 0 && catalog.Full +} + +func createFolders(paths []string) error { + var errs []error + for _, path := range paths { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + err = os.MkdirAll(path, 0755) + if err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +func digestOfFilter(catalog v2alpha1.Operator) (string, error) { + pkgs, err := json.Marshal(catalog.IncludeConfig.Packages) + if err != nil { + return "", err + } + return fmt.Sprintf("%x", md5.Sum(pkgs))[0:32], nil +} + +func (o FilterCollector) isAlreadyFiltered(ctx context.Context, srcImage, filteredImageDigest string) bool { + + imgSpec, err := image.ParseRef(srcImage) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return false + } + + sourceCtx, err := o.Opts.SrcImage.NewSystemContext() + if err != nil { + return false + } + // OCPBUGS-37948 : No TLS verification when getting manifests from the cache registry + if strings.Contains(srcImage, o.Opts.LocalStorageFQDN) { // when copying from cache, use HTTP + sourceCtx.DockerInsecureSkipTLSVerify = types.OptionalBoolTrue + } + + catalogDigest, err := o.Manifest.GetDigest(ctx, sourceCtx, imgSpec.ReferenceWithTransport) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return false + } + return filteredImageDigest == catalogDigest +} diff --git a/v2/internal/pkg/release/graph_test.go b/v2/internal/pkg/release/graph_test.go index 36c16e735..25a449412 100644 --- a/v2/internal/pkg/release/graph_test.go +++ b/v2/internal/pkg/release/graph_test.go @@ -211,6 +211,6 @@ func (o mockImageBuilder) ProcessImageIndex(ctx context.Context, idx v1.ImageInd return nil, nil } -func (o mockImageBuilder) RebuildCatalogs(ctx context.Context, collectorSchema v2alpha1.CollectorSchema) ([]v2alpha1.CopyImageSchema, []v2alpha1.Image, error) { - return []v2alpha1.CopyImageSchema{}, []v2alpha1.Image{}, nil +func (o mockImageBuilder) RebuildCatalogs(ctx context.Context, collectorSchema v2alpha1.CollectorSchema) ([]v2alpha1.CopyImageSchema, error) { + return []v2alpha1.CopyImageSchema{}, nil }