diff --git a/cnb_image.go b/cnb_image.go index b6267d26..af8c78a8 100644 --- a/cnb_image.go +++ b/cnb_image.go @@ -393,11 +393,11 @@ func (i *CNBImageCore) ReuseLayer(diffID string) error { } idx, err := getLayerIndex(diffID, i.previousImage) if err != nil { - return err + return fmt.Errorf("failed to get layer index: %w", err) } previousHistory, err := getHistory(idx, i.previousImage) if err != nil { - return err + return fmt.Errorf("failed to get history: %w", err) } return i.ReuseLayerWithHistory(diffID, previousHistory) } @@ -405,11 +405,11 @@ func (i *CNBImageCore) ReuseLayer(diffID string) error { func getLayerIndex(forDiffID string, fromImage v1.Image) (int, error) { layerHash, err := v1.NewHash(forDiffID) if err != nil { - return -1, err + return -1, fmt.Errorf("failed to get layer hash: %w", err) } configFile, err := getConfigFile(fromImage) if err != nil { - return -1, err + return -1, fmt.Errorf("failed to get config file: %w", err) } for idx, configHash := range configFile.RootFS.DiffIDs { if layerHash.String() == configHash.String() { @@ -433,11 +433,11 @@ func getHistory(forIndex int, fromImage v1.Image) (v1.History, error) { func (i *CNBImageCore) ReuseLayerWithHistory(diffID string, history v1.History) error { layerHash, err := v1.NewHash(diffID) if err != nil { - return err + return fmt.Errorf("failed to get layer hash: %w", err) } layer, err := i.previousImage.LayerByDiffID(layerHash) if err != nil { - return err + return fmt.Errorf("failed to get layer by diffID: %w", err) } if i.preserveHistory { history.Created = v1.Time{Time: i.createdAt} diff --git a/layout/layout_test.go b/layout/layout_test.go index c8e53b67..8e7b2d01 100644 --- a/layout/layout_test.go +++ b/layout/layout_test.go @@ -255,6 +255,16 @@ func testImage(t *testing.T, when spec.G, it spec.S) { h.AssertError(t, err, "has no layers") }) }) + + when("existing config has extra fields", func() { + it("returns an unmodified digest", func() { + img, err := layout.NewImage(imagePath, layout.FromBaseImagePath(filepath.Join("testdata", "layout", "busybox-latest"))) + h.AssertNil(t, err) + digest, err := img.Digest() + h.AssertNil(t, err) + h.AssertEq(t, digest.String(), "sha256:f75f3d1a317fc82c793d567de94fc8df2bece37acd5f2bd364a0d91a0d1f3dab") + }) + }) }) when("#WithMediaTypes", func() { @@ -273,6 +283,25 @@ func testImage(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, img.Save()) h.AssertDockerMediaTypes(t, img) // after saving }) + + when("using a sparse image", func() { + it("sets the requested media types", func() { + img, err := layout.NewImage( + imagePath, + layout.FromBaseImagePath(sparseBaseImagePath), + layout.WithMediaTypes(imgutil.OCITypes), + ) + h.AssertNil(t, err) + h.AssertOCIMediaTypes(t, img) // before saving + // add a random layer + path, diffID, _ := h.RandomLayer(t, tmpDir) + err = img.AddLayerWithDiffID(path, diffID) + h.AssertNil(t, err) + h.AssertOCIMediaTypes(t, img) // after adding a layer + h.AssertNil(t, img.Save()) + h.AssertOCIMediaTypes(t, img) // after saving + }) + }) }) when("#WithPreviousImage", func() { diff --git a/layout/new.go b/layout/new.go index 9b6e61e6..9d8586d1 100644 --- a/layout/new.go +++ b/layout/new.go @@ -13,25 +13,33 @@ func NewImage(path string, ops ...ImageOption) (*Image, error) { for _, op := range ops { op(options) } + options.Platform = processDefaultPlatformOption(options.Platform) var err error - if options.PreviousImageRepoName != "" { - options.PreviousImage, err = newImageFromPath(options.PreviousImageRepoName, options.Platform, imgutil.DefaultTypes) + + if options.BaseImage == nil && options.BaseImageRepoName != "" { // options.BaseImage supersedes options.BaseImageRepoName + options.BaseImage, err = newImageFromPath(options.BaseImageRepoName, options.Platform) + if err != nil { + return nil, err + } + } + options.MediaTypes = imgutil.GetPreferredMediaTypes(*options) + if options.BaseImage != nil { + options.BaseImage, err = newImageFacadeFrom(options.BaseImage, options.MediaTypes) if err != nil { return nil, err } } - if options.BaseImage == nil && options.BaseImageRepoName != "" { // options.BaseImage supersedes options.BaseImageRepoName - options.BaseImage, err = newImageFromPath(options.BaseImageRepoName, options.Platform, imgutil.DefaultTypes) + if options.PreviousImageRepoName != "" { + options.PreviousImage, err = newImageFromPath(options.PreviousImageRepoName, options.Platform) if err != nil { return nil, err } } - options.MediaTypes = imgutil.GetPreferredMediaTypes(*options) - if options.BaseImage != nil { - options.BaseImage, err = imgutil.EnsureMediaTypes(options.BaseImage, options.MediaTypes) // FIXME: this can move into imgutil constructor + if options.PreviousImage != nil { + options.PreviousImage, err = newImageFacadeFrom(options.PreviousImage, options.MediaTypes) if err != nil { return nil, err } @@ -63,7 +71,7 @@ func processDefaultPlatformOption(requestedPlatform imgutil.Platform) imgutil.Pl // newImageFromPath creates a layout image from the given path. // * If an image index for multiple platforms exists, it will try to select the image according to the platform provided. // * If the image does not exist, then nothing is returned. -func newImageFromPath(path string, withPlatform imgutil.Platform, withMediaTypes imgutil.MediaTypes) (v1.Image, error) { +func newImageFromPath(path string, withPlatform imgutil.Platform) (v1.Image, error) { if !imageExists(path) { return nil, nil } @@ -80,19 +88,7 @@ func newImageFromPath(path string, withPlatform imgutil.Platform, withMediaTypes if err != nil { return nil, fmt.Errorf("failed to load image from index: %w", err) } - - // ensure layers will not error when accessed if there is no underlying data - manifestFile, err := image.Manifest() - if err != nil { - return nil, err - } - configFile, err := image.ConfigFile() - if err != nil { - return nil, err - } - return imgutil.EnsureMediaTypesAndLayers(image, withMediaTypes, func(idx int, layer v1.Layer) (v1.Layer, error) { - return newLayerOrFacadeFrom(*configFile, *manifestFile, idx, layer) - }) + return image, nil } // imageFromIndex creates a v1.Image from the given Image Index, selecting the image manifest diff --git a/layout/v1_facade.go b/layout/v1_facade.go index 71890276..8a7fa530 100644 --- a/layout/v1_facade.go +++ b/layout/v1_facade.go @@ -6,8 +6,102 @@ import ( "io" v1 "github.com/google/go-containerregistry/pkg/v1" + + "github.com/buildpacks/imgutil" ) +type v1ImageFacade struct { + v1.Image + diffIDMap map[v1.Hash]v1.Layer + digestMap map[v1.Hash]v1.Layer +} + +func newImageFacadeFrom(original v1.Image, withMediaTypes imgutil.MediaTypes) (v1.Image, error) { + configFile, err := original.ConfigFile() + if err != nil { + return nil, fmt.Errorf("failed to get config: %w", err) + } + manifestFile, err := original.Manifest() + if err != nil { + return nil, fmt.Errorf("failed to get manifest: %w", err) + } + originalLayers, err := original.Layers() + if err != nil { + return nil, fmt.Errorf("failed to get layers: %w", err) + } + + ensureLayers := func(idx int, layer v1.Layer) (v1.Layer, error) { + return newLayerOrFacadeFrom(*configFile, *manifestFile, idx, layer) + } + // first, ensure media types + image, mutated, err := imgutil.EnsureMediaTypesAndLayers(original, withMediaTypes, ensureLayers) // if no media types are requested, this does nothing + if err != nil { + return nil, fmt.Errorf("failed to ensure media types: %w", err) + } + // then, ensure layers + if mutated { + // layers are wrapped in a facade, it is possible to call layer.Compressed or layer.Uncompressed without error + return image, nil + } + // we didn't mutate the image (possibly to preserve the digest), we must wrap the image in a facade + facade := &v1ImageFacade{ + Image: original, + diffIDMap: make(map[v1.Hash]v1.Layer), + digestMap: make(map[v1.Hash]v1.Layer), + } + for idx, l := range originalLayers { + layer, err := newLayerOrFacadeFrom(*configFile, *manifestFile, idx, l) + if err != nil { + return nil, err + } + diffID, err := layer.DiffID() + if err != nil { + return nil, err + } + facade.diffIDMap[diffID] = layer + digest, err := layer.Digest() + if err != nil { + return nil, err + } + facade.digestMap[digest] = layer + } + + return facade, nil +} + +func (i *v1ImageFacade) Layers() ([]v1.Layer, error) { + var layers []v1.Layer + configFile, err := i.ConfigFile() + if err != nil { + return nil, err + } + if configFile == nil { + return nil, nil + } + for _, diffID := range configFile.RootFS.DiffIDs { + l, err := i.LayerByDiffID(diffID) + if err != nil { + return nil, err + } + layers = append(layers, l) + } + return layers, nil +} + +func (i *v1ImageFacade) LayerByDiffID(h v1.Hash) (v1.Layer, error) { + if layer, ok := i.diffIDMap[h]; ok { + return layer, nil + } + return nil, fmt.Errorf("failed to find layer with diffID %s", h) // shouldn't get here +} + +func (i *v1ImageFacade) LayerByDigest(h v1.Hash) (v1.Layer, error) { + if layer, ok := i.digestMap[h]; ok { + return layer, nil + } + return nil, fmt.Errorf("failed to find layer with digest %s", h) // shouldn't get here +} + type v1LayerFacade struct { v1.Layer diffID v1.Hash diff --git a/new.go b/new.go index 92177b93..5c47085d 100644 --- a/new.go +++ b/new.go @@ -34,8 +34,6 @@ func NewCNBImage(options ImageOptions) (*CNBImageCore, error) { } } - // FIXME: we can call EnsureMediaTypesAndLayers here when locallayout supports replacing the underlying image - // ensure windows if err = prepareNewWindowsImageIfNeeded(image); err != nil { return nil, err @@ -130,113 +128,102 @@ func emptyV1(withPlatform Platform, withMediaTypes MediaTypes) (v1.Image, error) if err != nil { return nil, err } - return EnsureMediaTypes(image, withMediaTypes) + image, _, err = EnsureMediaTypesAndLayers(image, withMediaTypes, PreserveLayers) + return image, err } -func PreserveLayers(idx int, layer v1.Layer) (v1.Layer, error) { +func PreserveLayers(_ int, layer v1.Layer) (v1.Layer, error) { return layer, nil } -// EnsureMediaTypes replaces the provided image with a new image that has the desired media types. +// EnsureMediaTypesAndLayers replaces the provided image with a new image that has the desired media types. // It does this by constructing a manifest and config from the provided image, // and adding the layers from the provided image to the new image with the right media type. // If requested types are missing or default, it does nothing. -func EnsureMediaTypes(image v1.Image, requestedTypes MediaTypes) (v1.Image, error) { +// While adding the layers, each layer can be additionally mutated by providing a "mutate layer" function. +func EnsureMediaTypesAndLayers(image v1.Image, requestedTypes MediaTypes, mutateLayer func(idx int, layer v1.Layer) (v1.Layer, error)) (v1.Image, bool, error) { if requestedTypes == MissingTypes || requestedTypes == DefaultTypes { - return image, nil + return image, false, nil } - return EnsureMediaTypesAndLayers(image, requestedTypes, PreserveLayers) -} - -// EnsureMediaTypesAndLayers replaces the provided image with a new image that has the desired media types. -// It does this by constructing a manifest and config from the provided image, -// and adding the layers from the provided image to the new image with the right media type. -// While adding the layers, each layer can be additionally mutated by providing a "mutate layer" function. -func EnsureMediaTypesAndLayers(image v1.Image, requestedTypes MediaTypes, mutateLayer func(idx int, layer v1.Layer) (v1.Layer, error)) (v1.Image, error) { // (1) get data from the original image // manifest beforeManifest, err := image.Manifest() if err != nil { - return nil, err + return nil, false, fmt.Errorf("failed to get manifest: %w", err) } // config beforeConfig, err := image.ConfigFile() if err != nil { - return nil, err + return nil, false, fmt.Errorf("failed to get config: %w", err) } // layers beforeLayers, err := image.Layers() if err != nil { - return nil, err + return nil, false, fmt.Errorf("failed to get layers: %w", err) } - layersToSet := make([]v1.Layer, len(beforeLayers)) - for idx, layer := range beforeLayers { - mutatedLayer, err := mutateLayer(idx, layer) + var layersToAdd []v1.Layer + for idx, l := range beforeLayers { + layer, err := mutateLayer(idx, l) if err != nil { - return nil, err + return nil, false, fmt.Errorf("failed to mutate layer: %w", err) } - layersToSet[idx] = mutatedLayer + layersToAdd = append(layersToAdd, layer) } - // (2) construct a new image with the right manifest media type + // (2) construct a new image manifest with the right media type manifestType := requestedTypes.ManifestType() if manifestType == "" { manifestType = beforeManifest.MediaType } retImage := mutate.MediaType(empty.Image, manifestType) - // (3) set config media type + // (3) set config with the right media type configType := requestedTypes.ConfigType() if configType == "" { configType = beforeManifest.Config.MediaType } - // zero out history and diff IDs, as these will be updated when we call `mutate.Append` to add the layers + // zero out diff IDs and history, these will be added back when we append the layers beforeHistory := beforeConfig.History beforeConfig.History = []v1.History{} - beforeConfig.RootFS.DiffIDs = make([]v1.Hash, 0) - // set config + beforeConfig.RootFS.DiffIDs = []v1.Hash{} retImage, err = mutate.ConfigFile(retImage, beforeConfig) if err != nil { - return nil, err + return nil, false, fmt.Errorf("failed to set config: %w", err) } retImage = mutate.ConfigMediaType(retImage, configType) + // (4) set layers with the right media type - additions := layersAddendum(layersToSet, beforeHistory, requestedTypes.LayerType()) + additions := layersAddendum(layersToAdd, beforeHistory, requestedTypes.LayerType()) if err != nil { - return nil, err + return nil, false, err } retImage, err = mutate.Append(retImage, additions...) if err != nil { - return nil, err + return nil, false, fmt.Errorf("failed to append layers: %w", err) } - afterLayers, err := retImage.Layers() - if err != nil { - return nil, err - } - if len(afterLayers) != len(beforeLayers) { - return nil, fmt.Errorf("found %d layers for image; expected %d", len(afterLayers), len(beforeLayers)) - } - return retImage, nil + + return retImage, true, nil } // layersAddendum creates an Addendum array with the given layers // and the desired media type func layersAddendum(layers []v1.Layer, history []v1.History, requestedType types.MediaType) []mutate.Addendum { addendums := make([]mutate.Addendum, 0) + history = NormalizedHistory(history, len(layers)) if len(history) != len(layers) { history = make([]v1.History, len(layers)) } var err error - for idx, layer := range layers { + for idx, l := range layers { layerType := requestedType if requestedType == "" { // try to get a non-empty media type - if layerType, err = layer.MediaType(); err != nil { + if layerType, err = l.MediaType(); err != nil { layerType = "" } } addendums = append(addendums, mutate.Addendum{ - Layer: layer, + Layer: l, History: history[idx], MediaType: layerType, }) diff --git a/remote/new.go b/remote/new.go index 74ebdc7b..a392de57 100644 --- a/remote/new.go +++ b/remote/new.go @@ -292,7 +292,7 @@ func (i *Image) setUnderlyingImage(base v1.Image) error { return nil } // provided v1.Image media types differ from requested, override them - newBase, err := imgutil.EnsureMediaTypes(base, i.requestedMediaTypes) + newBase, _, err := imgutil.EnsureMediaTypesAndLayers(base, i.requestedMediaTypes, imgutil.PreserveLayers) if err != nil { return err }