From 94e644420a95c3eca03bc9bbdb1f2c6b17e05269 Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Mon, 2 Nov 2020 10:00:15 -0500 Subject: [PATCH] Add reader/writer to for oci-archive multi image Add read/writer with helpers to allow podman save/load multi oci-archive images. Allow read oci-archive using source_index to point to the an inedx from oci-archive manifest. Signed-off-by: Qi Wang --- docs/containers-transports.5.md | 8 +- oci/archive/oci_dest.go | 5 +- oci/archive/oci_transport.go | 29 ++++-- oci/archive/oci_transport_test.go | 34 +++++-- oci/archive/reader.go | 98 +++++++++++++++++++ oci/archive/writer.go | 54 ++++++++++ oci/internal/oci_util.go | 22 ++++- oci/internal/oci_util_test.go | 21 +++- .../fixtures/two_names_manifest/index.json | 25 +++++ oci/layout/oci_dest.go | 3 + oci/layout/oci_transport.go | 47 +++++++-- oci/layout/oci_transport_test.go | 13 +++ 12 files changed, 332 insertions(+), 27 deletions(-) create mode 100644 oci/archive/reader.go create mode 100644 oci/archive/writer.go create mode 100644 oci/layout/fixtures/two_names_manifest/index.json diff --git a/docs/containers-transports.5.md b/docs/containers-transports.5.md index c87283fb84..1b68a9eb26 100644 --- a/docs/containers-transports.5.md +++ b/docs/containers-transports.5.md @@ -57,14 +57,16 @@ An image stored in the docker daemon's internal storage. The image must be specified as a _docker-reference_ or in an alternative _algo:digest_ format when being used as an image source. The _algo:digest_ refers to the image ID reported by docker-inspect(1). -### **oci:**_path[:tag]_ +### **oci:**_path[:tag|@source-index]_ An image compliant with the "Open Container Image Layout Specification" at _path_. Using a _tag_ is optional and allows for storing multiple images at the same _path_. +@_source-index_ is a zero-based index in manifest (to access untagged images). +If neither tag nor @_source_index is specified when reading an image, the path must contain exactly one image. -### **oci-archive:**_path[:tag]_ +### **oci-archive:**_path[:{tag|@source-index}]_ -An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_. +An image compliant with the "Open Container Image Layout Specification" stored as a tar(1) archive at _path_. For reading archives, @_source-index_ is a zero-based index in archive manifest (to access untagged images). If neither tag nor @_source_index is specified when reading an archive, the archive must contain exactly one image. ### **ostree:**_docker-reference[@/absolute/repo/path]_ diff --git a/oci/archive/oci_dest.go b/oci/archive/oci_dest.go index 23d4713252..3288e058dd 100644 --- a/oci/archive/oci_dest.go +++ b/oci/archive/oci_dest.go @@ -20,7 +20,10 @@ type ociArchiveImageDestination struct { // newImageDestination returns an ImageDestination for writing to an existing directory. func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) { - tempDirRef, err := createOCIRef(sys, ref.image) + if ref.sourceIndex != -1 { + return nil, errors.Errorf("invalid sourceIndex %d for creating image destination", ref.sourceIndex) + } + tempDirRef, err := createOCIRef(sys, ref.image, -1) if err != nil { return nil, errors.Wrapf(err, "error creating oci reference") } diff --git a/oci/archive/oci_transport.go b/oci/archive/oci_transport.go index c808539d23..eb01fad265 100644 --- a/oci/archive/oci_transport.go +++ b/oci/archive/oci_transport.go @@ -35,6 +35,7 @@ type ociArchiveReference struct { file string resolvedFile string image string + sourceIndex int } func (t ociArchiveTransport) Name() string { @@ -55,11 +56,19 @@ func (t ociArchiveTransport) ValidatePolicyConfigurationScope(scope string) erro // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference. func ParseReference(reference string) (types.ImageReference, error) { file, image := internal.SplitPathAndImage(reference) - return NewReference(file, image) + image, index, err := internal.ParseOCIReferenceName(image) + if err != nil { + return nil, err + } + return newReference(file, image, index) } -// NewReference returns an OCI reference for a file and a image. +// NewReference returns an OCI reference for a file and an image. func NewReference(file, image string) (types.ImageReference, error) { + return newReference(file, image, -1) +} + +func newReference(file, image string, sourceIndex int) (types.ImageReference, error) { resolved, err := explicitfilepath.ResolvePathToFullyExplicit(file) if err != nil { return nil, err @@ -73,7 +82,10 @@ func NewReference(file, image string) (types.ImageReference, error) { return nil, err } - return ociArchiveReference{file: file, resolvedFile: resolved, image: image}, nil + if sourceIndex != -1 && sourceIndex < 0 { + return nil, errors.Errorf("Invalid oci archive: reference: index @%d must not be negative", sourceIndex) + } + return ociArchiveReference{file: file, resolvedFile: resolved, image: image, sourceIndex: sourceIndex}, nil } func (ref ociArchiveReference) Transport() types.ImageTransport { @@ -83,7 +95,10 @@ func (ref ociArchiveReference) Transport() types.ImageTransport { // StringWithinTransport returns a string representation of the reference, which MUST be such that // reference.Transport().ParseReference(reference.StringWithinTransport()) returns an equivalent reference. func (ref ociArchiveReference) StringWithinTransport() string { - return fmt.Sprintf("%s:%s", ref.file, ref.image) + if ref.sourceIndex == -1 { + return fmt.Sprintf("%s:%s", ref.file, ref.image) + } + return fmt.Sprintf("%s:@%d", ref.file, ref.sourceIndex) } // DockerReference returns a Docker reference associated with this reference @@ -160,12 +175,12 @@ func (t *tempDirOCIRef) deleteTempDir() error { // createOCIRef creates the oci reference of the image // If SystemContext.BigFilesTemporaryDir not "", overrides the temporary directory to use for storing big files -func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error) { +func createOCIRef(sys *types.SystemContext, image string, sourceIndex int) (tempDirOCIRef, error) { dir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "oci") if err != nil { return tempDirOCIRef{}, errors.Wrapf(err, "error creating temp directory") } - ociRef, err := ocilayout.NewReference(dir, image) + ociRef, err := ocilayout.NewReferenceWithIndex(dir, image, sourceIndex) if err != nil { return tempDirOCIRef{}, err } @@ -176,7 +191,7 @@ func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error) // creates the temporary directory and copies the tarred content to it func createUntarTempDir(sys *types.SystemContext, ref ociArchiveReference) (tempDirOCIRef, error) { - tempDirRef, err := createOCIRef(sys, ref.image) + tempDirRef, err := createOCIRef(sys, ref.image, ref.sourceIndex) if err != nil { return tempDirOCIRef{}, errors.Wrap(err, "error creating oci reference") } diff --git a/oci/archive/oci_transport_test.go b/oci/archive/oci_transport_test.go index 046e2c2b9c..8bdfe3d60b 100644 --- a/oci/archive/oci_transport_test.go +++ b/oci/archive/oci_transport_test.go @@ -60,11 +60,18 @@ func testParseReference(t *testing.T, fn func(string) (types.ImageReference, err "relativepath", tmpDir + "/thisdoesnotexist", } { - for _, image := range []struct{ suffix, image string }{ - {":notlatest:image", "notlatest:image"}, - {":latestimage", "latestimage"}, - {":", ""}, - {"", ""}, + for _, image := range []struct { + suffix, image string + expectedSourceIndex int + }{ + {":notlatest:image", "notlatest:image", -1}, + {":latestimage", "latestimage", -1}, + {":", "", -1}, + {"", "", -1}, + {":@0", "", 0}, + {":@10", "", 10}, + {":@999999", "", 999999}, + {":busybox@0", "busybox@0", -1}, } { input := path + image.suffix ref, err := fn(input) @@ -73,11 +80,23 @@ func testParseReference(t *testing.T, fn func(string) (types.ImageReference, err require.True(t, ok) assert.Equal(t, path, ociArchRef.file, input) assert.Equal(t, image.image, ociArchRef.image, input) + assert.Equal(t, ociArchRef.sourceIndex, image.expectedSourceIndex, input) } } - _, err = fn(tmpDir + ":invalid'image!value@") - assert.Error(t, err) + for _, imageSuffix := range []string{ + ":invalid'image!value@", + ":@", + ":@-1", + ":@-2", + ":@busybox", + ":@0:buxybox", + } { + input := tmpDir + imageSuffix + ref, err := fn(input) + assert.Equal(t, ref, nil) + assert.Error(t, err) + } } func TestNewReference(t *testing.T) { @@ -194,6 +213,7 @@ func TestReferenceStringWithinTransport(t *testing.T) { for _, c := range []struct{ input, result string }{ {"/dir1:notlatest:notlatest", "/dir1:notlatest:notlatest"}, // Explicit image {"/dir3:", "/dir3:"}, // No image + {"/dir1:@1", "/dir1:@1"}, // Explicit sourceIndex of image } { ref, err := ParseReference(tmpDir + c.input) require.NoError(t, err, c.input) diff --git a/oci/archive/reader.go b/oci/archive/reader.go new file mode 100644 index 0000000000..f6728c25f2 --- /dev/null +++ b/oci/archive/reader.go @@ -0,0 +1,98 @@ +package archive + +import ( + "context" + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + + "github.com/containers/image/v5/internal/tmpdir" + "github.com/containers/image/v5/oci/layout" + "github.com/containers/image/v5/types" + "github.com/containers/storage/pkg/archive" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// Reader keeps the temp directory the oci archive will be untarred to and the manifest of the images +type Reader struct { + manifest *imgspecv1.Index + tempDirectory string +} + +// NewReader creates the temp directory that keeps the untarred archive from src. +// The caller should call .Close() on the returned object. +func NewReader(ctx context.Context, src string, sys *types.SystemContext) (*Reader, error) { + // TODO: This can take quite some time, and should ideally be cancellable using a context.Context. + arch, err := os.Open(src) + if err != nil { + return nil, err + } + defer arch.Close() + + dst, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "oci") + if err != nil { + return nil, errors.Wrap(err, "error creating temp directory") + } + + reader := Reader{ + tempDirectory: dst, + } + + if err := archive.NewDefaultArchiver().Untar(arch, dst, &archive.TarOptions{NoLchown: true}); err != nil { + if err := reader.Close(); err != nil { + return nil, errors.Wrapf(err, "error deleting temp directory %q", dst) + } + return nil, errors.Wrapf(err, "error untarring file %q", dst) + } + + indexJSON, err := os.Open(filepath.Join(dst, "index.json")) + if err != nil { + return nil, err + } + defer indexJSON.Close() + reader.manifest = &imgspecv1.Index{} + if err := json.NewDecoder(indexJSON).Decode(reader.manifest); err != nil { + return nil, err + } + + return &reader, nil +} + +// List returns a (name, reference) map for images in the reader +// the name will be used to determin reference name of the dest image. +// the ImageReferences are valid only until the Reader is closed. +func (r *Reader) List() (map[string]types.ImageReference, error) { + res := make(map[string]types.ImageReference) + var ( + ref types.ImageReference + err error + ) + for i, md := range r.manifest.Manifests { + if md.MediaType != imgspecv1.MediaTypeImageManifest && md.MediaType != imgspecv1.MediaTypeImageIndex { + continue + } + refName, ok := md.Annotations[imgspecv1.AnnotationRefName] + if !ok { + refName = "@" + md.Digest.Encoded() + if ref, err = layout.NewIndexReference(r.tempDirectory, i); err != nil { + return nil, err + } + } else { + if ref, err = layout.NewReference(r.tempDirectory, refName); err != nil { + return nil, err + } + } + if _, ok := res[refName]; ok { + return nil, errors.Errorf("image descriptor %s conflict", refName) + } + res[refName] = ref + } + return res, nil +} + +// Close deletes temporary files associated with the Reader, if any. +func (r *Reader) Close() error { + return os.RemoveAll(r.tempDirectory) +} diff --git a/oci/archive/writer.go b/oci/archive/writer.go new file mode 100644 index 0000000000..bab02af03c --- /dev/null +++ b/oci/archive/writer.go @@ -0,0 +1,54 @@ +package archive + +import ( + "io/ioutil" + "os" + + "github.com/containers/image/v5/directory/explicitfilepath" + "github.com/containers/image/v5/internal/tmpdir" + "github.com/containers/image/v5/oci/layout" + "github.com/containers/image/v5/types" + "github.com/pkg/errors" +) + +// Writer keeps the tempDir for creating oci archive and archive destination +type Writer struct { + // tempDir will be tarred to oci archive + tempDir string + // user-specified path + path string +} + +// NewWriter creates a temp directory will be tarred to oci-archive. +// The caller should call .Close() on the returned object. +func NewWriter(sys *types.SystemContext, file string) (*Writer, error) { + dir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "oci") + if err != nil { + return nil, errors.Wrapf(err, "error creating temp directory") + } + dst, err := explicitfilepath.ResolvePathToFullyExplicit(file) + if err != nil { + return nil, err + } + ociWriter := &Writer{ + tempDir: dir, + path: dst, + } + return ociWriter, nil +} + +// NewReference returns an ImageReference that allows adding an image to Writer, +// with an optional image name +func (w *Writer) NewReference(name string) (types.ImageReference, error) { + return layout.NewReference(w.tempDir, name) +} + +// Close converts the data about images in the temp directory to the archive and +// deletes temporary files associated with the Writer +func (w *Writer) Close() error { + err := tarDirectory(w.tempDir, w.path) + if err2 := os.RemoveAll(w.tempDir); err2 != nil && err == nil { + err = err2 + } + return err +} diff --git a/oci/internal/oci_util.go b/oci/internal/oci_util.go index c2012e50e0..1e9689ed71 100644 --- a/oci/internal/oci_util.go +++ b/oci/internal/oci_util.go @@ -1,11 +1,13 @@ package internal import ( - "github.com/pkg/errors" "path/filepath" "regexp" "runtime" + "strconv" "strings" + + "github.com/pkg/errors" ) // annotation spex from https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys @@ -124,3 +126,21 @@ func validateScopeNonWindows(scope string) error { return nil } + +// ParseOCIReferenceName parses the image from the oci reference that contains an index. +func ParseOCIReferenceName(image string) (img string, index int, err error) { + index = -1 + if strings.HasPrefix(image, "@") { + idx, err := strconv.Atoi(image[1:]) + if err != nil { + return "", index, errors.Wrapf(err, "Invalid source index @%s: not an integer", image[1:]) + } + if idx < 0 { + return "", index, errors.Errorf("Invalid source index @%d: must not be negative", idx) + } + index = idx + } else { + img = image + } + return img, index, nil +} diff --git a/oci/internal/oci_util_test.go b/oci/internal/oci_util_test.go index b10071d2d7..6231e60cf1 100644 --- a/oci/internal/oci_util_test.go +++ b/oci/internal/oci_util_test.go @@ -2,8 +2,9 @@ package internal import ( "fmt" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) type testDataSplitReference struct { @@ -60,3 +61,21 @@ func TestValidateScopeWindows(t *testing.T) { } } } + +func TestParseOCIReferenceName(t *testing.T) { + image, idx, err := ParseOCIReferenceName("@0") + assert.NoError(t, err) + assert.Equal(t, image, "") + assert.Equal(t, idx, 0) + + image, idx, err = ParseOCIReferenceName("notlatest@1") + assert.NoError(t, err) + assert.Equal(t, image, "notlatest@1") + assert.Equal(t, idx, -1) + + _, _, err = ParseOCIReferenceName("@-5") + assert.NotEmpty(t, err) + + _, _, err = ParseOCIReferenceName("@invalidIndex") + assert.NotEmpty(t, err) +} diff --git a/oci/layout/fixtures/two_names_manifest/index.json b/oci/layout/fixtures/two_names_manifest/index.json new file mode 100644 index 0000000000..8947602ddb --- /dev/null +++ b/oci/layout/fixtures/two_names_manifest/index.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7143, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + "platform": { + "architecture": "ppc64le", + "os": "linux" + }, + "annotations": { + "org.opencontainers.image.ref.name": "imageValue0" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:62ae1939cdb49e93f32cdb29d61ad276552532c6ae7fd543de7bf22511e4965e", + "size": 348, + "annotations": { + "org.opencontainers.image.ref.name": "imageValue1" + } + } + ] +} diff --git a/oci/layout/oci_dest.go b/oci/layout/oci_dest.go index 0c88e1ef0c..4c8ebfa2c6 100644 --- a/oci/layout/oci_dest.go +++ b/oci/layout/oci_dest.go @@ -26,6 +26,9 @@ type ociImageDestination struct { // newImageDestination returns an ImageDestination for writing to an existing directory. func newImageDestination(sys *types.SystemContext, ref ociReference) (types.ImageDestination, error) { + if ref.sourceIndex != -1 { + return nil, errors.Errorf("invalid sourceIndex %d for creating image destination", ref.sourceIndex) + } var index *imgspecv1.Index if indexExists(ref) { var err error diff --git a/oci/layout/oci_transport.go b/oci/layout/oci_transport.go index a99b631584..8ffa6048bd 100644 --- a/oci/layout/oci_transport.go +++ b/oci/layout/oci_transport.go @@ -63,20 +63,27 @@ type ociReference struct { resolvedDir string // Absolute path with no symlinks, at least at the time of its creation. Primarily used for policy namespaces. // If image=="", it means the "only image" in the index.json is used in the case it is a source // for destinations, the image name annotation "image.ref.name" is not added to the index.json - image string + image string + // Zero-based source manifest index. If sourceIndex==-1, the index will not be valid to point out the source image, only image field will be used. + sourceIndex int } // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an OCI ImageReference. func ParseReference(reference string) (types.ImageReference, error) { dir, image := internal.SplitPathAndImage(reference) - return NewReference(dir, image) + image, index, err := internal.ParseOCIReferenceName(image) + if err != nil { + return nil, err + } + return NewReferenceWithIndex(dir, image, index) } -// NewReference returns an OCI reference for a directory and a image. +// NewReferenceWithIndex returns an OCI reference for a directory and a imageļ¼Œ sourceIndex points to an image. // +// If sourceIndex==-1, the index will not be valid to point out the source image, only image will be used. // We do not expose an API supplying the resolvedDir; we could, but recomputing it // is generally cheap enough that we prefer being confident about the properties of resolvedDir. -func NewReference(dir, image string) (types.ImageReference, error) { +func NewReferenceWithIndex(dir, image string, sourceIndex int) (types.ImageReference, error) { resolved, err := explicitfilepath.ResolvePathToFullyExplicit(dir) if err != nil { return nil, err @@ -90,7 +97,23 @@ func NewReference(dir, image string) (types.ImageReference, error) { return nil, err } - return ociReference{dir: dir, resolvedDir: resolved, image: image}, nil + if sourceIndex != -1 && sourceIndex < 0 { + return nil, errors.Errorf("Invalid oci layout: reference: index @%d must not be negative", sourceIndex) + } + return ociReference{dir: dir, resolvedDir: resolved, image: image, sourceIndex: sourceIndex}, nil +} + +// NewIndexReference returns an OCI reference for a path and a zero-based source manifest index. +func NewIndexReference(dir string, sourceIndex int) (types.ImageReference, error) { + return NewReferenceWithIndex(dir, "", -1) +} + +// NewReference returns an OCI reference for a directory and a image. +// +// We do not expose an API supplying the resolvedDir; we could, but recomputing it +// is generally cheap enough that we prefer being confident about the properties of resolvedDir. +func NewReference(dir, image string) (types.ImageReference, error) { + return NewReferenceWithIndex(dir, image, -1) } func (ref ociReference) Transport() types.ImageTransport { @@ -103,7 +126,11 @@ func (ref ociReference) Transport() types.ImageTransport { // e.g. default attribute values omitted by the user may be filled in in the return value, or vice versa. // WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix. func (ref ociReference) StringWithinTransport() string { - return fmt.Sprintf("%s:%s", ref.dir, ref.image) + if ref.sourceIndex == -1 { + return fmt.Sprintf("%s:%s", ref.dir, ref.image) + } + return fmt.Sprintf("%s:@%d", ref.dir, ref.sourceIndex) + } // DockerReference returns a Docker reference associated with this reference @@ -182,8 +209,14 @@ func (ref ociReference) getManifestDescriptor() (imgspecv1.Descriptor, error) { if err != nil { return imgspecv1.Descriptor{}, err } - var d *imgspecv1.Descriptor + if ref.sourceIndex != -1 { + if len(index.Manifests) > ref.sourceIndex { + d = &index.Manifests[ref.sourceIndex] + return *d, nil + } + return imgspecv1.Descriptor{}, fmt.Errorf("index %d is too large, only %d entries available", ref.sourceIndex, len(index.Manifests)) + } if ref.image == "" { // return manifest if only one image is in the oci directory if len(index.Manifests) == 1 { diff --git a/oci/layout/oci_transport_test.go b/oci/layout/oci_transport_test.go index a795e9dd3a..f2f8375701 100644 --- a/oci/layout/oci_transport_test.go +++ b/oci/layout/oci_transport_test.go @@ -23,6 +23,18 @@ func TestGetManifestDescriptor(t *testing.T) { _, err = imageRef.(ociReference).getManifestDescriptor() assert.EqualError(t, err, ErrMoreThanOneImage.Error()) + + imageRef, err = NewReferenceWithIndex("fixtures/two_names_manifest", "imageValue0", -1) + require.NoError(t, err) + manDescriptor, err := imageRef.(ociReference).getManifestDescriptor() + require.NoError(t, err) + assert.Equal(t, manDescriptor.Annotations["org.opencontainers.image.ref.name"], "imageValue0") + + imageRef, err = NewReferenceWithIndex("fixtures/two_names_manifest", "", 1) + require.NoError(t, err) + manDescriptor, err = imageRef.(ociReference).getManifestDescriptor() + require.NoError(t, err) + assert.Equal(t, manDescriptor.Annotations["org.opencontainers.image.ref.name"], "imageValue1") } func TestTransportName(t *testing.T) { @@ -171,6 +183,7 @@ func TestReferenceStringWithinTransport(t *testing.T) { for _, c := range []struct{ input, result string }{ {"/dir1:notlatest:notlatest", "/dir1:notlatest:notlatest"}, // Explicit image {"/dir3:", "/dir3:"}, // No image + {"/dir1:@1", "/dir1:@1"}, // Explicit sourceIndex of image } { ref, err := ParseReference(tmpDir + c.input) require.NoError(t, err, c.input)