From 8165cba2ffa0966d744886b558f58c85fe70d09e Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Thu, 28 Mar 2019 17:26:34 +0800 Subject: [PATCH] exporter: support unpack opt for image exporter It is enhancement which allows to unpack image into containerd snapshotter storage by `--output type=image,<.>=<.>,unpack=true`. In order to support this feature, we needs to extend the Snapshotter witwh `Name() string` function. Because we needs to set gc label for snapshotter which need snapshotter name. fix: #908 Signed-off-by: Wei Fu --- cache/contenthash/checksum_test.go | 32 +++++----- cache/manager_test.go | 14 ++--- exporter/containerimage/export.go | 95 ++++++++++++++++++++++++++++++ exporter/containerimage/writer.go | 9 +++ snapshot/containerd/snapshotter.go | 4 +- snapshot/snapshotter.go | 13 +++- source/git/gitsource_test.go | 2 +- source/http/httpsource_test.go | 2 +- vendor/modules.txt | 2 +- worker/base/worker.go | 1 + worker/containerd/containerd.go | 2 +- worker/runc/runc.go | 2 +- 12 files changed, 146 insertions(+), 32 deletions(-) diff --git a/cache/contenthash/checksum_test.go b/cache/contenthash/checksum_test.go index d7d6cd622067b..9770b6375a31c 100644 --- a/cache/contenthash/checksum_test.go +++ b/cache/contenthash/checksum_test.go @@ -37,7 +37,7 @@ func TestChecksumHardlinks(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -120,7 +120,7 @@ func TestChecksumWildcard(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -175,7 +175,7 @@ func TestSymlinksNoFollow(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -234,7 +234,7 @@ func TestChecksumBasicFile(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -384,7 +384,7 @@ func TestHandleChange(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -462,7 +462,7 @@ func TestHandleRecursiveDir(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -511,7 +511,7 @@ func TestChecksumUnorderedFiles(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -564,7 +564,7 @@ func TestSymlinkInPathScan(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -595,7 +595,7 @@ func TestSymlinkNeedsScan(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -628,7 +628,7 @@ func TestSymlinkAbsDirSuffix(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -655,7 +655,7 @@ func TestSymlinkThroughParent(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -710,7 +710,7 @@ func TestSymlinkInPathHandleChange(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -773,7 +773,7 @@ func TestPersistence(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := setupCacheManager(t, tmpdir, snapshotter) + cm := setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ch := []string{ @@ -810,7 +810,7 @@ func TestPersistence(t *testing.T) { // we can't close snapshotter and open it twice (especially, its internal bbolt store) cm.Close() getDefaultManager().lru.Purge() - cm = setupCacheManager(t, tmpdir, snapshotter) + cm = setupCacheManager(t, tmpdir, "native", snapshotter) defer cm.Close() ref, err = cm.Get(context.TODO(), id) @@ -843,12 +843,12 @@ func createRef(t *testing.T, cm cache.Manager, files []string) cache.ImmutableRe return ref } -func setupCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshotter) cache.Manager { +func setupCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) cache.Manager { md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db")) require.NoError(t, err) cm, err := cache.NewManager(cache.ManagerOpt{ - Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter), + Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter), MetadataStore: md, }) require.NoError(t, err) diff --git a/cache/manager_test.go b/cache/manager_test.go index 3468bce1111cd..9eaedfddcc3a5 100644 --- a/cache/manager_test.go +++ b/cache/manager_test.go @@ -28,7 +28,7 @@ func TestManager(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := getCacheManager(t, tmpdir, snapshotter) + cm := getCacheManager(t, tmpdir, "native", snapshotter) _, err = cm.Get(ctx, "foobar") require.Error(t, err) @@ -152,7 +152,7 @@ func TestPrune(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := getCacheManager(t, tmpdir, snapshotter) + cm := getCacheManager(t, tmpdir, "native", snapshotter) active, err := cm.New(ctx, nil) require.NoError(t, err) @@ -250,7 +250,7 @@ func TestLazyCommit(t *testing.T) { snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) require.NoError(t, err) - cm := getCacheManager(t, tmpdir, snapshotter) + cm := getCacheManager(t, tmpdir, "native", snapshotter) active, err := cm.New(ctx, nil, CachePolicyRetain) require.NoError(t, err) @@ -332,7 +332,7 @@ func TestLazyCommit(t *testing.T) { require.NoError(t, err) // we can't close snapshotter and open it twice (especially, its internal bbolt store) - cm = getCacheManager(t, tmpdir, snapshotter) + cm = getCacheManager(t, tmpdir, "native", snapshotter) snap2, err = cm.Get(ctx, snap.ID()) require.NoError(t, err) @@ -353,7 +353,7 @@ func TestLazyCommit(t *testing.T) { err = cm.Close() require.NoError(t, err) - cm = getCacheManager(t, tmpdir, snapshotter) + cm = getCacheManager(t, tmpdir, "native", snapshotter) snap2, err = cm.Get(ctx, snap.ID()) require.NoError(t, err) @@ -369,12 +369,12 @@ func TestLazyCommit(t *testing.T) { require.Equal(t, errNotFound, errors.Cause(err)) } -func getCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshotter) Manager { +func getCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) Manager { md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db")) require.NoError(t, err) cm, err := NewManager(ManagerOpt{ - Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter), + Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter), MetadataStore: md, }) require.NoError(t, err, fmt.Sprintf("error: %+v", err)) diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 6e8f952540e10..e2dd2eee5f708 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -2,16 +2,24 @@ package containerimage import ( "context" + "fmt" "strconv" "strings" "time" + "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/rootfs" "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/session" + "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/util/push" "github.com/moby/buildkit/util/resolver" + digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -20,6 +28,7 @@ const ( keyPush = "push" keyPushByDigest = "push-by-digest" keyInsecure = "registry.insecure" + keyUnpack = "unpack" ociTypes = "oci-mediatypes" ) @@ -79,6 +88,16 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp return nil, errors.Wrapf(err, "non-bool value specified for %s", k) } i.insecure = b + case keyUnpack: + if v == "" { + i.unpack = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.unpack = b case ociTypes: if v == "" { i.ociTypes = true @@ -104,6 +123,7 @@ type imageExporterInstance struct { targetName string push bool pushByDigest bool + unpack bool insecure bool ociTypes bool meta map[string][]byte @@ -156,6 +176,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) } } tagDone(nil) + + if e.unpack { + if err := e.unpackImage(ctx, img); err != nil { + return nil, err + } + } } if e.push { if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.ResolverOpt, e.pushByDigest); err != nil { @@ -169,3 +195,72 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source) resp["containerimage.digest"] = desc.Digest.String() return resp, nil } + +func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image) (err0 error) { + unpackDone := oneOffProgress(ctx, "unpacking to "+img.Name) + defer func() { + unpackDone(err0) + }() + + var ( + contentStore = e.opt.ImageWriter.ContentStore() + applier = e.opt.ImageWriter.Applier() + snapshotter = e.opt.ImageWriter.Snapshotter() + ) + + // fetch manifest by default platform + manifest, err := images.Manifest(ctx, contentStore, img.Target, platforms.Default()) + if err != nil { + return err + } + + layers, err := getLayers(ctx, contentStore, manifest) + if err != nil { + return err + } + + // get containerd snapshotter + ctrdSnapshotter, release := snapshot.NewContainerdSnapshotter(snapshotter) + defer release() + + var chain []digest.Digest + for _, layer := range layers { + if _, err := rootfs.ApplyLayer(ctx, layer, chain, ctrdSnapshotter, applier); err != nil { + return err + } + chain = append(chain, layer.Diff.Digest) + } + + var ( + keyGCLabel = fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotter.Name()) + valueGCLabel = identity.ChainID(chain).String() + ) + + cinfo := content.Info{ + Digest: manifest.Config.Digest, + Labels: map[string]string{keyGCLabel: valueGCLabel}, + } + _, err = contentStore.Update(ctx, cinfo, fmt.Sprintf("labels.%s", keyGCLabel)) + return err +} + +func getLayers(ctx context.Context, contentStore content.Store, manifest ocispec.Manifest) ([]rootfs.Layer, error) { + diffIDs, err := images.RootFS(ctx, contentStore, manifest.Config) + if err != nil { + return nil, errors.Wrap(err, "failed to resolve rootfs") + } + + if len(diffIDs) != len(manifest.Layers) { + return nil, errors.Errorf("mismatched image rootfs and manifest layers") + } + + layers := make([]rootfs.Layer, len(diffIDs)) + for i := range diffIDs { + layers[i].Diff = ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageLayer, + Digest: diffIDs[i], + } + layers[i].Blob = manifest.Layers[i] + } + return layers, nil +} diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index f980afdc0eb84..0efc2d6f7c4de 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -33,6 +33,7 @@ const ( type WriterOpt struct { Snapshotter snapshot.Snapshotter ContentStore content.Store + Applier diff.Applier Differ diff.Comparer } @@ -290,6 +291,14 @@ func (ic *ImageWriter) ContentStore() content.Store { return ic.opt.ContentStore } +func (ic *ImageWriter) Snapshotter() snapshot.Snapshotter { + return ic.opt.Snapshotter +} + +func (ic *ImageWriter) Applier() diff.Applier { + return ic.opt.Applier +} + func emptyImageConfig() ([]byte, error) { img := ocispec.Image{ Architecture: runtime.GOARCH, diff --git a/snapshot/containerd/snapshotter.go b/snapshot/containerd/snapshotter.go index c4b6d1282cbbc..0dd5fbff6662b 100644 --- a/snapshot/containerd/snapshotter.go +++ b/snapshot/containerd/snapshotter.go @@ -13,10 +13,10 @@ import ( "github.com/moby/buildkit/snapshot/blobmapping" ) -func NewSnapshotter(snapshotter ctdsnapshot.Snapshotter, store content.Store, mdstore *metadata.Store, ns string, gc func(context.Context) error) snapshot.Snapshotter { +func NewSnapshotter(name string, snapshotter ctdsnapshot.Snapshotter, store content.Store, mdstore *metadata.Store, ns string, gc func(context.Context) error) snapshot.Snapshotter { return blobmapping.NewSnapshotter(blobmapping.Opt{ Content: store, - Snapshotter: snapshot.FromContainerdSnapshotter(&nsSnapshotter{ns, snapshotter, gc}), + Snapshotter: snapshot.FromContainerdSnapshotter(name, &nsSnapshotter{ns, snapshotter, gc}), MetadataStore: mdstore, }) } diff --git a/snapshot/snapshotter.go b/snapshot/snapshotter.go index 920407f45b279..585f0a3595ae5 100644 --- a/snapshot/snapshotter.go +++ b/snapshot/snapshotter.go @@ -16,6 +16,7 @@ type Mountable interface { } type SnapshotterBase interface { + Name() string Mounts(ctx context.Context, key string) (Mountable, error) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error View(ctx context.Context, key, parent string, opts ...snapshots.Opt) (Mountable, error) @@ -40,14 +41,22 @@ type Blobmapper interface { SetBlob(ctx context.Context, key string, diffID, blob digest.Digest) error } -func FromContainerdSnapshotter(s snapshots.Snapshotter) SnapshotterBase { - return &fromContainerd{Snapshotter: s} +func FromContainerdSnapshotter(name string, s snapshots.Snapshotter) SnapshotterBase { + return &fromContainerd{ + name: name, + Snapshotter: s, + } } type fromContainerd struct { + name string snapshots.Snapshotter } +func (s *fromContainerd) Name() string { + return s.name +} + func (s *fromContainerd) Mounts(ctx context.Context, key string) (Mountable, error) { mounts, err := s.Snapshotter.Mounts(ctx, key) if err != nil { diff --git a/source/git/gitsource_test.go b/source/git/gitsource_test.go index 545c2f1e87dba..8ba93607cb2db 100644 --- a/source/git/gitsource_test.go +++ b/source/git/gitsource_test.go @@ -297,7 +297,7 @@ func setupGitSource(t *testing.T, tmpdir string) source.Source { assert.NoError(t, err) cm, err := cache.NewManager(cache.ManagerOpt{ - Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter), + Snapshotter: snapshot.FromContainerdSnapshotter("native", snapshotter), MetadataStore: md, }) assert.NoError(t, err) diff --git a/source/http/httpsource_test.go b/source/http/httpsource_test.go index 1265f58219d1d..f1de58857a8bb 100644 --- a/source/http/httpsource_test.go +++ b/source/http/httpsource_test.go @@ -315,7 +315,7 @@ func newHTTPSource(tmpdir string) (source.Source, error) { } cm, err := cache.NewManager(cache.ManagerOpt{ - Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter), + Snapshotter: snapshot.FromContainerdSnapshotter("native", snapshotter), MetadataStore: md, }) if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index b035a207fc421..cfa19d03076ed 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -46,13 +46,13 @@ github.com/containerd/containerd/oci github.com/containerd/containerd/containers github.com/containerd/containerd/namespaces github.com/containerd/containerd/errdefs +github.com/containerd/containerd/rootfs github.com/containerd/containerd/images/oci github.com/containerd/containerd/api/services/content/v1 github.com/containerd/containerd/content/proxy github.com/containerd/containerd/services/content/contentserver github.com/containerd/containerd/reference github.com/containerd/containerd/remotes/docker/schema1 -github.com/containerd/containerd/rootfs github.com/containerd/containerd/images/archive github.com/containerd/containerd/archive github.com/containerd/containerd/archive/compression diff --git a/worker/base/worker.go b/worker/base/worker.go index b051390d3d3a2..bbe4d2eecbd71 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -152,6 +152,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) { iw, err := imageexporter.NewImageWriter(imageexporter.WriterOpt{ Snapshotter: opt.Snapshotter, ContentStore: opt.ContentStore, + Applier: opt.Applier, Differ: opt.Differ, }) if err != nil { diff --git a/worker/containerd/containerd.go b/worker/containerd/containerd.go index 94321d961a979..6c3b5dc701429 100644 --- a/worker/containerd/containerd.go +++ b/worker/containerd/containerd.go @@ -106,7 +106,7 @@ func newContainerd(root string, client *containerd.Client, snapshotterName, ns s Labels: xlabels, MetadataStore: md, Executor: containerdexecutor.New(client, root, "", network.Default()), - Snapshotter: containerdsnapshot.NewSnapshotter(client.SnapshotService(snapshotterName), cs, md, ns, gc), + Snapshotter: containerdsnapshot.NewSnapshotter(snapshotterName, client.SnapshotService(snapshotterName), cs, md, ns, gc), ContentStore: cs, Applier: winlayers.NewFileSystemApplierWithWindows(cs, df), Differ: winlayers.NewWalkingDiffWithWindows(cs, df), diff --git a/worker/runc/runc.go b/worker/runc/runc.go index 743e7f9b947bf..e302cfe9d15ae 100644 --- a/worker/runc/runc.go +++ b/worker/runc/runc.go @@ -101,7 +101,7 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc Labels: xlabels, MetadataStore: md, Executor: exe, - Snapshotter: containerdsnapshot.NewSnapshotter(mdb.Snapshotter(snFactory.Name), c, md, "buildkit", gc), + Snapshotter: containerdsnapshot.NewSnapshotter(snFactory.Name, mdb.Snapshotter(snFactory.Name), c, md, "buildkit", gc), ContentStore: c, Applier: winlayers.NewFileSystemApplierWithWindows(c, apply.NewFileSystemApplier(c)), Differ: winlayers.NewWalkingDiffWithWindows(c, walking.NewWalkingDiff(c)),