Skip to content

Commit

Permalink
exporter: support unpack opt for image exporter
Browse files Browse the repository at this point in the history 
It is enhancement which allows to unpack image into containerd
snapshotter storage by `--output type=image,<.>=<.>,unpack=true`.

In order to support this feature, we needs to extend the Snapshotter
witwh `Name() string` function. Because we needs to set gc label for
snapshotter which need snapshotter name.

fix: #908

Signed-off-by: Wei Fu <[email protected]>
  • Loading branch information
@fuweid
fuweid committed Mar 29, 2019
1 parent 33bb70c commit 8165cba
Show file tree
Hide file tree
Showing 12 changed files with 146 additions and 32 deletions.
32 changes: 16 additions & 16 deletions cache/contenthash/checksum_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func TestChecksumHardlinks(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -120,7 +120,7 @@ func TestChecksumWildcard(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -175,7 +175,7 @@ func TestSymlinksNoFollow(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -234,7 +234,7 @@ func TestChecksumBasicFile(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -384,7 +384,7 @@ func TestHandleChange(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -462,7 +462,7 @@ func TestHandleRecursiveDir(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -511,7 +511,7 @@ func TestChecksumUnorderedFiles(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -564,7 +564,7 @@ func TestSymlinkInPathScan(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -595,7 +595,7 @@ func TestSymlinkNeedsScan(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -628,7 +628,7 @@ func TestSymlinkAbsDirSuffix(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand All @@ -655,7 +655,7 @@ func TestSymlinkThroughParent(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -710,7 +710,7 @@ func TestSymlinkInPathHandleChange(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -773,7 +773,7 @@ func TestPersistence(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -810,7 +810,7 @@ func TestPersistence(t *testing.T) {
// we can't close snapshotter and open it twice (especially, its internal bbolt store)
cm.Close()
getDefaultManager().lru.Purge()
cm = setupCacheManager(t, tmpdir, snapshotter)
cm = setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ref, err = cm.Get(context.TODO(), id)
Expand Down Expand Up @@ -843,12 +843,12 @@ func createRef(t *testing.T, cm cache.Manager, files []string) cache.ImmutableRe
return ref
}

func setupCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshotter) cache.Manager {
func setupCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) cache.Manager {
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
require.NoError(t, err)

cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter),
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter),
MetadataStore: md,
})
require.NoError(t, err)
Expand Down
14 changes: 7 additions & 7 deletions cache/manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func TestManager(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)
cm := getCacheManager(t, tmpdir, "native", snapshotter)

_, err = cm.Get(ctx, "foobar")
require.Error(t, err)
Expand Down Expand Up @@ -152,7 +152,7 @@ func TestPrune(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)
cm := getCacheManager(t, tmpdir, "native", snapshotter)

active, err := cm.New(ctx, nil)
require.NoError(t, err)
Expand Down Expand Up @@ -250,7 +250,7 @@ func TestLazyCommit(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)
cm := getCacheManager(t, tmpdir, "native", snapshotter)

active, err := cm.New(ctx, nil, CachePolicyRetain)
require.NoError(t, err)
Expand Down Expand Up @@ -332,7 +332,7 @@ func TestLazyCommit(t *testing.T) {
require.NoError(t, err)

// we can't close snapshotter and open it twice (especially, its internal bbolt store)
cm = getCacheManager(t, tmpdir, snapshotter)
cm = getCacheManager(t, tmpdir, "native", snapshotter)

snap2, err = cm.Get(ctx, snap.ID())
require.NoError(t, err)
Expand All @@ -353,7 +353,7 @@ func TestLazyCommit(t *testing.T) {
err = cm.Close()
require.NoError(t, err)

cm = getCacheManager(t, tmpdir, snapshotter)
cm = getCacheManager(t, tmpdir, "native", snapshotter)

snap2, err = cm.Get(ctx, snap.ID())
require.NoError(t, err)
Expand All @@ -369,12 +369,12 @@ func TestLazyCommit(t *testing.T) {
require.Equal(t, errNotFound, errors.Cause(err))
}

func getCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshotter) Manager {
func getCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) Manager {
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
require.NoError(t, err)

cm, err := NewManager(ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter),
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter),
MetadataStore: md,
})
require.NoError(t, err, fmt.Sprintf("error: %+v", err))
Expand Down
95 changes: 95 additions & 0 deletions exporter/containerimage/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,24 @@ package containerimage

import (
"context"
"fmt"
"strconv"
"strings"
"time"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/rootfs"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/push"
"github.com/moby/buildkit/util/resolver"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)

Expand All @@ -20,6 +28,7 @@ const (
keyPush = "push"
keyPushByDigest = "push-by-digest"
keyInsecure = "registry.insecure"
keyUnpack = "unpack"
ociTypes = "oci-mediatypes"
)

Expand Down Expand Up @@ -79,6 +88,16 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
}
i.insecure = b
case keyUnpack:
if v == "" {
i.unpack = true
continue
}
b, err := strconv.ParseBool(v)
if err != nil {
return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
}
i.unpack = b
case ociTypes:
if v == "" {
i.ociTypes = true
Expand All @@ -104,6 +123,7 @@ type imageExporterInstance struct {
targetName string
push bool
pushByDigest bool
unpack bool
insecure bool
ociTypes bool
meta map[string][]byte
Expand Down Expand Up @@ -156,6 +176,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
}
}
tagDone(nil)

if e.unpack {
if err := e.unpackImage(ctx, img); err != nil {
return nil, err
}
}
}
if e.push {
if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.ResolverOpt, e.pushByDigest); err != nil {
Expand All @@ -169,3 +195,72 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
resp["containerimage.digest"] = desc.Digest.String()
return resp, nil
}

func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image) (err0 error) {
unpackDone := oneOffProgress(ctx, "unpacking to "+img.Name)
defer func() {
unpackDone(err0)
}()

var (
contentStore = e.opt.ImageWriter.ContentStore()
applier = e.opt.ImageWriter.Applier()
snapshotter = e.opt.ImageWriter.Snapshotter()
)

// fetch manifest by default platform
manifest, err := images.Manifest(ctx, contentStore, img.Target, platforms.Default())
if err != nil {
return err
}

layers, err := getLayers(ctx, contentStore, manifest)
if err != nil {
return err
}

// get containerd snapshotter
ctrdSnapshotter, release := snapshot.NewContainerdSnapshotter(snapshotter)
defer release()

var chain []digest.Digest
for _, layer := range layers {
if _, err := rootfs.ApplyLayer(ctx, layer, chain, ctrdSnapshotter, applier); err != nil {
return err
}
chain = append(chain, layer.Diff.Digest)
}

var (
keyGCLabel = fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotter.Name())
valueGCLabel = identity.ChainID(chain).String()
)

cinfo := content.Info{
Digest: manifest.Config.Digest,
Labels: map[string]string{keyGCLabel: valueGCLabel},
}
_, err = contentStore.Update(ctx, cinfo, fmt.Sprintf("labels.%s", keyGCLabel))
return err
}

func getLayers(ctx context.Context, contentStore content.Store, manifest ocispec.Manifest) ([]rootfs.Layer, error) {
diffIDs, err := images.RootFS(ctx, contentStore, manifest.Config)
if err != nil {
return nil, errors.Wrap(err, "failed to resolve rootfs")
}

if len(diffIDs) != len(manifest.Layers) {
return nil, errors.Errorf("mismatched image rootfs and manifest layers")
}

layers := make([]rootfs.Layer, len(diffIDs))
for i := range diffIDs {
layers[i].Diff = ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageLayer,
Digest: diffIDs[i],
}
layers[i].Blob = manifest.Layers[i]
}
return layers, nil
}
9 changes: 9 additions & 0 deletions exporter/containerimage/writer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ const (
type WriterOpt struct {
Snapshotter snapshot.Snapshotter
ContentStore content.Store
Applier diff.Applier
Differ diff.Comparer
}

Expand Down Expand Up @@ -290,6 +291,14 @@ func (ic *ImageWriter) ContentStore() content.Store {
return ic.opt.ContentStore
}

func (ic *ImageWriter) Snapshotter() snapshot.Snapshotter {
return ic.opt.Snapshotter
}

func (ic *ImageWriter) Applier() diff.Applier {
return ic.opt.Applier
}

func emptyImageConfig() ([]byte, error) {
img := ocispec.Image{
Architecture: runtime.GOARCH,
Expand Down
4 changes: 2 additions & 2 deletions snapshot/containerd/snapshotter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ import (
"github.com/moby/buildkit/snapshot/blobmapping"
)

func NewSnapshotter(snapshotter ctdsnapshot.Snapshotter, store content.Store, mdstore *metadata.Store, ns string, gc func(context.Context) error) snapshot.Snapshotter {
func NewSnapshotter(name string, snapshotter ctdsnapshot.Snapshotter, store content.Store, mdstore *metadata.Store, ns string, gc func(context.Context) error) snapshot.Snapshotter {
return blobmapping.NewSnapshotter(blobmapping.Opt{
Content: store,
Snapshotter: snapshot.FromContainerdSnapshotter(&nsSnapshotter{ns, snapshotter, gc}),
Snapshotter: snapshot.FromContainerdSnapshotter(name, &nsSnapshotter{ns, snapshotter, gc}),
MetadataStore: mdstore,
})
}
Expand Down
Loading

0 comments on commit 8165cba

Please sign in to comment.