Skip to content

Commit

Permalink
Merge pull request #909 from fuweid/me-allow-unpack-action
Browse files Browse the repository at this point in the history 
exporter: support unpack opt for image exporter
  • Loading branch information
@tonistiigi
tonistiigi authored Apr 26, 2019
2 parents 37ae84b + ffad8ad commit 3c78a9c
Show file tree
Hide file tree
Showing 13 changed files with 159 additions and 34 deletions.
32 changes: 16 additions & 16 deletions cache/contenthash/checksum_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func TestChecksumHardlinks(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -120,7 +120,7 @@ func TestChecksumWildcard(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -175,7 +175,7 @@ func TestSymlinksNoFollow(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -234,7 +234,7 @@ func TestChecksumBasicFile(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -384,7 +384,7 @@ func TestHandleChange(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -462,7 +462,7 @@ func TestHandleRecursiveDir(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -511,7 +511,7 @@ func TestChecksumUnorderedFiles(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -564,7 +564,7 @@ func TestSymlinkInPathScan(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -595,7 +595,7 @@ func TestSymlinkNeedsScan(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -628,7 +628,7 @@ func TestSymlinkAbsDirSuffix(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand All @@ -655,7 +655,7 @@ func TestSymlinkThroughParent(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -710,7 +710,7 @@ func TestSymlinkInPathHandleChange(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -773,7 +773,7 @@ func TestPersistence(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
cm := setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -810,7 +810,7 @@ func TestPersistence(t *testing.T) {
// we can't close snapshotter and open it twice (especially, its internal bbolt store)
cm.Close()
getDefaultManager().lru.Purge()
cm = setupCacheManager(t, tmpdir, snapshotter)
cm = setupCacheManager(t, tmpdir, "native", snapshotter)
defer cm.Close()

ref, err = cm.Get(context.TODO(), id)
Expand Down Expand Up @@ -843,12 +843,12 @@ func createRef(t *testing.T, cm cache.Manager, files []string) cache.ImmutableRe
return ref
}

func setupCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshotter) cache.Manager {
func setupCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) cache.Manager {
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
require.NoError(t, err)

cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil),
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter, nil),
MetadataStore: md,
})
require.NoError(t, err)
Expand Down
14 changes: 7 additions & 7 deletions cache/manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func TestManager(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)
cm := getCacheManager(t, tmpdir, "native", snapshotter)

_, err = cm.Get(ctx, "foobar")
require.Error(t, err)
Expand Down Expand Up @@ -152,7 +152,7 @@ func TestPrune(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)
cm := getCacheManager(t, tmpdir, "native", snapshotter)

active, err := cm.New(ctx, nil)
require.NoError(t, err)
Expand Down Expand Up @@ -250,7 +250,7 @@ func TestLazyCommit(t *testing.T) {

snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)
cm := getCacheManager(t, tmpdir, "native", snapshotter)

active, err := cm.New(ctx, nil, CachePolicyRetain)
require.NoError(t, err)
Expand Down Expand Up @@ -332,7 +332,7 @@ func TestLazyCommit(t *testing.T) {
require.NoError(t, err)

// we can't close snapshotter and open it twice (especially, its internal bbolt store)
cm = getCacheManager(t, tmpdir, snapshotter)
cm = getCacheManager(t, tmpdir, "native", snapshotter)

snap2, err = cm.Get(ctx, snap.ID())
require.NoError(t, err)
Expand All @@ -353,7 +353,7 @@ func TestLazyCommit(t *testing.T) {
err = cm.Close()
require.NoError(t, err)

cm = getCacheManager(t, tmpdir, snapshotter)
cm = getCacheManager(t, tmpdir, "native", snapshotter)

snap2, err = cm.Get(ctx, snap.ID())
require.NoError(t, err)
Expand All @@ -369,12 +369,12 @@ func TestLazyCommit(t *testing.T) {
require.Equal(t, errNotFound, errors.Cause(err))
}

func getCacheManager(t *testing.T, tmpdir string, snapshotter snapshots.Snapshotter) Manager {
func getCacheManager(t *testing.T, tmpdir string, snapshotterName string, snapshotter snapshots.Snapshotter) Manager {
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
require.NoError(t, err)

cm, err := NewManager(ManagerOpt{
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotter, nil),
Snapshotter: snapshot.FromContainerdSnapshotter(snapshotterName, snapshotter, nil),
MetadataStore: md,
})
require.NoError(t, err, fmt.Sprintf("error: %+v", err))
Expand Down
18 changes: 16 additions & 2 deletions cmd/buildctl/build_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -82,7 +83,15 @@ func testBuildContainerdExporter(t *testing.T, sb integration.Sandbox) {
rdr, err := marshal(st.Root())
require.NoError(t, err)

cmd := sb.Cmd("build --progress=plain --exporter=image --exporter-opt name=example.com/moby/imageexporter:test")
imageName := "example.com/moby/imageexporter:test"

buildCmd := []string{
"build", "--progress=plain",
"--exporter=image", "--exporter-opt", "unpack=true",
"--exporter-opt", "name=" + imageName,
}

cmd := sb.Cmd(strings.Join(buildCmd, " "))
cmd.Stdin = rdr
err = cmd.Run()
require.NoError(t, err)
Expand All @@ -93,8 +102,13 @@ func testBuildContainerdExporter(t *testing.T, sb integration.Sandbox) {

ctx := namespaces.WithNamespace(context.Background(), "buildkit")

_, err = client.ImageService().Get(ctx, "example.com/moby/imageexporter:test")
img, err := client.GetImage(ctx, imageName)
require.NoError(t, err)

// NOTE: by default, it is overlayfs
ok, err := img.IsUnpacked(ctx, "overlayfs")
require.NoError(t, err)
require.Equal(t, ok, true)
}

func marshal(st llb.State) (io.Reader, error) {
Expand Down
95 changes: 95 additions & 0 deletions exporter/containerimage/export.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,24 @@ package containerimage

import (
"context"
"fmt"
"strconv"
"strings"
"time"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/rootfs"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/push"
"github.com/moby/buildkit/util/resolver"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)

Expand All @@ -20,6 +28,7 @@ const (
keyPush = "push"
keyPushByDigest = "push-by-digest"
keyInsecure = "registry.insecure"
keyUnpack = "unpack"
ociTypes = "oci-mediatypes"
)

Expand Down Expand Up @@ -79,6 +88,16 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
}
i.insecure = b
case keyUnpack:
if v == "" {
i.unpack = true
continue
}
b, err := strconv.ParseBool(v)
if err != nil {
return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
}
i.unpack = b
case ociTypes:
if v == "" {
i.ociTypes = true
Expand All @@ -104,6 +123,7 @@ type imageExporterInstance struct {
targetName string
push bool
pushByDigest bool
unpack bool
insecure bool
ociTypes bool
meta map[string][]byte
Expand Down Expand Up @@ -156,6 +176,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
}
}
tagDone(nil)

if e.unpack {
if err := e.unpackImage(ctx, img); err != nil {
return nil, err
}
}
}
if e.push {
if err := push.Push(ctx, e.opt.SessionManager, e.opt.ImageWriter.ContentStore(), desc.Digest, targetName, e.insecure, e.opt.ResolverOpt, e.pushByDigest); err != nil {
Expand All @@ -169,3 +195,72 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source)
resp["containerimage.digest"] = desc.Digest.String()
return resp, nil
}

func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image) (err0 error) {
unpackDone := oneOffProgress(ctx, "unpacking to "+img.Name)
defer func() {
unpackDone(err0)
}()

var (
contentStore = e.opt.ImageWriter.ContentStore()
applier = e.opt.ImageWriter.Applier()
snapshotter = e.opt.ImageWriter.Snapshotter()
)

// fetch manifest by default platform
manifest, err := images.Manifest(ctx, contentStore, img.Target, platforms.Default())
if err != nil {
return err
}

layers, err := getLayers(ctx, contentStore, manifest)
if err != nil {
return err
}

// get containerd snapshotter
ctrdSnapshotter, release := snapshot.NewContainerdSnapshotter(snapshotter)
defer release()

var chain []digest.Digest
for _, layer := range layers {
if _, err := rootfs.ApplyLayer(ctx, layer, chain, ctrdSnapshotter, applier); err != nil {
return err
}
chain = append(chain, layer.Diff.Digest)
}

var (
keyGCLabel = fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotter.Name())
valueGCLabel = identity.ChainID(chain).String()
)

cinfo := content.Info{
Digest: manifest.Config.Digest,
Labels: map[string]string{keyGCLabel: valueGCLabel},
}
_, err = contentStore.Update(ctx, cinfo, fmt.Sprintf("labels.%s", keyGCLabel))
return err
}

func getLayers(ctx context.Context, contentStore content.Store, manifest ocispec.Manifest) ([]rootfs.Layer, error) {
diffIDs, err := images.RootFS(ctx, contentStore, manifest.Config)
if err != nil {
return nil, errors.Wrap(err, "failed to resolve rootfs")
}

if len(diffIDs) != len(manifest.Layers) {
return nil, errors.Errorf("mismatched image rootfs and manifest layers")
}

layers := make([]rootfs.Layer, len(diffIDs))
for i := range diffIDs {
layers[i].Diff = ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageLayer,
Digest: diffIDs[i],
}
layers[i].Blob = manifest.Layers[i]
}
return layers, nil
}
Loading

0 comments on commit 3c78a9c

Please sign in to comment.