Merge pull request #560 from ChengyuZhu6/layer_block

Support sharing image on the host for layer block mode

pkg/filesystem/tarfs_adaptor.go

@@ -82,3 +82,10 @@ func (fs *Filesystem) GetTarfsImageDiskFilePath(id string) (string, error) {
 	}
 	return fs.tarfsMgr.ImageDiskFilePath(id), nil
 }
+
+func (fs *Filesystem) GetTarfsLayerDiskFilePath(id string) (string, error) {
+	if fs.tarfsMgr == nil {
+		return "", errors.New("tarfs mode is not enabled")
+	}
+	return fs.tarfsMgr.LayerDiskFilePath(id), nil
+}

pkg/tarfs/tarfs.go

@@ -466,10 +466,17 @@ func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[
 	updateFields := []string{}
 
 	wholeImage, exportDisk, withVerity := config.GetTarfsExportFlags()
+
+	log.L.Debugf("ExportBlockData wholeImage = %v, exportDisk = %v, withVerity = %v, perLayer = %v", wholeImage, exportDisk, withVerity, perLayer)
 	// Nothing to do for this case, all needed datum are ready.
 	if !exportDisk && !withVerity {
 		return updateFields, nil
 	} else if !wholeImage != perLayer {
+		// Special handling for `layer_block` mode
+		if exportDisk && !withVerity && !perLayer {
+			labels[label.NydusLayerBlockInfo] = ""
+			updateFields = append(updateFields, "labels."+label.NydusLayerBlockInfo)
+		}
 		return updateFields, nil
 	}
 
@@ -505,7 +512,7 @@ func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[
 		diskFileName = t.ImageDiskFilePath(blobID)
 	} else {
 		metaFileName = t.layerMetaFilePath(storageLocater(snapshotID))
-		diskFileName = t.layerDiskFilePath(blobID)
+		diskFileName = t.LayerDiskFilePath(blobID)
 	}
 
 	// Do not regenerate if the disk image already exists.
@@ -821,7 +828,7 @@ func (t *Manager) layerTarFilePath(blobID string) string {
 	return filepath.Join(t.cacheDirPath, blobID)
 }
 
-func (t *Manager) layerDiskFilePath(blobID string) string {
+func (t *Manager) LayerDiskFilePath(blobID string) string {
 	return filepath.Join(t.cacheDirPath, blobID+"."+TarfsLayerDiskName)
 }
 

snapshot/mount_option.go

@@ -17,11 +17,13 @@ import (
 
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/containerd/snapshots/storage"
 	"github.com/containerd/nydus-snapshotter/config/daemonconfig"
 	"github.com/containerd/nydus-snapshotter/pkg/label"
 	"github.com/containerd/nydus-snapshotter/pkg/layout"
 	"github.com/containerd/nydus-snapshotter/pkg/rafs"
+	"github.com/containerd/nydus-snapshotter/pkg/snapshot"
 	"github.com/pkg/errors"
 )
 
@@ -101,7 +103,7 @@ func (o *snapshotter) remoteMountWithExtraOptions(ctx context.Context, s storage
 	}, nil
 }
 
-func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overlayOptions []string) ([]mount.Mount, error) {
+func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overlayOptions []string, key string) ([]mount.Mount, error) {
 	hasVolume := false
 	rafs := rafs.RafsGlobalCache.Get(id)
 	if rafs == nil {
@@ -122,7 +124,7 @@ func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overla
 
 	// Insert Kata volume for tarfs
 	if blobID, ok := rafs.Annotations[label.NydusTarfsLayer]; ok {
-		options, err := o.mountWithTarfsVolume(*rafs, blobID)
+		options, err := o.mountWithTarfsVolume(ctx, *rafs, blobID, key)
 		if err != nil {
 			return []mount.Mount{}, errors.Wrapf(err, "create kata volume for tarfs")
 		}
@@ -149,66 +151,103 @@ func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overla
 
 func (o *snapshotter) mountWithProxyVolume(rafs rafs.Rafs) ([]string, error) {
 	options := []string{}
+	source := rafs.Annotations[label.CRIImageRef]
 	for k, v := range rafs.Annotations {
 		options = append(options, fmt.Sprintf("%s=%s", k, v))
 	}
-
-	volume := &KataVirtualVolume{
-		VolumeType: KataVirtualVolumeImageGuestPullType,
-		Source:     "",
-		FSType:     "",
-		Options:    options,
-		ImagePull:  &ImagePullVolume{Metadata: rafs.Annotations},
-	}
-	if !volume.Validate() {
-		return []string{}, errors.Errorf("got invalid kata volume, %v", volume)
-	}
-
-	info, err := EncodeKataVirtualVolumeToBase64(*volume)
+	opt, err := o.prepareKataVirtualVolume(label.NydusProxyMode, source, KataVirtualVolumeImageGuestPullType, "", options, rafs.Annotations)
 	if err != nil {
-		return []string{}, errors.Errorf("failed to encoding Kata Volume info %v", volume)
+		return options, errors.Wrapf(err, "failed to prepare KataVirtualVolume")
 	}
-	opt := fmt.Sprintf("%s=%s", KataVirtualVolumeOptionName, info)
-
 	return []string{opt}, nil
 }
 
-func (o *snapshotter) mountWithTarfsVolume(rafs rafs.Rafs, blobID string) ([]string, error) {
-	var volume *KataVirtualVolume
-
+func (o *snapshotter) mountWithTarfsVolume(ctx context.Context, rafs rafs.Rafs, blobID, key string) ([]string, error) {
+	options := []string{}
 	if info, ok := rafs.Annotations[label.NydusImageBlockInfo]; ok {
 		path, err := o.fs.GetTarfsImageDiskFilePath(blobID)
 		if err != nil {
 			return []string{}, errors.Wrapf(err, "get tarfs image disk file path")
 		}
-		volume = &KataVirtualVolume{
-			VolumeType: KataVirtualVolumeImageRawBlockType,
-			Source:     path,
-			FSType:     "erofs",
-			Options:    []string{"ro"},
+		log.L.Debugf("mountWithTarfsVolume info %v", info)
+		opt, err := o.prepareKataVirtualVolume(label.NydusImageBlockInfo, path, KataVirtualVolumeImageRawBlockType, "erofs", []string{"ro"}, rafs.Annotations)
+		if err != nil {
+			return options, errors.Wrapf(err, "failed to prepare KataVirtualVolume for image_raw_block")
 		}
-		if len(info) > 0 {
-			dmverity, err := parseTarfsDmVerityInfo(info)
+
+		options = append(options, opt)
+		log.L.Debugf("mountWithTarfsVolume type=%v, options %v", KataVirtualVolumeImageRawBlockType, options)
+		return options, nil
+	}
+
+	if _, ok := rafs.Annotations[label.NydusLayerBlockInfo]; ok {
+		for {
+			pID, pInfo, _, pErr := snapshot.GetSnapshotInfo(ctx, o.ms, key)
+			log.G(ctx).Debugf("mountWithKataVolume pID= %v, pInfo = %v", pID, pInfo)
+
+			if pErr != nil {
+				return options, errors.Wrapf(pErr, "failed to get snapshot info")
+			}
+			if pInfo.Kind == snapshots.KindActive {
+				key = pInfo.Parent
+				continue
+			}
+
+			blobID = pInfo.Labels[label.NydusTarfsLayer]
+			path, err := o.fs.GetTarfsLayerDiskFilePath(blobID)
 			if err != nil {
-				return []string{}, err
+				return options, errors.Wrapf(err, "get tarfs image disk file path")
 			}
-			volume.DmVerity = &dmverity
+
+			opt, err := o.prepareKataVirtualVolume(label.NydusLayerBlockInfo, path, KataVirtualVolumeLayerRawBlockType, "erofs", []string{"ro"}, pInfo.Labels)
+			if err != nil {
+				return options, errors.Wrapf(err, "failed to prepare KataVirtualVolume for layer_raw_block")
+			}
+
+			options = append(options, opt)
+
+			if pInfo.Parent == "" {
+				break
+			}
+			key = pInfo.Parent
 		}
+		log.L.Debugf("mountWithTarfsVolume type=%v, options %v", KataVirtualVolumeLayerRawBlockType, options)
+		return options, nil
 	}
 
-	if volume != nil {
-		if !volume.Validate() {
-			return []string{}, errors.Errorf("got invalid kata volume, %v", volume)
-		}
-		info, err := EncodeKataVirtualVolumeToBase64(*volume)
-		if err != nil {
-			return []string{}, errors.Errorf("failed to encoding Kata Volume info %v", volume)
+	// If Neither image_raw_block or layer_raw_block, return empty strings
+	return options, nil
+}
+
+func (o *snapshotter) prepareKataVirtualVolume(blockType, source, volumeType, fsType string, options []string, labels map[string]string) (string, error) {
+	volume := &KataVirtualVolume{
+		VolumeType: volumeType,
+		Source:     source,
+		FSType:     fsType,
+		Options:    options,
+	}
+	if blockType == label.NydusImageBlockInfo || blockType == label.NydusLayerBlockInfo {
+		dmverityInfo := labels[blockType]
+		if len(dmverityInfo) > 0 {
+			dmverity, err := parseTarfsDmVerityInfo(dmverityInfo)
+			if err != nil {
+				return "", err
+			}
+			volume.DmVerity = &dmverity
 		}
-		opt := fmt.Sprintf("%s=%s", KataVirtualVolumeOptionName, info)
-		return []string{opt}, nil
+	} else if blockType == label.NydusProxyMode {
+		volume.ImagePull = &ImagePullVolume{Metadata: labels}
 	}
 
-	return []string{}, nil
+	if !volume.Validate() {
+		return "", errors.Errorf("got invalid kata volume, %v", volume)
+	}
+	info, err := EncodeKataVirtualVolumeToBase64(*volume)
+	if err != nil {
+		return "", errors.Errorf("failed to encoding Kata Volume info %v", volume)
+	}
+	opt := fmt.Sprintf("%s=%s", KataVirtualVolumeOptionName, info)
+	return opt, nil
 }
 
 func parseTarfsDmVerityInfo(info string) (DmVerityInfo, error) {

snapshot/process.go

@@ -52,7 +52,7 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry,
 			}
 
 			logger.Infof("Nydus remote snapshot %s is ready", id)
-			mounts, err := sn.mountRemote(ctx, labels, s, id)
+			mounts, err := sn.mountRemote(ctx, labels, s, id, key)
 			return false, mounts, err
 		}
 	}

snapshot/snapshot.go

@@ -433,7 +433,7 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er
 	}
 
 	if needRemoteMounts {
-		return o.mountRemote(ctx, info.Labels, *snap, metaSnapshotID)
+		return o.mountRemote(ctx, info.Labels, *snap, metaSnapshotID, key)
 	}
 
 	return o.mountNative(ctx, info.Labels, *snap)
@@ -531,7 +531,7 @@ func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snap
 	}
 
 	if needRemoteMounts {
-		return o.mountRemote(ctx, base.Labels, s, metaSnapshotID)
+		return o.mountRemote(ctx, base.Labels, s, metaSnapshotID, key)
 	}
 	return o.mountNative(ctx, base.Labels, s)
 }
@@ -840,7 +840,7 @@ func overlayMount(options []string) []mount.Mount {
 
 // `s` is the upmost snapshot and `id` refers to the nydus meta snapshot
 // `s` and `id` can represent a different layer, it's useful when View an image
-func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, s storage.Snapshot, id string) ([]mount.Mount, error) {
+func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, s storage.Snapshot, id, key string) ([]mount.Mount, error) {
 	var overlayOptions []string
 	if _, ok := labels[label.OverlayfsVolatileOpt]; ok {
 		overlayOptions = append(overlayOptions, "volatile")
@@ -871,7 +871,7 @@ func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string,
 	log.G(ctx).Infof("remote mount options %v", overlayOptions)
 
 	if o.enableKataVolume {
-		return o.mountWithKataVolume(ctx, id, overlayOptions)
+		return o.mountWithKataVolume(ctx, id, overlayOptions, key)
 	}
 	// Add `extraoption` if NydusOverlayFS is enable or daemonMode is `None`
 	if o.enableNydusOverlayFS || config.GetDaemonMode() == config.DaemonModeNone {