From 49f347e68f0eb47fe29a69698fdef8c4151453ef Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Mon, 4 Dec 2023 16:44:33 +0800 Subject: [PATCH 1/5] tarfs: Introduce a function to get the path of layer disk file Introduce a function to get the path of layer disk file. Fixes: #559 Signed-off-by: ChengyuZhu6 --- pkg/filesystem/tarfs_adaptor.go | 7 +++++++ pkg/tarfs/tarfs.go | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/filesystem/tarfs_adaptor.go b/pkg/filesystem/tarfs_adaptor.go index 3e47295b38..5d95ad51cb 100755 --- a/pkg/filesystem/tarfs_adaptor.go +++ b/pkg/filesystem/tarfs_adaptor.go @@ -82,3 +82,10 @@ func (fs *Filesystem) GetTarfsImageDiskFilePath(id string) (string, error) { } return fs.tarfsMgr.ImageDiskFilePath(id), nil } + +func (fs *Filesystem) GetTarfsLayerDiskFilePath(id string) (string, error) { + if fs.tarfsMgr == nil { + return "", errors.New("tarfs mode is not enabled") + } + return fs.tarfsMgr.LayerDiskFilePath(id), nil +} diff --git a/pkg/tarfs/tarfs.go b/pkg/tarfs/tarfs.go index 543dc5ceda..d4088d7027 100755 --- a/pkg/tarfs/tarfs.go +++ b/pkg/tarfs/tarfs.go @@ -505,7 +505,7 @@ func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[ diskFileName = t.ImageDiskFilePath(blobID) } else { metaFileName = t.layerMetaFilePath(storageLocater(snapshotID)) - diskFileName = t.layerDiskFilePath(blobID) + diskFileName = t.LayerDiskFilePath(blobID) } // Do not regenerate if the disk image already exists. @@ -821,7 +821,7 @@ func (t *Manager) layerTarFilePath(blobID string) string { return filepath.Join(t.cacheDirPath, blobID) } -func (t *Manager) layerDiskFilePath(blobID string) string { +func (t *Manager) LayerDiskFilePath(blobID string) string { return filepath.Join(t.cacheDirPath, blobID+"."+TarfsLayerDiskName) } From 685b70863b25e0a5c73b694648187c8bca9b2feb Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Mon, 4 Dec 2023 17:32:53 +0800 Subject: [PATCH 2/5] tarfs: fix error when export block data for layer block When the layer_block mode is enabled, the remote snapshot does not contain the `containerd.io/snapshot/nydus-layer-block` label, which prevents the creation of a kata volume. Therefore, we need to add the `containerd.io/snapshot/nydus-layer-block` label to the remote snapshot. Signed-off-by: ChengyuZhu6 --- pkg/tarfs/tarfs.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/tarfs/tarfs.go b/pkg/tarfs/tarfs.go index d4088d7027..afe96839ee 100755 --- a/pkg/tarfs/tarfs.go +++ b/pkg/tarfs/tarfs.go @@ -466,10 +466,17 @@ func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[ updateFields := []string{} wholeImage, exportDisk, withVerity := config.GetTarfsExportFlags() + + log.L.Debugf("ExportBlockData wholeImage = %v, exportDisk = %v, withVerity = %v, perLayer = %v", wholeImage, exportDisk, withVerity, perLayer) // Nothing to do for this case, all needed datum are ready. if !exportDisk && !withVerity { return updateFields, nil } else if !wholeImage != perLayer { + // Special handling for `layer_block` mode + if exportDisk && !withVerity && !perLayer { + labels[label.NydusLayerBlockInfo] = "" + updateFields = append(updateFields, "labels."+label.NydusLayerBlockInfo) + } return updateFields, nil } From 7ead21fa5b03838dd82caea3a2589682f47b9fd2 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Mon, 4 Dec 2023 17:36:21 +0800 Subject: [PATCH 3/5] snapshot: Introduce a function to prepare KataVirtualVolume Introduce a function to prepare KataVirtualVolume for both guest pull image and host sharing image modes to eliminate redundant code. Signed-off-by: ChengyuZhu6 --- snapshot/mount_option.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/snapshot/mount_option.go b/snapshot/mount_option.go index 0e161629a6..b245ca4f35 100644 --- a/snapshot/mount_option.go +++ b/snapshot/mount_option.go @@ -211,6 +211,37 @@ func (o *snapshotter) mountWithTarfsVolume(rafs rafs.Rafs, blobID string) ([]str return []string{}, nil } +func (o *snapshotter) prepareKataVirtualVolume(blockType, source, volumeType, fsType string, options []string, labels map[string]string) (string, error) { + volume := &KataVirtualVolume{ + VolumeType: volumeType, + Source: source, + FSType: fsType, + Options: options, + } + if blockType == label.NydusImageBlockInfo || blockType == label.NydusLayerBlockInfo { + dmverityInfo := labels[blockType] + if len(dmverityInfo) > 0 { + dmverity, err := parseTarfsDmVerityInfo(dmverityInfo) + if err != nil { + return "", err + } + volume.DmVerity = &dmverity + } + } else if blockType == label.NydusProxyMode { + volume.ImagePull = &ImagePullVolume{Metadata: labels} + } + + if !volume.Validate() { + return "", errors.Errorf("got invalid kata volume, %v", volume) + } + info, err := EncodeKataVirtualVolumeToBase64(*volume) + if err != nil { + return "", errors.Errorf("failed to encoding Kata Volume info %v", volume) + } + opt := fmt.Sprintf("%s=%s", KataVirtualVolumeOptionName, info) + return opt, nil +} + func parseTarfsDmVerityInfo(info string) (DmVerityInfo, error) { var dataBlocks, hashOffset uint64 var rootHash string From 671c4140e2a10d5e0d1695c0a70379ff54b601b1 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Mon, 4 Dec 2023 17:41:50 +0800 Subject: [PATCH 4/5] snapshot: support to handle kata volume for layer block Support to handle kata volume for layer block and reconstruct functions about handling kata virtual volumes. Signed-off-by: ChengyuZhu6 --- snapshot/mount_option.go | 91 +++++++++++++++++++++------------------- snapshot/process.go | 2 +- snapshot/snapshot.go | 8 ++-- 3 files changed, 54 insertions(+), 47 deletions(-) diff --git a/snapshot/mount_option.go b/snapshot/mount_option.go index b245ca4f35..6fbb418bd1 100644 --- a/snapshot/mount_option.go +++ b/snapshot/mount_option.go @@ -17,11 +17,13 @@ import ( "github.com/containerd/containerd/log" "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/snapshots" "github.com/containerd/containerd/snapshots/storage" "github.com/containerd/nydus-snapshotter/config/daemonconfig" "github.com/containerd/nydus-snapshotter/pkg/label" "github.com/containerd/nydus-snapshotter/pkg/layout" "github.com/containerd/nydus-snapshotter/pkg/rafs" + "github.com/containerd/nydus-snapshotter/pkg/snapshot" "github.com/pkg/errors" ) @@ -101,7 +103,7 @@ func (o *snapshotter) remoteMountWithExtraOptions(ctx context.Context, s storage }, nil } -func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overlayOptions []string) ([]mount.Mount, error) { +func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overlayOptions []string, key string) ([]mount.Mount, error) { hasVolume := false rafs := rafs.RafsGlobalCache.Get(id) if rafs == nil { @@ -122,7 +124,7 @@ func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overla // Insert Kata volume for tarfs if blobID, ok := rafs.Annotations[label.NydusTarfsLayer]; ok { - options, err := o.mountWithTarfsVolume(*rafs, blobID) + options, err := o.mountWithTarfsVolume(ctx, *rafs, blobID, key) if err != nil { return []mount.Mount{}, errors.Wrapf(err, "create kata volume for tarfs") } @@ -152,63 +154,68 @@ func (o *snapshotter) mountWithProxyVolume(rafs rafs.Rafs) ([]string, error) { for k, v := range rafs.Annotations { options = append(options, fmt.Sprintf("%s=%s", k, v)) } - - volume := &KataVirtualVolume{ - VolumeType: KataVirtualVolumeImageGuestPullType, - Source: "", - FSType: "", - Options: options, - ImagePull: &ImagePullVolume{Metadata: rafs.Annotations}, - } - if !volume.Validate() { - return []string{}, errors.Errorf("got invalid kata volume, %v", volume) - } - - info, err := EncodeKataVirtualVolumeToBase64(*volume) + opt, err := o.prepareKataVirtualVolume(label.NydusProxyMode, "", KataVirtualVolumeImageGuestPullType, "", options, rafs.Annotations) if err != nil { - return []string{}, errors.Errorf("failed to encoding Kata Volume info %v", volume) + return options, errors.Wrapf(err, "failed to prepare KataVirtualVolume") } - opt := fmt.Sprintf("%s=%s", KataVirtualVolumeOptionName, info) - return []string{opt}, nil } -func (o *snapshotter) mountWithTarfsVolume(rafs rafs.Rafs, blobID string) ([]string, error) { - var volume *KataVirtualVolume - +func (o *snapshotter) mountWithTarfsVolume(ctx context.Context, rafs rafs.Rafs, blobID, key string) ([]string, error) { + options := []string{} if info, ok := rafs.Annotations[label.NydusImageBlockInfo]; ok { path, err := o.fs.GetTarfsImageDiskFilePath(blobID) if err != nil { return []string{}, errors.Wrapf(err, "get tarfs image disk file path") } - volume = &KataVirtualVolume{ - VolumeType: KataVirtualVolumeImageRawBlockType, - Source: path, - FSType: "erofs", - Options: []string{"ro"}, + log.L.Debugf("mountWithTarfsVolume info %v", info) + opt, err := o.prepareKataVirtualVolume(label.NydusImageBlockInfo, path, KataVirtualVolumeImageRawBlockType, "erofs", []string{"ro"}, rafs.Annotations) + if err != nil { + return options, errors.Wrapf(err, "failed to prepare KataVirtualVolume for image_raw_block") } - if len(info) > 0 { - dmverity, err := parseTarfsDmVerityInfo(info) + + options = append(options, opt) + log.L.Debugf("mountWithTarfsVolume type=%v, options %v", KataVirtualVolumeImageRawBlockType, options) + return options, nil + } + + if _, ok := rafs.Annotations[label.NydusLayerBlockInfo]; ok { + for { + pID, pInfo, _, pErr := snapshot.GetSnapshotInfo(ctx, o.ms, key) + log.G(ctx).Debugf("mountWithKataVolume pID= %v, pInfo = %v", pID, pInfo) + + if pErr != nil { + return options, errors.Wrapf(pErr, "failed to get snapshot info") + } + if pInfo.Kind == snapshots.KindActive { + key = pInfo.Parent + continue + } + + blobID = pInfo.Labels[label.NydusTarfsLayer] + path, err := o.fs.GetTarfsLayerDiskFilePath(blobID) if err != nil { - return []string{}, err + return options, errors.Wrapf(err, "get tarfs image disk file path") } - volume.DmVerity = &dmverity - } - } - if volume != nil { - if !volume.Validate() { - return []string{}, errors.Errorf("got invalid kata volume, %v", volume) - } - info, err := EncodeKataVirtualVolumeToBase64(*volume) - if err != nil { - return []string{}, errors.Errorf("failed to encoding Kata Volume info %v", volume) + opt, err := o.prepareKataVirtualVolume(label.NydusLayerBlockInfo, path, KataVirtualVolumeLayerRawBlockType, "erofs", []string{"ro"}, pInfo.Labels) + if err != nil { + return options, errors.Wrapf(err, "failed to prepare KataVirtualVolume for layer_raw_block") + } + + options = append(options, opt) + + if pInfo.Parent == "" { + break + } + key = pInfo.Parent } - opt := fmt.Sprintf("%s=%s", KataVirtualVolumeOptionName, info) - return []string{opt}, nil + log.L.Debugf("mountWithTarfsVolume type=%v, options %v", KataVirtualVolumeLayerRawBlockType, options) + return options, nil } - return []string{}, nil + // If Neither image_raw_block or layer_raw_block, return empty strings + return options, nil } func (o *snapshotter) prepareKataVirtualVolume(blockType, source, volumeType, fsType string, options []string, labels map[string]string) (string, error) { diff --git a/snapshot/process.go b/snapshot/process.go index ce90f5baf6..9f0da66f2d 100644 --- a/snapshot/process.go +++ b/snapshot/process.go @@ -52,7 +52,7 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, } logger.Infof("Nydus remote snapshot %s is ready", id) - mounts, err := sn.mountRemote(ctx, labels, s, id) + mounts, err := sn.mountRemote(ctx, labels, s, id, key) return false, mounts, err } } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 108ab055d3..c2e7f43523 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -433,7 +433,7 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er } if needRemoteMounts { - return o.mountRemote(ctx, info.Labels, *snap, metaSnapshotID) + return o.mountRemote(ctx, info.Labels, *snap, metaSnapshotID, key) } return o.mountNative(ctx, info.Labels, *snap) @@ -531,7 +531,7 @@ func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snap } if needRemoteMounts { - return o.mountRemote(ctx, base.Labels, s, metaSnapshotID) + return o.mountRemote(ctx, base.Labels, s, metaSnapshotID, key) } return o.mountNative(ctx, base.Labels, s) } @@ -840,7 +840,7 @@ func overlayMount(options []string) []mount.Mount { // `s` is the upmost snapshot and `id` refers to the nydus meta snapshot // `s` and `id` can represent a different layer, it's useful when View an image -func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, s storage.Snapshot, id string) ([]mount.Mount, error) { +func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, s storage.Snapshot, id, key string) ([]mount.Mount, error) { var overlayOptions []string if _, ok := labels[label.OverlayfsVolatileOpt]; ok { overlayOptions = append(overlayOptions, "volatile") @@ -871,7 +871,7 @@ func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, log.G(ctx).Infof("remote mount options %v", overlayOptions) if o.enableKataVolume { - return o.mountWithKataVolume(ctx, id, overlayOptions) + return o.mountWithKataVolume(ctx, id, overlayOptions, key) } // Add `extraoption` if NydusOverlayFS is enable or daemonMode is `None` if o.enableNydusOverlayFS || config.GetDaemonMode() == config.DaemonModeNone { From 44733280e5ab6361ed4546411495ee446e4675e5 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Tue, 5 Dec 2023 09:02:47 +0800 Subject: [PATCH 5/5] snapshot: get image reference from cri annotations for guest pull The sandbox image (pause image) name cannot be stored in annotations by containerd, so snapshotter is unable to retrieve it. This means that the source field of `KataVirtualVolume` has to be set to "" to support guest pull in proxy mode now. However, once containerd fixes this bug (https://github.com/containerd/containerd/pull/9419), I think the nydus snapshotter could be able to get the sandbox image name from annotations directly. I recommend that we can support to obtain the image name through "containerd.io/snapshot/cri.image-ref" in snapshotter. If this value is empty, the source should be set to "", otherwise it should be set to the image name. Signed-off-by: ChengyuZhu6 --- snapshot/mount_option.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/snapshot/mount_option.go b/snapshot/mount_option.go index 6fbb418bd1..50624e73de 100644 --- a/snapshot/mount_option.go +++ b/snapshot/mount_option.go @@ -151,10 +151,11 @@ func (o *snapshotter) mountWithKataVolume(ctx context.Context, id string, overla func (o *snapshotter) mountWithProxyVolume(rafs rafs.Rafs) ([]string, error) { options := []string{} + source := rafs.Annotations[label.CRIImageRef] for k, v := range rafs.Annotations { options = append(options, fmt.Sprintf("%s=%s", k, v)) } - opt, err := o.prepareKataVirtualVolume(label.NydusProxyMode, "", KataVirtualVolumeImageGuestPullType, "", options, rafs.Annotations) + opt, err := o.prepareKataVirtualVolume(label.NydusProxyMode, source, KataVirtualVolumeImageGuestPullType, "", options, rafs.Annotations) if err != nil { return options, errors.Wrapf(err, "failed to prepare KataVirtualVolume") }