nydus-image: Fix blob unpacking bug in specified blob directory

rafs/src/metadata/cached_v5.rs

@@ -460,6 +460,26 @@ impl RafsInode for CachedInodeV5 {
         }
     }
 
+    #[inline]
+    fn is_blkdev(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFBLK as u32
+    }
+
+    #[inline]
+    fn is_chrdev(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFCHR as u32
+    }
+
+    #[inline]
+    fn is_sock(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFSOCK as u32
+    }
+
+    #[inline]
+    fn is_fifo(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFIFO as u32
+    }
+
     #[inline]
     fn is_dir(&self) -> bool {
         self.i_mode & libc::S_IFMT as u32 == libc::S_IFDIR as u32

rafs/src/metadata/direct_v5.rs

@@ -714,6 +714,10 @@ impl RafsInode for OndiskInodeWrapper {
         self
     }
 
+    impl_inode_wrapper!(is_blkdev, bool);
+    impl_inode_wrapper!(is_chrdev, bool);
+    impl_inode_wrapper!(is_sock, bool);
+    impl_inode_wrapper!(is_fifo, bool);
     impl_inode_wrapper!(is_dir, bool);
     impl_inode_wrapper!(is_reg, bool);
     impl_inode_wrapper!(is_symlink, bool);

rafs/src/metadata/direct_v6.rs

@@ -946,6 +946,26 @@ impl RafsInode for OndiskInodeWrapper {
         0
     }
 
+    #[inline]
+    fn is_blkdev(&self) -> bool {
+        self.mode_format_bits() == libc::S_IFBLK as u32
+    }
+
+    #[inline]
+    fn is_chrdev(&self) -> bool {
+        self.mode_format_bits() == libc::S_IFCHR as u32
+    }
+
+    #[inline]
+    fn is_sock(&self) -> bool {
+        self.mode_format_bits() == libc::S_IFSOCK as u32
+    }
+
+    #[inline]
+    fn is_fifo(&self) -> bool {
+        self.mode_format_bits() == libc::S_IFIFO as u32
+    }
+
     fn is_dir(&self) -> bool {
         self.mode_format_bits() == libc::S_IFDIR as u32
     }

rafs/src/metadata/inode.rs

@@ -156,7 +156,7 @@ impl InodeWrapper {
         match self {
             InodeWrapper::V5(i) => i.is_chrdev(),
             InodeWrapper::V6(i) => i.is_chrdev(),
-            InodeWrapper::Ref(_i) => unimplemented!(),
+            InodeWrapper::Ref(i) => i.as_inode().is_chrdev(),
         }
     }
 
@@ -183,7 +183,7 @@ impl InodeWrapper {
         match self {
             InodeWrapper::V5(i) => i.is_sock(),
             InodeWrapper::V6(i) => i.is_sock(),
-            InodeWrapper::Ref(_i) => unimplemented!(),
+            InodeWrapper::Ref(i) => i.as_inode().is_dir(),
         }
     }
 
@@ -322,7 +322,7 @@ impl InodeWrapper {
         match self {
             InodeWrapper::V5(i) => i.i_uid,
             InodeWrapper::V6(i) => i.i_uid,
-            InodeWrapper::Ref(_i) => unimplemented!(),
+            InodeWrapper::Ref(i) => i.as_inode().get_attr().uid,
         }
     }
 
@@ -341,7 +341,7 @@ impl InodeWrapper {
         match self {
             InodeWrapper::V5(i) => i.i_gid,
             InodeWrapper::V6(i) => i.i_gid,
-            InodeWrapper::Ref(_i) => unimplemented!(),
+            InodeWrapper::Ref(i) => i.as_inode().get_attr().gid,
         }
     }
 

rafs/src/metadata/mod.rs

@@ -176,6 +176,18 @@ pub trait RafsInode: Any {
     /// Posix: get project id associated with the inode.
     fn projid(&self) -> u32;
 
+    /// Mode: check whether the inode is a block device.
+    fn is_blkdev(&self) -> bool;
+
+    /// Mode: check whether the inode is a char device.
+    fn is_chrdev(&self) -> bool;
+
+    /// Mode: check whether the inode is a sock.
+    fn is_sock(&self) -> bool;
+
+    /// Mode: check whether the inode is a fifo.
+    fn is_fifo(&self) -> bool;
+
     /// Mode: check whether the inode is a directory.
     fn is_dir(&self) -> bool;
 

rafs/src/mock/mock_inode.rs

@@ -175,6 +175,26 @@ impl RafsInode for MockInode {
             .collect::<Vec<XattrName>>())
     }
 
+    #[inline]
+    fn is_blkdev(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFBLK as u32
+    }
+
+    #[inline]
+    fn is_chrdev(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFCHR as u32
+    }
+
+    #[inline]
+    fn is_sock(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFSOCK as u32
+    }
+
+    #[inline]
+    fn is_fifo(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFIFO as u32
+    }
+
     fn is_dir(&self) -> bool {
         self.i_mode & libc::S_IFMT as u32 == libc::S_IFDIR as u32
     }

src/bin/nydus-image/main.rs

@@ -1305,7 +1305,16 @@ impl Command {
 
                 Some(Arc::new(local_fs))
             }
-            None => Self::get_backend(matches, "unpacker").ok(),
+            None => {
+                if let Some(backend) = &config.backend {
+                    Some(BlobFactory::new_backend(&backend, "unpacker")?)
+                } else {
+                    match Self::get_backend(matches, "unpacker") {
+                        Ok(backend) => Some(backend),
+                        Err(_) => bail!("one of `--blob`, `--blob-dir` and `--backend-config` must be specified"),
+                    }
+                }
+            }
         };
 
         OCIUnpacker::new(bootstrap, backend, output)

src/bin/nydus-image/unpack/mod.rs

@@ -2,6 +2,7 @@
 // Copyright (C) 2022 Alibaba Cloud. All rights reserved.
 //
 // SPDX-License-Identifier: Apache-2.0
+use std::collections::HashMap;
 use std::fs::{File, OpenOptions};
 use std::io::Read;
 use std::path::{Path, PathBuf};
@@ -115,8 +116,7 @@ impl OCITarBuilderFactory {
     ) -> Result<Box<dyn TarBuilder>> {
         let writer = self.create_writer(output_path)?;
 
-        let blob = meta.superblock.get_blob_infos().pop();
-        let builders = self.create_builders(blob, blob_backend)?;
+        let builders = self.create_builders(meta, blob_backend)?;
 
         let builder = OCITarBuilder::new(builders, writer);
 
@@ -139,7 +139,7 @@ impl OCITarBuilderFactory {
 
     fn create_builders(
         &self,
-        blob: Option<Arc<BlobInfo>>,
+        meta: &RafsSuper,
         blob_backend: &Option<Arc<dyn BlobBackend + Send + Sync>>,
     ) -> Result<Vec<Box<dyn SectionBuilder>>> {
         // PAX basic builders
@@ -155,7 +155,8 @@ impl OCITarBuilderFactory {
         let fifo_builder = OCIFifoBuilder::new(special_builder.clone());
         let char_builder = OCICharBuilder::new(special_builder.clone());
         let block_builder = OCIBlockBuilder::new(special_builder);
-        let reg_builder = self.create_reg_builder(blob, blob_backend)?;
+        let blobs = meta.superblock.get_blob_infos();
+        let reg_builder = self.create_reg_builder(blobs, blob_backend)?;
 
         // The order counts.
         let builders = vec![
@@ -174,29 +175,28 @@ impl OCITarBuilderFactory {
 
     fn create_reg_builder(
         &self,
-        blob: Option<Arc<BlobInfo>>,
+        blobs: Vec<Arc<BlobInfo>>,
         blob_backend: &Option<Arc<dyn BlobBackend + Send + Sync>>,
     ) -> Result<OCIRegBuilder> {
-        let (reader, compressor) = match blob {
-            None => (None, None),
-            Some(ref blob) => {
-                let blob_backend = blob_backend
-                    .as_deref()
-                    .with_context(|| "both blob path or blob backend config are not specified")?;
-                let reader = blob_backend
-                    .get_reader("unpacker")
-                    .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?;
-
-                let compressor = blob.compressor();
-
-                (Some(reader), Some(compressor))
-            }
-        };
+        let mut readers = HashMap::new();
+        let mut compressors = HashMap::new();
+        for blob in blobs {
+            let blob_backend = blob_backend
+                .as_deref()
+                .with_context(|| "both blob path or blob backend config are not specified")?;
+            let reader = blob_backend
+                .get_reader(blob.blob_id().as_str())
+                .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?;
+
+            let compressor = blob.compressor();
+            readers.insert(blob.blob_index(), reader);
+            compressors.insert(blob.blob_index(), compressor);
+        }
 
         Ok(OCIRegBuilder::new(
             Rc::new(PAXExtensionSectionBuilder::new()),
-            reader,
-            compressor,
+            readers,
+            compressors,
         ))
     }
 }

src/bin/nydus-image/unpack/pax.rs

@@ -167,37 +167,34 @@ impl SectionBuilder for OCIDirBuilder {
 
 pub struct OCIRegBuilder {
     ext_builder: Rc<PAXExtensionSectionBuilder>,
-    reader: Option<Arc<dyn BlobReader>>,
-    compressor: Option<Algorithm>,
+    readers: HashMap<u32, Arc<dyn BlobReader>>,
+    compressors: HashMap<u32, Algorithm>,
 }
 
 impl OCIRegBuilder {
     pub fn new(
         ext_builder: Rc<PAXExtensionSectionBuilder>,
-        reader: Option<Arc<dyn BlobReader>>,
-        compressor: Option<Algorithm>,
+        readers: HashMap<u32, Arc<dyn BlobReader>>,
+        compressors: HashMap<u32, Algorithm>,
     ) -> Self {
         OCIRegBuilder {
             ext_builder,
-            reader,
-            compressor,
+            readers,
+            compressors,
         }
     }
 
     fn build_data(&self, inode: &dyn RafsInodeExt) -> Box<dyn Read> {
-        if self.reader.is_none() {
-            return Box::new(io::empty());
-        }
-
         let chunks = (0..inode.get_chunk_count())
             .map(|i| inode.get_chunk_info(i).unwrap())
             .collect();
 
-        let reader = ChunkReader::new(
-            *self.compressor.as_ref().unwrap(),
-            self.reader.as_ref().unwrap().clone(),
-            chunks,
-        );
+        let mut compressors = HashMap::new();
+        compressors.clone_from(&self.compressors);
+
+        let mut readers = HashMap::new();
+        readers.clone_from(&self.readers);
+        let reader = ChunkReader::new(compressors, readers, chunks);
 
         Box::new(reader)
     }
@@ -702,30 +699,36 @@ impl Util {
 }
 
 struct ChunkReader {
-    compressor: Algorithm,
-    reader: Arc<dyn BlobReader>,
+    compressors: HashMap<u32, Algorithm>,
+    readers: HashMap<u32, Arc<dyn BlobReader>>,
 
     chunks: IntoIter<Arc<dyn BlobChunkInfo>>,
     chunk: Cursor<Vec<u8>>,
 }
 
 impl ChunkReader {
     fn new(
-        compressor: Algorithm,
-        reader: Arc<dyn BlobReader>,
+        compressors: HashMap<u32, Algorithm>,
+        readers: HashMap<u32, Arc<dyn BlobReader>>,
         chunks: Vec<Arc<dyn BlobChunkInfo>>,
     ) -> Self {
         Self {
-            compressor,
-            reader,
+            compressors,
+            readers,
             chunks: chunks.into_iter(),
             chunk: Cursor::new(Vec::new()),
         }
     }
 
     fn load_chunk(&mut self, chunk: &dyn BlobChunkInfo) -> Result<()> {
         let mut buf = alloc_buf(chunk.compressed_size() as usize);
-        self.reader
+
+        let reader = self
+            .readers
+            .get(&chunk.blob_index())
+            .expect("No valid reader")
+            .clone();
+        reader
             .read(buf.as_mut_slice(), chunk.compressed_offset())
             .map_err(|err| {
                 error!("fail to read chunk, error: {:?}", err);
@@ -737,8 +740,13 @@ impl ChunkReader {
             return Ok(());
         }
 
+        let compressor = *self
+            .compressors
+            .get(&chunk.blob_index())
+            .expect("No valid compressor");
+
         let mut data = vec![0u8; chunk.uncompressed_size() as usize];
-        compress::decompress(buf.as_mut_slice(), data.as_mut_slice(), self.compressor)
+        compress::decompress(buf.as_mut_slice(), data.as_mut_slice(), compressor)
             .with_context(|| "fail to decompress")?;
 
         self.chunk = Cursor::new(data);