diff --git a/rafs/src/metadata/cached_v5.rs b/rafs/src/metadata/cached_v5.rs index 597e1ddb06a..828fc1abb9f 100644 --- a/rafs/src/metadata/cached_v5.rs +++ b/rafs/src/metadata/cached_v5.rs @@ -460,6 +460,26 @@ impl RafsInode for CachedInodeV5 { } } + #[inline] + fn is_blkdev(&self) -> bool { + self.i_mode & libc::S_IFMT as u32 == libc::S_IFBLK as u32 + } + + #[inline] + fn is_chrdev(&self) -> bool { + self.i_mode & libc::S_IFMT as u32 == libc::S_IFCHR as u32 + } + + #[inline] + fn is_sock(&self) -> bool { + self.i_mode & libc::S_IFMT as u32 == libc::S_IFSOCK as u32 + } + + #[inline] + fn is_fifo(&self) -> bool { + self.i_mode & libc::S_IFMT as u32 == libc::S_IFIFO as u32 + } + #[inline] fn is_dir(&self) -> bool { self.i_mode & libc::S_IFMT as u32 == libc::S_IFDIR as u32 diff --git a/rafs/src/metadata/direct_v5.rs b/rafs/src/metadata/direct_v5.rs index 2b7221525eb..900ce17eb46 100644 --- a/rafs/src/metadata/direct_v5.rs +++ b/rafs/src/metadata/direct_v5.rs @@ -714,6 +714,10 @@ impl RafsInode for OndiskInodeWrapper { self } + impl_inode_wrapper!(is_blkdev, bool); + impl_inode_wrapper!(is_chrdev, bool); + impl_inode_wrapper!(is_sock, bool); + impl_inode_wrapper!(is_fifo, bool); impl_inode_wrapper!(is_dir, bool); impl_inode_wrapper!(is_reg, bool); impl_inode_wrapper!(is_symlink, bool); diff --git a/rafs/src/metadata/direct_v6.rs b/rafs/src/metadata/direct_v6.rs index f7731aa1c9c..1803475a368 100644 --- a/rafs/src/metadata/direct_v6.rs +++ b/rafs/src/metadata/direct_v6.rs @@ -946,6 +946,26 @@ impl RafsInode for OndiskInodeWrapper { 0 } + #[inline] + fn is_blkdev(&self) -> bool { + self.mode_format_bits() == libc::S_IFBLK as u32 + } + + #[inline] + fn is_chrdev(&self) -> bool { + self.mode_format_bits() == libc::S_IFCHR as u32 + } + + #[inline] + fn is_sock(&self) -> bool { + self.mode_format_bits() == libc::S_IFSOCK as u32 + } + + #[inline] + fn is_fifo(&self) -> bool { + self.mode_format_bits() == libc::S_IFIFO as u32 + } + fn is_dir(&self) -> bool { self.mode_format_bits() == libc::S_IFDIR as u32 } diff --git a/rafs/src/metadata/inode.rs b/rafs/src/metadata/inode.rs index 84b35dbf4ec..780401a9102 100644 --- a/rafs/src/metadata/inode.rs +++ b/rafs/src/metadata/inode.rs @@ -156,7 +156,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.is_chrdev(), InodeWrapper::V6(i) => i.is_chrdev(), - InodeWrapper::Ref(_i) => unimplemented!(), + InodeWrapper::Ref(i) => i.as_inode().is_chrdev(), } } @@ -183,7 +183,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.is_sock(), InodeWrapper::V6(i) => i.is_sock(), - InodeWrapper::Ref(_i) => unimplemented!(), + InodeWrapper::Ref(i) => i.as_inode().is_dir(), } } @@ -322,7 +322,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_uid, InodeWrapper::V6(i) => i.i_uid, - InodeWrapper::Ref(_i) => unimplemented!(), + InodeWrapper::Ref(i) => i.as_inode().get_attr().uid, } } @@ -341,7 +341,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_gid, InodeWrapper::V6(i) => i.i_gid, - InodeWrapper::Ref(_i) => unimplemented!(), + InodeWrapper::Ref(i) => i.as_inode().get_attr().gid, } } diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs index c04c3b4da1d..c9f767b8ddd 100644 --- a/rafs/src/metadata/mod.rs +++ b/rafs/src/metadata/mod.rs @@ -176,6 +176,18 @@ pub trait RafsInode: Any { /// Posix: get project id associated with the inode. fn projid(&self) -> u32; + /// Mode: check whether the inode is a block device. + fn is_blkdev(&self) -> bool; + + /// Mode: check whether the inode is a char device. + fn is_chrdev(&self) -> bool; + + /// Mode: check whether the inode is a sock. + fn is_sock(&self) -> bool; + + /// Mode: check whether the inode is a fifo. + fn is_fifo(&self) -> bool; + /// Mode: check whether the inode is a directory. fn is_dir(&self) -> bool; diff --git a/rafs/src/mock/mock_inode.rs b/rafs/src/mock/mock_inode.rs index 6db91fb2d70..6c62180a150 100644 --- a/rafs/src/mock/mock_inode.rs +++ b/rafs/src/mock/mock_inode.rs @@ -175,6 +175,26 @@ impl RafsInode for MockInode { .collect::>()) } + #[inline] + fn is_blkdev(&self) -> bool { + self.i_mode & libc::S_IFMT as u32 == libc::S_IFBLK as u32 + } + + #[inline] + fn is_chrdev(&self) -> bool { + self.i_mode & libc::S_IFMT as u32 == libc::S_IFCHR as u32 + } + + #[inline] + fn is_sock(&self) -> bool { + self.i_mode & libc::S_IFMT as u32 == libc::S_IFSOCK as u32 + } + + #[inline] + fn is_fifo(&self) -> bool { + self.i_mode & libc::S_IFMT as u32 == libc::S_IFIFO as u32 + } + fn is_dir(&self) -> bool { self.i_mode & libc::S_IFMT as u32 == libc::S_IFDIR as u32 } diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index b42c110b3b6..524f56e256a 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -1305,7 +1305,16 @@ impl Command { Some(Arc::new(local_fs)) } - None => Self::get_backend(matches, "unpacker").ok(), + None => { + if let Some(backend) = &config.backend { + Some(BlobFactory::new_backend(&backend, "unpacker")?) + } else { + match Self::get_backend(matches, "unpacker") { + Ok(backend) => Some(backend), + Err(_) => bail!("one of `--blob`, `--blob-dir` and `--backend-config` must be specified"), + } + } + } }; OCIUnpacker::new(bootstrap, backend, output) diff --git a/src/bin/nydus-image/unpack/mod.rs b/src/bin/nydus-image/unpack/mod.rs index 0dad0a810d4..81587172c7c 100644 --- a/src/bin/nydus-image/unpack/mod.rs +++ b/src/bin/nydus-image/unpack/mod.rs @@ -2,6 +2,7 @@ // Copyright (C) 2022 Alibaba Cloud. All rights reserved. // // SPDX-License-Identifier: Apache-2.0 +use std::collections::HashMap; use std::fs::{File, OpenOptions}; use std::io::Read; use std::path::{Path, PathBuf}; @@ -115,8 +116,7 @@ impl OCITarBuilderFactory { ) -> Result> { let writer = self.create_writer(output_path)?; - let blob = meta.superblock.get_blob_infos().pop(); - let builders = self.create_builders(blob, blob_backend)?; + let builders = self.create_builders(meta, blob_backend)?; let builder = OCITarBuilder::new(builders, writer); @@ -139,7 +139,7 @@ impl OCITarBuilderFactory { fn create_builders( &self, - blob: Option>, + meta: &RafsSuper, blob_backend: &Option>, ) -> Result>> { // PAX basic builders @@ -155,7 +155,8 @@ impl OCITarBuilderFactory { let fifo_builder = OCIFifoBuilder::new(special_builder.clone()); let char_builder = OCICharBuilder::new(special_builder.clone()); let block_builder = OCIBlockBuilder::new(special_builder); - let reg_builder = self.create_reg_builder(blob, blob_backend)?; + let blobs = meta.superblock.get_blob_infos(); + let reg_builder = self.create_reg_builder(blobs, blob_backend)?; // The order counts. let builders = vec![ @@ -174,29 +175,28 @@ impl OCITarBuilderFactory { fn create_reg_builder( &self, - blob: Option>, + blobs: Vec>, blob_backend: &Option>, ) -> Result { - let (reader, compressor) = match blob { - None => (None, None), - Some(ref blob) => { - let blob_backend = blob_backend - .as_deref() - .with_context(|| "both blob path or blob backend config are not specified")?; - let reader = blob_backend - .get_reader("unpacker") - .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?; - - let compressor = blob.compressor(); - - (Some(reader), Some(compressor)) - } - }; + let mut readers = HashMap::new(); + let mut compressors = HashMap::new(); + for blob in blobs { + let blob_backend = blob_backend + .as_deref() + .with_context(|| "both blob path or blob backend config are not specified")?; + let reader = blob_backend + .get_reader(blob.blob_id().as_str()) + .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?; + + let compressor = blob.compressor(); + readers.insert(blob.blob_index(), reader); + compressors.insert(blob.blob_index(), compressor); + } Ok(OCIRegBuilder::new( Rc::new(PAXExtensionSectionBuilder::new()), - reader, - compressor, + readers, + compressors, )) } } diff --git a/src/bin/nydus-image/unpack/pax.rs b/src/bin/nydus-image/unpack/pax.rs index b609ba15ae9..f9d1736d134 100644 --- a/src/bin/nydus-image/unpack/pax.rs +++ b/src/bin/nydus-image/unpack/pax.rs @@ -167,37 +167,34 @@ impl SectionBuilder for OCIDirBuilder { pub struct OCIRegBuilder { ext_builder: Rc, - reader: Option>, - compressor: Option, + readers: HashMap>, + compressors: HashMap, } impl OCIRegBuilder { pub fn new( ext_builder: Rc, - reader: Option>, - compressor: Option, + readers: HashMap>, + compressors: HashMap, ) -> Self { OCIRegBuilder { ext_builder, - reader, - compressor, + readers, + compressors, } } fn build_data(&self, inode: &dyn RafsInodeExt) -> Box { - if self.reader.is_none() { - return Box::new(io::empty()); - } - let chunks = (0..inode.get_chunk_count()) .map(|i| inode.get_chunk_info(i).unwrap()) .collect(); - let reader = ChunkReader::new( - *self.compressor.as_ref().unwrap(), - self.reader.as_ref().unwrap().clone(), - chunks, - ); + let mut compressors = HashMap::new(); + compressors.clone_from(&self.compressors); + + let mut readers = HashMap::new(); + readers.clone_from(&self.readers); + let reader = ChunkReader::new(compressors, readers, chunks); Box::new(reader) } @@ -702,8 +699,8 @@ impl Util { } struct ChunkReader { - compressor: Algorithm, - reader: Arc, + compressors: HashMap, + readers: HashMap>, chunks: IntoIter>, chunk: Cursor>, @@ -711,13 +708,13 @@ struct ChunkReader { impl ChunkReader { fn new( - compressor: Algorithm, - reader: Arc, + compressors: HashMap, + readers: HashMap>, chunks: Vec>, ) -> Self { Self { - compressor, - reader, + compressors, + readers, chunks: chunks.into_iter(), chunk: Cursor::new(Vec::new()), } @@ -725,7 +722,13 @@ impl ChunkReader { fn load_chunk(&mut self, chunk: &dyn BlobChunkInfo) -> Result<()> { let mut buf = alloc_buf(chunk.compressed_size() as usize); - self.reader + + let reader = self + .readers + .get(&chunk.blob_index()) + .expect("No valid reader") + .clone(); + reader .read(buf.as_mut_slice(), chunk.compressed_offset()) .map_err(|err| { error!("fail to read chunk, error: {:?}", err); @@ -737,8 +740,13 @@ impl ChunkReader { return Ok(()); } + let compressor = *self + .compressors + .get(&chunk.blob_index()) + .expect("No valid compressor"); + let mut data = vec![0u8; chunk.uncompressed_size() as usize]; - compress::decompress(buf.as_mut_slice(), data.as_mut_slice(), self.compressor) + compress::decompress(buf.as_mut_slice(), data.as_mut_slice(), compressor) .with_context(|| "fail to decompress")?; self.chunk = Cursor::new(data); diff --git a/src/bin/nydus-image/unpack/pax/test.rs b/src/bin/nydus-image/unpack/pax/test.rs index 35c43be9068..fcb86574ad5 100644 --- a/src/bin/nydus-image/unpack/pax/test.rs +++ b/src/bin/nydus-image/unpack/pax/test.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::{io::Read, sync::Arc}; use nydus_storage::backend::{BackendResult, BlobReader}; @@ -79,7 +80,7 @@ impl BlobChunkInfo for MockChunkInfo { } fn blob_index(&self) -> u32 { - todo!(); + 0 } fn compressed_offset(&self) -> u64 { @@ -206,7 +207,13 @@ fn create_compress_chunk_reader() -> ChunkReader { let blob_reader = Arc::new(MockBlobReader::new(compressed_chunk.into_owned())); - ChunkReader::new(Algorithm::GZip, blob_reader, vec![meta]) + let mut readers: HashMap> = HashMap::new(); + readers.insert(meta.blob_index(), blob_reader); + + let mut compressors: HashMap = HashMap::new(); + compressors.insert(meta.blob_index(), Algorithm::GZip); + + ChunkReader::new(compressors, readers, vec![meta]) } fn create_default_chunk_reader() -> ChunkReader { @@ -230,5 +237,13 @@ fn create_default_chunk_reader() -> ChunkReader { let blob_reader = Arc::new(MockBlobReader::new([chunk1, chunk2].concat())); - ChunkReader::new(Algorithm::None, blob_reader, vec![chunk_meta1, chunk_meta2]) + let mut readers: HashMap> = HashMap::new(); + readers.insert(chunk_meta1.blob_index(), blob_reader.clone()); + readers.insert(chunk_meta2.blob_index(), blob_reader); + + let mut compressors: HashMap = HashMap::new(); + compressors.insert(chunk_meta1.blob_index(), Algorithm::None); + compressors.insert(chunk_meta2.blob_index(), Algorithm::None); + + ChunkReader::new(compressors, readers, vec![chunk_meta1, chunk_meta2]) }