Skip to content

Commit

Permalink
nydus-image: Fix blobs unpack bug
Browse files Browse the repository at this point in the history
Signed-off-by: Junduo Dong <[email protected]>
  • Loading branch information
An-DJ committed Sep 21, 2023
1 parent d2fcfcd commit 1b4bb61
Show file tree
Hide file tree
Showing 10 changed files with 161 additions and 53 deletions.
20 changes: 20 additions & 0 deletions rafs/src/metadata/cached_v5.rs
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,26 @@ impl RafsInode for CachedInodeV5 {
}
}

#[inline]
fn is_blkdev(&self) -> bool {
self.i_mode & libc::S_IFMT as u32 == libc::S_IFBLK as u32
}

#[inline]
fn is_chrdev(&self) -> bool {
self.i_mode & libc::S_IFMT as u32 == libc::S_IFCHR as u32
}

#[inline]
fn is_sock(&self) -> bool {
self.i_mode & libc::S_IFMT as u32 == libc::S_IFSOCK as u32
}

#[inline]
fn is_fifo(&self) -> bool {
self.i_mode & libc::S_IFMT as u32 == libc::S_IFIFO as u32
}

#[inline]
fn is_dir(&self) -> bool {
self.i_mode & libc::S_IFMT as u32 == libc::S_IFDIR as u32
Expand Down
4 changes: 4 additions & 0 deletions rafs/src/metadata/direct_v5.rs
Original file line number Diff line number Diff line change
Expand Up @@ -714,6 +714,10 @@ impl RafsInode for OndiskInodeWrapper {
self
}

impl_inode_wrapper!(is_blkdev, bool);
impl_inode_wrapper!(is_chrdev, bool);
impl_inode_wrapper!(is_sock, bool);
impl_inode_wrapper!(is_fifo, bool);
impl_inode_wrapper!(is_dir, bool);
impl_inode_wrapper!(is_reg, bool);
impl_inode_wrapper!(is_symlink, bool);
Expand Down
20 changes: 20 additions & 0 deletions rafs/src/metadata/direct_v6.rs
Original file line number Diff line number Diff line change
Expand Up @@ -946,6 +946,26 @@ impl RafsInode for OndiskInodeWrapper {
0
}

#[inline]
fn is_blkdev(&self) -> bool {
self.mode_format_bits() == libc::S_IFBLK as u32
}

#[inline]
fn is_chrdev(&self) -> bool {
self.mode_format_bits() == libc::S_IFCHR as u32
}

#[inline]
fn is_sock(&self) -> bool {
self.mode_format_bits() == libc::S_IFSOCK as u32
}

#[inline]
fn is_fifo(&self) -> bool {
self.mode_format_bits() == libc::S_IFIFO as u32
}

fn is_dir(&self) -> bool {
self.mode_format_bits() == libc::S_IFDIR as u32
}
Expand Down
8 changes: 4 additions & 4 deletions rafs/src/metadata/inode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ impl InodeWrapper {
match self {
InodeWrapper::V5(i) => i.is_chrdev(),
InodeWrapper::V6(i) => i.is_chrdev(),
InodeWrapper::Ref(_i) => unimplemented!(),
InodeWrapper::Ref(i) => i.as_inode().is_chrdev(),
}
}

Expand All @@ -183,7 +183,7 @@ impl InodeWrapper {
match self {
InodeWrapper::V5(i) => i.is_sock(),
InodeWrapper::V6(i) => i.is_sock(),
InodeWrapper::Ref(_i) => unimplemented!(),
InodeWrapper::Ref(i) => i.as_inode().is_dir(),
}
}

Expand Down Expand Up @@ -322,7 +322,7 @@ impl InodeWrapper {
match self {
InodeWrapper::V5(i) => i.i_uid,
InodeWrapper::V6(i) => i.i_uid,
InodeWrapper::Ref(_i) => unimplemented!(),
InodeWrapper::Ref(i) => i.as_inode().get_attr().uid,
}
}

Expand All @@ -341,7 +341,7 @@ impl InodeWrapper {
match self {
InodeWrapper::V5(i) => i.i_gid,
InodeWrapper::V6(i) => i.i_gid,
InodeWrapper::Ref(_i) => unimplemented!(),
InodeWrapper::Ref(i) => i.as_inode().get_attr().gid,
}
}

Expand Down
12 changes: 12 additions & 0 deletions rafs/src/metadata/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,18 @@ pub trait RafsInode: Any {
/// Posix: get project id associated with the inode.
fn projid(&self) -> u32;

/// Mode: check whether the inode is a block device.
fn is_blkdev(&self) -> bool;

/// Mode: check whether the inode is a char device.
fn is_chrdev(&self) -> bool;

/// Mode: check whether the inode is a sock.
fn is_sock(&self) -> bool;

/// Mode: check whether the inode is a fifo.
fn is_fifo(&self) -> bool;

/// Mode: check whether the inode is a directory.
fn is_dir(&self) -> bool;

Expand Down
20 changes: 20 additions & 0 deletions rafs/src/mock/mock_inode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,26 @@ impl RafsInode for MockInode {
.collect::<Vec<XattrName>>())
}

#[inline]
fn is_blkdev(&self) -> bool {
self.i_mode & libc::S_IFMT as u32 == libc::S_IFBLK as u32
}

#[inline]
fn is_chrdev(&self) -> bool {
self.i_mode & libc::S_IFMT as u32 == libc::S_IFCHR as u32
}

#[inline]
fn is_sock(&self) -> bool {
self.i_mode & libc::S_IFMT as u32 == libc::S_IFSOCK as u32
}

#[inline]
fn is_fifo(&self) -> bool {
self.i_mode & libc::S_IFMT as u32 == libc::S_IFIFO as u32
}

fn is_dir(&self) -> bool {
self.i_mode & libc::S_IFMT as u32 == libc::S_IFDIR as u32
}
Expand Down
11 changes: 10 additions & 1 deletion src/bin/nydus-image/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1321,7 +1321,16 @@ impl Command {

Some(Arc::new(local_fs))
}
None => Self::get_backend(matches, "unpacker").ok(),
None => {
if let Some(backend) = &config.backend {
Some(BlobFactory::new_backend(&backend, "unpacker")?)
} else {
match Self::get_backend(matches, "unpacker") {
Ok(backend) => Some(backend),
Err(_) => bail!("one of `--blob`, `--blob-dir` and `--backend-config` must be specified"),
}
}
}
};

OCIUnpacker::new(bootstrap, backend, output)
Expand Down
44 changes: 22 additions & 22 deletions src/bin/nydus-image/unpack/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Copyright (C) 2022 Alibaba Cloud. All rights reserved.
//
// SPDX-License-Identifier: Apache-2.0
use std::collections::HashMap;
use std::fs::{File, OpenOptions};
use std::io::Read;
use std::path::{Path, PathBuf};
Expand Down Expand Up @@ -115,8 +116,7 @@ impl OCITarBuilderFactory {
) -> Result<Box<dyn TarBuilder>> {
let writer = self.create_writer(output_path)?;

let blob = meta.superblock.get_blob_infos().pop();
let builders = self.create_builders(blob, blob_backend)?;
let builders = self.create_builders(meta, blob_backend)?;

let builder = OCITarBuilder::new(builders, writer);

Expand All @@ -139,7 +139,7 @@ impl OCITarBuilderFactory {

fn create_builders(
&self,
blob: Option<Arc<BlobInfo>>,
meta: &RafsSuper,
blob_backend: &Option<Arc<dyn BlobBackend + Send + Sync>>,
) -> Result<Vec<Box<dyn SectionBuilder>>> {
// PAX basic builders
Expand All @@ -155,7 +155,8 @@ impl OCITarBuilderFactory {
let fifo_builder = OCIFifoBuilder::new(special_builder.clone());
let char_builder = OCICharBuilder::new(special_builder.clone());
let block_builder = OCIBlockBuilder::new(special_builder);
let reg_builder = self.create_reg_builder(blob, blob_backend)?;
let blobs = meta.superblock.get_blob_infos();
let reg_builder = self.create_reg_builder(blobs, blob_backend)?;

// The order counts.
let builders = vec![
Expand All @@ -174,29 +175,28 @@ impl OCITarBuilderFactory {

fn create_reg_builder(
&self,
blob: Option<Arc<BlobInfo>>,
blobs: Vec<Arc<BlobInfo>>,
blob_backend: &Option<Arc<dyn BlobBackend + Send + Sync>>,
) -> Result<OCIRegBuilder> {
let (reader, compressor) = match blob {
None => (None, None),
Some(ref blob) => {
let blob_backend = blob_backend
.as_deref()
.with_context(|| "both blob path or blob backend config are not specified")?;
let reader = blob_backend
.get_reader("unpacker")
.map_err(|err| anyhow!("fail to get reader, error {:?}", err))?;

let compressor = blob.compressor();

(Some(reader), Some(compressor))
}
};
let mut readers = HashMap::new();
let mut compressors = HashMap::new();
for blob in blobs {
let blob_backend = blob_backend
.as_deref()
.with_context(|| "both blob path or blob backend config are not specified")?;
let reader = blob_backend
.get_reader(blob.blob_id().as_str())
.map_err(|err| anyhow!("fail to get reader, error {:?}", err))?;

let compressor = blob.compressor();
readers.insert(blob.blob_index(), reader);
compressors.insert(blob.blob_index(), compressor);
}

Ok(OCIRegBuilder::new(
Rc::new(PAXExtensionSectionBuilder::new()),
reader,
compressor,
readers,
compressors,
))
}
}
Expand Down
54 changes: 31 additions & 23 deletions src/bin/nydus-image/unpack/pax.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,37 +167,34 @@ impl SectionBuilder for OCIDirBuilder {

pub struct OCIRegBuilder {
ext_builder: Rc<PAXExtensionSectionBuilder>,
reader: Option<Arc<dyn BlobReader>>,
compressor: Option<Algorithm>,
readers: HashMap<u32, Arc<dyn BlobReader>>,
compressors: HashMap<u32, Algorithm>,
}

impl OCIRegBuilder {
pub fn new(
ext_builder: Rc<PAXExtensionSectionBuilder>,
reader: Option<Arc<dyn BlobReader>>,
compressor: Option<Algorithm>,
readers: HashMap<u32, Arc<dyn BlobReader>>,
compressors: HashMap<u32, Algorithm>,
) -> Self {
OCIRegBuilder {
ext_builder,
reader,
compressor,
readers,
compressors,
}
}

fn build_data(&self, inode: &dyn RafsInodeExt) -> Box<dyn Read> {
if self.reader.is_none() {
return Box::new(io::empty());
}

let chunks = (0..inode.get_chunk_count())
.map(|i| inode.get_chunk_info(i).unwrap())
.collect();

let reader = ChunkReader::new(
*self.compressor.as_ref().unwrap(),
self.reader.as_ref().unwrap().clone(),
chunks,
);
let mut compressors = HashMap::new();
compressors.clone_from(&self.compressors);

let mut readers = HashMap::new();
readers.clone_from(&self.readers);
let reader = ChunkReader::new(compressors, readers, chunks);

Box::new(reader)
}
Expand Down Expand Up @@ -702,30 +699,36 @@ impl Util {
}

struct ChunkReader {
compressor: Algorithm,
reader: Arc<dyn BlobReader>,
compressors: HashMap<u32, Algorithm>,
readers: HashMap<u32, Arc<dyn BlobReader>>,

chunks: IntoIter<Arc<dyn BlobChunkInfo>>,
chunk: Cursor<Vec<u8>>,
}

impl ChunkReader {
fn new(
compressor: Algorithm,
reader: Arc<dyn BlobReader>,
compressors: HashMap<u32, Algorithm>,
readers: HashMap<u32, Arc<dyn BlobReader>>,
chunks: Vec<Arc<dyn BlobChunkInfo>>,
) -> Self {
Self {
compressor,
reader,
compressors,
readers,
chunks: chunks.into_iter(),
chunk: Cursor::new(Vec::new()),
}
}

fn load_chunk(&mut self, chunk: &dyn BlobChunkInfo) -> Result<()> {
let mut buf = alloc_buf(chunk.compressed_size() as usize);
self.reader

let reader = self
.readers
.get(&chunk.blob_index())
.expect("No valid reader")
.clone();
reader
.read(buf.as_mut_slice(), chunk.compressed_offset())
.map_err(|err| {
error!("fail to read chunk, error: {:?}", err);
Expand All @@ -737,8 +740,13 @@ impl ChunkReader {
return Ok(());
}

let compressor = *self
.compressors
.get(&chunk.blob_index())
.expect("No valid compressor");

let mut data = vec![0u8; chunk.uncompressed_size() as usize];
compress::decompress(buf.as_mut_slice(), data.as_mut_slice(), self.compressor)
compress::decompress(buf.as_mut_slice(), data.as_mut_slice(), compressor)
.with_context(|| "fail to decompress")?;

self.chunk = Cursor::new(data);
Expand Down
Loading

0 comments on commit 1b4bb61

Please sign in to comment.