Skip to content

Commit

Permalink
Merge pull request #1156 from jiangliu/rafs-v6-inode
Browse files Browse the repository at this point in the history 
rafs: define dedicated RafsV6Inode to reduce memory consumption
  • Loading branch information
@imeoer
imeoer authored Mar 20, 2023
2 parents c9d9b43 + d2fa7d5 commit 65127af
Show file tree
Hide file tree
Showing 10 changed files with 397 additions and 209 deletions.
9 changes: 6 additions & 3 deletions rafs/src/builder/core/bootstrap.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,10 @@ impl Bootstrap {
// binary search.
tree.children
.sort_by_key(|child| child.node.name().to_os_string());
parent.inode.set_child_index(index);
parent.inode.set_child_count(tree.children.len() as u32);
if ctx.fs_version.is_v6() {
if ctx.fs_version.is_v5() {
parent.inode.set_child_index(index);
} else if ctx.fs_version.is_v6() {
let d_size = tree.node.v6_dirent_size(ctx, tree)?;
parent.v6_set_dir_offset(bootstrap_ctx, d_size, block_size)?;
}
Expand All @@ -177,7 +178,9 @@ impl Bootstrap {
for child in tree.children.iter_mut() {
let index = nodes.len() as u64 + 1;
child.node.index = index;
child.node.inode.set_parent(parent_ino);
if ctx.fs_version.is_v5() {
child.node.inode.set_parent(parent_ino);
}

// Handle hardlink.
// All hardlink nodes' ino and nlink should be the same.
Expand Down
12 changes: 9 additions & 3 deletions rafs/src/builder/core/node.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,13 +173,12 @@ impl Display for Node {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(
f,
"{} {:?}: index {} ino {} real_ino {} i_parent {} child_index {} child_count {} i_nlink {} i_size {} i_blocks {} i_name_size {} i_symlink_size {} has_xattr {} link {:?} i_mtime {} i_mtime_nsec {}",
"{} {:?}: index {} ino {} real_ino {} child_index {} child_count {} i_nlink {} i_size {} i_blocks {} i_name_size {} i_symlink_size {} has_xattr {} link {:?} i_mtime {} i_mtime_nsec {}",
self.file_type(),
self.target(),
self.index,
self.inode.ino(),
self.info.src_ino,
self.inode.parent(),
self.inode.child_index(),
self.inode.child_count(),
self.inode.nlink(),
Expand Down Expand Up @@ -609,7 +608,11 @@ impl Node {
}

fn build_inode(&mut self, chunk_size: u32) -> Result<()> {
self.inode.set_name_size(self.name().byte_size());
let size = self.name().byte_size();
if size > u16::MAX as usize {
bail!("file name length 0x{:x} is too big", size,);
}
self.inode.set_name_size(size);

// NOTE: Always retrieve xattr before attr so that we can know the size of xattr pairs.
self.build_inode_xattr()
Expand All @@ -631,6 +634,9 @@ impl Node {
})?;
let symlink: OsString = target_path.into();
let size = symlink.byte_size();
if size > u16::MAX as usize {
bail!("symlink content size 0x{:x} is too big", size);
}
self.inode.set_symlink_size(size);
self.set_symlink(symlink);
}
Expand Down
11 changes: 5 additions & 6 deletions rafs/src/builder/core/v6.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@ use super::chunk_dict::DigestWithBlobIndex;
use super::node::Node;
use crate::builder::{Bootstrap, BootstrapContext, BuildContext, ConversionType, Tree};
use crate::metadata::chunk::ChunkWrapper;
use crate::metadata::inode::new_v6_inode;
use crate::metadata::layout::v6::{
align_offset, calculate_nid, RafsV6BlobTable, RafsV6Device, RafsV6Dirent, RafsV6InodeChunkAddr,
RafsV6InodeChunkHeader, RafsV6OndiskInode, RafsV6SuperBlock, RafsV6SuperBlockExt,
EROFS_BLOCK_BITS_9, EROFS_BLOCK_SIZE_4096, EROFS_BLOCK_SIZE_512, EROFS_DEVTABLE_OFFSET,
EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE, EROFS_INODE_FLAT_PLAIN,
EROFS_INODE_SLOT_SIZE, EROFS_SUPER_BLOCK_SIZE, EROFS_SUPER_OFFSET,
align_offset, calculate_nid, new_v6_inode, RafsV6BlobTable, RafsV6Device, RafsV6Dirent,
RafsV6InodeChunkAddr, RafsV6InodeChunkHeader, RafsV6OndiskInode, RafsV6SuperBlock,
RafsV6SuperBlockExt, EROFS_BLOCK_BITS_9, EROFS_BLOCK_SIZE_4096, EROFS_BLOCK_SIZE_512,
EROFS_DEVTABLE_OFFSET, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE,
EROFS_INODE_FLAT_PLAIN, EROFS_INODE_SLOT_SIZE, EROFS_SUPER_BLOCK_SIZE, EROFS_SUPER_OFFSET,
};
use crate::metadata::RafsStore;
use crate::RafsIoWrite;
Expand Down
79 changes: 47 additions & 32 deletions rafs/src/builder/stargz.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ use super::core::overlay::Overlay;
use super::core::tree::Tree;
use super::{build_bootstrap, Builder};
use crate::metadata::chunk::ChunkWrapper;
use crate::metadata::inode::InodeWrapper;
use crate::metadata::layout::v5::{RafsV5ChunkInfo, RafsV5Inode, RafsV5InodeFlags};
use crate::metadata::inode::{InodeWrapper, RafsInodeFlags, RafsV6Inode};
use crate::metadata::layout::v5::{RafsV5ChunkInfo, RafsV5Inode};
use crate::metadata::layout::RafsXAttrs;
use crate::metadata::{Inode, RafsVersion};

Expand Down Expand Up @@ -543,16 +543,16 @@ impl StargzTreeBuilder {
let entry_path = entry.path()?;
let mut file_size = entry.size;
let mut flags = match version {
RafsVersion::V5 => RafsV5InodeFlags::default(),
RafsVersion::V6 => RafsV5InodeFlags::default(),
RafsVersion::V5 => RafsInodeFlags::default(),
RafsVersion::V6 => RafsInodeFlags::default(),
};

// Parse symlink
let (symlink, symlink_size) = if entry.is_symlink() {
let symlink_link_path = entry.symlink_link_path();
let symlink_size = symlink_link_path.byte_size() as u16;
file_size = symlink_size.into();
flags |= RafsV5InodeFlags::SYMLINK;
flags |= RafsInodeFlags::SYMLINK;
(Some(symlink_link_path.as_os_str().to_owned()), symlink_size)
} else {
(None, 0)
Expand All @@ -562,7 +562,7 @@ impl StargzTreeBuilder {
let mut xattrs = RafsXAttrs::new();
if entry.has_xattr() {
for (name, value) in entry.xattrs.iter() {
flags |= RafsV5InodeFlags::XATTR;
flags |= RafsInodeFlags::XATTR;
let value = base64::decode(value).with_context(|| {
format!(
"parse xattr name {:?} of file {:?} failed",
Expand All @@ -576,7 +576,7 @@ impl StargzTreeBuilder {
// Handle hardlink ino
let mut ino = (self.path_inode_map.len() + 1) as Inode;
if entry.is_hardlink() {
flags |= RafsV5InodeFlags::HARDLINK;
flags |= RafsInodeFlags::HARDLINK;
if let Some(_ino) = self.path_inode_map.get(&entry.hardlink_link_path()) {
ino = *_ino;
} else {
Expand All @@ -591,32 +591,47 @@ impl StargzTreeBuilder {
let uid = if explicit_uidgid { entry.uid } else { 0 };
let gid = if explicit_uidgid { entry.gid } else { 0 };

// Parse inode info
let v5_inode = RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: entry.mode(),
i_size: file_size,
i_nlink: entry.num_link,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: 0,
i_name_size: name_size,
i_symlink_size: symlink_size,
i_rdev: entry.rdev(),
// TODO: add mtime from entry.ModTime()
i_mtime: 0,
i_mtime_nsec: 0,
i_reserved: [0; 8],
};
let inode = match version {
RafsVersion::V5 => InodeWrapper::V5(v5_inode),
RafsVersion::V6 => InodeWrapper::V6(v5_inode),
RafsVersion::V5 => InodeWrapper::V5(RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: entry.mode(),
i_size: file_size,
i_nlink: entry.num_link,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: 0,
i_name_size: name_size,
i_symlink_size: symlink_size,
i_rdev: entry.rdev(),
// TODO: add mtime from entry.ModTime()
i_mtime: 0,
i_mtime_nsec: 0,
i_reserved: [0; 8],
}),
RafsVersion::V6 => InodeWrapper::V6(RafsV6Inode {
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: entry.mode(),
i_size: file_size,
i_nlink: entry.num_link,
i_blocks: 0,
i_flags: flags,
i_child_count: 0,
i_name_size: name_size,
i_symlink_size: symlink_size,
i_rdev: entry.rdev(),
// TODO: add mtime from entry.ModTime()
i_mtime: 0,
i_mtime_nsec: 0,
}),
};

let path = entry.path()?;
Expand Down
77 changes: 47 additions & 30 deletions rafs/src/builder/tarball.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ use super::core::node::{Node, NodeInfo};
use super::core::overlay::Overlay;
use super::core::tree::Tree;
use super::{build_bootstrap, dump_bootstrap, finalize_blob, Builder};
use crate::metadata::inode::InodeWrapper;
use crate::metadata::layout::v5::{RafsV5Inode, RafsV5InodeFlags};
use crate::metadata::inode::{InodeWrapper, RafsInodeFlags, RafsV6Inode};
use crate::metadata::layout::v5::RafsV5Inode;
use crate::metadata::layout::RafsXAttrs;
use crate::metadata::{Inode, RafsVersion};

Expand Down Expand Up @@ -229,8 +229,8 @@ impl<'a> TarballTreeBuilder<'a> {
let (uid, gid) = Self::get_uid_gid(self.ctx, header)?;
let mtime = header.mtime().unwrap_or_default();
let mut flags = match self.ctx.fs_version {
RafsVersion::V5 => RafsV5InodeFlags::default(),
RafsVersion::V6 => RafsV5InodeFlags::default(),
RafsVersion::V5 => RafsInodeFlags::default(),
RafsVersion::V6 => RafsInodeFlags::default(),
};

// Parse special files
Expand Down Expand Up @@ -262,7 +262,7 @@ impl<'a> TarballTreeBuilder<'a> {
bail!("tarball: symlink target from tar entry is too big");
}
file_size = symlink_size as u64;
flags |= RafsV5InodeFlags::SYMLINK;
flags |= RafsInodeFlags::SYMLINK;
(
Some(symlink_link_path.as_os_str().to_owned()),
symlink_size as u16,
Expand Down Expand Up @@ -299,7 +299,7 @@ impl<'a> TarballTreeBuilder<'a> {
path.as_ref().display()
);
}
flags |= RafsV5InodeFlags::HARDLINK;
flags |= RafsInodeFlags::HARDLINK;
} else {
self.path_inode_map
.insert(path.as_ref().to_path_buf(), (ino, nodes.len()));
Expand Down Expand Up @@ -328,30 +328,45 @@ impl<'a> TarballTreeBuilder<'a> {
}
}

let v5_inode = RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: mode,
i_size: file_size,
i_nlink: 1,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: child_count as u32,
i_name_size: name.len() as u16,
i_symlink_size: symlink_size,
i_rdev: rdev,
i_mtime: mtime,
i_mtime_nsec: 0,
i_reserved: [0; 8],
};
let mut inode = match self.ctx.fs_version {
RafsVersion::V5 => InodeWrapper::V5(v5_inode),
RafsVersion::V6 => InodeWrapper::V6(v5_inode),
RafsVersion::V5 => InodeWrapper::V5(RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: mode,
i_size: file_size,
i_nlink: 1,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: child_count as u32,
i_name_size: name.len() as u16,
i_symlink_size: symlink_size,
i_rdev: rdev,
i_mtime: mtime,
i_mtime_nsec: 0,
i_reserved: [0; 8],
}),
RafsVersion::V6 => InodeWrapper::V6(RafsV6Inode {
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: mode,
i_size: file_size,
i_nlink: 1,
i_blocks: 0,
i_flags: flags,
i_child_count: child_count as u32,
i_name_size: name.len() as u16,
i_symlink_size: symlink_size,
i_rdev: rdev,
i_mtime: mtime,
i_mtime_nsec: 0,
}),
};
inode.set_has_xattr(!xattrs.is_empty());

Expand Down Expand Up @@ -391,7 +406,9 @@ impl<'a> TarballTreeBuilder<'a> {
// the associated regular file.
if entry_type.is_hard_link() {
let n = &nodes[index];
node.inode.set_digest(*n.inode.digest());
if n.inode.is_v5() {
node.inode.set_digest(*n.inode.digest());
}
node.inode.set_size(n.inode.size());
node.inode.set_child_count(n.inode.child_count());
node.chunks = n.chunks.clone();
Expand Down
9 changes: 5 additions & 4 deletions rafs/src/metadata/cached_v5.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,10 @@ use nydus_storage::device::{BlobChunkFlags, BlobChunkInfo, BlobDevice, BlobInfo}
use nydus_utils::digest::RafsDigest;
use nydus_utils::ByteSize;

use crate::metadata::inode::RafsInodeFlags;
use crate::metadata::layout::v5::{
rafsv5_alloc_bio_vecs, rafsv5_validate_inode, RafsV5BlobTable, RafsV5ChunkInfo, RafsV5Inode,
RafsV5InodeChunkOps, RafsV5InodeFlags, RafsV5InodeOps, RafsV5XAttrsTable, RAFSV5_ALIGNMENT,
RafsV5InodeChunkOps, RafsV5InodeOps, RafsV5XAttrsTable, RAFSV5_ALIGNMENT,
};
use crate::metadata::layout::{bytes_to_os_str, parse_xattr, RAFS_V5_ROOT_INODE};
use crate::metadata::{
Expand Down Expand Up @@ -236,7 +237,7 @@ pub struct CachedInodeV5 {
i_projid: u32,
i_uid: u32,
i_gid: u32,
i_flags: RafsV5InodeFlags,
i_flags: RafsInodeFlags,
i_size: u64,
i_blocks: u64,
i_nlink: u32,
Expand Down Expand Up @@ -481,7 +482,7 @@ impl RafsInode for CachedInodeV5 {

#[inline]
fn has_xattr(&self) -> bool {
self.i_flags.contains(RafsV5InodeFlags::XATTR)
self.i_flags.contains(RafsInodeFlags::XATTR)
}

#[inline]
Expand Down Expand Up @@ -664,7 +665,7 @@ impl RafsV5InodeOps for CachedInodeV5 {
}

fn has_hole(&self) -> bool {
self.i_flags.contains(RafsV5InodeFlags::HAS_HOLE)
self.i_flags.contains(RafsInodeFlags::HAS_HOLE)
}
}

Expand Down
Loading

0 comments on commit 65127af

Please sign in to comment.