Skip to content

Commit

Permalink
rafs: define dedicated RafsV6Inode to reduce memory consumption
Browse files Browse the repository at this point in the history 
There are several unused fields in RafsV5Inode when used for v6,
so define dedicated RafsV6Inode to reduce memory consumption.

Signed-off-by: Jiang Liu <[email protected]>
  • Loading branch information
@jiangliu
jiangliu committed Mar 17, 2023
1 parent c9d9b43 commit 3bc46ce
Show file tree
Hide file tree
Showing 10 changed files with 401 additions and 209 deletions.
9 changes: 6 additions & 3 deletions rafs/src/builder/core/bootstrap.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,10 @@ impl Bootstrap {
// binary search.
tree.children
.sort_by_key(|child| child.node.name().to_os_string());
parent.inode.set_child_index(index);
parent.inode.set_child_count(tree.children.len() as u32);
if ctx.fs_version.is_v6() {
if ctx.fs_version.is_v5() {
parent.inode.set_child_index(index);
} else if ctx.fs_version.is_v6() {
let d_size = tree.node.v6_dirent_size(ctx, tree)?;
parent.v6_set_dir_offset(bootstrap_ctx, d_size, block_size)?;
}
Expand All @@ -177,7 +178,9 @@ impl Bootstrap {
for child in tree.children.iter_mut() {
let index = nodes.len() as u64 + 1;
child.node.index = index;
child.node.inode.set_parent(parent_ino);
if ctx.fs_version.is_v5() {
child.node.inode.set_parent(parent_ino);
}

// Handle hardlink.
// All hardlink nodes' ino and nlink should be the same.
Expand Down
16 changes: 13 additions & 3 deletions rafs/src/builder/core/node.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,13 +173,12 @@ impl Display for Node {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(
f,
"{} {:?}: index {} ino {} real_ino {} i_parent {} child_index {} child_count {} i_nlink {} i_size {} i_blocks {} i_name_size {} i_symlink_size {} has_xattr {} link {:?} i_mtime {} i_mtime_nsec {}",
"{} {:?}: index {} ino {} real_ino {} child_index {} child_count {} i_nlink {} i_size {} i_blocks {} i_name_size {} i_symlink_size {} has_xattr {} link {:?} i_mtime {} i_mtime_nsec {}",
self.file_type(),
self.target(),
self.index,
self.inode.ino(),
self.info.src_ino,
self.inode.parent(),
self.inode.child_index(),
self.inode.child_count(),
self.inode.nlink(),
Expand Down Expand Up @@ -609,7 +608,15 @@ impl Node {
}

fn build_inode(&mut self, chunk_size: u32) -> Result<()> {
self.inode.set_name_size(self.name().byte_size());
let size = self.name().byte_size();
if size > u16::MAX as usize {
bail!(
"file name length 0x{:x} is too big, max 0x{:x}",
size,
u16::MAX
);
}
self.inode.set_name_size(size);

// NOTE: Always retrieve xattr before attr so that we can know the size of xattr pairs.
self.build_inode_xattr()
Expand All @@ -631,6 +638,9 @@ impl Node {
})?;
let symlink: OsString = target_path.into();
let size = symlink.byte_size();
if size > u16::MAX as usize {
bail!("symlink content size 0x{:x} is too big", size);
}
self.inode.set_symlink_size(size);
self.set_symlink(symlink);
}
Expand Down
11 changes: 5 additions & 6 deletions rafs/src/builder/core/v6.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@ use super::chunk_dict::DigestWithBlobIndex;
use super::node::Node;
use crate::builder::{Bootstrap, BootstrapContext, BuildContext, ConversionType, Tree};
use crate::metadata::chunk::ChunkWrapper;
use crate::metadata::inode::new_v6_inode;
use crate::metadata::layout::v6::{
align_offset, calculate_nid, RafsV6BlobTable, RafsV6Device, RafsV6Dirent, RafsV6InodeChunkAddr,
RafsV6InodeChunkHeader, RafsV6OndiskInode, RafsV6SuperBlock, RafsV6SuperBlockExt,
EROFS_BLOCK_BITS_9, EROFS_BLOCK_SIZE_4096, EROFS_BLOCK_SIZE_512, EROFS_DEVTABLE_OFFSET,
EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE, EROFS_INODE_FLAT_PLAIN,
EROFS_INODE_SLOT_SIZE, EROFS_SUPER_BLOCK_SIZE, EROFS_SUPER_OFFSET,
align_offset, calculate_nid, new_v6_inode, RafsV6BlobTable, RafsV6Device, RafsV6Dirent,
RafsV6InodeChunkAddr, RafsV6InodeChunkHeader, RafsV6OndiskInode, RafsV6SuperBlock,
RafsV6SuperBlockExt, EROFS_BLOCK_BITS_9, EROFS_BLOCK_SIZE_4096, EROFS_BLOCK_SIZE_512,
EROFS_DEVTABLE_OFFSET, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE,
EROFS_INODE_FLAT_PLAIN, EROFS_INODE_SLOT_SIZE, EROFS_SUPER_BLOCK_SIZE, EROFS_SUPER_OFFSET,
};
use crate::metadata::RafsStore;
use crate::RafsIoWrite;
Expand Down
79 changes: 47 additions & 32 deletions rafs/src/builder/stargz.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ use super::core::overlay::Overlay;
use super::core::tree::Tree;
use super::{build_bootstrap, Builder};
use crate::metadata::chunk::ChunkWrapper;
use crate::metadata::inode::InodeWrapper;
use crate::metadata::layout::v5::{RafsV5ChunkInfo, RafsV5Inode, RafsV5InodeFlags};
use crate::metadata::inode::{InodeWrapper, RafsInodeFlags, RafsV6Inode};
use crate::metadata::layout::v5::{RafsV5ChunkInfo, RafsV5Inode};
use crate::metadata::layout::RafsXAttrs;
use crate::metadata::{Inode, RafsVersion};

Expand Down Expand Up @@ -543,16 +543,16 @@ impl StargzTreeBuilder {
let entry_path = entry.path()?;
let mut file_size = entry.size;
let mut flags = match version {
RafsVersion::V5 => RafsV5InodeFlags::default(),
RafsVersion::V6 => RafsV5InodeFlags::default(),
RafsVersion::V5 => RafsInodeFlags::default(),
RafsVersion::V6 => RafsInodeFlags::default(),
};

// Parse symlink
let (symlink, symlink_size) = if entry.is_symlink() {
let symlink_link_path = entry.symlink_link_path();
let symlink_size = symlink_link_path.byte_size() as u16;
file_size = symlink_size.into();
flags |= RafsV5InodeFlags::SYMLINK;
flags |= RafsInodeFlags::SYMLINK;
(Some(symlink_link_path.as_os_str().to_owned()), symlink_size)
} else {
(None, 0)
Expand All @@ -562,7 +562,7 @@ impl StargzTreeBuilder {
let mut xattrs = RafsXAttrs::new();
if entry.has_xattr() {
for (name, value) in entry.xattrs.iter() {
flags |= RafsV5InodeFlags::XATTR;
flags |= RafsInodeFlags::XATTR;
let value = base64::decode(value).with_context(|| {
format!(
"parse xattr name {:?} of file {:?} failed",
Expand All @@ -576,7 +576,7 @@ impl StargzTreeBuilder {
// Handle hardlink ino
let mut ino = (self.path_inode_map.len() + 1) as Inode;
if entry.is_hardlink() {
flags |= RafsV5InodeFlags::HARDLINK;
flags |= RafsInodeFlags::HARDLINK;
if let Some(_ino) = self.path_inode_map.get(&entry.hardlink_link_path()) {
ino = *_ino;
} else {
Expand All @@ -591,32 +591,47 @@ impl StargzTreeBuilder {
let uid = if explicit_uidgid { entry.uid } else { 0 };
let gid = if explicit_uidgid { entry.gid } else { 0 };

// Parse inode info
let v5_inode = RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: entry.mode(),
i_size: file_size,
i_nlink: entry.num_link,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: 0,
i_name_size: name_size,
i_symlink_size: symlink_size,
i_rdev: entry.rdev(),
// TODO: add mtime from entry.ModTime()
i_mtime: 0,
i_mtime_nsec: 0,
i_reserved: [0; 8],
};
let inode = match version {
RafsVersion::V5 => InodeWrapper::V5(v5_inode),
RafsVersion::V6 => InodeWrapper::V6(v5_inode),
RafsVersion::V5 => InodeWrapper::V5(RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: entry.mode(),
i_size: file_size,
i_nlink: entry.num_link,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: 0,
i_name_size: name_size,
i_symlink_size: symlink_size,
i_rdev: entry.rdev(),
// TODO: add mtime from entry.ModTime()
i_mtime: 0,
i_mtime_nsec: 0,
i_reserved: [0; 8],
}),
RafsVersion::V6 => InodeWrapper::V6(RafsV6Inode {
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: entry.mode(),
i_size: file_size,
i_nlink: entry.num_link,
i_blocks: 0,
i_flags: flags,
i_child_count: 0,
i_name_size: name_size,
i_symlink_size: symlink_size,
i_rdev: entry.rdev(),
// TODO: add mtime from entry.ModTime()
i_mtime: 0,
i_mtime_nsec: 0,
}),
};

let path = entry.path()?;
Expand Down
77 changes: 47 additions & 30 deletions rafs/src/builder/tarball.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ use super::core::node::{Node, NodeInfo};
use super::core::overlay::Overlay;
use super::core::tree::Tree;
use super::{build_bootstrap, dump_bootstrap, finalize_blob, Builder};
use crate::metadata::inode::InodeWrapper;
use crate::metadata::layout::v5::{RafsV5Inode, RafsV5InodeFlags};
use crate::metadata::inode::{InodeWrapper, RafsInodeFlags, RafsV6Inode};
use crate::metadata::layout::v5::RafsV5Inode;
use crate::metadata::layout::RafsXAttrs;
use crate::metadata::{Inode, RafsVersion};

Expand Down Expand Up @@ -229,8 +229,8 @@ impl<'a> TarballTreeBuilder<'a> {
let (uid, gid) = Self::get_uid_gid(self.ctx, header)?;
let mtime = header.mtime().unwrap_or_default();
let mut flags = match self.ctx.fs_version {
RafsVersion::V5 => RafsV5InodeFlags::default(),
RafsVersion::V6 => RafsV5InodeFlags::default(),
RafsVersion::V5 => RafsInodeFlags::default(),
RafsVersion::V6 => RafsInodeFlags::default(),
};

// Parse special files
Expand Down Expand Up @@ -262,7 +262,7 @@ impl<'a> TarballTreeBuilder<'a> {
bail!("tarball: symlink target from tar entry is too big");
}
file_size = symlink_size as u64;
flags |= RafsV5InodeFlags::SYMLINK;
flags |= RafsInodeFlags::SYMLINK;
(
Some(symlink_link_path.as_os_str().to_owned()),
symlink_size as u16,
Expand Down Expand Up @@ -299,7 +299,7 @@ impl<'a> TarballTreeBuilder<'a> {
path.as_ref().display()
);
}
flags |= RafsV5InodeFlags::HARDLINK;
flags |= RafsInodeFlags::HARDLINK;
} else {
self.path_inode_map
.insert(path.as_ref().to_path_buf(), (ino, nodes.len()));
Expand Down Expand Up @@ -328,30 +328,45 @@ impl<'a> TarballTreeBuilder<'a> {
}
}

let v5_inode = RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: mode,
i_size: file_size,
i_nlink: 1,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: child_count as u32,
i_name_size: name.len() as u16,
i_symlink_size: symlink_size,
i_rdev: rdev,
i_mtime: mtime,
i_mtime_nsec: 0,
i_reserved: [0; 8],
};
let mut inode = match self.ctx.fs_version {
RafsVersion::V5 => InodeWrapper::V5(v5_inode),
RafsVersion::V6 => InodeWrapper::V6(v5_inode),
RafsVersion::V5 => InodeWrapper::V5(RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: mode,
i_size: file_size,
i_nlink: 1,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: child_count as u32,
i_name_size: name.len() as u16,
i_symlink_size: symlink_size,
i_rdev: rdev,
i_mtime: mtime,
i_mtime_nsec: 0,
i_reserved: [0; 8],
}),
RafsVersion::V6 => InodeWrapper::V6(RafsV6Inode {
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: mode,
i_size: file_size,
i_nlink: 1,
i_blocks: 0,
i_flags: flags,
i_child_count: child_count as u32,
i_name_size: name.len() as u16,
i_symlink_size: symlink_size,
i_rdev: rdev,
i_mtime: mtime,
i_mtime_nsec: 0,
}),
};
inode.set_has_xattr(!xattrs.is_empty());

Expand Down Expand Up @@ -391,7 +406,9 @@ impl<'a> TarballTreeBuilder<'a> {
// the associated regular file.
if entry_type.is_hard_link() {
let n = &nodes[index];
node.inode.set_digest(*n.inode.digest());
if n.inode.is_v5() {
node.inode.set_digest(*n.inode.digest());
}
node.inode.set_size(n.inode.size());
node.inode.set_child_count(n.inode.child_count());
node.chunks = n.chunks.clone();
Expand Down
9 changes: 5 additions & 4 deletions rafs/src/metadata/cached_v5.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,10 @@ use nydus_storage::device::{BlobChunkFlags, BlobChunkInfo, BlobDevice, BlobInfo}
use nydus_utils::digest::RafsDigest;
use nydus_utils::ByteSize;

use crate::metadata::inode::RafsInodeFlags;
use crate::metadata::layout::v5::{
rafsv5_alloc_bio_vecs, rafsv5_validate_inode, RafsV5BlobTable, RafsV5ChunkInfo, RafsV5Inode,
RafsV5InodeChunkOps, RafsV5InodeFlags, RafsV5InodeOps, RafsV5XAttrsTable, RAFSV5_ALIGNMENT,
RafsV5InodeChunkOps, RafsV5InodeOps, RafsV5XAttrsTable, RAFSV5_ALIGNMENT,
};
use crate::metadata::layout::{bytes_to_os_str, parse_xattr, RAFS_V5_ROOT_INODE};
use crate::metadata::{
Expand Down Expand Up @@ -236,7 +237,7 @@ pub struct CachedInodeV5 {
i_projid: u32,
i_uid: u32,
i_gid: u32,
i_flags: RafsV5InodeFlags,
i_flags: RafsInodeFlags,
i_size: u64,
i_blocks: u64,
i_nlink: u32,
Expand Down Expand Up @@ -481,7 +482,7 @@ impl RafsInode for CachedInodeV5 {

#[inline]
fn has_xattr(&self) -> bool {
self.i_flags.contains(RafsV5InodeFlags::XATTR)
self.i_flags.contains(RafsInodeFlags::XATTR)
}

#[inline]
Expand Down Expand Up @@ -664,7 +665,7 @@ impl RafsV5InodeOps for CachedInodeV5 {
}

fn has_hole(&self) -> bool {
self.i_flags.contains(RafsV5InodeFlags::HAS_HOLE)
self.i_flags.contains(RafsInodeFlags::HAS_HOLE)
}
}

Expand Down
Loading

0 comments on commit 3bc46ce

Please sign in to comment.