Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

rafs: define dedicated RafsV6Inode to reduce memory consumption #1156

Merged
merged 1 commit into from
Mar 20, 2023
Merged

rafs: define dedicated RafsV6Inode to reduce memory consumption #1156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions rafs/src/builder/core/bootstrap.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,10 @@ impl Bootstrap {
// binary search.
tree.children
.sort_by_key(|child| child.node.name().to_os_string());
parent.inode.set_child_index(index);
parent.inode.set_child_count(tree.children.len() as u32);
if ctx.fs_version.is_v6() {
if ctx.fs_version.is_v5() {
parent.inode.set_child_index(index);
} else if ctx.fs_version.is_v6() {
let d_size = tree.node.v6_dirent_size(ctx, tree)?;
parent.v6_set_dir_offset(bootstrap_ctx, d_size, block_size)?;
}
Expand All @@ -177,7 +178,9 @@ impl Bootstrap {
for child in tree.children.iter_mut() {
let index = nodes.len() as u64 + 1;
child.node.index = index;
child.node.inode.set_parent(parent_ino);
if ctx.fs_version.is_v5() {
child.node.inode.set_parent(parent_ino);
}

// Handle hardlink.
// All hardlink nodes' ino and nlink should be the same.
Expand Down
12 changes: 9 additions & 3 deletions rafs/src/builder/core/node.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,13 +173,12 @@ impl Display for Node {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(
f,
"{} {:?}: index {} ino {} real_ino {} i_parent {} child_index {} child_count {} i_nlink {} i_size {} i_blocks {} i_name_size {} i_symlink_size {} has_xattr {} link {:?} i_mtime {} i_mtime_nsec {}",
"{} {:?}: index {} ino {} real_ino {} child_index {} child_count {} i_nlink {} i_size {} i_blocks {} i_name_size {} i_symlink_size {} has_xattr {} link {:?} i_mtime {} i_mtime_nsec {}",
self.file_type(),
self.target(),
self.index,
self.inode.ino(),
self.info.src_ino,
self.inode.parent(),
self.inode.child_index(),
self.inode.child_count(),
self.inode.nlink(),
Expand Down Expand Up @@ -609,7 +608,11 @@ impl Node {
}

fn build_inode(&mut self, chunk_size: u32) -> Result<()> {
self.inode.set_name_size(self.name().byte_size());
let size = self.name().byte_size();
if size > u16::MAX as usize {
bail!("file name length 0x{:x} is too big", size,);
}
self.inode.set_name_size(size);

// NOTE: Always retrieve xattr before attr so that we can know the size of xattr pairs.
self.build_inode_xattr()
Expand All @@ -631,6 +634,9 @@ impl Node {
})?;
let symlink: OsString = target_path.into();
let size = symlink.byte_size();
if size > u16::MAX as usize {
bail!("symlink content size 0x{:x} is too big", size);
}
self.inode.set_symlink_size(size);
self.set_symlink(symlink);
}
Expand Down
11 changes: 5 additions & 6 deletions rafs/src/builder/core/v6.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@ use super::chunk_dict::DigestWithBlobIndex;
use super::node::Node;
use crate::builder::{Bootstrap, BootstrapContext, BuildContext, ConversionType, Tree};
use crate::metadata::chunk::ChunkWrapper;
use crate::metadata::inode::new_v6_inode;
use crate::metadata::layout::v6::{
align_offset, calculate_nid, RafsV6BlobTable, RafsV6Device, RafsV6Dirent, RafsV6InodeChunkAddr,
RafsV6InodeChunkHeader, RafsV6OndiskInode, RafsV6SuperBlock, RafsV6SuperBlockExt,
EROFS_BLOCK_BITS_9, EROFS_BLOCK_SIZE_4096, EROFS_BLOCK_SIZE_512, EROFS_DEVTABLE_OFFSET,
EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE, EROFS_INODE_FLAT_PLAIN,
EROFS_INODE_SLOT_SIZE, EROFS_SUPER_BLOCK_SIZE, EROFS_SUPER_OFFSET,
align_offset, calculate_nid, new_v6_inode, RafsV6BlobTable, RafsV6Device, RafsV6Dirent,
RafsV6InodeChunkAddr, RafsV6InodeChunkHeader, RafsV6OndiskInode, RafsV6SuperBlock,
RafsV6SuperBlockExt, EROFS_BLOCK_BITS_9, EROFS_BLOCK_SIZE_4096, EROFS_BLOCK_SIZE_512,
EROFS_DEVTABLE_OFFSET, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE,
EROFS_INODE_FLAT_PLAIN, EROFS_INODE_SLOT_SIZE, EROFS_SUPER_BLOCK_SIZE, EROFS_SUPER_OFFSET,
};
use crate::metadata::RafsStore;
use crate::RafsIoWrite;
Expand Down
79 changes: 47 additions & 32 deletions rafs/src/builder/stargz.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ use super::core::overlay::Overlay;
use super::core::tree::Tree;
use super::{build_bootstrap, Builder};
use crate::metadata::chunk::ChunkWrapper;
use crate::metadata::inode::InodeWrapper;
use crate::metadata::layout::v5::{RafsV5ChunkInfo, RafsV5Inode, RafsV5InodeFlags};
use crate::metadata::inode::{InodeWrapper, RafsInodeFlags, RafsV6Inode};
use crate::metadata::layout::v5::{RafsV5ChunkInfo, RafsV5Inode};
use crate::metadata::layout::RafsXAttrs;
use crate::metadata::{Inode, RafsVersion};

Expand Down Expand Up @@ -543,16 +543,16 @@ impl StargzTreeBuilder {
let entry_path = entry.path()?;
let mut file_size = entry.size;
let mut flags = match version {
RafsVersion::V5 => RafsV5InodeFlags::default(),
RafsVersion::V6 => RafsV5InodeFlags::default(),
RafsVersion::V5 => RafsInodeFlags::default(),
RafsVersion::V6 => RafsInodeFlags::default(),
};

// Parse symlink
let (symlink, symlink_size) = if entry.is_symlink() {
let symlink_link_path = entry.symlink_link_path();
let symlink_size = symlink_link_path.byte_size() as u16;
file_size = symlink_size.into();
flags |= RafsV5InodeFlags::SYMLINK;
flags |= RafsInodeFlags::SYMLINK;
(Some(symlink_link_path.as_os_str().to_owned()), symlink_size)
} else {
(None, 0)
Expand All @@ -562,7 +562,7 @@ impl StargzTreeBuilder {
let mut xattrs = RafsXAttrs::new();
if entry.has_xattr() {
for (name, value) in entry.xattrs.iter() {
flags |= RafsV5InodeFlags::XATTR;
flags |= RafsInodeFlags::XATTR;
let value = base64::decode(value).with_context(|| {
format!(
"parse xattr name {:?} of file {:?} failed",
Expand All @@ -576,7 +576,7 @@ impl StargzTreeBuilder {
// Handle hardlink ino
let mut ino = (self.path_inode_map.len() + 1) as Inode;
if entry.is_hardlink() {
flags |= RafsV5InodeFlags::HARDLINK;
flags |= RafsInodeFlags::HARDLINK;
if let Some(_ino) = self.path_inode_map.get(&entry.hardlink_link_path()) {
ino = *_ino;
} else {
Expand All @@ -591,32 +591,47 @@ impl StargzTreeBuilder {
let uid = if explicit_uidgid { entry.uid } else { 0 };
let gid = if explicit_uidgid { entry.gid } else { 0 };

// Parse inode info
let v5_inode = RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: entry.mode(),
i_size: file_size,
i_nlink: entry.num_link,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: 0,
i_name_size: name_size,
i_symlink_size: symlink_size,
i_rdev: entry.rdev(),
// TODO: add mtime from entry.ModTime()
i_mtime: 0,
i_mtime_nsec: 0,
i_reserved: [0; 8],
};
let inode = match version {
RafsVersion::V5 => InodeWrapper::V5(v5_inode),
RafsVersion::V6 => InodeWrapper::V6(v5_inode),
RafsVersion::V5 => InodeWrapper::V5(RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: entry.mode(),
i_size: file_size,
i_nlink: entry.num_link,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: 0,
i_name_size: name_size,
i_symlink_size: symlink_size,
i_rdev: entry.rdev(),
// TODO: add mtime from entry.ModTime()
i_mtime: 0,
i_mtime_nsec: 0,
i_reserved: [0; 8],
}),
RafsVersion::V6 => InodeWrapper::V6(RafsV6Inode {
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: entry.mode(),
i_size: file_size,
i_nlink: entry.num_link,
i_blocks: 0,
i_flags: flags,
i_child_count: 0,
i_name_size: name_size,
i_symlink_size: symlink_size,
i_rdev: entry.rdev(),
// TODO: add mtime from entry.ModTime()
i_mtime: 0,
i_mtime_nsec: 0,
}),
};

let path = entry.path()?;
Expand Down
77 changes: 47 additions & 30 deletions rafs/src/builder/tarball.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ use super::core::node::{Node, NodeInfo};
use super::core::overlay::Overlay;
use super::core::tree::Tree;
use super::{build_bootstrap, dump_bootstrap, finalize_blob, Builder};
use crate::metadata::inode::InodeWrapper;
use crate::metadata::layout::v5::{RafsV5Inode, RafsV5InodeFlags};
use crate::metadata::inode::{InodeWrapper, RafsInodeFlags, RafsV6Inode};
use crate::metadata::layout::v5::RafsV5Inode;
use crate::metadata::layout::RafsXAttrs;
use crate::metadata::{Inode, RafsVersion};

Expand Down Expand Up @@ -229,8 +229,8 @@ impl<'a> TarballTreeBuilder<'a> {
let (uid, gid) = Self::get_uid_gid(self.ctx, header)?;
let mtime = header.mtime().unwrap_or_default();
let mut flags = match self.ctx.fs_version {
RafsVersion::V5 => RafsV5InodeFlags::default(),
RafsVersion::V6 => RafsV5InodeFlags::default(),
RafsVersion::V5 => RafsInodeFlags::default(),
RafsVersion::V6 => RafsInodeFlags::default(),
};

// Parse special files
Expand Down Expand Up @@ -262,7 +262,7 @@ impl<'a> TarballTreeBuilder<'a> {
bail!("tarball: symlink target from tar entry is too big");
}
file_size = symlink_size as u64;
flags |= RafsV5InodeFlags::SYMLINK;
flags |= RafsInodeFlags::SYMLINK;
(
Some(symlink_link_path.as_os_str().to_owned()),
symlink_size as u16,
Expand Down Expand Up @@ -299,7 +299,7 @@ impl<'a> TarballTreeBuilder<'a> {
path.as_ref().display()
);
}
flags |= RafsV5InodeFlags::HARDLINK;
flags |= RafsInodeFlags::HARDLINK;
} else {
self.path_inode_map
.insert(path.as_ref().to_path_buf(), (ino, nodes.len()));
Expand Down Expand Up @@ -328,30 +328,45 @@ impl<'a> TarballTreeBuilder<'a> {
}
}

let v5_inode = RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: mode,
i_size: file_size,
i_nlink: 1,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: child_count as u32,
i_name_size: name.len() as u16,
i_symlink_size: symlink_size,
i_rdev: rdev,
i_mtime: mtime,
i_mtime_nsec: 0,
i_reserved: [0; 8],
};
let mut inode = match self.ctx.fs_version {
RafsVersion::V5 => InodeWrapper::V5(v5_inode),
RafsVersion::V6 => InodeWrapper::V6(v5_inode),
RafsVersion::V5 => InodeWrapper::V5(RafsV5Inode {
i_digest: RafsDigest::default(),
i_parent: 0,
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: mode,
i_size: file_size,
i_nlink: 1,
i_blocks: 0,
i_flags: flags,
i_child_index: 0,
i_child_count: child_count as u32,
i_name_size: name.len() as u16,
i_symlink_size: symlink_size,
i_rdev: rdev,
i_mtime: mtime,
i_mtime_nsec: 0,
i_reserved: [0; 8],
}),
RafsVersion::V6 => InodeWrapper::V6(RafsV6Inode {
i_ino: ino,
i_projid: 0,
i_uid: uid,
i_gid: gid,
i_mode: mode,
i_size: file_size,
i_nlink: 1,
i_blocks: 0,
i_flags: flags,
i_child_count: child_count as u32,
i_name_size: name.len() as u16,
i_symlink_size: symlink_size,
i_rdev: rdev,
i_mtime: mtime,
i_mtime_nsec: 0,
}),
};
inode.set_has_xattr(!xattrs.is_empty());

Expand Down Expand Up @@ -391,7 +406,9 @@ impl<'a> TarballTreeBuilder<'a> {
// the associated regular file.
if entry_type.is_hard_link() {
let n = &nodes[index];
node.inode.set_digest(*n.inode.digest());
if n.inode.is_v5() {
node.inode.set_digest(*n.inode.digest());
}
node.inode.set_size(n.inode.size());
node.inode.set_child_count(n.inode.child_count());
node.chunks = n.chunks.clone();
Expand Down
9 changes: 5 additions & 4 deletions rafs/src/metadata/cached_v5.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,10 @@ use nydus_storage::device::{BlobChunkFlags, BlobChunkInfo, BlobDevice, BlobInfo}
use nydus_utils::digest::RafsDigest;
use nydus_utils::ByteSize;

use crate::metadata::inode::RafsInodeFlags;
use crate::metadata::layout::v5::{
rafsv5_alloc_bio_vecs, rafsv5_validate_inode, RafsV5BlobTable, RafsV5ChunkInfo, RafsV5Inode,
RafsV5InodeChunkOps, RafsV5InodeFlags, RafsV5InodeOps, RafsV5XAttrsTable, RAFSV5_ALIGNMENT,
RafsV5InodeChunkOps, RafsV5InodeOps, RafsV5XAttrsTable, RAFSV5_ALIGNMENT,
};
use crate::metadata::layout::{bytes_to_os_str, parse_xattr, RAFS_V5_ROOT_INODE};
use crate::metadata::{
Expand Down Expand Up @@ -236,7 +237,7 @@ pub struct CachedInodeV5 {
i_projid: u32,
i_uid: u32,
i_gid: u32,
i_flags: RafsV5InodeFlags,
i_flags: RafsInodeFlags,
i_size: u64,
i_blocks: u64,
i_nlink: u32,
Expand Down Expand Up @@ -481,7 +482,7 @@ impl RafsInode for CachedInodeV5 {

#[inline]
fn has_xattr(&self) -> bool {
self.i_flags.contains(RafsV5InodeFlags::XATTR)
self.i_flags.contains(RafsInodeFlags::XATTR)
}

#[inline]
Expand Down Expand Up @@ -664,7 +665,7 @@ impl RafsV5InodeOps for CachedInodeV5 {
}

fn has_hole(&self) -> bool {
self.i_flags.contains(RafsV5InodeFlags::HAS_HOLE)
self.i_flags.contains(RafsInodeFlags::HAS_HOLE)
}
}

Expand Down
Loading