Introduce new tarfs mode to Nydus

rafs/src/builder/core/blob.rs

@@ -55,6 +55,7 @@ impl Blob {
                 Self::finalize_blob_data(ctx, blob_mgr, blob_writer)?;
             }
             ConversionType::TarToRef
+            | ConversionType::TarToTarfs
             | ConversionType::TargzToRef
             | ConversionType::EStargzToRef => {
                 // Use `sha256(tarball)` as `blob_id` for ref-type conversions.
@@ -68,6 +69,9 @@ impl Blob {
                         }
                     } else if let Some(tar_reader) = &ctx.blob_tar_reader {
                         blob_ctx.compressed_blob_size = tar_reader.position();
+                        if ctx.conversion_type == ConversionType::TarToTarfs {
+                            blob_ctx.uncompressed_blob_size = blob_ctx.compressed_blob_size;
+                        }
                         if blob_ctx.blob_id.is_empty() {
                             let hash = tar_reader.get_hash_object();
                             blob_ctx.blob_id = format!("{:x}", hash.finalize());

rafs/src/builder/core/bootstrap.rs

@@ -152,6 +152,7 @@ impl Bootstrap {
         let index = nodes.len() as u32 + 1;
         let parent = &mut nodes[tree.node.index as usize - 1];
         let parent_ino = parent.inode.ino();
+        let block_size = ctx.v6_block_size();
 
         // Maybe the parent is not a directory in multi-layers build scenario, so we check here.
         if parent.is_dir() {
@@ -162,7 +163,8 @@ impl Bootstrap {
             parent.inode.set_child_index(index);
             parent.inode.set_child_count(tree.children.len() as u32);
             if ctx.fs_version.is_v6() {
-                parent.v6_set_dir_offset(bootstrap_ctx, tree.node.v6_dirent_size(tree)?)?;
+                let d_size = tree.node.v6_dirent_size(ctx, tree)?;
+                parent.v6_set_dir_offset(bootstrap_ctx, d_size, block_size)?;
             }
         }
 
@@ -216,7 +218,7 @@ impl Bootstrap {
             if !child.node.is_dir() && ctx.fs_version.is_v6() {
                 child
                     .node
-                    .v6_set_offset(bootstrap_ctx, v6_hardlink_offset)?;
+                    .v6_set_offset(bootstrap_ctx, v6_hardlink_offset, block_size)?;
             }
 
             // Store node for bootstrap & blob dump.

rafs/src/builder/core/context.rs

@@ -30,7 +30,7 @@ use nydus_storage::meta::{
     BlobMetaChunkInfo, ZranContextGenerator,
 };
 use nydus_utils::digest::DigestData;
-use nydus_utils::{compress, digest, div_round_up, round_down_4k, BufReaderInfo};
+use nydus_utils::{compress, digest, div_round_up, round_down, BufReaderInfo};
 
 use super::node::{ChunkSource, Node};
 use crate::builder::{
@@ -62,6 +62,7 @@ pub enum ConversionType {
     TarToStargz,
     TarToRafs,
     TarToRef,
+    TarToTarfs,
 }
 
 impl Default for ConversionType {
@@ -85,6 +86,7 @@ impl FromStr for ConversionType {
             "targz-ref" => Ok(Self::TargzToRef),
             "tar-rafs" => Ok(Self::TarToRafs),
             "tar-stargz" => Ok(Self::TarToStargz),
+            "tar-tarfs" => Ok(Self::TarToTarfs),
             // kept for backward compatibility
             "directory" => Ok(Self::DirectoryToRafs),
             "stargz_index" => Ok(Self::EStargzIndexToRef),
@@ -106,8 +108,9 @@ impl fmt::Display for ConversionType {
             ConversionType::TargzToStargz => write!(f, "targz-ref"),
             ConversionType::TargzToRef => write!(f, "targz-ref"),
             ConversionType::TarToRafs => write!(f, "tar-rafs"),
-            ConversionType::TarToStargz => write!(f, "tar-stargz"),
             ConversionType::TarToRef => write!(f, "tar-ref"),
+            ConversionType::TarToStargz => write!(f, "tar-stargz"),
+            ConversionType::TarToTarfs => write!(f, "tar-tarfs"),
         }
     }
 }
@@ -120,6 +123,7 @@ impl ConversionType {
                 | ConversionType::EStargzIndexToRef
                 | ConversionType::TargzToRef
                 | ConversionType::TarToRef
+                | ConversionType::TarToTarfs
         )
     }
 }
@@ -478,6 +482,9 @@ impl BlobContext {
         blob_ctx
             .blob_meta_header
             .set_cap_tar_toc(features.contains(BlobFeatures::CAP_TAR_TOC));
+        blob_ctx
+            .blob_meta_header
+            .set_tarfs(features.contains(BlobFeatures::TARFS));
 
         blob_ctx
     }
@@ -703,6 +710,7 @@ impl BlobContext {
     /// Get offset of compressed blob, since current_compressed_offset
     /// is always >= compressed_blob_size, we can safely subtract here.
     pub fn compressed_offset(&self) -> u64 {
+        assert!(self.current_compressed_offset >= self.compressed_blob_size);
         self.current_compressed_offset - self.compressed_blob_size
     }
 }
@@ -745,7 +753,9 @@ impl BlobManager {
             ctx.digester,
         );
         blob_ctx.set_chunk_size(ctx.chunk_size);
-        blob_ctx.set_meta_info_enabled(ctx.fs_version == RafsVersion::V6);
+        blob_ctx.set_meta_info_enabled(
+            ctx.fs_version == RafsVersion::V6 && ctx.conversion_type != ConversionType::TarToTarfs,
+        );
 
         Ok(blob_ctx)
     }
@@ -990,19 +1000,22 @@ impl BootstrapContext {
     // Try to find an used block with no less than `size` space left.
     // If found it, return the offset where we can store data.
     // If not, return 0.
-    pub(crate) fn allocate_available_block(&mut self, size: u64) -> u64 {
-        if size >= EROFS_BLOCK_SIZE_4096 {
+    pub(crate) fn allocate_available_block(&mut self, size: u64, block_size: u64) -> u64 {
+        if size >= block_size {
             return 0;
         }
 
         let min_idx = div_round_up(size, EROFS_INODE_SLOT_SIZE as u64) as usize;
-        let max_idx = div_round_up(EROFS_BLOCK_SIZE_4096, EROFS_INODE_SLOT_SIZE as u64) as usize;
+        let max_idx = div_round_up(block_size, EROFS_INODE_SLOT_SIZE as u64) as usize;
 
         for idx in min_idx..max_idx {
             let blocks = &mut self.v6_available_blocks[idx];
             if let Some(mut offset) = blocks.pop_front() {
-                offset += EROFS_BLOCK_SIZE_4096 - (idx * EROFS_INODE_SLOT_SIZE) as u64;
-                self.append_available_block(offset + (min_idx * EROFS_INODE_SLOT_SIZE) as u64);
+                offset += block_size - (idx * EROFS_INODE_SLOT_SIZE) as u64;
+                self.append_available_block(
+                    offset + (min_idx * EROFS_INODE_SLOT_SIZE) as u64,
+                    block_size,
+                );
                 return offset;
             }
         }
@@ -1011,11 +1024,11 @@ impl BootstrapContext {
     }
 
     // Append the block that `offset` belongs to corresponding deque.
-    pub(crate) fn append_available_block(&mut self, offset: u64) {
-        if offset % EROFS_BLOCK_SIZE_4096 != 0 {
-            let avail = EROFS_BLOCK_SIZE_4096 - offset % EROFS_BLOCK_SIZE_4096;
+    pub(crate) fn append_available_block(&mut self, offset: u64, block_size: u64) {
+        if offset % block_size != 0 {
+            let avail = block_size - offset % block_size;
             let idx = avail as usize / EROFS_INODE_SLOT_SIZE;
-            self.v6_available_blocks[idx].push_back(round_down_4k(offset));
+            self.v6_available_blocks[idx].push_back(round_down(offset, block_size));
         }
     }
 }
@@ -1114,6 +1127,9 @@ impl BuildContext {
             blob_features |= BlobFeatures::HAS_TOC;
             blob_features |= BlobFeatures::HAS_TAR_HEADER;
         }
+        if conversion_type == ConversionType::TarToTarfs {
+            blob_features |= BlobFeatures::TARFS;
+        }
 
         BuildContext {
             blob_id,

rafs/src/builder/core/node.rs

@@ -24,7 +24,9 @@ use nydus_utils::digest::{DigestHasher, RafsDigest};
 use nydus_utils::{div_round_up, event_tracer, root_tracer, try_round_up_4k, ByteSize};
 use sha2::digest::Digest;
 
-use crate::builder::{ArtifactWriter, BlobContext, BlobManager, BuildContext, ChunkDict, Overlay};
+use crate::builder::{
+    ArtifactWriter, BlobContext, BlobManager, BuildContext, ChunkDict, ConversionType, Overlay,
+};
 use crate::metadata::chunk::ChunkWrapper;
 use crate::metadata::inode::InodeWrapper;
 use crate::metadata::layout::v6::EROFS_INODE_FLAT_PLAIN;
@@ -270,35 +272,45 @@ impl Node {
             };
 
             let chunk_data = &mut data_buf[0..uncompressed_size as usize];
-            let (chunk, chunk_info) = self.read_file_chunk(ctx, reader, chunk_data)?;
+            let (mut chunk, chunk_info) = self.read_file_chunk(ctx, reader, chunk_data)?;
             if let Some(h) = inode_hasher.as_mut() {
                 h.digest_update(chunk.id().as_ref());
             }
 
-            let mut chunk = match self.deduplicate_chunk(
-                ctx,
-                blob_mgr,
-                file_offset,
-                uncompressed_size,
-                chunk,
-            )? {
-                None => continue,
-                Some(c) => c,
-            };
+            // No need to perform chunk deduplication for tar-tarfs case.
+            if ctx.conversion_type != ConversionType::TarToTarfs {
+                chunk = match self.deduplicate_chunk(
+                    ctx,
+                    blob_mgr,
+                    file_offset,
+                    uncompressed_size,
+                    chunk,
+                )? {
+                    None => continue,
+                    Some(c) => c,
+                };
+            }
 
             let (blob_index, blob_ctx) = blob_mgr.get_or_create_current_blob(ctx)?;
             let chunk_index = blob_ctx.alloc_chunk_index()?;
             chunk.set_blob_index(blob_index);
             chunk.set_index(chunk_index);
             chunk.set_file_offset(file_offset);
-            self.dump_file_chunk(ctx, blob_ctx, blob_writer, chunk_data, &mut chunk)?;
+            if ctx.conversion_type == ConversionType::TarToTarfs {
+                chunk.set_uncompressed_offset(chunk.compressed_offset());
+                chunk.set_uncompressed_size(chunk.compressed_size());
+            } else {
+                self.dump_file_chunk(ctx, blob_ctx, blob_writer, chunk_data, &mut chunk)?;
+            }
 
             let chunk = Arc::new(chunk);
             blob_size += chunk.compressed_size() as u64;
-            blob_ctx.add_chunk_meta_info(&chunk, chunk_info)?;
-            blob_mgr
-                .layered_chunk_dict
-                .add_chunk(chunk.clone(), ctx.digester);
+            if ctx.conversion_type != ConversionType::TarToTarfs {
+                blob_ctx.add_chunk_meta_info(&chunk, chunk_info)?;
+                blob_mgr
+                    .layered_chunk_dict
+                    .add_chunk(chunk.clone(), ctx.digester);
+            }
             self.chunks.push(NodeChunk {
                 source: ChunkSource::Build,
                 inner: chunk,
@@ -347,7 +359,11 @@ impl Node {
                 .with_context(|| format!("failed to read node file {:?}", self.path()))?;
         }
 
-        chunk.set_id(RafsDigest::from_buf(buf, ctx.digester));
+        // For tar-tarfs case, no need to compute chunk id.
+        if ctx.conversion_type != ConversionType::TarToTarfs {
+            chunk.set_id(RafsDigest::from_buf(buf, ctx.digester));
+        }
+
         Ok((chunk, chunk_info))
     }