From c015e3ebe63dd787b75827fefb63ffb6032a0bbf Mon Sep 17 00:00:00 2001 From: hijackthe2 <2948278083@qq.com> Date: Fri, 3 Nov 2023 10:05:52 +0800 Subject: [PATCH] builder: add some unit test cases Some unit test cases are added for compact.rs, lib.rs, merge.rs, stargz.rs, core/context.rs, and core/node.rs in builder/src to increase code coverage. Signed-off-by: hijackthe2 <2948278083@qq.com> --- builder/src/compact.rs | 654 ++++++++++++++++++++++++++++++++++++ builder/src/core/context.rs | 68 ++++ builder/src/core/node.rs | 217 ++++++++++++ builder/src/lib.rs | 43 +++ builder/src/merge.rs | 98 ++++++ builder/src/stargz.rs | 114 ++++++- 6 files changed, 1188 insertions(+), 6 deletions(-) diff --git a/builder/src/compact.rs b/builder/src/compact.rs index 5847b97371d..97cd6594b17 100644 --- a/builder/src/compact.rs +++ b/builder/src/compact.rs @@ -659,3 +659,657 @@ impl BlobCompactor { )?)) } } + +#[cfg(test)] +mod tests { + use crate::core::node::Node; + use crate::HashChunkDict; + use crate::{NodeChunk, Overlay}; + + use super::*; + use nydus_api::ConfigV2; + use nydus_rafs::metadata::RafsSuperConfig; + use nydus_storage::backend::{BackendResult, BlobReader}; + use nydus_storage::device::v5::BlobV5ChunkInfo; + use nydus_storage::device::{BlobChunkFlags, BlobChunkInfo, BlobFeatures}; + use nydus_storage::RAFS_DEFAULT_CHUNK_SIZE; + use nydus_utils::crypt::Algorithm; + use nydus_utils::metrics::BackendMetrics; + use nydus_utils::{compress, crypt}; + use std::any::Any; + use vmm_sys_util::tempdir::TempDir; + use vmm_sys_util::tempfile::TempFile; + + #[doc(hidden)] + #[macro_export] + macro_rules! impl_getter { + ($G: ident, $F: ident, $U: ty) => { + fn $G(&self) -> $U { + self.$F + } + }; + } + + #[derive(Default, Clone)] + struct MockChunkInfo { + pub block_id: RafsDigest, + pub blob_index: u32, + pub flags: BlobChunkFlags, + pub compress_size: u32, + pub uncompress_size: u32, + pub compress_offset: u64, + pub uncompress_offset: u64, + pub file_offset: u64, + pub index: u32, + #[allow(unused)] + pub reserved: u32, + } + + impl BlobChunkInfo for MockChunkInfo { + fn chunk_id(&self) -> &RafsDigest { + &self.block_id + } + fn id(&self) -> u32 { + self.index + } + fn is_compressed(&self) -> bool { + self.flags.contains(BlobChunkFlags::COMPRESSED) + } + + fn is_encrypted(&self) -> bool { + false + } + + fn as_any(&self) -> &dyn Any { + self + } + + impl_getter!(blob_index, blob_index, u32); + impl_getter!(compressed_offset, compress_offset, u64); + impl_getter!(compressed_size, compress_size, u32); + impl_getter!(uncompressed_offset, uncompress_offset, u64); + impl_getter!(uncompressed_size, uncompress_size, u32); + } + + impl BlobV5ChunkInfo for MockChunkInfo { + fn as_base(&self) -> &dyn BlobChunkInfo { + self + } + + impl_getter!(index, index, u32); + impl_getter!(file_offset, file_offset, u64); + impl_getter!(flags, flags, BlobChunkFlags); + } + + struct MockBackend { + pub metrics: Arc, + } + + impl BlobReader for MockBackend { + fn blob_size(&self) -> BackendResult { + Ok(1) + } + + fn try_read(&self, buf: &mut [u8], _offset: u64) -> BackendResult { + let mut i = 0; + while i < buf.len() { + buf[i] = i as u8; + i += 1; + } + Ok(i) + } + + fn metrics(&self) -> &BackendMetrics { + // Safe because nydusd must have backend attached with id, only image builder can no id + // but use backend instance to upload blob. + &self.metrics + } + } + + unsafe impl Send for MockBackend {} + unsafe impl Sync for MockBackend {} + + impl BlobBackend for MockBackend { + fn shutdown(&self) {} + + fn metrics(&self) -> &BackendMetrics { + // Safe because nydusd must have backend attached with id, only image builder can no id + // but use backend instance to upload blob. + &self.metrics + } + + fn get_reader(&self, _blob_id: &str) -> BackendResult> { + Ok(Arc::new(MockBackend { + metrics: self.metrics.clone(), + })) + } + } + + #[test] + #[should_panic = "not implemented: unsupport ChunkWrapper::Ref(c)"] + fn test_chunk_key_from() { + let cw = ChunkWrapper::new(RafsVersion::V5); + matches!(ChunkKey::from(&cw), ChunkKey::Digest(_)); + + let cw = ChunkWrapper::new(RafsVersion::V6); + matches!(ChunkKey::from(&cw), ChunkKey::Offset(_, _)); + + let chunk = Arc::new(MockChunkInfo { + block_id: Default::default(), + blob_index: 2, + flags: BlobChunkFlags::empty(), + compress_size: 0x800, + uncompress_size: 0x1000, + compress_offset: 0x800, + uncompress_offset: 0x1000, + file_offset: 0x1000, + index: 1, + reserved: 0, + }) as Arc; + let cw = ChunkWrapper::Ref(chunk); + ChunkKey::from(&cw); + } + + #[test] + fn test_chunk_set() { + let mut chunk_set1 = ChunkSet::new(); + + let mut chunk_wrapper1 = ChunkWrapper::new(RafsVersion::V5); + chunk_wrapper1.set_id(RafsDigest { data: [1u8; 32] }); + chunk_wrapper1.set_compressed_size(8); + let mut chunk_wrapper2 = ChunkWrapper::new(RafsVersion::V6); + chunk_wrapper2.set_compressed_size(16); + + chunk_set1.add_chunk(&chunk_wrapper1); + chunk_set1.add_chunk(&chunk_wrapper2); + assert_eq!(chunk_set1.total_size, 24); + + let chunk_key2 = ChunkKey::from(&chunk_wrapper2); + assert_eq!( + format!("{:?}", Some(chunk_wrapper2)), + format!("{:?}", chunk_set1.get_chunk(&chunk_key2)) + ); + + let mut chunk_wrapper3 = ChunkWrapper::new(RafsVersion::V5); + chunk_wrapper3.set_id(RafsDigest { data: [3u8; 32] }); + chunk_wrapper3.set_compressed_size(32); + + let mut chunk_set2 = ChunkSet::new(); + chunk_set2.add_chunk(&chunk_wrapper3); + chunk_set2.merge(chunk_set1); + assert_eq!(chunk_set2.total_size, 56); + assert_eq!(chunk_set2.chunks.len(), 3); + + let build_ctx = BuildContext::default(); + let tmp_file = TempFile::new().unwrap(); + let blob_storage = ArtifactStorage::SingleFile(PathBuf::from(tmp_file.as_path())); + let cipher_object = Algorithm::Aes256Xts.new_cipher().unwrap(); + let mut new_blob_ctx = BlobContext::new( + "blob_id".to_owned(), + 0, + BlobFeatures::all(), + compress::Algorithm::Lz4Block, + digest::Algorithm::Sha256, + crypt::Algorithm::Aes256Xts, + Arc::new(cipher_object), + None, + ); + let ori_blob_ids = ["1".to_owned(), "2".to_owned()]; + let backend = Arc::new(MockBackend { + metrics: BackendMetrics::new("id", "backend_type"), + }) as Arc; + + let mut res = chunk_set2 + .dump( + &build_ctx, + blob_storage, + &ori_blob_ids, + &mut new_blob_ctx, + 0, + true, + &backend, + ) + .unwrap(); + + res.sort_by(|a, b| a.0.id().data.cmp(&b.0.id().data)); + + assert_eq!(res.len(), 3); + assert_eq!( + format!("{:?}", res[0].1.id()), + format!("{:?}", RafsDigest { data: [0u8; 32] }) + ); + assert_eq!( + format!("{:?}", res[1].1.id()), + format!("{:?}", RafsDigest { data: [1u8; 32] }) + ); + assert_eq!( + format!("{:?}", res[2].1.id()), + format!("{:?}", RafsDigest { data: [3u8; 32] }) + ); + } + + #[test] + fn test_state() { + let state = State::Rebuild(ChunkSet::new()); + assert!(state.is_rebuild()); + let state = State::ChunkDict; + assert!(state.is_from_dict()); + let state = State::default(); + assert!(state.is_invalid()); + + let mut chunk_set1 = ChunkSet::new(); + let mut chunk_wrapper1 = ChunkWrapper::new(RafsVersion::V5); + chunk_wrapper1.set_id(RafsDigest { data: [1u8; 32] }); + chunk_wrapper1.set_compressed_size(8); + chunk_set1.add_chunk(&chunk_wrapper1); + let mut state1 = State::Original(chunk_set1); + assert_eq!(state1.chunk_total_size().unwrap(), 8); + + let mut chunk_wrapper2 = ChunkWrapper::new(RafsVersion::V6); + chunk_wrapper2.set_compressed_size(16); + let mut chunk_set2 = ChunkSet::new(); + chunk_set2.add_chunk(&chunk_wrapper2); + let mut state2 = State::Rebuild(chunk_set2); + assert_eq!(state2.chunk_total_size().unwrap(), 16); + + assert!(state1.merge_blob(state2.clone()).is_err()); + assert!(state2.merge_blob(state1).is_ok()); + assert!(state2.merge_blob(State::Invalid).is_err()); + + assert_eq!(state2.chunk_total_size().unwrap(), 24); + assert!(State::Delete.chunk_total_size().is_err()); + } + + #[test] + fn test_apply_chunk_change() { + let mut chunk_wrapper1 = ChunkWrapper::new(RafsVersion::V5); + chunk_wrapper1.set_id(RafsDigest { data: [1u8; 32] }); + chunk_wrapper1.set_uncompressed_size(8); + chunk_wrapper1.set_compressed_size(8); + + let mut chunk_wrapper2 = ChunkWrapper::new(RafsVersion::V6); + chunk_wrapper2.set_uncompressed_size(16); + chunk_wrapper2.set_compressed_size(16); + + assert!(apply_chunk_change(&chunk_wrapper1, &mut chunk_wrapper2).is_err()); + chunk_wrapper2.set_uncompressed_size(8); + assert!(apply_chunk_change(&chunk_wrapper1, &mut chunk_wrapper2).is_err()); + + chunk_wrapper2.set_compressed_size(8); + chunk_wrapper1.set_blob_index(0x10); + chunk_wrapper1.set_index(0x20); + chunk_wrapper1.set_uncompressed_offset(0x30); + chunk_wrapper1.set_compressed_offset(0x40); + assert!(apply_chunk_change(&chunk_wrapper1, &mut chunk_wrapper2).is_ok()); + assert_eq!(chunk_wrapper2.blob_index(), 0x10); + assert_eq!(chunk_wrapper2.index(), 0x20); + assert_eq!(chunk_wrapper2.uncompressed_offset(), 0x30); + assert_eq!(chunk_wrapper2.compressed_offset(), 0x40); + } + + fn create_blob_compactor() -> Result { + let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); + let mut source_path = PathBuf::from(root_dir); + source_path.push("../tests/texture/bootstrap/rafs-v5.boot"); + let path = source_path.to_str().unwrap(); + let rafs_config = RafsSuperConfig { + version: RafsVersion::V5, + compressor: compress::Algorithm::Lz4Block, + digester: digest::Algorithm::Blake3, + chunk_size: 0x100000, + batch_size: 0, + explicit_uidgid: true, + is_tarfs_mode: false, + }; + let dict = + HashChunkDict::from_commandline_arg(path, Arc::new(ConfigV2::default()), &rafs_config) + .unwrap(); + + let mut ori_blob_mgr = BlobManager::new(digest::Algorithm::Sha256); + ori_blob_mgr.set_chunk_dict(dict); + + let backend = Arc::new(MockBackend { + metrics: BackendMetrics::new("id", "backend_type"), + }); + + let tmpdir = TempDir::new()?; + let tmpfile = TempFile::new_in(tmpdir.as_path())?; + let node = Node::from_fs_object( + RafsVersion::V6, + tmpdir.as_path().to_path_buf(), + tmpfile.as_path().to_path_buf(), + Overlay::UpperAddition, + RAFS_DEFAULT_CHUNK_SIZE as u32, + true, + false, + )?; + let tree = Tree::new(node); + let bootstrap = Bootstrap::new(tree)?; + + BlobCompactor::new( + RafsVersion::V6, + ori_blob_mgr, + backend, + digest::Algorithm::Sha256, + &bootstrap, + ) + } + + #[test] + fn test_blob_compactor_new() { + let compactor = create_blob_compactor(); + assert!(compactor.is_ok()); + assert!(compactor.unwrap().is_v6()); + } + + #[test] + fn test_blob_compactor_load_chunk_dict_blobs() { + let mut compactor = create_blob_compactor().unwrap(); + let chunk_dict = compactor.get_chunk_dict(); + let n = chunk_dict.get_blobs().len(); + for i in 0..n { + chunk_dict.set_real_blob_idx(i as u32, i as u32); + } + compactor.states = vec![State::default(); n + 1]; + compactor.load_chunk_dict_blobs(); + + assert_eq!(compactor.states.len(), n + 1); + assert!(compactor.states[0].is_from_dict()); + assert!(compactor.states[n >> 1].is_from_dict()); + assert!(compactor.states[n - 1].is_from_dict()); + assert!(!compactor.states[n].is_from_dict()); + } + + fn blob_compactor_load_and_dedup_chunks() -> Result { + let mut compactor = create_blob_compactor()?; + + let mut chunk1 = ChunkWrapper::new(RafsVersion::V5); + chunk1.set_id(RafsDigest { data: [1u8; 32] }); + chunk1.set_uncompressed_size(0); + chunk1.set_compressed_offset(0x11); + chunk1.set_blob_index(1); + let node_chunk1 = NodeChunk { + source: crate::ChunkSource::Dict, + inner: Arc::new(chunk1.clone()), + }; + let mut chunk2 = ChunkWrapper::new(RafsVersion::V6); + chunk2.set_id(RafsDigest { data: [2u8; 32] }); + chunk2.set_uncompressed_size(0x20); + chunk2.set_compressed_offset(0x22); + chunk2.set_blob_index(2); + let node_chunk2 = NodeChunk { + source: crate::ChunkSource::Dict, + inner: Arc::new(chunk2.clone()), + }; + let mut chunk3 = ChunkWrapper::new(RafsVersion::V6); + chunk3.set_id(RafsDigest { data: [3u8; 32] }); + chunk3.set_uncompressed_size(0x20); + chunk3.set_compressed_offset(0x22); + chunk3.set_blob_index(2); + let node_chunk3 = NodeChunk { + source: crate::ChunkSource::Dict, + inner: Arc::new(chunk3.clone()), + }; + + let mut chunk_dict = HashChunkDict::new(digest::Algorithm::Sha256); + chunk_dict.add_chunk( + Arc::new(ChunkWrapper::new(RafsVersion::V5)), + digest::Algorithm::Sha256, + ); + chunk_dict.add_chunk(Arc::new(chunk1.clone()), digest::Algorithm::Sha256); + compactor.ori_blob_mgr.set_chunk_dict(Arc::new(chunk_dict)); + + compactor.states = vec![State::ChunkDict; 5]; + + let tmpdir = TempDir::new()?; + let tmpfile = TempFile::new_in(tmpdir.as_path())?; + let node = Node::from_fs_object( + RafsVersion::V6, + tmpdir.as_path().to_path_buf(), + tmpfile.as_path().to_path_buf(), + Overlay::UpperAddition, + RAFS_DEFAULT_CHUNK_SIZE as u32, + true, + false, + )?; + let mut tree = Tree::new(node); + let tmpfile2 = TempFile::new_in(tmpdir.as_path())?; + let mut node = Node::from_fs_object( + RafsVersion::V6, + tmpdir.as_path().to_path_buf(), + tmpfile2.as_path().to_path_buf(), + Overlay::UpperAddition, + RAFS_DEFAULT_CHUNK_SIZE as u32, + true, + false, + )?; + node.chunks.push(node_chunk1); + node.chunks.push(node_chunk2); + node.chunks.push(node_chunk3); + let tree2 = Tree::new(node); + tree.insert_child(tree2); + + let bootstrap = Bootstrap::new(tree)?; + + assert!(compactor.load_and_dedup_chunks(&bootstrap).is_ok()); + assert_eq!(compactor.c2nodes.len(), 2); + assert_eq!(compactor.b2nodes.len(), 2); + + let chunk_key1 = ChunkKey::from(&chunk1); + assert!(compactor.c2nodes.get(&chunk_key1).is_some()); + assert_eq!(compactor.c2nodes.get(&chunk_key1).unwrap().len(), 1); + assert!(compactor.b2nodes.get(&chunk2.blob_index()).is_some()); + assert_eq!( + compactor.b2nodes.get(&chunk2.blob_index()).unwrap().len(), + 2 + ); + + Ok(compactor) + } + + #[test] + fn test_blob_compactor_load_and_dedup_chunks() { + assert!(blob_compactor_load_and_dedup_chunks().is_ok()); + } + + #[test] + fn test_blob_compactor_dump_new_blobs() { + let tmp_dir = TempDir::new().unwrap(); + let build_ctx = BuildContext::new( + "build_ctx".to_string(), + false, + 0, + compress::Algorithm::Lz4Block, + digest::Algorithm::Sha256, + true, + WhiteoutSpec::None, + ConversionType::DirectoryToRafs, + PathBuf::from(tmp_dir.as_path()), + Default::default(), + None, + false, + Features::new(), + false, + ); + + let mut compactor = blob_compactor_load_and_dedup_chunks().unwrap(); + + let blob_ctx1 = BlobContext::new( + "blob_id1".to_owned(), + 0, + build_ctx.blob_features, + build_ctx.compressor, + build_ctx.digester, + build_ctx.cipher, + Default::default(), + None, + ); + let blob_ctx2 = BlobContext::new( + "blob_id2".to_owned(), + 0, + build_ctx.blob_features, + build_ctx.compressor, + build_ctx.digester, + build_ctx.cipher, + Default::default(), + None, + ); + let blob_ctx3 = BlobContext::new( + "blob_id3".to_owned(), + 0, + build_ctx.blob_features, + build_ctx.compressor, + build_ctx.digester, + build_ctx.cipher, + Default::default(), + None, + ); + let blob_ctx4 = BlobContext::new( + "blob_id4".to_owned(), + 0, + build_ctx.blob_features, + build_ctx.compressor, + build_ctx.digester, + build_ctx.cipher, + Default::default(), + None, + ); + let blob_ctx5 = BlobContext::new( + "blob_id5".to_owned(), + 0, + build_ctx.blob_features, + build_ctx.compressor, + build_ctx.digester, + build_ctx.cipher, + Default::default(), + None, + ); + compactor.ori_blob_mgr.add_blob(blob_ctx1); + compactor.ori_blob_mgr.add_blob(blob_ctx2); + compactor.ori_blob_mgr.add_blob(blob_ctx3); + compactor.ori_blob_mgr.add_blob(blob_ctx4); + compactor.ori_blob_mgr.add_blob(blob_ctx5); + + compactor.states[0] = State::Invalid; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.as_path().to_str().unwrap(); + assert!(compactor.dump_new_blobs(&build_ctx, dir, true).is_err()); + + compactor.states = vec![ + State::Delete, + State::ChunkDict, + State::Original(ChunkSet::new()), + State::Rebuild(ChunkSet::new()), + State::Delete, + ]; + assert!(compactor.dump_new_blobs(&build_ctx, dir, true).is_ok()); + assert_eq!(compactor.ori_blob_mgr.len(), 3); + } + + #[test] + fn test_blob_compactor_do_compact() { + let mut compactor = blob_compactor_load_and_dedup_chunks().unwrap(); + + let tmp_dir = TempDir::new().unwrap(); + let build_ctx = BuildContext::new( + "build_ctx".to_string(), + false, + 0, + compress::Algorithm::Lz4Block, + digest::Algorithm::Sha256, + true, + WhiteoutSpec::None, + ConversionType::DirectoryToRafs, + PathBuf::from(tmp_dir.as_path()), + Default::default(), + None, + false, + Features::new(), + false, + ); + let mut blob_ctx1 = BlobContext::new( + "blob_id1".to_owned(), + 0, + build_ctx.blob_features, + build_ctx.compressor, + build_ctx.digester, + build_ctx.cipher, + Default::default(), + None, + ); + blob_ctx1.compressed_blob_size = 2; + let mut blob_ctx2 = BlobContext::new( + "blob_id2".to_owned(), + 0, + build_ctx.blob_features, + build_ctx.compressor, + build_ctx.digester, + build_ctx.cipher, + Default::default(), + None, + ); + blob_ctx2.compressed_blob_size = 0; + let blob_ctx3 = BlobContext::new( + "blob_id3".to_owned(), + 0, + build_ctx.blob_features, + build_ctx.compressor, + build_ctx.digester, + build_ctx.cipher, + Default::default(), + None, + ); + let blob_ctx4 = BlobContext::new( + "blob_id4".to_owned(), + 0, + build_ctx.blob_features, + build_ctx.compressor, + build_ctx.digester, + build_ctx.cipher, + Default::default(), + None, + ); + let blob_ctx5 = BlobContext::new( + "blob_id5".to_owned(), + 0, + build_ctx.blob_features, + build_ctx.compressor, + build_ctx.digester, + build_ctx.cipher, + Default::default(), + None, + ); + compactor.ori_blob_mgr.add_blob(blob_ctx1); + compactor.ori_blob_mgr.add_blob(blob_ctx2); + compactor.ori_blob_mgr.add_blob(blob_ctx3); + compactor.ori_blob_mgr.add_blob(blob_ctx4); + compactor.ori_blob_mgr.add_blob(blob_ctx5); + + let mut chunk_set1 = ChunkSet::new(); + chunk_set1.total_size = 4; + let mut chunk_set2 = ChunkSet::new(); + chunk_set2.total_size = 6; + let mut chunk_set3 = ChunkSet::new(); + chunk_set3.total_size = 5; + + compactor.states = vec![ + State::Original(chunk_set1), + State::Original(chunk_set2), + State::Rebuild(chunk_set3), + State::ChunkDict, + State::Invalid, + ]; + + let cfg = Config { + min_used_ratio: 50, + compact_blob_size: 10, + max_compact_size: 8, + layers_to_compact: 0, + blobs_dir: "blobs_dir".to_string(), + }; + + assert!(compactor.do_compact(&cfg).is_ok()); + assert!(!compactor.states.last().unwrap().is_invalid()); + } +} diff --git a/builder/src/core/context.rs b/builder/src/core/context.rs index 1c21dae4a0e..b9290d7015f 100644 --- a/builder/src/core/context.rs +++ b/builder/src/core/context.rs @@ -1464,3 +1464,71 @@ impl BuildOutput { }) } } + +#[cfg(test)] +mod tests { + use std::sync::atomic::AtomicBool; + + use nydus_api::{BackendConfigV2, ConfigV2Internal, LocalFsConfig}; + + use super::*; + + #[test] + fn test_blob_context_from() { + let mut blob = BlobInfo::new( + 1, + "blob_id".to_string(), + 16, + 8, + 4, + 2, + BlobFeatures::INLINED_FS_META | BlobFeatures::SEPARATE | BlobFeatures::HAS_TOC, + ); + let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); + let mut source_path = PathBuf::from(root_dir); + source_path.push("../tests/texture/blobs/be7d77eeb719f70884758d1aa800ed0fb09d701aaec469964e9d54325f0d5fef"); + assert!(blob + .set_blob_id_from_meta_path(source_path.as_path()) + .is_ok()); + blob.set_blob_meta_size(2); + blob.set_blob_toc_size(2); + blob.set_blob_meta_digest([32u8; 32]); + blob.set_blob_toc_digest([64u8; 32]); + blob.set_blob_meta_info(1, 2, 4, 8); + + let mut ctx = BuildContext::default(); + ctx.configuration.internal.set_blob_accessible(true); + let config = ConfigV2 { + version: 2, + backend: Some(BackendConfigV2 { + backend_type: "localfs".to_owned(), + localdisk: None, + localfs: Some(LocalFsConfig { + blob_file: source_path.to_str().unwrap().to_owned(), + dir: "/tmp".to_owned(), + alt_dirs: vec!["/var/nydus/cache".to_owned()], + }), + oss: None, + s3: None, + registry: None, + http_proxy: None, + }), + id: "id".to_owned(), + cache: None, + rafs: None, + internal: ConfigV2Internal { + blob_accessible: Arc::new(AtomicBool::new(true)), + }, + }; + ctx.set_configuration(config.into()); + + let chunk_source = ChunkSource::Dict; + + let blob_ctx = BlobContext::from(&ctx, &blob, chunk_source); + + assert!(blob_ctx.is_ok()); + let blob_ctx = blob_ctx.unwrap(); + assert_eq!(blob_ctx.uncompressed_blob_size, 16); + assert!(blob_ctx.blob_meta_info_enabled); + } +} diff --git a/builder/src/core/node.rs b/builder/src/core/node.rs index 5e100813287..06fe9190971 100644 --- a/builder/src/core/node.rs +++ b/builder/src/core/node.rs @@ -868,3 +868,220 @@ impl Node { self.info = Arc::new(info); } } + +#[cfg(test)] +mod tests { + use std::io::BufReader; + + use nydus_utils::{digest, BufReaderInfo}; + use vmm_sys_util::tempfile::TempFile; + + use crate::{ArtifactWriter, BlobCacheGenerator, HashChunkDict}; + + use super::*; + + #[test] + fn test_node_chunk() { + let chunk_wrapper1 = ChunkWrapper::new(RafsVersion::V5); + let mut chunk = NodeChunk { + source: ChunkSource::Build, + inner: Arc::new(chunk_wrapper1), + }; + println!("NodeChunk: {}", chunk); + matches!(chunk.inner.deref().clone(), ChunkWrapper::V5(_)); + + let chunk_wrapper2 = ChunkWrapper::new(RafsVersion::V6); + chunk.copy_from(&chunk_wrapper2); + matches!(chunk.inner.deref().clone(), ChunkWrapper::V6(_)); + + chunk.set_index(0x10); + assert_eq!(chunk.inner.index(), 0x10); + chunk.set_blob_index(0x20); + assert_eq!(chunk.inner.blob_index(), 0x20); + chunk.set_compressed_size(0x30); + assert_eq!(chunk.inner.compressed_size(), 0x30); + chunk.set_file_offset(0x40); + assert_eq!(chunk.inner.file_offset(), 0x40); + } + + #[test] + fn test_node_dump_node_data() { + let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); + let mut source_path = PathBuf::from(root_dir); + source_path.push("../tests/texture/blobs/be7d77eeb719f70884758d1aa800ed0fb09d701aaec469964e9d54325f0d5fef"); + + let mut inode = InodeWrapper::new(RafsVersion::V5); + inode.set_child_count(2); + inode.set_size(20); + let info = NodeInfo { + explicit_uidgid: true, + src_ino: 1, + src_dev: u64::MAX, + rdev: u64::MAX, + path: source_path.clone(), + source: PathBuf::from("/"), + target: source_path.clone(), + target_vec: vec![OsString::from(source_path)], + symlink: Some(OsString::from("symlink")), + xattrs: RafsXAttrs::new(), + v6_force_extended_inode: false, + }; + let mut node = Node::new(inode, info, 1); + + let mut ctx = BuildContext::default(); + ctx.set_chunk_size(2); + ctx.conversion_type = ConversionType::TarToRef; + ctx.cipher = crypt::Algorithm::Aes128Xts; + let tmp_file1 = TempFile::new().unwrap(); + std::fs::write( + tmp_file1.as_path(), + "This is a test!\n".repeat(32).as_bytes(), + ) + .unwrap(); + let buf_reader = BufReader::new(tmp_file1.into_file()); + ctx.blob_tar_reader = Some(BufReaderInfo::from_buf_reader(buf_reader)); + let tmp_file2 = TempFile::new().unwrap(); + ctx.blob_cache_generator = Some( + BlobCacheGenerator::new(crate::ArtifactStorage::SingleFile(PathBuf::from( + tmp_file2.as_path(), + ))) + .unwrap(), + ); + + let mut blob_mgr = BlobManager::new(digest::Algorithm::Sha256); + let mut chunk_dict = HashChunkDict::new(digest::Algorithm::Sha256); + let mut chunk_wrapper = ChunkWrapper::new(RafsVersion::V5); + chunk_wrapper.set_id(RafsDigest { + data: [ + 209, 217, 144, 116, 135, 113, 3, 121, 133, 92, 96, 25, 219, 145, 151, 219, 119, 47, + 96, 147, 90, 51, 78, 44, 193, 149, 6, 102, 13, 173, 138, 191, + ], + }); + chunk_wrapper.set_uncompressed_size(2); + chunk_dict.add_chunk(Arc::new(chunk_wrapper), digest::Algorithm::Sha256); + blob_mgr.set_chunk_dict(Arc::new(chunk_dict)); + + let tmp_file3 = TempFile::new().unwrap(); + let mut blob_writer = ArtifactWriter::new(crate::ArtifactStorage::SingleFile( + PathBuf::from(tmp_file3.as_path()), + )) + .unwrap(); + + let mut chunk_data_buf = [1u8; 32]; + + node.inode.set_mode(0o755 | libc::S_IFDIR as u32); + let data_size = + node.dump_node_data(&ctx, &mut blob_mgr, &mut blob_writer, &mut chunk_data_buf); + assert!(data_size.is_ok()); + assert_eq!(data_size.unwrap(), 0); + + node.inode.set_mode(0o755 | libc::S_IFLNK as u32); + let data_size = + node.dump_node_data(&ctx, &mut blob_mgr, &mut blob_writer, &mut chunk_data_buf); + assert!(data_size.is_ok()); + assert_eq!(data_size.unwrap(), 0); + + node.inode.set_mode(0o755 | libc::S_IFBLK as u32); + let data_size = + node.dump_node_data(&ctx, &mut blob_mgr, &mut blob_writer, &mut chunk_data_buf); + assert!(data_size.is_ok()); + assert_eq!(data_size.unwrap(), 0); + + node.inode.set_mode(0o755 | libc::S_IFREG as u32); + let data_size = + node.dump_node_data(&ctx, &mut blob_mgr, &mut blob_writer, &mut chunk_data_buf); + assert!(data_size.is_ok()); + assert_eq!(data_size.unwrap(), 18); + } + + #[test] + fn test_node() { + let inode = InodeWrapper::new(RafsVersion::V5); + let info = NodeInfo { + explicit_uidgid: true, + src_ino: 1, + src_dev: u64::MAX, + rdev: u64::MAX, + path: PathBuf::new(), + source: PathBuf::new(), + target: PathBuf::new(), + target_vec: vec![OsString::new()], + symlink: None, + xattrs: RafsXAttrs::new(), + v6_force_extended_inode: false, + }; + + let mut inode1 = inode.clone(); + inode1.set_size(1 << 60); + inode1.set_mode(0o755 | libc::S_IFREG as u32); + let node = Node::new(inode1, info.clone(), 1); + assert!(node.chunk_count(2).is_err()); + + let mut inode2 = inode.clone(); + inode2.set_mode(0o755 | libc::S_IFCHR as u32); + let node = Node::new(inode2, info.clone(), 1); + assert!(node.chunk_count(2).is_ok()); + assert_eq!(node.chunk_count(2).unwrap(), 0); + + let mut inode3 = inode.clone(); + inode3.set_mode(0o755 | libc::S_IFLNK as u32); + let node = Node::new(inode3, info.clone(), 1); + assert_eq!(node.file_type(), "symlink"); + let mut inode4 = inode.clone(); + inode4.set_mode(0o755 | libc::S_IFDIR as u32); + let node = Node::new(inode4, info.clone(), 1); + assert_eq!(node.file_type(), "dir"); + let mut inode5 = inode.clone(); + inode5.set_mode(0o755 | libc::S_IFREG as u32); + let node = Node::new(inode5, info.clone(), 1); + assert_eq!(node.file_type(), "file"); + + let mut info1 = info.clone(); + info1.target_vec = vec![OsString::from("1"), OsString::from("2")]; + let node = Node::new(inode.clone(), info1, 1); + assert_eq!(node.name(), OsString::from("2").as_os_str()); + let mut info2 = info.clone(); + info2.target_vec = vec![]; + info2.path = PathBuf::from("/"); + info2.source = PathBuf::from("/"); + let node = Node::new(inode.clone(), info2, 1); + assert_eq!(node.name(), OsStr::from_bytes(ROOT_PATH_NAME)); + let mut info3 = info.clone(); + info3.target_vec = vec![]; + info3.path = PathBuf::from("/1"); + info3.source = PathBuf::from("/11"); + let node = Node::new(inode.clone(), info3, 1); + assert_eq!(node.name(), OsStr::new("1")); + + let target = PathBuf::from("/root/child"); + assert_eq!( + Node::generate_target_vec(&target), + vec![ + OsString::from("/"), + OsString::from("root"), + OsString::from("child") + ] + ); + + let mut node = Node::new(inode, info, 1); + node.set_symlink(OsString::from("symlink")); + assert_eq!(node.info.deref().symlink, Some(OsString::from("symlink"))); + + let mut xatter = RafsXAttrs::new(); + assert!(xatter + .add(OsString::from("user.key"), [1u8; 16].to_vec()) + .is_ok()); + assert!(xatter + .add( + OsString::from("system.posix_acl_default.key"), + [2u8; 8].to_vec() + ) + .is_ok()); + node.set_xattr(xatter); + node.inode.set_has_xattr(true); + node.remove_xattr(OsStr::new("user.key")); + assert!(node.inode.has_xattr()); + node.remove_xattr(OsStr::new("system.posix_acl_default.key")); + assert!(!node.inode.has_xattr()); + } +} diff --git a/builder/src/lib.rs b/builder/src/lib.rs index ed933c978f2..7d785ea3f88 100644 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -353,3 +353,46 @@ impl TarBuilder { || path == Path::new("/.no.prefetch.landmark") } } + +#[cfg(test)] +mod tests { + use vmm_sys_util::tempdir::TempDir; + + use super::*; + + #[test] + fn test_tar_builder_is_stargz_special_files() { + let builder = TarBuilder::new(true, 0, RafsVersion::V6); + + let path = Path::new("/stargz.index.json"); + assert!(builder.is_stargz_special_files(&path)); + let path = Path::new("/.prefetch.landmark"); + assert!(builder.is_stargz_special_files(&path)); + let path = Path::new("/.no.prefetch.landmark"); + assert!(builder.is_stargz_special_files(&path)); + + let path = Path::new("/no.prefetch.landmark"); + assert!(!builder.is_stargz_special_files(&path)); + let path = Path::new("/prefetch.landmark"); + assert!(!builder.is_stargz_special_files(&path)); + let path = Path::new("/tar.index.json"); + assert!(!builder.is_stargz_special_files(&path)); + } + + #[test] + fn test_tar_builder_create_directory() { + let tmp_dir = TempDir::new().unwrap(); + let target_paths = [OsString::from(tmp_dir.as_path())]; + let mut builder = TarBuilder::new(true, 0, RafsVersion::V6); + + let node = builder.create_directory(&target_paths); + assert!(node.is_ok()); + let node = node.unwrap(); + println!("Node: {}", node); + assert_eq!(node.file_type(), "dir"); + assert_eq!(node.target(), tmp_dir.as_path()); + + assert_eq!(builder.next_ino, 1); + assert_eq!(builder.next_ino(), 2); + } +} diff --git a/builder/src/merge.rs b/builder/src/merge.rs index 56b0c8d3a94..917c8860159 100644 --- a/builder/src/merge.rs +++ b/builder/src/merge.rs @@ -315,3 +315,101 @@ impl Merger { BuildOutput::new(&blob_mgr, &bootstrap_storage) } } + +#[cfg(test)] +mod tests { + use nydus_utils::digest; + use vmm_sys_util::tempfile::TempFile; + + use super::*; + + #[test] + fn test_merger_get_string_from_list() { + let res = Merger::get_string_from_list(&None, 1); + assert!(res.is_ok()); + assert!(res.unwrap().is_none()); + + let original_ids = vec!["string1".to_owned(), "string2".to_owned()]; + let original_ids = Some(original_ids); + + let res = Merger::get_string_from_list(&original_ids, 0); + assert!(res.is_ok()); + assert_eq!(res.unwrap(), Some("string1".to_owned())); + assert!(Merger::get_string_from_list(&original_ids, 2).is_err()); + } + + #[test] + fn test_merger_get_digest_from_list() { + let res = Merger::get_digest_from_list(&None, 1); + assert!(res.is_ok()); + assert!(res.unwrap().is_none()); + + let original_ids = vec!["string1".to_owned(), "12ab".repeat(16)]; + let original_ids = Some(original_ids); + + let res = Merger::get_digest_from_list(&original_ids, 1); + assert!(res.is_ok()); + assert_eq!( + res.unwrap(), + Some([ + 18u8, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171, + 18, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171 + ]) + ); + assert!(Merger::get_digest_from_list(&original_ids, 0).is_err()); + assert!(Merger::get_digest_from_list(&original_ids, 2).is_err()); + } + + #[test] + fn test_merger_get_size_from_list() { + let res = Merger::get_size_from_list(&None, 1); + assert!(res.is_ok()); + assert!(res.unwrap().is_none()); + + let original_ids = vec![1u64, 2, 3, 4]; + let original_ids = Some(original_ids); + let res = Merger::get_size_from_list(&original_ids, 1); + assert!(res.is_ok()); + assert_eq!(res.unwrap(), Some(2u64)); + assert!(Merger::get_size_from_list(&original_ids, 4).is_err()); + } + + #[test] + fn test_merger_merge() { + let mut ctx = BuildContext::default(); + ctx.configuration.internal.set_blob_accessible(false); + ctx.digester = digest::Algorithm::Sha256; + + let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); + let mut source_path1 = PathBuf::from(root_dir); + source_path1.push("../tests/texture/bootstrap/rafs-v6-2.2.boot"); + let mut source_path2 = PathBuf::from(root_dir); + source_path2.push("../tests/texture/bootstrap/rafs-v6-2.2.boot"); + + let tmp_file = TempFile::new().unwrap(); + let target = ArtifactStorage::SingleFile(tmp_file.as_path().to_path_buf()); + + let blob_toc_digests = Some(vec![ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".to_owned(), + "4cf0c409788fc1c149afbf4c81276b92427ae41e46412334ca495991b8526650".to_owned(), + ]); + + let build_output = Merger::merge( + &mut ctx, + None, + vec![source_path1, source_path2], + Some(vec!["a70f".repeat(16), "9bd3".repeat(16)]), + Some(vec!["blob_id".to_owned(), "blob_id2".to_owned()]), + Some(vec![16u64, 32u64]), + blob_toc_digests, + Some(vec![64u64, 128]), + target, + None, + Arc::new(ConfigV2::new("config_v2")), + ); + assert!(build_output.is_ok()); + let build_output = build_output.unwrap(); + println!("BuildOutpu: {}", build_output); + assert_eq!(build_output.blob_size, Some(16)); + } +} diff --git a/builder/src/stargz.rs b/builder/src/stargz.rs index b473d6d0516..f711ddb5bf2 100644 --- a/builder/src/stargz.rs +++ b/builder/src/stargz.rs @@ -913,11 +913,10 @@ mod tests { use super::*; use crate::{ArtifactStorage, ConversionType, Features, Prefetch, WhiteoutSpec}; - #[ignore] #[test] fn test_build_stargz_toc() { let tmp_dir = vmm_sys_util::tempdir::TempDir::new().unwrap(); - let tmp_dir = tmp_dir.as_path().to_path_buf(); + let mut tmp_dir = tmp_dir.as_path().to_path_buf(); let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); let source_path = PathBuf::from(root_dir).join("../tests/texture/stargz/estargz_sample.json"); @@ -939,13 +938,116 @@ mod tests { false, ); ctx.fs_version = RafsVersion::V6; + ctx.conversion_type = ConversionType::EStargzToRafs; let mut bootstrap_mgr = - BootstrapManager::new(Some(ArtifactStorage::FileDir(tmp_dir)), None); + BootstrapManager::new(Some(ArtifactStorage::FileDir(tmp_dir.clone())), None); let mut blob_mgr = BlobManager::new(digest::Algorithm::Sha256); let mut builder = StargzBuilder::new(0x1000000, &ctx); - builder - .build(&mut ctx, &mut bootstrap_mgr, &mut blob_mgr) - .unwrap(); + let builder = builder.build(&mut ctx, &mut bootstrap_mgr, &mut blob_mgr); + assert!(builder.is_ok()); + let builder = builder.unwrap(); + assert_eq!( + builder.blobs, + vec![String::from( + "bd4eff3fe6f5a352457c076d2133583e43db895b4af08d717b3fbcaeca89834e" + )] + ); + assert_eq!(builder.blob_size, Some(4128)); + tmp_dir.push("e60676aef5cc0d5caca9f4c8031f5b0c8392a0611d44c8e1bbc46dbf7fe7bfef"); + assert_eq!( + builder.bootstrap_path.unwrap(), + tmp_dir.to_str().unwrap().to_string() + ) + } + + #[test] + fn test_toc_entry() { + let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); + let source_path = PathBuf::from(root_dir).join("../tests/texture/tar/all-entry-type.tar"); + + let mut entry = TocEntry { + name: source_path, + toc_type: "".to_string(), + size: 0x10, + link_name: PathBuf::from("link_name"), + mode: 0, + uid: 1, + gid: 1, + uname: "user_name".to_string(), + gname: "group_name".to_string(), + dev_major: 255, + dev_minor: 33, + xattrs: Default::default(), + digest: Default::default(), + offset: 0, + chunk_offset: 0, + chunk_size: 0, + chunk_digest: "sha256:".to_owned(), + inner_offset: 0, + }; + entry.chunk_digest.extend(vec!['a'; 64].iter()); + + entry.toc_type = "dir".to_owned(); + assert!(entry.is_dir()); + assert!(entry.is_supported()); + assert_eq!(entry.mode(), libc::S_IFDIR); + assert_eq!(entry.rdev(), u32::MAX); + + entry.toc_type = "req".to_owned(); + assert!(!entry.is_reg()); + entry.toc_type = "reg".to_owned(); + assert!(entry.is_reg()); + assert!(entry.is_supported()); + assert_eq!(entry.mode(), libc::S_IFREG); + assert_eq!(entry.size(), 0x10); + + entry.toc_type = "symlink".to_owned(); + assert!(entry.is_symlink()); + assert!(entry.is_supported()); + assert_eq!(entry.mode(), libc::S_IFLNK); + assert_eq!(entry.symlink_link_path(), Path::new("link_name")); + assert!(entry.normalize().is_ok()); + + entry.toc_type = "hardlink".to_owned(); + assert!(entry.is_supported()); + assert!(entry.is_hardlink()); + assert_eq!(entry.mode(), libc::S_IFREG); + assert_eq!(entry.hardlink_link_path(), Path::new("link_name")); + assert!(entry.normalize().is_ok()); + + entry.toc_type = "chunk".to_owned(); + assert!(entry.is_supported()); + assert!(entry.is_chunk()); + assert_eq!(entry.mode(), 0); + assert_eq!(entry.size(), 0); + assert!(entry.normalize().is_err()); + + entry.toc_type = "block".to_owned(); + assert!(entry.is_special()); + assert!(entry.is_blockdev()); + assert_eq!(entry.mode(), libc::S_IFBLK); + + entry.toc_type = "char".to_owned(); + assert!(entry.is_special()); + assert!(entry.is_chardev()); + assert_eq!(entry.mode(), libc::S_IFCHR); + assert_ne!(entry.size(), 0x10); + + entry.toc_type = "fifo".to_owned(); + assert!(entry.is_fifo()); + assert!(entry.is_special()); + assert_eq!(entry.mode(), libc::S_IFIFO); + assert_eq!(entry.rdev(), 65313); + + assert_eq!(entry.name().unwrap().to_str(), Some("all-entry-type.tar")); + entry.name = PathBuf::from("/"); + assert_eq!(entry.name().unwrap().to_str(), Some("/")); + assert_ne!(entry.path(), Path::new("all-entry-type.tar")); + + assert_eq!(entry.block_id().unwrap().data, [0xaa as u8; 32]); + + entry.name = PathBuf::from(""); + assert!(entry.normalize().is_err()); } }