From 2efc850e67ddaeea1c9218a7711f2f91fd03b9e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Wed, 3 Jan 2024 08:19:07 +0000 Subject: [PATCH 1/7] fix: residual fuse mountpoint after graceful shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Case1: Fuse server exits in thread not main. There is possibility that process finishes before shutdown of server. 2. Case2: Fuse server exits in thread of state machine. There is possibiltiy that state machine not responses to signal catch thread. Then dead lock happens. Process exits before shutdown of server. This pr aims to seperator shutdown actions from signal catch handler. It only notifies controller. Controller exits with shutdown of fuse server. No race. No deadlock. Signed-off-by: 泰友 --- service/src/daemon.rs | 8 ++++++-- src/bin/nydusd/main.rs | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/service/src/daemon.rs b/service/src/daemon.rs index 26d11bb6338..1a3c7384f01 100644 --- a/service/src/daemon.rs +++ b/service/src/daemon.rs @@ -416,12 +416,16 @@ impl DaemonController { self.fs_service.lock().unwrap().clone() } - /// Shutdown all services managed by the controller. - pub fn shutdown(&self) { + /// Notify controller shutdown + pub fn notify_shutdown(&self){ // Marking exiting state. self.active.store(false, Ordering::Release); // Signal the `run_loop()` working thread to exit. let _ = self.waker.wake(); + } + + /// Shutdown all services managed by the controller. + pub fn shutdown(&self) { let daemon = self.daemon.lock().unwrap().take(); if let Some(d) = daemon { diff --git a/src/bin/nydusd/main.rs b/src/bin/nydusd/main.rs index 63a4db0265e..693d656a6fb 100644 --- a/src/bin/nydusd/main.rs +++ b/src/bin/nydusd/main.rs @@ -600,7 +600,7 @@ fn process_singleton_arguments( } extern "C" fn sig_exit(_sig: std::os::raw::c_int) { - DAEMON_CONTROLLER.shutdown(); + DAEMON_CONTROLLER.notify_shutdown(); } fn main() -> Result<()> { From 01a59730752f12df0531040857b5642a4ae26f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Thu, 1 Feb 2024 08:44:34 +0000 Subject: [PATCH 2/7] fix: compatibility to image without ext table for blob cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are scenes that cache file is smaller than expect size. Such as: 1. Nydusd 1.6 generates cache file by prefetch, which is smaller than size in boot. 2. Nydusd 2.2 generates cache file by prefetch, when image not provide ext blob tables. 3. Nydusd not have enough time to fill cache for blob. Equality check for size is too much strict for both 1.6 compatibility and 2.2 concurrency. This pr ensures blob size smaller or equal than expect size. It also truncates blob cache when smaller than expect size. Signed-off-by: 泰友 --- storage/src/cache/filecache/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/storage/src/cache/filecache/mod.rs b/storage/src/cache/filecache/mod.rs index 6c7cb8f0b77..b7793146034 100644 --- a/storage/src/cache/filecache/mod.rs +++ b/storage/src/cache/filecache/mod.rs @@ -232,7 +232,10 @@ impl FileCacheEntry { } else { blob_info.uncompressed_size() }; - if file_size == 0 { + // Compatibility + // 1. Nydusd 1.6 generates cache file by prefetch, which is smaller than size in boot. + // 2. Nydusd 2.2 generates cache file by prefetch, when image not provide ext blob table. + if file_size == 0 || file_size < cached_file_size { file.set_len(cached_file_size)?; } else if cached_file_size != 0 && file_size != cached_file_size { let msg = format!( From 02bd6a76e97570b6c108c90e67beda712c4547ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Tue, 23 Jan 2024 07:49:46 +0000 Subject: [PATCH 3/7] feat: reuse blob cache of old version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nydus2.2 has same format for blob cache and bitmap on disk. However, 2.2 nydusd not use blob cache of 1.6 nydusd. This pr counts ready chunk number for both 1.6 and 2.2, instead of thinking it as not filled at all. Signed-off-by: 泰友 --- service/src/daemon.rs | 3 +- storage/src/cache/cachedfile.rs | 16 ++++++++ storage/src/cache/filecache/mod.rs | 6 +-- storage/src/cache/state/persist_map.rs | 51 ++++++++++++-------------- 4 files changed, 42 insertions(+), 34 deletions(-) diff --git a/service/src/daemon.rs b/service/src/daemon.rs index 1a3c7384f01..0c1910eae76 100644 --- a/service/src/daemon.rs +++ b/service/src/daemon.rs @@ -417,7 +417,7 @@ impl DaemonController { } /// Notify controller shutdown - pub fn notify_shutdown(&self){ + pub fn notify_shutdown(&self) { // Marking exiting state. self.active.store(false, Ordering::Release); // Signal the `run_loop()` working thread to exit. @@ -426,7 +426,6 @@ impl DaemonController { /// Shutdown all services managed by the controller. pub fn shutdown(&self) { - let daemon = self.daemon.lock().unwrap().take(); if let Some(d) = daemon { if let Err(e) = d.trigger_stop() { diff --git a/storage/src/cache/cachedfile.rs b/storage/src/cache/cachedfile.rs index 14f09905384..a160848fbce 100644 --- a/storage/src/cache/cachedfile.rs +++ b/storage/src/cache/cachedfile.rs @@ -1436,6 +1436,22 @@ impl FileIoMergeState { tag: BlobIoTag, chunk: Option>, ) -> Result<()> { + // Make sure user io of same region continuous + if !self.regions.is_empty() && self.joinable(region_type) { + let region = &self.regions[self.regions.len() - 1]; + if !region.seg.is_empty() && tag.is_user_io() { + if let BlobIoTag::User(ref seg) = tag { + // Usually seg.offset = 0, for semantic integrity here. + if seg.offset as u64 + start + != region.blob_address + region.seg.offset as u64 + region.seg.len as u64 + { + // Stop append for non-continuous user io segment. + self.commit(); + } + } + } + } + if self.regions.is_empty() || !self.joinable(region_type) { self.regions.push(Region::new(region_type)); self.last_region_joinable = true; diff --git a/storage/src/cache/filecache/mod.rs b/storage/src/cache/filecache/mod.rs index b7793146034..331928c811f 100644 --- a/storage/src/cache/filecache/mod.rs +++ b/storage/src/cache/filecache/mod.rs @@ -215,11 +215,7 @@ impl FileCacheEntry { ); // Set cache file to its expected size. - let suffix = if mgr.cache_raw_data { - ".blob.raw" - } else { - ".blob.data" - }; + let suffix = if mgr.cache_raw_data { ".blob.raw" } else { "" }; let blob_data_file_path = blob_file_path.clone() + suffix; let file = OpenOptions::new() .create(true) diff --git a/storage/src/cache/state/persist_map.rs b/storage/src/cache/state/persist_map.rs index bf434174607..bfe39734d0f 100644 --- a/storage/src/cache/state/persist_map.rs +++ b/storage/src/cache/state/persist_map.rs @@ -111,36 +111,33 @@ impl PersistMap { } let header = filemap.get_mut::
(0)?; - let mut not_ready_count = chunk_count; - if header.version >= 1 { - if header.magic2 != MAGIC2 { - return Err(einval!(format!( - "invalid blob chunk_map file header: {:?}", - filename - ))); + if header.version >= 1 && header.magic2 != MAGIC2 { + return Err(einval!(format!( + "invalid blob chunk_map file header: {:?}", + filename + ))); + } + let not_ready_count = if new_content { + chunk_count + } else if header.version >= 1 && header.all_ready == MAGIC_ALL_READY { + 0 + } else { + let mut ready_count = 0; + for idx in HEADER_SIZE..expected_size as usize { + let current = filemap.get_ref::(idx)?; + let val = current.load(Ordering::Acquire); + ready_count += val.count_ones() as u32; } - if header.all_ready == MAGIC_ALL_READY { - not_ready_count = 0; - } else if new_content { - not_ready_count = chunk_count; - } else { - let mut ready_count = 0; - for idx in HEADER_SIZE..expected_size as usize { - let current = filemap.get_ref::(idx)?; - let val = current.load(Ordering::Acquire); - ready_count += val.count_ones() as u32; - } - if ready_count >= chunk_count { - let header = filemap.get_mut::
(0)?; - header.all_ready = MAGIC_ALL_READY; - let _ = file.sync_all(); - not_ready_count = 0; - } else { - not_ready_count = chunk_count - ready_count; - } + if ready_count >= chunk_count { + let header = filemap.get_mut::
(0)?; + header.all_ready = MAGIC_ALL_READY; + let _ = file.sync_all(); + 0 + } else { + chunk_count - ready_count } - } + }; readahead(file.as_raw_fd(), 0, expected_size); if !persist { From 58de577302f127b6b59665698171aa294da30903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Mon, 4 Mar 2024 10:02:36 +0000 Subject: [PATCH 4/7] fix: bad read by wrong data region MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User io may involve discontinuous segments in different chunks. Bad read is produced by merging them into continuous one. That is what Region does. This pr separate discontinuous segments into different regions, avoiding merging forcibly. Signed-off-by: 泰友 --- storage/src/cache/cachedfile.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/src/cache/cachedfile.rs b/storage/src/cache/cachedfile.rs index a160848fbce..6c6730cb99a 100644 --- a/storage/src/cache/cachedfile.rs +++ b/storage/src/cache/cachedfile.rs @@ -1580,7 +1580,7 @@ mod tests { let tag = BlobIoTag::User(BlobIoSegment { offset: 0x1800, - len: 0x1800, + len: 0x800, }); state .push(RegionType::CacheFast, 0x1000, 0x2000, tag, None) @@ -1597,8 +1597,8 @@ mod tests { assert_eq!(state.regions.len(), 1); let tag = BlobIoTag::User(BlobIoSegment { - offset: 0x0000, - len: 0x2000, + offset: 0x0001, + len: 0x1fff, }); state .push(RegionType::CacheSlow, 0x5000, 0x2000, tag, None) From 900e17e18c66c1463f2ed8e890ddcf4a4d4a7103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Wed, 27 Mar 2024 07:41:35 +0000 Subject: [PATCH 5/7] fix: vulnerability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upgrade mio to 0.8.11 Upgrade h2 to 0.3.25 Signed-off-by: 泰友 --- Cargo.lock | 136 +++++++++++++++++++++++++++++++++++++++++++---------- Cargo.toml | 4 +- 2 files changed, 114 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a7d879bfd1..7bbe54b63ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -376,6 +376,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.2.8" @@ -415,7 +421,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -634,9 +640,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.18" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" dependencies = [ "bytes", "fnv", @@ -644,7 +650,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -657,6 +663,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + [[package]] name = "heck" version = "0.4.1" @@ -813,7 +825,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", ] [[package]] @@ -832,7 +854,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -884,9 +906,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.147" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libz-sys" @@ -1003,14 +1025,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.5" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", "wasi", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1205,7 +1227,7 @@ dependencies = [ "hex", "hyper", "hyperlocal", - "indexmap", + "indexmap 1.9.1", "lazy_static", "libc", "log", @@ -1424,7 +1446,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1621,7 +1643,7 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1636,7 +1658,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1814,7 +1836,7 @@ dependencies = [ "fastrand", "redox_syscall", "rustix", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1888,7 +1910,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -2250,13 +2272,37 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.0", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm 0.42.0", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -2265,42 +2311,84 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_i686_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_x86_64_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "winreg" version = "0.10.1" diff --git a/Cargo.toml b/Cargo.toml index 39fa0986cf2..f2c8c7ba26a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ indexmap = "1" lazy_static = "1" libc = "0.2" log = "0.4.8" -mio = { version = "0.8", features = ["os-poll", "os-ext"] } +mio = { version = "0.8.11", features = ["os-poll", "os-ext"] } nix = "0.24.0" rlimit = "0.9.0" serde = { version = "1.0.110", features = ["serde_derive", "rc"] } @@ -97,4 +97,4 @@ backend-registry = ["nydus-storage/backend-registry"] backend-s3 = ["nydus-storage/backend-s3"] [workspace] -members = ["api", "app", "blobfs", "clib", "error", "rafs", "storage", "service", "utils"] \ No newline at end of file +members = ["api", "app", "blobfs", "clib", "error", "rafs", "storage", "service", "utils"] From 7182c36450b677d972f35b633b4f5b716a981e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Wed, 27 Mar 2024 08:49:43 +0000 Subject: [PATCH 6/7] feat: upgrade golanglint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 泰友 --- .github/workflows/smoke.yml | 2 +- smoke/tests/tool/context.go | 2 +- smoke/tests/tool/iterator.go | 5 ++--- smoke/tests/tool/layer.go | 6 +++--- smoke/tests/tool/verify.go | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 231dcee020d..c802c117218 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -125,7 +125,7 @@ jobs: export NYDUS_NYDUSIFY_$version_export=/usr/bin/nydus-$version/nydusify done - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.51.2 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.57.1 sudo -E make smoke-only nydus-unit-test: diff --git a/smoke/tests/tool/context.go b/smoke/tests/tool/context.go index d8f6a6ad4fb..1d6548647d8 100644 --- a/smoke/tests/tool/context.go +++ b/smoke/tests/tool/context.go @@ -105,6 +105,6 @@ func (ctx *Context) PrepareWorkDir(t *testing.T) { } } -func (ctx *Context) Destroy(t *testing.T) { +func (ctx *Context) Destroy(_ *testing.T) { os.RemoveAll(ctx.Env.WorkDir) } diff --git a/smoke/tests/tool/iterator.go b/smoke/tests/tool/iterator.go index bf49e32ef1e..b5bc0099770 100644 --- a/smoke/tests/tool/iterator.go +++ b/smoke/tests/tool/iterator.go @@ -129,10 +129,9 @@ func (c *DescartesIterator) calNext() { carried = true cursors[idx]++ break - } else { - carried = false - cursors[idx] = 0 } + carried = false + cursors[idx] = 0 } if !carried { c.noNext() diff --git a/smoke/tests/tool/layer.go b/smoke/tests/tool/layer.go index 7902d5d38ae..3f741005f11 100644 --- a/smoke/tests/tool/layer.go +++ b/smoke/tests/tool/layer.go @@ -191,7 +191,7 @@ func (l *Layer) PackRef(t *testing.T, ctx Context, blobDir string, compress bool return ociBlobDigest, rafsBlobDigest } -func (l *Layer) Overlay(t *testing.T, upper *Layer) *Layer { +func (l *Layer) Overlay(_ *testing.T, upper *Layer) *Layer { // Handle whiteout/opaque files for upperName := range upper.FileTree { name := filepath.Base(upperName) @@ -231,14 +231,14 @@ func (l *Layer) Overlay(t *testing.T, upper *Layer) *Layer { func (l *Layer) recordFileTree(t *testing.T) { l.FileTree = map[string]*File{} - filepath.Walk(l.workDir, func(path string, fi os.FileInfo, err error) error { + filepath.Walk(l.workDir, func(path string, _ os.FileInfo, _ error) error { targetPath := l.TargetPath(t, path) l.FileTree[targetPath] = NewFile(t, path, targetPath) return nil }) } -func (l *Layer) toOCITar(t *testing.T) io.ReadCloser { +func (l *Layer) toOCITar(_ *testing.T) io.ReadCloser { return archive.Diff(context.Background(), "", l.workDir) } diff --git a/smoke/tests/tool/verify.go b/smoke/tests/tool/verify.go index d2d4ad277d2..b3cd90a3b40 100644 --- a/smoke/tests/tool/verify.go +++ b/smoke/tests/tool/verify.go @@ -43,7 +43,7 @@ func Verify(t *testing.T, ctx Context, expectedFiles map[string]*File) { }() actualFiles := map[string]*File{} - err = filepath.WalkDir(ctx.Env.MountDir, func(path string, entry fs.DirEntry, err error) error { + err = filepath.WalkDir(ctx.Env.MountDir, func(path string, _ fs.DirEntry, err error) error { require.Nil(t, err) targetPath, err := filepath.Rel(ctx.Env.MountDir, path) From 3f76fe722bb66c3655ea5ebb4ead37846a301e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Wed, 27 Mar 2024 11:19:27 +0000 Subject: [PATCH 7/7] feat: upgrade rust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 1.66 to 1.70 Signed-off-by: 泰友 --- rafs/src/lib.rs | 2 +- rafs/src/metadata/direct_v6.rs | 4 ++-- rafs/src/metadata/mod.rs | 18 ++++-------------- rust-toolchain | 2 +- src/bin/nydus-image/builder/stargz.rs | 13 ++++++++++--- src/bin/nydus-image/builder/tarball.rs | 20 +++++++++++++++++--- src/bin/nydus-image/core/blob.rs | 8 ++++---- src/bin/nydus-image/core/bootstrap.rs | 14 +++++++------- src/bin/nydus-image/core/node.rs | 8 ++++---- src/bin/nydus-image/inspect.rs | 2 +- src/bin/nydus-image/main.rs | 4 ++-- src/bin/nydus-image/merge.rs | 4 ++-- src/bin/nydus-image/trace.rs | 2 +- src/bin/nydus-image/unpack/pax.rs | 2 +- storage/src/factory.rs | 2 +- storage/src/meta/toc.rs | 2 +- 16 files changed, 59 insertions(+), 48 deletions(-) diff --git a/rafs/src/lib.rs b/rafs/src/lib.rs index 02ccf465619..3597f9d24cf 100644 --- a/rafs/src/lib.rs +++ b/rafs/src/lib.rs @@ -112,7 +112,7 @@ pub trait RafsIoWrite: Write + Seek + 'static { fn validate_alignment(&mut self, size: usize, alignment: usize) -> Result { if alignment != 0 { - let cur = self.seek(SeekFrom::Current(0))?; + let cur = self.stream_position()?; if (size & (alignment - 1) != 0) || (cur & (alignment as u64 - 1) != 0) { return Err(einval!("unaligned data")); diff --git a/rafs/src/metadata/direct_v6.rs b/rafs/src/metadata/direct_v6.rs index caa12b343b4..d69ccebbe8d 100644 --- a/rafs/src/metadata/direct_v6.rs +++ b/rafs/src/metadata/direct_v6.rs @@ -652,9 +652,9 @@ impl OndiskInodeWrapper { Ok(()) } - fn get_entry_count<'a>( + fn get_entry_count( &self, - state: &'a Guard>, + state: &Guard>, inode: &dyn RafsV6OndiskInode, block_index: usize, ) -> Result { diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs index 7ba097d0507..bf1e2efeddd 100644 --- a/rafs/src/metadata/mod.rs +++ b/rafs/src/metadata/mod.rs @@ -534,20 +534,15 @@ impl Default for RafsSuperMeta { } /// RAFS filesystem versions. -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Default)] pub enum RafsVersion { /// RAFS v5 + #[default] V5, /// RAFS v6 V6, } -impl Default for RafsVersion { - fn default() -> Self { - RafsVersion::V5 - } -} - impl TryFrom for RafsVersion { type Error = Error; @@ -583,20 +578,15 @@ impl RafsVersion { } /// Rafs metadata working mode. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Default)] pub enum RafsMode { /// Directly mapping and accessing metadata into process by mmap(). + #[default] Direct, /// Read metadata into memory before using, for RAFS v5. Cached, } -impl Default for RafsMode { - fn default() -> Self { - RafsMode::Direct - } -} - impl FromStr for RafsMode { type Err = Error; diff --git a/rust-toolchain b/rust-toolchain index 0403bed10c3..bfe79d0bddb 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.66.1 +1.70 diff --git a/src/bin/nydus-image/builder/stargz.rs b/src/bin/nydus-image/builder/stargz.rs index 844cee27fa6..15f7965d353 100644 --- a/src/bin/nydus-image/builder/stargz.rs +++ b/src/bin/nydus-image/builder/stargz.rs @@ -218,7 +218,14 @@ impl TocEntry { mode |= libc::S_IFIFO; } - self.mode | mode as u32 + #[cfg(target_os = "macos")] + { + self.mode | mode as u32 + } + #[cfg(not(target_os = "macos"))] + { + self.mode | mode + } } /// Get real device id associated with the `TocEntry`. @@ -379,9 +386,9 @@ impl StargzTreeBuilder { flags: BlobChunkFlags::COMPRESSED, compressed_size: 0, uncompressed_size: uncompress_size as u32, - compressed_offset: entry.offset as u64, + compressed_offset: entry.offset, uncompressed_offset: uncompress_offset, - file_offset: entry.chunk_offset as u64, + file_offset: entry.chunk_offset, index: 0, reserved: 0, }); diff --git a/src/bin/nydus-image/builder/tarball.rs b/src/bin/nydus-image/builder/tarball.rs index 86741c0f200..2c5eb6930c8 100644 --- a/src/bin/nydus-image/builder/tarball.rs +++ b/src/bin/nydus-image/builder/tarball.rs @@ -164,7 +164,7 @@ impl<'a> TarballTreeBuilder<'a> { let path = PathBuf::from("/").join(path); let path = path.components().as_path(); if !self.is_special_files(path) { - self.make_lost_dirs(&path, &mut nodes)?; + self.make_lost_dirs(path, &mut nodes)?; let node = self.parse_entry(&nodes, &mut entry, path)?; nodes.push(node); } @@ -428,7 +428,14 @@ impl<'a> TarballTreeBuilder<'a> { EntryType::Fifo => libc::S_IFIFO, _ => bail!("unsupported tar entry type"), }; - Ok((mode & !libc::S_IFMT as u32) | ty as u32) + #[cfg(target_os = "macos")] + { + Ok((mode & !libc::S_IFMT as u32) | ty as u32) + } + #[cfg(not(target_os = "macos"))] + { + Ok((mode & !libc::S_IFMT) | ty) + } } fn get_file_name(path: &Path) -> Result<&OsStr> { @@ -468,7 +475,14 @@ impl<'a> TarballTreeBuilder<'a> { let name = Self::get_file_name(path)?; let mut inode = InodeWrapper::new(self.ctx.fs_version); inode.set_ino(ino); - inode.set_mode(0o755 | libc::S_IFDIR as u32); + #[cfg(target_os = "macos")] + { + inode.set_mode(0o755 | libc::S_IFDIR as u32); + } + #[cfg(not(target_os = "macos"))] + { + inode.set_mode(0o755 | libc::S_IFDIR); + } inode.set_nlink(2); inode.set_name_size(name.len()); inode.set_rdev(u32::MAX); diff --git a/src/bin/nydus-image/core/blob.rs b/src/bin/nydus-image/core/blob.rs index 0b8e32582bb..a3f07873b62 100644 --- a/src/bin/nydus-image/core/blob.rs +++ b/src/bin/nydus-image/core/blob.rs @@ -176,8 +176,8 @@ impl Blob { header.set_ci_compressor(compressor); header.set_ci_entries(blob_meta_info.len() as u32); header.set_ci_compressed_offset(compressed_offset); - header.set_ci_compressed_size(compressed_size as u64); - header.set_ci_uncompressed_size(uncompressed_size as u64); + header.set_ci_compressed_size(compressed_size); + header.set_ci_uncompressed_size(uncompressed_size); header.set_4k_aligned(true); match blob_meta_info { BlobMetaChunkArray::V1(_) => header.set_chunk_info_v2(false), @@ -223,8 +223,8 @@ impl Blob { compressor, hasher.digest_finalize(), compressed_offset, - compressed_size as u64, - uncompressed_size as u64, + compressed_size, + uncompressed_size, )?; let mut hasher = RafsDigest::hasher(digest::Algorithm::Sha256); diff --git a/src/bin/nydus-image/core/bootstrap.rs b/src/bin/nydus-image/core/bootstrap.rs index e97ce7fbe0f..791d37ad848 100644 --- a/src/bin/nydus-image/core/bootstrap.rs +++ b/src/bin/nydus-image/core/bootstrap.rs @@ -580,7 +580,7 @@ impl Bootstrap { let blob_table_size = blob_table.size() as u64; let blob_table_offset = align_offset( (EROFS_DEVTABLE_OFFSET as u64) + devtable_len as u64, - EROFS_BLOCK_SIZE as u64, + EROFS_BLOCK_SIZE, ); let blob_table_entries = blobs.len(); assert!(blob_table_entries < u16::MAX as usize); @@ -614,7 +614,7 @@ impl Bootstrap { let meta_addr = if blob_table_size > 0 { align_offset( blob_table_offset + blob_table_size + prefetch_table_size as u64, - EROFS_BLOCK_SIZE as u64, + EROFS_BLOCK_SIZE, ) } else { orig_meta_addr @@ -664,7 +664,7 @@ impl Bootstrap { // Dump blob table bootstrap_ctx .writer - .seek_offset(blob_table_offset as u64) + .seek_offset(blob_table_offset) .context("failed seek for extended blob table offset")?; blob_table .store(bootstrap_ctx.writer.as_mut()) @@ -707,7 +707,7 @@ impl Bootstrap { ext_sb.set_prefetch_table_size(prefetch_table_size); bootstrap_ctx .writer - .seek_offset(prefetch_table_offset as u64) + .seek_offset(prefetch_table_offset) .context("failed seek prefetch table offset")?; pt.store(bootstrap_ctx.writer.as_mut()).unwrap(); } @@ -718,7 +718,7 @@ impl Bootstrap { .writer .seek_to_end() .context("failed to seek to bootstrap's end for chunk table")?; - let padding = align_offset(pos, EROFS_BLOCK_SIZE as u64) - pos; + let padding = align_offset(pos, EROFS_BLOCK_SIZE) - pos; bootstrap_ctx .writer .write_all(&WRITE_PADDING_DATA[0..padding as usize]) @@ -758,9 +758,9 @@ impl Bootstrap { .context("failed to seek to bootstrap's end")?; debug!( "align bootstrap to 4k {}", - align_offset(pos, EROFS_BLOCK_SIZE as u64) + align_offset(pos, EROFS_BLOCK_SIZE) ); - let padding = align_offset(pos, EROFS_BLOCK_SIZE as u64) - pos; + let padding = align_offset(pos, EROFS_BLOCK_SIZE) - pos; bootstrap_ctx .writer .write_all(&WRITE_PADDING_DATA[0..padding as usize]) diff --git a/src/bin/nydus-image/core/node.rs b/src/bin/nydus-image/core/node.rs index 3394d8956bb..9209eb3839e 100644 --- a/src/bin/nydus-image/core/node.rs +++ b/src/bin/nydus-image/core/node.rs @@ -961,7 +961,7 @@ impl Node { let len = c.as_bytes().len() + size_of::(); // erofs disk format requires dirent to be aligned to block size. if (d_size % EROFS_BLOCK_SIZE) + len as u64 > EROFS_BLOCK_SIZE { - d_size = round_up(d_size as u64, EROFS_BLOCK_SIZE); + d_size = round_up(d_size, EROFS_BLOCK_SIZE); } d_size += len as u64; } @@ -976,7 +976,7 @@ impl Node { let len = child.node.name().as_bytes().len() + size_of::(); // erofs disk format requires dirent to be aligned to block size. if (d_size % EROFS_BLOCK_SIZE) + len as u64 > EROFS_BLOCK_SIZE { - d_size = round_up(d_size as u64, EROFS_BLOCK_SIZE); + d_size = round_up(d_size, EROFS_BLOCK_SIZE); } d_size += len as u64; } @@ -1227,7 +1227,7 @@ impl Node { } f_bootstrap - .seek(SeekFrom::Start(dirent_off as u64)) + .seek(SeekFrom::Start(dirent_off)) .context("failed seek for dir inode")?; f_bootstrap .write(dir_data.as_slice()) @@ -1284,7 +1284,7 @@ impl Node { _ => bail!("unsupported RAFS v6 inode layout for directory"), }; f_bootstrap - .seek(SeekFrom::Start(tail_off as u64)) + .seek(SeekFrom::Start(tail_off)) .context("failed seek for dir inode")?; f_bootstrap .write(dir_data.as_slice()) diff --git a/src/bin/nydus-image/inspect.rs b/src/bin/nydus-image/inspect.rs index 7e348da81ef..73db42b4e1f 100644 --- a/src/bin/nydus-image/inspect.rs +++ b/src/bin/nydus-image/inspect.rs @@ -381,7 +381,7 @@ RAFS Blob Size: {rafs_size} } } } else { - let file_path = self.rafs_meta.path_from_ino(ino as u64)?; + let file_path = self.rafs_meta.path_from_ino(ino)?; file_paths.push(file_path); }; Ok(file_paths) diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 70aa84e5097..1355f85965d 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -200,7 +200,7 @@ fn prepare_cmd_args(bti_string: &'static str) -> App { .long("bootstrap") .short('B') .help("File path to save the generated RAFS metadata blob") - .required_unless_present_any(&["blob-dir", "blob-inline-meta"]) + .required_unless_present_any(["blob-dir", "blob-inline-meta"]) .conflicts_with("blob-inline-meta"), ) .arg( @@ -214,7 +214,7 @@ fn prepare_cmd_args(bti_string: &'static str) -> App { .long("blob") .short('b') .help("File path to save the generated RAFS data blob") - .required_unless_present_any(&["type", "blob-dir"]), + .required_unless_present_any(["type", "blob-dir"]), ) .arg( Arg::new("blob-inline-meta") diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs index 6e4be8f1ab7..73018f5b2ed 100644 --- a/src/bin/nydus-image/merge.rs +++ b/src/bin/nydus-image/merge.rs @@ -152,7 +152,7 @@ impl Merger { tree = Some(Tree::from_bootstrap(&rs, &mut ())?); let blobs = rs.superblock.get_blob_infos(); for blob in &blobs { - let blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?; + let blob_ctx = BlobContext::from(ctx, blob, ChunkSource::Parent)?; blob_idx_map.insert(blob_ctx.blob_id.clone(), blob_mgr.len()); blob_mgr.add(blob_ctx); } @@ -190,7 +190,7 @@ impl Merger { let mut parent_blob_added = false; let blobs = &rs.superblock.get_blob_infos(); for blob in blobs { - let mut blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?; + let mut blob_ctx = BlobContext::from(ctx, blob, ChunkSource::Parent)?; if let Some(chunk_size) = chunk_size { ensure!( chunk_size == blob_ctx.chunk_size, diff --git a/src/bin/nydus-image/trace.rs b/src/bin/nydus-image/trace.rs index dde82b0dce2..d54f23d940a 100644 --- a/src/bin/nydus-image/trace.rs +++ b/src/bin/nydus-image/trace.rs @@ -122,7 +122,7 @@ impl BuildRootTracer { pub fn tracer(&self, class: TraceClass) -> Option> { let g = self.tracers.read().unwrap(); // Safe to unwrap because tracers should always be enabled - (&g).get(&class).cloned() + g.get(&class).cloned() } pub fn dump_summary_map(&self) -> Result> { diff --git a/src/bin/nydus-image/unpack/pax.rs b/src/bin/nydus-image/unpack/pax.rs index 932d2e27f4b..2126a1c8d6b 100644 --- a/src/bin/nydus-image/unpack/pax.rs +++ b/src/bin/nydus-image/unpack/pax.rs @@ -562,7 +562,7 @@ impl PAXUtil { let max_len = header.as_old().linkname.len(); if path.as_os_str().len() <= max_len { return header - .set_link_name(&path) + .set_link_name(path) .with_context(|| "fail to set short link for pax header") .map(|_| None); } diff --git a/storage/src/factory.rs b/storage/src/factory.rs index 288e53bbb97..cc37a4e913c 100644 --- a/storage/src/factory.rs +++ b/storage/src/factory.rs @@ -58,7 +58,7 @@ struct BlobCacheMgrKey { config: Arc, } -#[allow(clippy::derive_hash_xor_eq)] +#[allow(clippy::derived_hash_with_manual_eq)] impl Hash for BlobCacheMgrKey { fn hash(&self, state: &mut H) { self.config.id.hash(state); diff --git a/storage/src/meta/toc.rs b/storage/src/meta/toc.rs index e759bd95f8c..b678dcc17f4 100644 --- a/storage/src/meta/toc.rs +++ b/storage/src/meta/toc.rs @@ -56,7 +56,7 @@ impl TryFrom for TocEntryFlags { compress::Algorithm::None => Ok(Self::COMPRESSION_NONE), compress::Algorithm::Zstd => Ok(Self::COMPRESSION_ZSTD), compress::Algorithm::Lz4Block => Ok(Self::COMPRESSION_LZ4_BLOCK), - _ => return Err(eother!(format!("unsupported compressor {}", c,))), + _ => Err(eother!(format!("unsupported compressor {}", c,))), } } }