raftstore: avoid deleting snap files in raftstore thread (#17155)

close #8160

This commit offloads one case of snapshot deletion from the raftstore 
thread to the background cleanup thread introduced in #12415. This 
aligns with the principle of performing only performance-critical work 
in the raftstore thread.

Signed-off-by: Bisheng Huang <[email protected]>

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>

components/raftstore/src/store/fsm/peer.rs

@@ -2703,16 +2703,27 @@ where
 
         let is_snapshot = msg.get_message().has_snapshot();
 
-        // TODO: spin off the I/O code (delete_snapshot)
         let regions_to_destroy = match self.check_snapshot(&msg)? {
             Either::Left(key) => {
                 if let Some(key) = key {
                     // If the snapshot file is not used again, then it's OK to
                     // delete them here. If the snapshot file will be reused when
                     // receiving, then it will fail to pass the check again, so
                     // missing snapshot files should not be noticed.
-                    let s = self.ctx.snap_mgr.get_snapshot_for_applying(&key)?;
-                    self.ctx.snap_mgr.delete_snapshot(&key, s.as_ref(), false);
+                    let snap = self.ctx.snap_mgr.get_snapshot_for_gc(&key, false)?;
+                    if let Err(e) = self.ctx.cleanup_scheduler.schedule(CleanupTask::GcSnapshot(
+                        GcSnapshotTask::DeleteSnapshotFiles {
+                            key: key.clone(),
+                            snapshot: snap,
+                            check_entry: false,
+                        },
+                    )) {
+                        error!(
+                            "failed to schedule task to delete snap file";
+                            "key" => %key,
+                            "err" => %e,
+                        )
+                    }
                 }
                 return Ok(());
             }