feat: parallel item fetching & Some more fixes

AMTSupport · Jun 22, 2023 · 213ad05 · 213ad05
1 parent 22a6cc0
commit 213ad05
Show file tree

Hide file tree

Showing 9 changed files with 370 additions and 160 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/backup/Cargo.toml b/crates/backup/Cargo.toml
@@ -16,6 +16,7 @@ futures = "0.3.28" # Needed by opendal
 tokio-util = { version = "0.7.8", features = ["full"] }
 futures-util = "0.3.28"
 const_format = "0.2.31"
+rayon = "1.7"
 
 # For downloading the required cli
 bytes = "1.4.0"

diff --git a/crates/backup/src/sources/downloader.rs b/crates/backup/src/sources/downloader.rs
@@ -3,18 +3,21 @@ use crate::sources::download;
 use crate::sources::exporter::Exporter;
 use async_trait::async_trait;
 use futures_util::StreamExt;
+use indicatif::{MultiProgress, ProgressBar};
 use lib::anyhow::{anyhow, Context, Result};
 use lib::fs::create_parents;
-use tracing::{debug, trace};
 use lib::{anyhow, progress};
-use std::fs::{metadata, set_permissions, File};
+use std::fs::File;
 use std::io::copy;
 use std::path::PathBuf;
 use std::process::Command;
+use tracing::{debug, trace};
 
-use indicatif::{MultiProgress, ProgressBar};
 #[cfg(unix)]
-use std::os::unix::fs::PermissionsExt;
+use {
+    std::fs::{metadata, set_permissions},
+    std::os::unix::fs::PermissionsExt,
+};
 
 #[async_trait]
 pub trait Downloader: Exporter {