chore: cleanup, tests, and some fixes

crates/backup/Cargo.toml

@@ -9,7 +9,7 @@ keywords = ["backup"]
 lib = { path = "../lib" }
 filetime = "0.2"
 glob = "0.3.1"
-indicatif = { version = "0.17.4" }
+indicatif = { version = "0.17.4", features = ["improved_unicode","in_memory","rayon"] }
 serde_json = "1.0.96"
 async-trait = "0.1.68"
 futures = "0.3.28" # Needed by opendal
@@ -22,8 +22,10 @@ rayon = "1.7"
 bytes = "1.4.0"
 rand = "0.8.5"
 
+# Logging & Errors
 tracing = "0.1.37"
-tracing-subscriber = "0.3.17"
+thiserror = "1.0.40"
+anyhow = "1.0.71"
 
 [dependencies.zip]
 version = "0.6.6"
@@ -60,19 +62,15 @@ version = "4.2.7"
 default-features = false
 features = ["derive", "std", "help", "usage", "suggestions", "color"]
 
-#[dependencies.inquire]
-#version = "0.6.2"
-#default-features = false
-#features = ["crossterm"]
-
-#[patch.crates-io.inquire]
-#git = "https://github.com/anwarhahjjeffersongeorge/inquire-filepath/"
-#branch = "filepath-select"
-
 [dependencies.inquire]
 version = "0.6.2"
 default-features = false
 features = ["default"]
 
 [dependencies.value-bag]
 version = "1.4.0"
+
+
+[dev-dependencies]
+serde_test = "1.0.164"
+

crates/backup/src/application.rs

@@ -63,7 +63,7 @@ pub async fn main(destination: PathBuf, cli: Cli, is_interactive: bool) -> Resul
             let mut handles = vec![];
             for exporter in config.config.exporters.clone() {
                 let passed_progress = multi_bar.add(spinner());
-                passed_progress.set_message(format!("Running exporter: {}", exporter));
+                passed_progress.set_message(format!("Running exporter: {exporter}"));
 
                 let total_progress = total_progress.clone();
                 let config = config.clone();

crates/backup/src/config/rules.rs

@@ -1,4 +1,3 @@
-use clap::Parser;
 use lib::anyhow::Result;
 use serde::{Deserialize, Serialize};
 use std::path::Path;
@@ -10,18 +9,21 @@ pub struct Rules {
     pub auto_prune: AutoPrune,
 }
 
-#[derive(Parser, Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct AutoPrune {
     /// Whether or not the auto prune feature is enabled.
-    #[arg(long = "prune", action = clap::ArgAction::SetTrue)]
     pub enabled: bool,
 
-    /// How long backups should be kept for in days.
-    #[arg(long = "prune-keep-days", default_value = "28")]
-    pub keep_for: usize,
+    /// How many days of backups should be kept.
+    pub days: usize,
+
+    /// How many per week backups should be kept.
+    pub weeks: usize,
+
+    /// How many per month backups should be kept.
+    pub months: usize,
 
     /// The minimum number of backups to keep ignoring the keep_for duration.
-    #[arg(long = "prune-keep-count", default_value = "5")]
     pub keep_latest: usize,
 }
 
@@ -32,15 +34,17 @@ impl AutoPrune {
         let age = now.duration_since(mtime)?;
         let days = chrono::Duration::from_std(age)?.num_days();
 
-        Ok(days > self.keep_for as i64 && remaining_files > self.keep_latest)
+        Ok(days > self.days as i64 && remaining_files > self.keep_latest)
     }
 }
 
 impl Default for AutoPrune {
     fn default() -> Self {
         Self {
             enabled: false,
-            keep_for: 28,
+            days: 14,
+            weeks: 0,
+            months: 0,
             keep_latest: 5,
         }
     }

crates/backup/src/config/runtime.rs

@@ -160,7 +160,7 @@ impl RuntimeConfig {
             };
 
             if let Ok(days) = inquire::Text::new("How long do you want to retain backups for?")
-                .with_default(&prune.keep_for.to_string())
+                .with_default(&prune.days.to_string())
                 .with_validator(|v: &_| match usize::from_str(v).is_ok() {
                     true => Ok(Validation::Valid),
                     false => Ok(Validation::Invalid(
@@ -169,7 +169,7 @@ impl RuntimeConfig {
                 })
                 .prompt()
             {
-                prune.keep_for = usize::from_str(&days)?;
+                prune.days = usize::from_str(&days)?;
             }
 
             if let Ok(minimum) = inquire::Text::new(

crates/backup/src/main.rs

@@ -12,6 +12,7 @@ use std::path::PathBuf;
 pub async fn main() -> Result<()> {
     let cli = application::Cli::try_parse()?;
     lib::log::init("backup-interactive", &cli.flags)?;
+    rayon::ThreadPoolBuilder::new().num_threads(22).build_global().unwrap();
     application::main(select_location()?, cli, true).await
 
     // TODO :: Verify writable

crates/backup/src/sources/auto_prune.rs

@@ -1,8 +1,8 @@
 use crate::config::runtime::RuntimeConfig;
 use indicatif::MultiProgress;
 use lib::anyhow::Result;
-use tracing::debug;
 use std::path::PathBuf;
+use tracing::debug;
 
 // TODO :: Implement logic from cleaner crate to handle this!
 pub trait Prune {
@@ -19,6 +19,24 @@ pub trait Prune {
     /// # Returns
     /// A `Result` with the `Vec<PathBuf>` of the files which were removed.
     fn prune(&self, config: &RuntimeConfig, _progress_bar: &MultiProgress) -> Result<Vec<PathBuf>> {
+        // let files = self.files(config).sort_by(|a, b| {
+        //     fn chrono(path: &PathBuf) -> Result<DateTime<FixedOffset>> {
+        //         let meta = path.metadata().context("Get meta for comparing times")?;
+        //         let mtime = meta.modified().context("Get modified time for comparing times")?;
+        //
+        //         let now = SystemTime::now();
+        //         let age = now.duration_since(mtime).context("Get duration since modified time")?;
+        //         match Duration::from_std(age) {
+        //             Ok(d) => Ok(d)
+        //             Err(_) => {}
+        //         }
+        //         context("Convert std to chrono")
+        //     }
+        //
+        //     let a = a.metadata();
+        //     let b = b.metadata();
+        //     a.metadata()
+        // });
         let files = self.files(config);
         let mut files = files.iter();
         let mut removed_files = vec![];

crates/backup/src/sources/bitwarden.rs

@@ -10,12 +10,12 @@ use indicatif::{MultiProgress, ProgressBar};
 use lib::anyhow;
 use lib::anyhow::{anyhow, Context};
 use lib::fs::normalise_path;
-use tracing::{error, info};
 use serde::{Deserialize, Serialize};
 use std::env;
 use std::fmt::{Display, Formatter};
 use std::path::PathBuf;
 use std::process::Command;
+use tracing::{error, info};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct BitWardenCore {
@@ -184,29 +184,36 @@ impl Exporter for BitWardenCore {
         _main_bar: &ProgressBar,
         _progress_bar: &MultiProgress,
     ) -> Result<()> {
-        let output_file = self.backup_dir(config).join(format!(
-            "{org_id}_{date}.json",
-            org_id = &self.org_id,
-            date = chrono::Local::now().format("%Y-%m-%dT%H:%M:%SZ%z")
-        ));
-
-        let output_file = normalise_path(output_file);
-
-        let cmd = self
-            .command(config)
-            .arg("export")
-            .args(["--organizationid", &self.org_id])
-            .args(["--format", "csv"])
-            .args(["--output", output_file.to_str().unwrap()])
-            .output()
-            .context("Run BitWarden export command")?;
+        let export = |format: &str, ext: &str| -> Result<()> {
+            let output_file = normalise_path(self.backup_dir(config).join(format!(
+                "{org_id}_{date}-{format}.{ext}",
+                org_id = &self.org_id,
+                date = chrono::Local::now().format("%Y-%m-%dT%H:%M:%SZ%z")
+            )));
+
+            let cmd = self
+                .command(config)
+                .arg("export")
+                .args(["--organizationid", &self.org_id])
+                .args(["--format", format])
+                .args(["--output", output_file.to_str().unwrap()])
+                .output()
+                .context(format!("Create bitwarden export for {}", &self.org_name))?;
+
+            if !cmd.stderr.is_empty() {
+                let string = String::from_utf8(cmd.stderr)?;
+                return Err(anyhow!(
+                    "BitWarden export for {} failed: {string}",
+                    &self.org_name
+                ));
+            }
 
-        if !cmd.stderr.is_empty() {
-            let string = String::from_utf8(cmd.stderr)?;
-            return Err(anyhow!("BitWarden export failed: {string}"));
-        }
+            Ok(())
+        };
 
-        Ok(())
+        export("encrypted_json", "json")?;
+        export("json", "json")?;
+        export("csv", "csv")
     }
 }
 

crates/backup/src/sources/exporter.rs

@@ -1,6 +1,7 @@
 use crate::config::backend::Backend;
 use crate::config::runtime::RuntimeConfig;
 use crate::sources::bitwarden::BitWardenCore;
+use crate::sources::downloader::Downloader;
 use crate::sources::op::core::OnePasswordCore;
 use crate::sources::s3::S3Core;
 use async_trait::async_trait;
@@ -54,8 +55,24 @@ impl ExporterSource {
     pub async fn create(&self, config: &RuntimeConfig) -> Result<Vec<Backend>> {
         match self {
             Self::S3 => S3Core::interactive(config).await,
-            Self::BitWarden => BitWardenCore::interactive(config).await,
-            Self::OnePassword => OnePasswordCore::interactive(config).await,
+            Self::BitWarden => {
+                let bars = MultiProgress::new();
+                let main_bar = bars.add(ProgressBar::new_spinner());
+                main_bar.set_message("Setting up BitWarden CLI");
+                BitWardenCore::download_cli(config, &main_bar, &bars).await?;
+                main_bar.finish_and_clear();
+
+                BitWardenCore::interactive(config).await
+            }
+            Self::OnePassword => {
+                let bars = MultiProgress::new();
+                let main_bar = bars.add(ProgressBar::new_spinner());
+                main_bar.set_message("Setting up 1Password CLI");
+                OnePasswordCore::download_cli(config, &main_bar, &bars).await?;
+                main_bar.finish_and_clear();
+
+                OnePasswordCore::interactive(config).await
+            }
         }
     }
 }