implement sources / generates hash based task skipping

Co-authored-by: Bas Zalmstra <[email protected]>

Cargo.toml

@@ -29,9 +29,10 @@ assert_matches = "1.5.0"
 async-once-cell = "0.5.3"
 async-recursion = "1.0.5"
 async-scoped = { version = "0.9.0", features = ["use-tokio"] }
-cfg-if = "1.0"
-chrono = "0.4.35"
-clap = { version = "4.5.2", default-features = false, features = [
+blake3 = "1.5.0"
+cfg-if = "0.1"
+chrono = "0.4.34"
+clap = { version = "4.5.1", default-features = false, features = [
     "derive",
     "usage",
     "wrap_help",
@@ -43,6 +44,7 @@ clap = { version = "4.5.2", default-features = false, features = [
 clap-verbosity-flag = "2.2.0"
 clap_complete = "4.5.1"
 console = { version = "0.15.8", features = ["windows-console-colors"] }
+crossbeam-channel = "0.5.12"
 deno_task_shell = "0.14.4"
 dialoguer = "0.11.0"
 dirs = "5.0.1"
@@ -51,9 +53,11 @@ distribution-types = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 dunce = "1.0.4"
 flate2 = "1.0.28"
 futures = "0.3.30"
+globset = "0.4.14"
 http-cache-reqwest = "0.13.0"
 human_bytes = "0.4.3"
 humantime = "2.1.0"
+ignore = "0.4.22"
 indexmap = { version = "2.2.5", features = ["serde"] }
 indicatif = "0.17.8"
 insta = { version = "1.36.1", features = ["yaml"] }
@@ -129,6 +133,7 @@ uv-interpreter = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 uv-normalize = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 uv-resolver = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 uv-traits = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
+xxhash-rust = "0.8.10"
 zip = { version = "0.6.6", default-features = false, features = [
     "deflate",
     "time",

examples/cpp-sdl/CMakeLists.txt

@@ -11,4 +11,4 @@ target_link_libraries(
     ${PROJECT_NAME} PRIVATE
     SDL2::SDL2
     SDL2::SDL2main
-)
+) 

examples/cpp-sdl/pixi.toml

@@ -31,10 +31,19 @@ configure = { cmd = [
     # We wanna build in the .build directory
     "-B",
     ".build",
-] }
+], sources = ["CMakeLists.txt"] }
 
 # Build the executable but make sure CMake is configured first.
-build = { cmd = ["ninja", "-C", ".build"], depends_on = ["configure"] }
+[feature.build.tasks.build]
+cmd = ["ninja", "-C", ".build"]
+depends_on = ["configure"]
+sources = [
+    "CMakeLists.txt",
+    "src/*"
+]
+generates = [
+    ".build/bin/sdl_example"
+]
 
 [environments]
 build = ["build"]

src/cli/run.rs

@@ -144,6 +144,17 @@ pub async fn execute(args: Args) -> miette::Result<()> {
             );
         }
 
+        // check task cache
+        if executable_task
+            .can_skip(&lock_file)
+            .await
+            .into_diagnostic()?
+        {
+            eprintln!("Task can be skipped (cache hit) 🚀");
+            task_idx += 1;
+            continue;
+        }
+
         // If we don't have a command environment yet, we need to compute it. We lazily compute the
         // task environment because we only need the environment if a task is actually executed.
         let task_env: &_ = match task_envs.entry(executable_task.run_environment.clone()) {
@@ -169,6 +180,12 @@ pub async fn execute(args: Args) -> miette::Result<()> {
             }
             Err(err) => return Err(err.into()),
         }
+
+        // Update the task cache with the new hash
+        executable_task
+            .save_cache(&lock_file)
+            .await
+            .into_diagnostic()?;
     }
 
     Ok(())

src/cli/task.rs

@@ -123,6 +123,8 @@ impl From<AddArgs> for Task {
             Self::Execute(Execute {
                 cmd: CmdArgs::Single(cmd_args),
                 depends_on,
+                inputs: None,
+                outputs: None,
                 cwd: value.cwd,
             })
         }

src/consts.rs

@@ -8,6 +8,7 @@ pub const PREFIX_FILE_NAME: &str = "prefix";
 pub const ENVIRONMENTS_DIR: &str = "envs";
 pub const SOLVE_GROUP_ENVIRONMENTS_DIR: &str = "solve-group-envs";
 pub const PYPI_DEPENDENCIES: &str = "pypi-dependencies";
+pub const TASK_CACHE_DIR: &str = "task-cache";
 
 pub const DEFAULT_ENVIRONMENT_NAME: &str = "default";
 

src/project/mod.rs

@@ -457,6 +457,10 @@ impl Project {
         })
         .await
     }
+
+    pub(crate) fn task_cache_folder(&self) -> PathBuf {
+        self.pixi_dir().join(consts::TASK_CACHE_DIR)
+    }
 }
 
 fn build_reqwest_clients() -> (Client, ClientWithMiddleware) {

src/task/executable_task.rs

@@ -1,4 +1,5 @@
 use crate::consts::TASK_STYLE;
+use crate::lock_file::LockFileDerivedData;
 use crate::project::Environment;
 use crate::task::TaskName;
 use crate::{
@@ -20,6 +21,9 @@ use std::{
 use thiserror::Error;
 use tokio::task::JoinHandle;
 
+use super::task_hash::TaskCache;
+use super::TaskHash;
+
 /// Runs task in project.
 #[derive(Default, Debug)]
 pub struct RunOutput {
@@ -181,6 +185,68 @@ impl<'p> ExecutableTask<'p> {
             stderr: stderr_handle.await.unwrap(),
         })
     }
+
+    pub(crate) async fn can_skip(
+        &self,
+        lock_file: &LockFileDerivedData<'_>,
+    ) -> Result<bool, std::io::Error> {
+        tracing::info!("Checking if task can be skipped");
+        let task_cache_folder = self.project().task_cache_folder();
+
+        let project_name = self.project().name();
+        let environment_name = self.run_environment.name();
+
+        let cache_name = format!(
+            "{}-{}-{}.json",
+            project_name,
+            environment_name,
+            self.name().unwrap_or("default")
+        );
+
+        let cache_file = task_cache_folder.join(cache_name);
+        if cache_file.exists() {
+            let cache = std::fs::read_to_string(&cache_file).unwrap();
+            let cache: TaskCache = serde_json::from_str(&cache).unwrap();
+            let hash = TaskHash::from_task(self, &lock_file.lock_file).await;
+            if let Ok(Some(hash)) = hash {
+                return Ok(hash.computation_hash() == cache.hash);
+            }
+        }
+        Ok(false)
+    }
+
+    pub(crate) async fn save_cache(
+        &self,
+        lock_file: &LockFileDerivedData<'_>,
+    ) -> Result<(), std::io::Error> {
+        let task_cache_folder = self.project().task_cache_folder();
+        if !task_cache_folder.exists() {
+            std::fs::create_dir_all(&task_cache_folder)?;
+        }
+        let project_name = self.project().name();
+        let environment_name = self.run_environment.name();
+
+        let cache_name = format!(
+            "{}-{}-{}.json",
+            project_name,
+            environment_name,
+            self.name().unwrap_or("default")
+        );
+
+        let cache_file = task_cache_folder.join(cache_name);
+        if let Some(hash) = TaskHash::from_task(self, &lock_file.lock_file)
+            .await
+            .unwrap()
+        {
+            let cache = TaskCache {
+                hash: hash.computation_hash(),
+            };
+            let cache = serde_json::to_string(&cache).unwrap();
+            std::fs::write(&cache_file, cache)
+        } else {
+            Ok(())
+        }
+    }
 }
 
 /// A helper object that implements [`Display`] to display (with ascii color) the command of the