diff --git a/src/cli/add.rs b/src/cli/add.rs index a23772533..ab623e4c4 100644 --- a/src/cli/add.rs +++ b/src/cli/add.rs @@ -1,14 +1,15 @@ use crate::environment::{update_prefix, verify_prefix_location_unchanged}; use crate::prefix::Prefix; -use crate::project::SpecType; +use crate::project::DependencyType::CondaDependency; +use crate::project::{DependencyType, SpecType}; use crate::{ consts, lock_file::{load_lock_file, update_lock_file}, + project::python::PyPiRequirement, project::Project, virtual_packages::get_minimal_virtual_packages, }; use clap::Parser; -use console::style; use indexmap::IndexMap; use itertools::Itertools; use miette::{IntoDiagnostic, WrapErr}; @@ -20,6 +21,7 @@ use rattler_repodata_gateway::sparse::SparseRepoData; use rattler_solve::{resolvo, SolverImpl}; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; +use std::str::FromStr; /// Adds a dependency to the project #[derive(Parser, Debug, Default)] @@ -27,8 +29,9 @@ use std::path::PathBuf; pub struct Args { /// Specify the dependencies you wish to add to the project. /// - /// All dependencies should be defined as MatchSpec. If no specific version is - /// provided, the latest version compatible with your project will be chosen automatically. + /// The dependencies should be defined as MatchSpec for conda package, or a PyPI requirement + /// for the --pypi dependencies. If no specific version is provided, the latest version + /// compatible with your project will be chosen automatically or a * will be used. /// /// Example usage: /// @@ -50,21 +53,29 @@ pub struct Args { /// /// Mixing `--platform` and `--build`/`--host` flags is supported /// + /// The `--pypi` option will add the package as a pypi-dependency this can not be mixed with the conda dependencies + /// - `pixi add --pypi boto3` + /// - `pixi add --pypi "boto3==version" + /// #[arg(required = true)] - pub specs: Vec, + pub specs: Vec, /// The path to 'pixi.toml' #[arg(long)] pub manifest_path: Option, - /// This is a host dependency + /// The specified dependencies are host dependencies. Conflicts with `build` and `pypi` #[arg(long, conflicts_with = "build")] pub host: bool, - /// This is a build dependency + /// The specified dependencies are build dependencies. Conflicts with `host` and `pypi` #[arg(long, conflicts_with = "host")] pub build: bool, + /// The specified dependencies are pypi dependencies. Conflicts with `host` and `build` + #[arg(long, conflicts_with_all = ["host", "build"])] + pub pypi: bool, + /// Don't update lockfile, implies the no-install as well. #[clap(long, conflicts_with = "no_install")] pub no_lockfile_update: bool, @@ -78,22 +89,24 @@ pub struct Args { pub platform: Vec, } -impl SpecType { +impl DependencyType { pub fn from_args(args: &Args) -> Self { - if args.host { - Self::Host + if args.pypi { + Self::PypiDependency + } else if args.host { + CondaDependency(SpecType::Host) } else if args.build { - Self::Build + CondaDependency(SpecType::Build) } else { - Self::Run + CondaDependency(SpecType::Run) } } } pub async fn execute(args: Args) -> miette::Result<()> { let mut project = Project::load_or_else_discover(args.manifest_path.as_deref())?; - let spec_type = SpecType::from_args(&args); - let spec_platforms = args.platform; + let dependency_type = DependencyType::from_args(&args); + let spec_platforms = &args.platform; // Sanity check of prefix location verify_prefix_location_unchanged( @@ -111,24 +124,115 @@ pub async fn execute(args: Args) -> miette::Result<()> { .collect::>(); project.add_platforms(platforms_to_add.iter())?; - add_specs_to_project( - &mut project, - args.specs, - spec_type, - args.no_install, - args.no_lockfile_update, - spec_platforms, - ) - .await + match dependency_type { + DependencyType::CondaDependency(spec_type) => { + let specs = args + .specs + .clone() + .into_iter() + .map(|s| MatchSpec::from_str(&s)) + .collect::, _>>() + .into_diagnostic()?; + add_conda_specs_to_project( + &mut project, + specs, + spec_type, + args.no_install, + args.no_lockfile_update, + spec_platforms, + ) + .await + } + DependencyType::PypiDependency => { + // Parse specs as pep508_rs requirements + let pep508_requirements = args + .specs + .clone() + .into_iter() + .map(|input| pep508_rs::Requirement::from_str(input.as_ref()).into_diagnostic()) + .collect::>>()?; + + // Move those requirements into our custom PyPiRequirement + let specs = pep508_requirements + .into_iter() + .map(|req| { + let name = rip::types::PackageName::from_str(req.name.as_str())?; + let requirement = PyPiRequirement::from(req); + Ok((name, requirement)) + }) + .collect::, rip::types::ParsePackageNameError>>() + .into_diagnostic()?; + + add_pypi_specs_to_project( + &mut project, + specs, + spec_platforms, + args.no_lockfile_update, + args.no_install, + ) + .await + } + }?; + + for package in args.specs { + eprintln!( + "{}Added {}", + console::style(console::Emoji("✔ ", "")).green(), + console::style(package).bold(), + ); + } + + // Print if it is something different from host and dep + if !matches!(dependency_type, CondaDependency(SpecType::Run)) { + eprintln!( + "Added these as {}.", + console::style(dependency_type.name()).bold() + ); + } + + // Print something if we've added for platforms + if !args.platform.is_empty() { + eprintln!( + "Added these only for platform(s): {}", + console::style(args.platform.iter().join(", ")).bold() + ) + } + + Ok(()) } -pub async fn add_specs_to_project( +pub async fn add_pypi_specs_to_project( + project: &mut Project, + specs: Vec<(rip::types::PackageName, PyPiRequirement)>, + specs_platforms: &Vec, + no_update_lockfile: bool, + no_install: bool, +) -> miette::Result<()> { + for (name, spec) in &specs { + // TODO: Get best version + // Add the dependency to the project + if specs_platforms.is_empty() { + project.add_pypi_dependency(name, spec)?; + } else { + for platform in specs_platforms.iter() { + project.add_target_pypi_dependency(*platform, name.clone(), spec)?; + } + } + } + project.save()?; + + update_lockfile(project, None, no_install, no_update_lockfile).await?; + + Ok(()) +} + +pub async fn add_conda_specs_to_project( project: &mut Project, specs: Vec, spec_type: SpecType, no_install: bool, no_update_lockfile: bool, - specs_platforms: Vec, + specs_platforms: &Vec, ) -> miette::Result<()> { // Split the specs into package name and version specifier let new_specs = specs @@ -150,7 +254,7 @@ pub async fn add_specs_to_project( let platforms = if specs_platforms.is_empty() { project.platforms() } else { - &specs_platforms + specs_platforms } .to_vec(); for platform in platforms { @@ -188,7 +292,6 @@ pub async fn add_specs_to_project( } // Update the specs passed on the command line with the best available versions. - let mut added_specs = Vec::new(); for (name, spec) in new_specs { let versions_seen = package_versions .get(&name) @@ -211,21 +314,29 @@ pub async fn add_specs_to_project( project.add_target_dependency(*platform, &spec, spec_type)?; } } - - added_specs.push(spec); } project.save()?; + update_lockfile( + project, + Some(sparse_repo_data), + no_install, + no_update_lockfile, + ) + .await?; + + Ok(()) +} + +async fn update_lockfile( + project: &Project, + sparse_repo_data: Option>, + no_install: bool, + no_update_lockfile: bool, +) -> miette::Result<()> { // Update the lock file let lock_file = if !no_update_lockfile { - Some( - update_lock_file( - project, - load_lock_file(project).await?, - Some(sparse_repo_data), - ) - .await?, - ) + Some(update_lock_file(project, load_lock_file(project).await?, sparse_repo_data).await?) } else { None }; @@ -248,37 +359,12 @@ pub async fn add_specs_to_project( ) .await?; } else { - eprintln!("{} skipping installation of environment because your platform ({platform}) is not supported by this project.", style("!").yellow().bold()) + eprintln!("{} skipping installation of environment because your platform ({platform}) is not supported by this project.", console::style("!").yellow().bold()) } } } - - for spec in added_specs { - eprintln!( - "{}Added {}", - console::style(console::Emoji("✔ ", "")).green(), - spec, - ); - } - - // Print if it is something different from host and dep - match spec_type { - SpecType::Host => eprintln!("Added these as host dependencies."), - SpecType::Build => eprintln!("Added these as build dependencies."), - SpecType::Run => {} - }; - - // Print something if we've added for platforms - if !specs_platforms.is_empty() { - eprintln!( - "Added these only for platform(s): {}", - specs_platforms.iter().join(", ") - ) - } - Ok(()) } - /// Given several specs determines the highest installable version for them. pub fn determine_best_version( new_specs: &HashMap, diff --git a/src/lock_file/python.rs b/src/lock_file/python.rs index a212fd533..7f72f5a2f 100644 --- a/src/lock_file/python.rs +++ b/src/lock_file/python.rs @@ -19,10 +19,10 @@ pub async fn resolve_pypi_dependencies<'p>( platform: Platform, conda_packages: &mut [RepoDataRecord], ) -> miette::Result>> { - let dependencies = match project.pypi_dependencies() { - Some(deps) if !deps.is_empty() => deps, - _ => return Ok(vec![]), - }; + let dependencies = project.pypi_dependencies(platform); + if dependencies.is_empty() { + return Ok(vec![]); + } // Amend the records with pypi purls if they are not present yet. let conda_forge_mapping = python_name_mapping::conda_pypi_name_mapping().await?; diff --git a/src/lock_file/satisfiability.rs b/src/lock_file/satisfiability.rs index b9a26f8b1..b9e7833ce 100644 --- a/src/lock_file/satisfiability.rs +++ b/src/lock_file/satisfiability.rs @@ -72,9 +72,9 @@ pub fn lock_file_satisfies_project( .collect::>(); let mut pypi_dependencies = project - .pypi_dependencies() + .pypi_dependencies(platform) .into_iter() - .flat_map(|deps| deps.into_iter().map(|(name, req)| req.as_pep508(name))) + .map(|(name, requirement)| requirement.as_pep508(name)) .map(DependencyKind::PyPi) .peekable(); diff --git a/src/project/manifest.rs b/src/project/manifest.rs index 6b7cd3184..1245a9f1e 100644 --- a/src/project/manifest.rs +++ b/src/project/manifest.rs @@ -7,6 +7,7 @@ use indexmap::IndexMap; use miette::{Context, IntoDiagnostic, LabeledSpan, NamedSource, Report}; use rattler_conda_types::{Channel, NamelessMatchSpec, Platform, Version}; use rattler_virtual_packages::{Archspec, Cuda, LibC, Linux, Osx, VirtualPackage}; +use rip::types::PackageName; use serde::Deserializer; use serde_with::de::DeserializeAsWrap; use serde_with::{serde_as, DeserializeAs, DisplayFromStr, PickFirst}; @@ -146,6 +147,17 @@ impl ProjectManifest { } } + /// Get the map of dependencies for a given spec type. + pub fn create_or_get_pypi_dependencies( + &mut self, + ) -> &mut IndexMap { + if let Some(ref mut deps) = self.pypi_dependencies { + deps + } else { + self.pypi_dependencies.insert(IndexMap::new()) + } + } + /// Remove dependency given a `SpecType`. pub fn remove_dependency( &mut self, @@ -259,6 +271,9 @@ pub struct TargetMetadata { #[serde_as(as = "Option>>")] pub build_dependencies: Option>, + #[serde(default, rename = "pypi-dependencies")] + pub pypi_dependencies: Option>, + /// Additional information to activate an environment. #[serde(default)] pub activation: Option, diff --git a/src/project/mod.rs b/src/project/mod.rs index d289fd33f..93f1d12a4 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -1,6 +1,6 @@ pub mod environment; pub mod manifest; -mod python; +pub(crate) mod python; mod serde; use indexmap::IndexMap; @@ -31,6 +31,22 @@ use crate::{ use toml_edit::{Array, Document, Item, Table, TomlError, Value}; use url::Url; +/// The dependency types we support +#[derive(Debug, Copy, Clone)] +pub enum DependencyType { + CondaDependency(SpecType), + PypiDependency, +} + +impl DependencyType { + /// Convert to a name used in the manifest + pub fn name(&self) -> &'static str { + match self { + DependencyType::CondaDependency(dep) => dep.name(), + DependencyType::PypiDependency => "pypi-dependencies", + } + } +} #[derive(Debug, Copy, Clone)] /// What kind of dependency spec do we have pub enum SpecType { @@ -452,8 +468,25 @@ impl Project { Ok(dependencies) } - pub fn pypi_dependencies(&self) -> Option<&IndexMap> { - self.manifest.pypi_dependencies.as_ref() + pub fn pypi_dependencies( + &self, + platform: Platform, + ) -> IndexMap<&rip::types::PackageName, &PyPiRequirement> { + // Get the base pypi dependencies (defined in the `[pypi-dependencies]` section) + let base_pypi_dependencies = self.manifest.pypi_dependencies.iter(); + + // Get the platform specific dependencies in the order they were defined. + let platform_specific = self + .target_specific_metadata(platform) + .flat_map(|target| target.pypi_dependencies.iter()); + + // Combine the specs. + // + // Note that if a dependency was specified twice the platform specific one "wins". + base_pypi_dependencies + .chain(platform_specific) + .flatten() + .collect::>() } /// Returns the Python index URLs to use for this project. @@ -541,6 +574,25 @@ impl Project { Ok((name, nameless)) } + fn add_pypi_dep_to_table( + deps_table: &mut Item, + name: &rip::types::PackageName, + requirement: &PyPiRequirement, + ) -> miette::Result<()> { + // If it doesn't exist create a proper table + if deps_table.is_none() { + *deps_table = Item::Table(Table::new()); + } + + // Cast the item into a table + let deps_table = deps_table.as_table_like_mut().ok_or_else(|| { + miette::miette!("dependencies in {} are malformed", consts::PROJECT_MANIFEST) + })?; + + deps_table.insert(name.as_str(), (*requirement).clone().into()); + Ok(()) + } + fn add_dep_to_target_table( &mut self, platform: Platform, @@ -565,7 +617,24 @@ impl Project { })?; platform_table.set_dotted(true); - let dependencies = platform_table[dep_type.as_str()] + let dependencies = platform_table[dep_type.as_str()].or_insert(Item::Table(Table::new())); + + Self::add_to_deps_table(dependencies, spec) + } + fn add_pypi_dep_to_target_table( + &mut self, + platform: Platform, + name: &rip::types::PackageName, + requirement: &PyPiRequirement, + ) -> miette::Result<()> { + let target = self.doc["target"] + .or_insert(Item::Table(Table::new())) + .as_table_mut() + .ok_or_else(|| { + miette::miette!("target table in {} is malformed", consts::PROJECT_MANIFEST) + })?; + target.set_dotted(true); + let platform_table = self.doc["target"][platform.as_str()] .or_insert(Item::Table(Table::new())) .as_table_mut() .ok_or_else(|| { @@ -574,25 +643,13 @@ impl Project { consts::PROJECT_MANIFEST ) })?; + platform_table.set_dotted(true); - // Determine the name of the package to add - let name = spec - .name - .clone() - .ok_or_else(|| miette::miette!("* package specifier is not supported"))?; - - // Format the requirement - // TODO: Do this smarter. E.g.: - // - split this into an object if exotic properties (like channel) are specified. - // - split the name from the rest of the requirement. - let nameless = NamelessMatchSpec::from(spec.to_owned()); - - // Store (or replace) in the document - dependencies.insert(name.as_source(), Item::Value(nameless.to_string().into())); + let dependencies = platform_table[DependencyType::PypiDependency.name()] + .or_insert(Item::Table(Table::new())); - Ok((name, nameless)) + Self::add_pypi_dep_to_table(dependencies, name, requirement) } - pub fn add_target_dependency( &mut self, platform: Platform, @@ -613,6 +670,25 @@ impl Project { Ok(()) } + pub fn add_target_pypi_dependency( + &mut self, + platform: Platform, + name: rip::types::PackageName, + requirement: &PyPiRequirement, + ) -> miette::Result<()> { + // Add to target table toml + self.add_pypi_dep_to_target_table(platform, &name.clone(), requirement)?; + // Add to manifest + self.manifest + .target + .entry(TargetSelector::Platform(platform).into()) + .or_insert(TargetMetadata::default()) + .pypi_dependencies + .get_or_insert(IndexMap::new()) + .insert(name, requirement.clone()); + Ok(()) + } + pub fn add_dependency(&mut self, spec: &MatchSpec, spec_type: SpecType) -> miette::Result<()> { // Find the dependencies table let deps = &mut self.doc[spec_type.name()]; @@ -625,6 +701,22 @@ impl Project { Ok(()) } + pub fn add_pypi_dependency( + &mut self, + name: &rip::types::PackageName, + requirement: &PyPiRequirement, + ) -> miette::Result<()> { + // Find the dependencies table + let deps = &mut self.doc[DependencyType::PypiDependency.name()]; + Project::add_pypi_dep_to_table(deps, name, requirement)?; + + self.manifest + .create_or_get_pypi_dependencies() + .insert(name.clone(), requirement.clone()); + + Ok(()) + } + /// Removes a dependency from `pixi.toml` based on `SpecType`. pub fn remove_dependency( &mut self, diff --git a/src/project/python.rs b/src/project/python.rs index 3bb284c34..80ce30b15 100644 --- a/src/project/python.rs +++ b/src/project/python.rs @@ -1,15 +1,15 @@ use pep440_rs::VersionSpecifiers; -use pep508_rs::VersionOrUrl; use serde::de::{Error, MapAccess, Visitor}; use serde::{de, Deserialize, Deserializer}; use std::fmt::Formatter; use std::str::FromStr; use thiserror::Error; +use toml_edit::Item; #[derive(Debug, Clone, Eq, PartialEq)] pub struct PyPiRequirement { - version: Option, - extras: Option>, + pub(crate) version: Option, + pub(crate) extras: Option>, } /// The type of parse error that occurred when parsing match spec. @@ -25,6 +25,42 @@ pub enum ParsePyPiRequirementError { MissingOperator(String), } +impl From for Item { + /// PyPiRequirement to a toml_edit item, to put in the manifest file. + fn from(val: PyPiRequirement) -> Item { + if val.extras.is_some() { + // If extras is defined use an inline table + let mut table = toml_edit::Table::new().into_inline_table(); + + // First add the version + if val.version.is_some() { + let v = val.version.expect("Expect a version here").to_string(); + table.insert( + "version", + toml_edit::Value::String(toml_edit::Formatted::new(v)), + ); + } else { + table.insert( + "version", + toml_edit::Value::String(toml_edit::Formatted::new("*".to_string())), + ); + } + // Add extras as an array. + table.insert( + "extras", + toml_edit::Value::Array(toml_edit::Array::from_iter(val.extras.unwrap())), + ); + Item::Value(toml_edit::Value::InlineTable(table)) + } else { + // Without extras use the string representation. + if val.version.is_some() { + Item::Value(val.version.unwrap().to_string().into()) + } else { + Item::Value("*".into()) + } + } + } +} impl FromStr for PyPiRequirement { type Err = ParsePyPiRequirementError; @@ -46,7 +82,7 @@ impl FromStr for PyPiRequirement { // From string can only parse the version specifier. Ok(Self { version: Some( - VersionSpecifiers::from_str(s) + pep440_rs::VersionSpecifiers::from_str(s) .map_err(ParsePyPiRequirementError::Pep440Error)?, ), extras: None, @@ -55,13 +91,34 @@ impl FromStr for PyPiRequirement { } } +/// Implement from [`pep508_rs::Requirement`] to make the conversion easier. +impl From for PyPiRequirement { + fn from(req: pep508_rs::Requirement) -> Self { + let version = if let Some(version_or_url) = req.version_or_url { + match version_or_url { + pep508_rs::VersionOrUrl::VersionSpecifier(v) => Some(v), + pep508_rs::VersionOrUrl::Url(_) => None, + } + } else { + None + }; + PyPiRequirement { + version, + extras: req.extras, + } + } +} + impl PyPiRequirement { /// Returns the requirements as [`pep508_rs::Requirement`]s. pub fn as_pep508(&self, name: &rip::types::PackageName) -> pep508_rs::Requirement { pep508_rs::Requirement { name: name.as_str().to_string(), extras: self.extras.clone(), - version_or_url: self.version.clone().map(VersionOrUrl::VersionSpecifier), + version_or_url: self + .version + .clone() + .map(pep508_rs::VersionOrUrl::VersionSpecifier), marker: None, } } @@ -91,13 +148,24 @@ impl<'de> Deserialize<'de> for PyPiRequirement { // Use a temp struct to deserialize into when it is a map. #[derive(Deserialize)] struct RawPyPiRequirement { - version: Option, + version: Option, extras: Option>, } let raw_requirement = RawPyPiRequirement::deserialize(de::value::MapAccessDeserializer::new(map))?; + + // Parse the * in version or allow for no version with extras. + let mut version = None; + if let Some(raw_version) = raw_requirement.version { + if raw_version != "*" { + version = Some( + VersionSpecifiers::from_str(raw_version.as_str()) + .map_err(A::Error::custom)?, + ); + } + } Ok(PyPiRequirement { - version: raw_requirement.version, + version, extras: raw_requirement.extras, }) } @@ -122,7 +190,7 @@ mod test { assert_eq!( requirement.first().unwrap().1, &PyPiRequirement { - version: Some(VersionSpecifiers::from_str(">=3.12").unwrap()), + version: Some(pep440_rs::VersionSpecifiers::from_str(">=3.12").unwrap()), extras: None } ); @@ -131,7 +199,7 @@ mod test { assert_eq!( requirement.first().unwrap().1, &PyPiRequirement { - version: Some(VersionSpecifiers::from_str("==3.12.0").unwrap()), + version: Some(pep440_rs::VersionSpecifiers::from_str("==3.12.0").unwrap()), extras: None } ); @@ -141,7 +209,7 @@ mod test { assert_eq!( requirement.first().unwrap().1, &PyPiRequirement { - version: Some(VersionSpecifiers::from_str("~=2.1.3").unwrap()), + version: Some(pep440_rs::VersionSpecifiers::from_str("~=2.1.3").unwrap()), extras: None } ); @@ -168,7 +236,7 @@ mod test { assert_eq!( requirement.first().unwrap().1, &PyPiRequirement { - version: Some(VersionSpecifiers::from_str(">=3.12").unwrap()), + version: Some(pep440_rs::VersionSpecifiers::from_str(">=3.12").unwrap()), extras: Some(vec!("bar".to_string())) } ); @@ -185,7 +253,7 @@ mod test { assert_eq!( requirement.first().unwrap().1, &PyPiRequirement { - version: Some(VersionSpecifiers::from_str(">=3.12,<3.13.0").unwrap()), + version: Some(pep440_rs::VersionSpecifiers::from_str(">=3.12,<3.13.0").unwrap()), extras: Some(vec!("bar".to_string(), "foo".to_string())) } ); diff --git a/src/project/snapshots/pixi__project__manifest__test__activation_scripts.snap b/src/project/snapshots/pixi__project__manifest__test__activation_scripts.snap index bd6a61da0..85450d578 100644 --- a/src/project/snapshots/pixi__project__manifest__test__activation_scripts.snap +++ b/src/project/snapshots/pixi__project__manifest__test__activation_scripts.snap @@ -52,6 +52,7 @@ ProjectManifest { dependencies: {}, host_dependencies: None, build_dependencies: None, + pypi_dependencies: None, activation: Some( Activation { scripts: Some( @@ -74,6 +75,7 @@ ProjectManifest { dependencies: {}, host_dependencies: None, build_dependencies: None, + pypi_dependencies: None, activation: Some( Activation { scripts: Some( diff --git a/src/project/snapshots/pixi__project__manifest__test__target_specific.snap b/src/project/snapshots/pixi__project__manifest__test__target_specific.snap index 1dfa4a5a5..6cd303ec5 100644 --- a/src/project/snapshots/pixi__project__manifest__test__target_specific.snap +++ b/src/project/snapshots/pixi__project__manifest__test__target_specific.snap @@ -72,6 +72,7 @@ ProjectManifest { }, host_dependencies: None, build_dependencies: None, + pypi_dependencies: None, activation: None, tasks: {}, }, @@ -106,6 +107,7 @@ ProjectManifest { }, host_dependencies: None, build_dependencies: None, + pypi_dependencies: None, activation: None, tasks: {}, }, diff --git a/src/project/snapshots/pixi__project__manifest__test__target_specific_tasks.snap b/src/project/snapshots/pixi__project__manifest__test__target_specific_tasks.snap index acb8cbbad..4db44368b 100644 --- a/src/project/snapshots/pixi__project__manifest__test__target_specific_tasks.snap +++ b/src/project/snapshots/pixi__project__manifest__test__target_specific_tasks.snap @@ -56,6 +56,7 @@ ProjectManifest { dependencies: {}, host_dependencies: None, build_dependencies: None, + pypi_dependencies: None, activation: None, tasks: { "test": Plain( @@ -74,6 +75,7 @@ ProjectManifest { dependencies: {}, host_dependencies: None, build_dependencies: None, + pypi_dependencies: None, activation: None, tasks: { "test": Plain( diff --git a/src/project/snapshots/pixi__project__tests__remove_dependencies.snap b/src/project/snapshots/pixi__project__tests__remove_dependencies.snap index 0a10389fc..78fc94f9b 100644 --- a/src/project/snapshots/pixi__project__tests__remove_dependencies.snap +++ b/src/project/snapshots/pixi__project__tests__remove_dependencies.snap @@ -69,6 +69,7 @@ ProjectManifest { }, host_dependencies: None, build_dependencies: None, + pypi_dependencies: None, activation: None, tasks: {}, }, @@ -99,6 +100,7 @@ ProjectManifest { }, }, ), + pypi_dependencies: None, activation: None, tasks: {}, }, diff --git a/src/project/snapshots/pixi__project__tests__remove_target_dependencies.snap b/src/project/snapshots/pixi__project__tests__remove_target_dependencies.snap index b01f78488..73f613ce8 100644 --- a/src/project/snapshots/pixi__project__tests__remove_target_dependencies.snap +++ b/src/project/snapshots/pixi__project__tests__remove_target_dependencies.snap @@ -83,6 +83,7 @@ ProjectManifest { }, host_dependencies: None, build_dependencies: None, + pypi_dependencies: None, activation: None, tasks: {}, }, @@ -99,6 +100,7 @@ ProjectManifest { build_dependencies: Some( {}, ), + pypi_dependencies: None, activation: None, tasks: {}, }, diff --git a/tests/add_tests.rs b/tests/add_tests.rs index 775a3297f..1bd036dfa 100644 --- a/tests/add_tests.rs +++ b/tests/add_tests.rs @@ -3,8 +3,9 @@ mod common; use crate::common::package_database::{Package, PackageDatabase}; use crate::common::LockFileExt; use crate::common::PixiControl; -use pixi::project::SpecType; -use rattler_conda_types::Platform; +use pixi::project::{DependencyType, SpecType}; +use rattler_conda_types::{PackageName, Platform}; +use std::str::FromStr; use tempfile::TempDir; /// Test add functionality for different types of packages. @@ -35,11 +36,11 @@ async fn add_functionality() { // Add a package pixi.add("rattler==1").await.unwrap(); pixi.add("rattler==2") - .set_type(SpecType::Host) + .set_type(DependencyType::CondaDependency(SpecType::Host)) .await .unwrap(); pixi.add("rattler==3") - .set_type(SpecType::Build) + .set_type(DependencyType::CondaDependency(SpecType::Build)) .await .unwrap(); @@ -76,10 +77,13 @@ async fn add_functionality_union() { // Add a package pixi.add("rattler").await.unwrap(); pixi.add("libcomputer") - .set_type(SpecType::Host) + .set_type(DependencyType::CondaDependency(SpecType::Host)) + .await + .unwrap(); + pixi.add("libidk") + .set_type(DependencyType::CondaDependency(SpecType::Build)) .await .unwrap(); - pixi.add("libidk").set_type(SpecType::Build).await.unwrap(); // Toml should contain the correct sections // We test if the toml file that is saved is correct @@ -134,10 +138,85 @@ async fn add_functionality_os() { // Add a package pixi.add("rattler==1") .set_platforms(&[Platform::LinuxS390X]) - .set_type(SpecType::Host) + .set_type(DependencyType::CondaDependency(SpecType::Host)) .await .unwrap(); let lock = pixi.lock_file().await.unwrap(); assert!(lock.contains_matchspec_for_platform("rattler==1", Platform::LinuxS390X)); } + +/// Test the `pixi add --pypi` functionality +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn add_pypi_functionality() { + let mut package_database = PackageDatabase::default(); + + // Add a package `foo` that depends on `bar` both set to version 1. + package_database.add_package(Package::build("python", "3.9").finish()); + + // Write the repodata to disk + let channel_dir = TempDir::new().unwrap(); + package_database + .write_repodata(channel_dir.path()) + .await + .unwrap(); + + let pixi = PixiControl::new().unwrap(); + + pixi.init() + .with_local_channel(channel_dir.path()) + .await + .unwrap(); + + // Add python + pixi.add("python") + .set_type(DependencyType::CondaDependency(SpecType::Run)) + .with_install(false) + .await + .unwrap(); + + // Add a pypi package + pixi.add("pipx") + .set_type(DependencyType::PypiDependency) + .with_install(false) + .await + .unwrap(); + + // Add a pypi package to a target + pixi.add("boto3>=1.33") + .set_type(DependencyType::PypiDependency) + .with_install(false) + .set_platforms(&[Platform::Osx64]) + .await + .unwrap(); + + // Add a pypi package to a target + pixi.add("pytest[all]") + .set_type(DependencyType::PypiDependency) + .set_platforms(&[Platform::Linux64]) + .with_install(false) + .await + .unwrap(); + + pixi.add("requests [security,tests] >= 2.8.1, == 2.8.*") + .set_type(DependencyType::PypiDependency) + .set_platforms(&[Platform::Linux64]) + .with_install(false) + .await + .unwrap(); + + let lock = pixi.lock_file().await.unwrap(); + assert!(lock.contains_package(&PackageName::from_str("pipx").unwrap())); + assert!(lock.contains_pep508_requirement_for_platform( + pep508_rs::Requirement::from_str("boto3>=1.33").unwrap(), + Platform::Osx64 + )); + assert!(lock.contains_pep508_requirement_for_platform( + pep508_rs::Requirement::from_str("pytest[all]").unwrap(), + Platform::Linux64 + )); + assert!(lock.contains_pep508_requirement_for_platform( + pep508_rs::Requirement::from_str("requests [security,tests] >= 2.8.1, == 2.8.*").unwrap(), + Platform::Linux64 + )); +} diff --git a/tests/common/builders.rs b/tests/common/builders.rs index 6d9371437..f43ada816 100644 --- a/tests/common/builders.rs +++ b/tests/common/builders.rs @@ -23,10 +23,9 @@ //! //! ``` -use crate::common::IntoMatchSpec; use futures::FutureExt; use pixi::cli::{add, init, install, project, task}; -use pixi::project::SpecType; +use pixi::project::{DependencyType, SpecType}; use rattler_conda_types::Platform; use std::future::{Future, IntoFuture}; use std::path::{Path, PathBuf}; @@ -79,25 +78,32 @@ pub struct AddBuilder { } impl AddBuilder { - pub fn with_spec(mut self, spec: impl IntoMatchSpec) -> Self { - self.args.specs.push(spec.into()); + pub fn with_spec(mut self, spec: &str) -> Self { + self.args.specs.push(spec.to_string()); self } /// Set as a host - pub fn set_type(mut self, t: SpecType) -> Self { + pub fn set_type(mut self, t: DependencyType) -> Self { match t { - SpecType::Host => { - self.args.host = true; - self.args.build = false; - } - SpecType::Build => { - self.args.host = false; - self.args.build = true; - } - SpecType::Run => { + DependencyType::CondaDependency(spec_type) => match spec_type { + SpecType::Host => { + self.args.host = true; + self.args.build = false; + } + SpecType::Build => { + self.args.host = false; + self.args.build = true; + } + SpecType::Run => { + self.args.host = false; + self.args.build = false; + } + }, + DependencyType::PypiDependency => { self.args.host = false; self.args.build = false; + self.args.pypi = true; } } self diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 4023e7f93..5c5e8f1e9 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -15,9 +15,11 @@ use pixi::cli::task::{AddArgs, AliasArgs}; use pixi::cli::{add, init, project, run, task}; use pixi::{consts, Project}; use rattler_conda_types::{MatchSpec, PackageName, Platform, Version}; -use rattler_lock::CondaLock; +use rattler_lock::{CondaLock, LockedDependencyKind}; +use std::collections::HashSet; use miette::IntoDiagnostic; +use pep508_rs::VersionOrUrl; use std::path::{Path, PathBuf}; use std::process::Output; use std::str::FromStr; @@ -61,6 +63,12 @@ pub trait LockFileExt { matchspec: impl IntoMatchSpec, platform: impl Into, ) -> bool; + /// Check if the pep508 requirement is contained in the lockfile for this platform + fn contains_pep508_requirement_for_platform( + &self, + requirement: pep508_rs::Requirement, + platform: impl Into, + ) -> bool; } impl LockFileExt for CondaLock { @@ -102,6 +110,46 @@ impl LockFileExt for CondaLock { && locked_dep.platform == platform }) } + + fn contains_pep508_requirement_for_platform( + &self, + requirement: pep508_rs::Requirement, + platform: impl Into, + ) -> bool { + let name = requirement.name; + let version: Option = + requirement + .version_or_url + .and_then(|version_or_url| match version_or_url { + VersionOrUrl::VersionSpecifier(version) => Some(version), + VersionOrUrl::Url(_) => unimplemented!(), + }); + + let platform = platform.into(); + self.package.iter().any(|locked_dep| { + let package_version = + pep440_rs::Version::from_str(&locked_dep.version).expect("could not parse version"); + + let req_extras = requirement + .extras + .as_ref() + .map(|extras| extras.iter().cloned().collect::>()) + .unwrap_or_default(); + + locked_dep.name == *name + && version + .as_ref() + .map_or(true, |v| v.contains(&package_version)) + && locked_dep.platform == platform + // Check if the extras are the same. + && match &locked_dep.kind { + LockedDependencyKind::Conda(_) => false, + LockedDependencyKind::Pypi(locked) => { + req_extras == locked.extras.iter().cloned().collect() + } + } + }) + } } impl PixiControl { @@ -151,16 +199,17 @@ impl PixiControl { /// Initialize pixi project inside a temporary directory. Returns a [`AddBuilder`]. To execute /// the command and await the result call `.await` on the return value. - pub fn add(&self, spec: impl IntoMatchSpec) -> AddBuilder { + pub fn add(&self, spec: &str) -> AddBuilder { AddBuilder { args: add::Args { manifest_path: Some(self.manifest_path()), host: false, - specs: vec![spec.into()], + specs: vec![spec.to_string()], build: false, no_install: true, no_lockfile_update: false, platform: Default::default(), + pypi: false, }, } } diff --git a/tests/snapshots/add_tests__add_pypi_functionality-2.snap b/tests/snapshots/add_tests__add_pypi_functionality-2.snap new file mode 100644 index 000000000..3032bfb6c --- /dev/null +++ b/tests/snapshots/add_tests__add_pypi_functionality-2.snap @@ -0,0 +1,125 @@ +--- +source: tests/add_tests.rs +expression: pixi.project().unwrap().manifest.target +--- +{ + PixiSpanned { + span: Some( + 303..309, + ), + value: Platform( + Osx64, + ), + }: TargetMetadata { + dependencies: {}, + host_dependencies: None, + build_dependencies: None, + pypi_dependencies: Some( + { + PackageName { + source: "boto3", + normalized: "boto3", + }: PyPiRequirement { + version: Some( + VersionSpecifiers( + [ + VersionSpecifier { + operator: GreaterThanEqual, + version: Version { + epoch: 0, + release: [ + 1, + 33, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + extras: None, + }, + }, + ), + activation: None, + tasks: {}, + }, + PixiSpanned { + span: Some( + 355..363, + ), + value: Platform( + Linux64, + ), + }: TargetMetadata { + dependencies: {}, + host_dependencies: None, + build_dependencies: None, + pypi_dependencies: Some( + { + PackageName { + source: "pytest", + normalized: "pytest", + }: PyPiRequirement { + version: None, + extras: Some( + [ + "all", + ], + ), + }, + PackageName { + source: "requests", + normalized: "requests", + }: PyPiRequirement { + version: Some( + VersionSpecifiers( + [ + VersionSpecifier { + operator: GreaterThanEqual, + version: Version { + epoch: 0, + release: [ + 2, + 8, + 1, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + VersionSpecifier { + operator: EqualStar, + version: Version { + epoch: 0, + release: [ + 2, + 8, + ], + pre: None, + post: None, + dev: None, + local: None, + }, + }, + ], + ), + ), + extras: Some( + [ + "security", + "tests", + ], + ), + }, + }, + ), + activation: None, + tasks: {}, + }, +} diff --git a/tests/snapshots/add_tests__add_pypi_functionality.snap b/tests/snapshots/add_tests__add_pypi_functionality.snap new file mode 100644 index 000000000..bb169a8a9 --- /dev/null +++ b/tests/snapshots/add_tests__add_pypi_functionality.snap @@ -0,0 +1,15 @@ +--- +source: tests/add_tests.rs +expression: pixi.project().unwrap().manifest.pypi_dependencies +--- +Some( + { + PackageName { + source: "pipx", + normalized: "pipx", + }: PyPiRequirement { + version: None, + extras: None, + }, + }, +)