diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9f98aca7a..00450ef2e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -103,10 +103,10 @@ jobs: fail-fast: false matrix: include: - - { name: "Linux", target: x86_64-unknown-linux-musl, os: ubuntu-20.04 } + - { name: "Linux", target: x86_64-unknown-linux-musl, os: 8core_ubuntu_latest_runner } - { name: "macOS", target: x86_64-apple-darwin, os: macos-13 } - { name: "macOS-arm", target: aarch64-apple-darwin, os: macos-14 } - - { name: "Windows", target: x86_64-pc-windows-msvc, os: windows-latest } + - { name: "Windows", target: x86_64-pc-windows-msvc, os: 16core_windows_latest_runner } steps: - uses: actions/checkout@v4 - uses: taiki-e/setup-cross-toolchain-action@v1 diff --git a/Cargo.lock b/Cargo.lock index 8587e3ae6..654d4bf56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,6 +227,21 @@ dependencies = [ "slab", ] +[[package]] +name = "async-fd-lock" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7569377d7062165f6f7834d9cb3051974a2d141433cc201c2f94c149e993cccf" +dependencies = [ + "async-trait", + "cfg-if", + "pin-project", + "rustix", + "thiserror", + "tokio", + "windows-sys 0.52.0", +] + [[package]] name = "async-fs" version = "2.1.2" @@ -1485,6 +1500,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs4" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c6b3bd49c37d2aa3f3f2220233b29a7cd23f79d1fe70e5337d25fb390793de" +dependencies = [ + "fs-err", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -3428,7 +3454,6 @@ dependencies = [ "serde_json", "serde_with", "serde_yaml", - "serial_test", "shlex", "signal-hook", "strsim", @@ -3974,9 +3999,9 @@ dependencies = [ [[package]] name = "rattler" -version = "0.27.6" +version = "0.27.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e94e9ac249e66d45f10216692c29750f39cb95f4ea701c623f0b3138c24fdb" +checksum = "2e25f40b548bffc452dad3e4664ae9c7fa20a33bd8f9889a7b0d93984bda9416" dependencies = [ "anyhow", "clap", @@ -4015,13 +4040,16 @@ dependencies = [ [[package]] name = "rattler_cache" -version = "0.1.8" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534344446506ef7b6035b2a3d75937b8375dea7a99da6eeb3631fd478782fa34" +checksum = "7ac5df432234cae7f70160ac221f94519cb9d9ad0896cc146db37dbec29fe566" dependencies = [ "anyhow", + "dashmap", "digest", "dirs", + "fs4", + "futures", "fxhash", "itertools 0.13.0", "parking_lot 0.12.3", @@ -4031,6 +4059,7 @@ dependencies = [ "rattler_package_streaming", "reqwest 0.12.5", "reqwest-middleware", + "simple_spawn_blocking", "thiserror", "tokio", "tracing", @@ -4039,9 +4068,9 @@ dependencies = [ [[package]] name = "rattler_conda_types" -version = "0.27.2" +version = "0.27.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4dc7a58ae237e3f57310095033da58b37d98ce7c167bc928b0f81951279c514" +checksum = "04c75121d007d0f8bd86af41dbc393c03da28ec6dff0c09081e59e708f3e95ee" dependencies = [ "chrono", "dirs", @@ -4092,9 +4121,9 @@ dependencies = [ [[package]] name = "rattler_lock" -version = "0.22.20" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce9290c3c0a702806b74b015a5c3dd638211255379f59d261e098ffc12d98d9" +checksum = "67d3cdbbd5159ed399c73ae56f0d1155e27540e1bcd0a6c0c522a259ec1a1762" dependencies = [ "chrono", "file_url", @@ -4125,9 +4154,9 @@ dependencies = [ [[package]] name = "rattler_networking" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c53c13a325db9d307886a6a31c3c88a3126b006fe618974b528b6dcf1943ece1" +checksum = "0a4c6c046bc414dd46f2f93ecbc78d661a06b9af123f836ea973e2080c62ee15" dependencies = [ "anyhow", "async-trait", @@ -4153,9 +4182,9 @@ dependencies = [ [[package]] name = "rattler_package_streaming" -version = "0.22.3" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4f972fe90d9ebbb055ca3cf3527d9206ff908fee1a39f880d5db209c729976" +checksum = "1c860f3eeb5d0bfe3ae0cbe8c6d1ef2f9711d72c558d9b9120db72fdb181c557" dependencies = [ "bzip2", "chrono", @@ -4192,12 +4221,13 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" -version = "0.21.8" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b4e6ee72bad488f18de5bdff0fb22f4a2d91a950e4d8b5b01700d4bfcae9c7" +checksum = "2a7bd0f0cf9418211588133da0590933789550b8b6fb47a8c9c594df258014b8" dependencies = [ "anyhow", "async-compression", + "async-fd-lock", "async-trait", "blake2", "bytes", @@ -4245,9 +4275,9 @@ dependencies = [ [[package]] name = "rattler_shell" -version = "0.21.6" +version = "0.21.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "470fb87026522cb8bdf914b7d044644889d62d91e18f57d3e01f90983a4d49b1" +checksum = "b63d167613b8c98aa9f337c6b3392a90ec21b29203b381d73e43a16321a87f22" dependencies = [ "enum_dispatch", "indexmap 2.3.0", @@ -4263,9 +4293,9 @@ dependencies = [ [[package]] name = "rattler_solve" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a2afc9a5f77abdc151bd99be9339078e1b6f9bfdbad04705bbb50c30ae11f50" +checksum = "8389dcf5c99f230122469552cd413895b5bf7bddc984be8a2ed000728d99484f" dependencies = [ "chrono", "futures", @@ -4282,9 +4312,9 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" -version = "1.0.4" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf264a6dff4b04c99c51a27b8008a1d250540a802179858b7e1b6f4b7c97689" +checksum = "025590e575da11dfe115387134dc1f347994111c7b117a64f0f941235dec63d5" dependencies = [ "archspec", "libloading", @@ -4918,15 +4948,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "scc" -version = "2.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d777f59627453628a9a5be1ee8d948745b94b1dfc2d0c3099cbd9e08ab89e7c" -dependencies = [ - "sdd", -] - [[package]] name = "schannel" version = "0.1.23" @@ -4997,12 +5018,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "sdd" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177258b64c0faaa9ffd3c65cd3262c2bc7e2588dbbd9c1641d0346145c1bbda8" - [[package]] name = "seahash" version = "4.1.0" @@ -5207,31 +5222,6 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "serial_test" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b4b487fe2acf240a021cf57c6b2b4903b1e78ca0ecd862a71b71d2a51fed77d" -dependencies = [ - "futures", - "log", - "once_cell", - "parking_lot 0.12.3", - "scc", - "serial_test_derive", -] - -[[package]] -name = "serial_test_derive" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", -] - [[package]] name = "sha-1" version = "0.10.1" diff --git a/Cargo.toml b/Cargo.toml index 2d325686d..543aa69bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,6 @@ serde_ignored = "0.1.10" serde_json = "1.0.116" serde_with = "3.7.0" serde_yaml = "0.9.34" -serial_test = "3.0.0" shlex = "1.3.0" signal-hook = "0.3.17" spdx = "0.10.4" @@ -90,16 +89,16 @@ typed-path = "0.9.1" # Rattler crates file_url = "0.1.4" -rattler = { version = "0.27.6", default-features = false } -rattler_cache = { version = "0.1.8", default-features = false } -rattler_conda_types = { version = "0.27.2", default-features = false } +rattler = { version = "0.27.8", default-features = false } +rattler_cache = { version = "0.2.0", default-features = false } +rattler_conda_types = { version = "0.27.4", default-features = false } rattler_digest = { version = "1.0.1", default-features = false } -rattler_lock = { version = "0.22.20", default-features = false } -rattler_networking = { version = "0.21.2", default-features = false } -rattler_repodata_gateway = { version = "0.21.8", default-features = false } -rattler_shell = { version = "0.21.6", default-features = false } -rattler_solve = { version = "1.0.3", default-features = false } -rattler_virtual_packages = { version = "1.0.4", default-features = false } +rattler_lock = { version = "0.22.22", default-features = false } +rattler_networking = { version = "0.21.3", default-features = false } +rattler_repodata_gateway = { version = "0.21.10", default-features = false } +rattler_shell = { version = "0.21.8", default-features = false } +rattler_solve = { version = "1.0.5", default-features = false } +rattler_virtual_packages = { version = "1.1.1", default-features = false } # Bumping this to a higher version breaks the Windows path handling. url = "2.5.0" @@ -314,7 +313,6 @@ http = { workspace = true } insta = { workspace = true, features = ["yaml", "glob"] } rstest = { workspace = true } serde_json = { workspace = true } -serial_test = { workspace = true } tokio = { workspace = true, features = ["rt"] } diff --git a/src/cli/exec.rs b/src/cli/exec.rs index cf21f7ef4..34156845d 100644 --- a/src/cli/exec.rs +++ b/src/cli/exec.rs @@ -12,7 +12,7 @@ use rattler::{ }; use rattler_conda_types::{GenericVirtualPackage, MatchSpec, PackageName, Platform}; use rattler_solve::{resolvo::Solver, SolverImpl, SolverTask}; -use rattler_virtual_packages::VirtualPackage; +use rattler_virtual_packages::{VirtualPackage, VirtualPackageOverrides}; use reqwest_middleware::ClientWithMiddleware; use crate::prefix::Prefix; @@ -185,7 +185,7 @@ pub async fn create_exec_prefix( .context("failed to get repodata")?; // Determine virtual packages of the current platform - let virtual_packages = VirtualPackage::current() + let virtual_packages = VirtualPackage::detect(&VirtualPackageOverrides::from_env()) .into_diagnostic() .context("failed to determine virtual packages")? .iter() diff --git a/src/cli/global/install.rs b/src/cli/global/install.rs index d87e23224..978759071 100644 --- a/src/cli/global/install.rs +++ b/src/cli/global/install.rs @@ -8,6 +8,8 @@ use clap::Parser; use indexmap::IndexMap; use itertools::Itertools; use miette::{Context, IntoDiagnostic}; +use pixi_config::{self, Config, ConfigCli}; +use pixi_progress::{await_in_progress, global_multi_progress, wrap_in_progress}; use pixi_utils::reqwest::build_reqwest_clients; use rattler::{ install::{DefaultProgressFormatter, IndicatifReporter, Installer}, @@ -21,16 +23,15 @@ use rattler_shell::{ shell::{Shell, ShellEnum}, }; use rattler_solve::{resolvo::Solver, SolverImpl, SolverTask}; -use rattler_virtual_packages::VirtualPackage; +use rattler_virtual_packages::{VirtualPackage, VirtualPackageOverrides}; use reqwest_middleware::ClientWithMiddleware; use super::common::{channel_name_from_prefix, find_designated_package, BinDir, BinEnvDir}; use crate::{ - cli::cli_config::ChannelsConfig, cli::has_specs::HasSpecs, prefix::Prefix, + cli::{cli_config::ChannelsConfig, has_specs::HasSpecs}, + prefix::Prefix, rlimit::try_increase_rlimit_to_sensible, }; -use pixi_config::{self, Config, ConfigCli}; -use pixi_progress::{await_in_progress, global_multi_progress, wrap_in_progress}; /// Installs the defined package in a global accessible location. #[derive(Parser, Debug)] @@ -313,7 +314,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { .into_diagnostic()?; // Determine virtual packages of the current platform - let virtual_packages = VirtualPackage::current() + let virtual_packages = VirtualPackage::detect(&VirtualPackageOverrides::from_env()) .into_diagnostic() .context("failed to determine virtual packages")? .iter() diff --git a/src/cli/global/upgrade.rs b/src/cli/global/upgrade.rs index dfbd2c85b..67b46889b 100644 --- a/src/cli/global/upgrade.rs +++ b/src/cli/global/upgrade.rs @@ -8,7 +8,7 @@ use miette::{Context, IntoDiagnostic, Report}; use pixi_utils::reqwest::build_reqwest_clients; use rattler_conda_types::{Channel, GenericVirtualPackage, MatchSpec, PackageName, Platform}; use rattler_solve::{resolvo::Solver, SolverImpl, SolverTask}; -use rattler_virtual_packages::VirtualPackage; +use rattler_virtual_packages::{VirtualPackage, VirtualPackageOverrides}; use tokio::task::JoinSet; use super::{common::find_installed_package, install::globally_install_package}; @@ -134,7 +134,7 @@ pub(super) async fn upgrade_packages( .collect(); // Determine virtual packages of the current platform - let virtual_packages = VirtualPackage::current() + let virtual_packages = VirtualPackage::detect(&VirtualPackageOverrides::from_env()) .into_diagnostic() .context("failed to determine virtual packages")? .iter() diff --git a/src/cli/info.rs b/src/cli/info.rs index 74a3643d3..5c52e4607 100644 --- a/src/cli/info.rs +++ b/src/cli/info.rs @@ -11,7 +11,7 @@ use pixi_manifest::{FeaturesExt, HasFeaturesIter}; use pixi_progress::await_in_progress; use rattler_conda_types::{GenericVirtualPackage, Platform}; use rattler_networking::authentication_storage; -use rattler_virtual_packages::VirtualPackage; +use rattler_virtual_packages::{VirtualPackage, VirtualPackageOverrides}; use serde::Serialize; use serde_with::{serde_as, DisplayFromStr}; use tokio::task::spawn_blocking; @@ -360,7 +360,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { }) .unwrap_or_default(); - let virtual_packages = VirtualPackage::current() + let virtual_packages = VirtualPackage::detect(&VirtualPackageOverrides::from_env()) .into_diagnostic()? .iter() .cloned() diff --git a/src/project/virtual_packages.rs b/src/project/virtual_packages.rs index 3291e47e4..42d3375fe 100644 --- a/src/project/virtual_packages.rs +++ b/src/project/virtual_packages.rs @@ -7,6 +7,7 @@ use pixi_manifest::{LibCSystemRequirement, SystemRequirements}; use rattler_conda_types::{GenericVirtualPackage, Platform, Version}; use rattler_virtual_packages::{ Archspec, Cuda, DetectVirtualPackageError, LibC, Linux, Osx, VirtualPackage, + VirtualPackageOverrides, }; use std::collections::HashMap; use thiserror::Error; @@ -128,7 +129,7 @@ pub(crate) fn verify_current_platform_has_required_virtual_packages( ))); } - let system_virtual_packages = VirtualPackage::current()? + let system_virtual_packages = VirtualPackage::detect(&VirtualPackageOverrides::from_env())? .iter() .cloned() .map(GenericVirtualPackage::from) diff --git a/tests/add_tests.rs b/tests/add_tests.rs index c49558fb1..c526c9b7d 100644 --- a/tests/add_tests.rs +++ b/tests/add_tests.rs @@ -12,7 +12,6 @@ use pixi_consts::consts; use pixi_manifest::pypi::VersionOrStar; use pixi_manifest::{pypi::PyPiPackageName, FeaturesExt, PyPiRequirement, SpecType}; use rattler_conda_types::{PackageName, Platform}; -use serial_test::serial; use tempfile::TempDir; use uv_normalize::ExtraName; @@ -240,7 +239,6 @@ async fn add_functionality_os() { /// Test the `pixi add --pypi` functionality #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] -#[serial] async fn add_pypi_functionality() { let pixi = PixiControl::new().unwrap(); @@ -363,7 +361,6 @@ async fn add_pypi_functionality() { /// Test the `pixi add --pypi` functionality with extras #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] -#[serial] async fn add_pypi_extra_functionality() { let pixi = PixiControl::new().unwrap(); @@ -449,7 +446,6 @@ async fn add_pypi_extra_functionality() { /// Test the sdist support for pypi packages #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] -#[serial] async fn add_sdist_functionality() { let pixi = PixiControl::new().unwrap(); diff --git a/tests/install_tests.rs b/tests/install_tests.rs index de929669c..ede9b7204 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -13,7 +13,6 @@ use pixi_config::{Config, DetachedEnvironments}; use pixi_consts::consts; use pixi_manifest::{FeatureName, FeaturesExt}; use rattler_conda_types::Platform; -use serial_test::serial; use std::{ fs::{create_dir_all, File}, io::Write, @@ -26,7 +25,6 @@ use uv_python::PythonEnvironment; /// Should add a python version to the environment and lock file that matches /// the specified version and run it #[tokio::test] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn install_run_python() { let pixi = PixiControl::new().unwrap(); @@ -153,7 +151,6 @@ async fn test_incremental_lock_file() { /// Test the `pixi install --locked` functionality. #[tokio::test] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn install_locked_with_config() { let pixi = PixiControl::new().unwrap(); @@ -249,7 +246,6 @@ async fn install_locked_with_config() { /// Test `pixi install/run --frozen` functionality #[tokio::test] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn install_frozen() { let pixi = PixiControl::new().unwrap(); @@ -303,7 +299,6 @@ fn create_uv_environment(prefix: &Path, cache: &uv_cache::Cache) -> PythonEnviro } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn pypi_reinstall_python() { let pixi = PixiControl::new().unwrap(); @@ -359,7 +354,6 @@ async fn pypi_reinstall_python() { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] // Check if we add and remove a pypi package that the site-packages is cleared async fn pypi_add_remove() { @@ -450,7 +444,6 @@ async fn test_channels_changed() { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn install_conda_meta_history() { let pixi = PixiControl::new().unwrap(); @@ -464,7 +457,6 @@ async fn install_conda_meta_history() { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn minimal_lockfile_update_pypi() { let pixi = PixiControl::new().unwrap(); @@ -509,7 +501,6 @@ async fn minimal_lockfile_update_pypi() { /// then change the installer back and see if it reinstalls the package /// with a new version #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn test_installer_name() { let pixi = PixiControl::new().unwrap(); @@ -567,7 +558,6 @@ async fn test_installer_name() { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] /// Test full prefix install for an old lock file to see if it still works. /// Makes sure the lockfile isn't touched and the environment is still installed. @@ -587,7 +577,6 @@ async fn test_old_lock_install() { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn test_no_build_isolation() { let current_platform = Platform::current(); @@ -662,7 +651,6 @@ setup( } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn test_many_linux_wheel_tag() { let pixi = PixiControl::new().unwrap(); diff --git a/tests/integration/test_main_cli.py b/tests/integration/test_main_cli.py index aaccad8d7..ffc272da8 100644 --- a/tests/integration/test_main_cli.py +++ b/tests/integration/test_main_cli.py @@ -366,9 +366,9 @@ def test_simple_project_setup(tmp_path: Path, pixi: Path) -> None: ) -def test_pixi_init_pyproject(tmp_path: Path) -> None: +def test_pixi_init_pyproject(tmp_path: Path, pixi: Path) -> None: manifest_path = tmp_path / "pyproject.toml" # Create a new project - verify_cli_command(f"pixi init {tmp_path} --pyproject-toml", ExitCode.SUCCESS) + verify_cli_command([pixi, "init", tmp_path, "--format", "pyproject"], ExitCode.SUCCESS) # Verify that install works - verify_cli_command(f"pixi install --manifest-path {manifest_path}", ExitCode.SUCCESS) + verify_cli_command([pixi, "install", "--manifest-path", manifest_path], ExitCode.SUCCESS) diff --git a/tests/solve_group_tests.rs b/tests/solve_group_tests.rs index 621cbc345..7535494da 100644 --- a/tests/solve_group_tests.rs +++ b/tests/solve_group_tests.rs @@ -7,7 +7,6 @@ use pypi_mapping::{self, PurlSource}; use rattler_conda_types::{PackageName, Platform, RepoDataRecord}; use rattler_lock::DEFAULT_ENVIRONMENT_NAME; use reqwest_middleware::ClientBuilder; -use serial_test::serial; use tempfile::TempDir; use url::Url; @@ -98,7 +97,6 @@ async fn conda_solve_group_functionality() { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[serial] async fn test_purl_are_added_for_pypi() { let pixi = PixiControl::new().unwrap(); pixi.init().await.unwrap();