From bed47a3b36f7260b5754eb82139fedd79fd0d91f Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Tue, 3 Sep 2024 10:03:17 +0200 Subject: [PATCH 1/7] fix: use two in memory indexes, for resolve and builds --- src/lock_file/resolve/pypi.rs | 13 +++++++++++-- src/lock_file/resolve/resolver_provider.rs | 14 +++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/lock_file/resolve/pypi.rs b/src/lock_file/resolve/pypi.rs index 6f76eedaf..5a8c5b863 100644 --- a/src/lock_file/resolve/pypi.rs +++ b/src/lock_file/resolve/pypi.rs @@ -385,9 +385,17 @@ pub async fn resolve_pypi( .into_diagnostic() .context("error creating requires-python for solver")?; + let registry_client2 = Arc::new( + RegistryClientBuilder::new(context.cache.clone()) + .client(context.client.clone()) + .index_urls(index_locations.index_urls()) + .keyring(context.keyring_provider) + .connectivity(Connectivity::Online) + .build(), + ); let fallback_provider = DefaultResolverProvider::new( DistributionDatabase::new( - ®istry_client, + ®istry_client2, &build_dispatch, context.concurrency.downloads, ), @@ -406,6 +414,7 @@ pub async fn resolve_pypi( package_requests: package_requests.clone(), }; + let in_memory_index2 = InMemoryIndex::default(); let python_version = PythonVersion::from_str(&interpreter_version.to_string()) .expect("could not get version from interpreter"); let resolution = Resolver::new_custom_io( @@ -414,7 +423,7 @@ pub async fn resolve_pypi( &context.hash_strategy, ResolverMarkers::SpecificEnvironment(marker_environment.into()), &PythonRequirement::from_python_version(&interpreter, &python_version), - &in_memory_index, + &in_memory_index2, &git_resolver, provider, EmptyInstalledPackages, diff --git a/src/lock_file/resolve/resolver_provider.rs b/src/lock_file/resolve/resolver_provider.rs index 19100870f..8dcdc816f 100644 --- a/src/lock_file/resolve/resolver_provider.rs +++ b/src/lock_file/resolve/resolver_provider.rs @@ -39,6 +39,13 @@ impl<'a, Context: BuildContext> ResolverProvider for CondaResolverProvider<'a, C ) -> impl Future + 'io { if let Some((repodata_record, identifier)) = self.conda_python_identifiers.get(package_name) { + let version = repodata_record.version().to_string(); + + tracing::debug!( + "overriding PyPI package version request {}=={}", + package_name, + version + ); // If we encounter a package that was installed by conda we simply return a single // available version in the form of a source distribution with the URL of the // conda package. @@ -60,11 +67,7 @@ impl<'a, Context: BuildContext> ResolverProvider for CondaResolverProvider<'a, C let source_dist = RegistrySourceDist { name: identifier.name.as_normalized().clone(), - version: repodata_record - .version() - .to_string() - .parse() - .expect("could not convert to pypi version"), + version: version.parse().expect("could not convert to pypi version"), file: Box::new(file), index: IndexUrl::Pypi(VerbatimUrl::from_url( consts::DEFAULT_PYPI_INDEX_URL.clone(), @@ -104,6 +107,7 @@ impl<'a, Context: BuildContext> ResolverProvider for CondaResolverProvider<'a, C ) -> impl Future + 'io { if let Dist::Source(SourceDist::Registry(RegistrySourceDist { name, .. })) = dist { if let Some((_, iden)) = self.conda_python_identifiers.get(name) { + tracing::debug!("overriding PyPI package metadata request {}", name); // If this is a Source dist and the package is actually installed by conda we // create fake metadata with no dependencies. We assume that all conda installed // packages are properly installed including its dependencies. From bb13ab5bb944a52bcf9595896c8e6b0c42541b3b Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Tue, 3 Sep 2024 10:27:47 +0200 Subject: [PATCH 2/7] fix: use one registry client again --- src/lock_file/resolve/pypi.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/lock_file/resolve/pypi.rs b/src/lock_file/resolve/pypi.rs index 5a8c5b863..587cd7efa 100644 --- a/src/lock_file/resolve/pypi.rs +++ b/src/lock_file/resolve/pypi.rs @@ -385,17 +385,9 @@ pub async fn resolve_pypi( .into_diagnostic() .context("error creating requires-python for solver")?; - let registry_client2 = Arc::new( - RegistryClientBuilder::new(context.cache.clone()) - .client(context.client.clone()) - .index_urls(index_locations.index_urls()) - .keyring(context.keyring_provider) - .connectivity(Connectivity::Online) - .build(), - ); let fallback_provider = DefaultResolverProvider::new( DistributionDatabase::new( - ®istry_client2, + ®istry_client, &build_dispatch, context.concurrency.downloads, ), From d4e67f25b734a426f6989a1af1dd4f74f0023094 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Tue, 3 Sep 2024 10:40:31 +0200 Subject: [PATCH 3/7] fix: better comments explaining the situation --- src/lock_file/resolve/pypi.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/lock_file/resolve/pypi.rs b/src/lock_file/resolve/pypi.rs index 587cd7efa..1adfb9383 100644 --- a/src/lock_file/resolve/pypi.rs +++ b/src/lock_file/resolve/pypi.rs @@ -285,7 +285,13 @@ pub async fn resolve_pypi( }; // Create a shared in-memory index. - let in_memory_index = InMemoryIndex::default(); + // We need two in-memory indexes, one for the build dispatch and one for the resolver. + // because we manually override requests for the resolver, + // but we don't want to override requests for the build dispatch. + // + // The BuildDispatch might resolve or install when building wheels which will be mostly + // with build isolation. In that case we want to use fresh non-tampered requests. + let build_dispatch_in_memory_index = InMemoryIndex::default(); let config_settings = ConfigSettings::default(); let env = PythonEnvironment::from_interpreter(interpreter.clone()); @@ -303,7 +309,7 @@ pub async fn resolve_pypi( &interpreter, &index_locations, &flat_index, - &in_memory_index, + &build_dispatch_in_memory_index, &git_resolver, &context.in_flight, IndexStrategy::default(), @@ -406,7 +412,9 @@ pub async fn resolve_pypi( package_requests: package_requests.clone(), }; - let in_memory_index2 = InMemoryIndex::default(); + // We need a new in-memory index for the resolver so that it does not conflict with the build dispatch + // one. As we have noted in the comment above. + let resolver_in_memory_index = InMemoryIndex::default(); let python_version = PythonVersion::from_str(&interpreter_version.to_string()) .expect("could not get version from interpreter"); let resolution = Resolver::new_custom_io( @@ -415,7 +423,7 @@ pub async fn resolve_pypi( &context.hash_strategy, ResolverMarkers::SpecificEnvironment(marker_environment.into()), &PythonRequirement::from_python_version(&interpreter, &python_version), - &in_memory_index2, + &resolver_in_memory_index, &git_resolver, provider, EmptyInstalledPackages, From d77123ba9c45f788f2b5b341a6534914f413286e Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Tue, 3 Sep 2024 11:07:46 +0200 Subject: [PATCH 4/7] feat: add test --- tests/install_tests.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/install_tests.rs b/tests/install_tests.rs index de929669c..457469e3b 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -661,6 +661,31 @@ setup( pixi.install().await.expect("cannot install project"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial] +#[cfg_attr(not(feature = "slow_integration_tests"), ignore)] +async fn test_setuptools_override_failure() { + // This was causing issues like: https://github.com/prefix-dev/pixi/issues/1686 + let manifest = format!( + r#" + [project] + channels = ["conda-forge"] + name = "pixi-source-problem" + platforms = ["{platform}"] + + [dependencies] + pip = ">=24.0,<25" + + # The transitive dependency of viser was causing issues + [pypi-dependencies] + viser = "==0.2.7" + "#, + platform = Platform::current() + ); + let pixi = PixiControl::from_manifest(&manifest).expect("cannot instantiate pixi project"); + pixi.install().await.expect("cannot install project"); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] From 3d8f991390caf51add25792f6dcb920cecee1da6 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Tue, 3 Sep 2024 12:12:53 +0200 Subject: [PATCH 5/7] Update tests/install_tests.rs Co-authored-by: Hofer-Julian <30049909+Hofer-Julian@users.noreply.github.com> --- tests/install_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/install_tests.rs b/tests/install_tests.rs index 457469e3b..e90b88f1e 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -676,7 +676,7 @@ async fn test_setuptools_override_failure() { [dependencies] pip = ">=24.0,<25" - # The transitive dependency of viser was causing issues + # The transitive dependencies of viser was causing issues [pypi-dependencies] viser = "==0.2.7" "#, From 46ae4197cb269888061358afdfdd034f256f0962 Mon Sep 17 00:00:00 2001 From: Hofer-Julian <30049909+Hofer-Julian@users.noreply.github.com> Date: Tue, 3 Sep 2024 12:15:24 +0200 Subject: [PATCH 6/7] Fix typo --- tests/install_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/install_tests.rs b/tests/install_tests.rs index e90b88f1e..ab579200e 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -676,7 +676,7 @@ async fn test_setuptools_override_failure() { [dependencies] pip = ">=24.0,<25" - # The transitive dependencies of viser was causing issues + # The transitive dependencies of viser were causing issues [pypi-dependencies] viser = "==0.2.7" "#, From bbcaef8282af14c0ac3db50e7746668ea4133b3f Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 3 Sep 2024 16:21:01 +0200 Subject: [PATCH 7/7] fix: remove serial --- tests/install_tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/install_tests.rs b/tests/install_tests.rs index 364c4add1..8396f2ab0 100644 --- a/tests/install_tests.rs +++ b/tests/install_tests.rs @@ -675,7 +675,6 @@ async fn test_setuptools_override_failure() { } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -#[serial] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn test_many_linux_wheel_tag() { let pixi = PixiControl::new().unwrap();