Skip to content

Commit

Permalink
fix: offline mapping should not request (#1968)
Browse files Browse the repository at this point in the history
  • Loading branch information
nichmor authored Sep 3, 2024
1 parent 1d33927 commit aca1bd4
Show file tree
Hide file tree
Showing 9 changed files with 185 additions and 40 deletions.
50 changes: 23 additions & 27 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ repository = "https://github.com/prefix-dev/pixi"
ahash = "0.8.11"
assert_matches = "1.5.0"
async-once-cell = "0.5.3"
async-trait = "0.1.82"
cfg-if = "1.0"
chrono = "0.4.38"
clap = { version = "4.5.9", default-features = false }
Expand All @@ -33,6 +34,7 @@ fd-lock = "4.0.2"
flate2 = "1.0.28"
fs_extra = "1.3.0"
futures = "0.3.30"
http = "1.1.0"
http-cache-reqwest = "0.14.0"
human_bytes = "0.4.3"
humantime = "2.1.0"
Expand Down Expand Up @@ -284,6 +286,7 @@ uv-types = { workspace = true }
xxhash-rust = { workspace = true }
zip = { workspace = true, features = ["deflate", "time"] }


[target.'cfg(unix)'.dependencies]
libc = { workspace = true, default-features = false }
nix = { workspace = true, features = ["fs", "signal", "term", "poll"] }
Expand All @@ -306,12 +309,15 @@ strip = false


[dev-dependencies]
async-trait = { workspace = true }
http = { workspace = true }
insta = { workspace = true, features = ["yaml", "glob"] }
rstest = { workspace = true }
serde_json = { workspace = true }
serial_test = { workspace = true }
tokio = { workspace = true, features = ["rt"] }


[patch.crates-io]
# For pyproject-toml
# pyproject-toml = { git = "https://github.com/tdejager/pyproject-toml-rs", branch = "feat/bump-508-440" }
Expand Down
23 changes: 20 additions & 3 deletions crates/pypi_mapping/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
use std::{collections::HashMap, path::PathBuf, str::FromStr, sync::Arc};
use std::{
collections::{BTreeSet, HashMap},
path::PathBuf,
str::FromStr,
sync::Arc,
};

use async_once_cell::OnceCell as AsyncCell;
use custom_pypi_mapping::fetch_mapping_from_path;
Expand Down Expand Up @@ -96,6 +101,7 @@ impl CustomMapping {
pub enum MappingSource {
Custom(Arc<CustomMapping>),
Prefix,
Disabled,
}

impl MappingSource {
Expand Down Expand Up @@ -130,7 +136,7 @@ impl PurlSource {
}

pub async fn amend_pypi_purls(
client: reqwest::Client,
client: ClientWithMiddleware,
mapping_source: &MappingSource,
conda_packages: &mut [RepoDataRecord],
reporter: Option<Arc<dyn Reporter>>,
Expand All @@ -148,7 +154,7 @@ pub async fn amend_pypi_purls(
options: HttpCacheOptions::default(),
});

let client = ClientBuilder::new(client)
let client = ClientBuilder::from_client(client)
.with(cache_strategy)
.with(retry_strategy)
.build();
Expand All @@ -161,6 +167,17 @@ pub async fn amend_pypi_purls(
MappingSource::Prefix => {
prefix_pypi_name_mapping::amend_pypi_purls(&client, conda_packages, reporter).await?;
}
MappingSource::Disabled => {
for record in conda_packages.iter_mut() {
if let Some(purl) = prefix_pypi_name_mapping::assume_conda_is_pypi(None, record) {
record
.package_record
.purls
.get_or_insert_with(BTreeSet::new)
.insert(purl);
}
}
}
}

Ok(())
Expand Down
22 changes: 16 additions & 6 deletions crates/pypi_mapping/src/prefix_pypi_name_mapping.rs
Original file line number Diff line number Diff line change
Expand Up @@ -236,12 +236,8 @@ pub fn amend_pypi_purls_for_record(

// package is not in our mapping yet
// so we assume that it is the same as the one from conda-forge
if purls.is_none() && is_conda_forge_record(record) {
// Convert the conda package names to pypi package names. If the conversion
// fails we just assume that its not a valid python package.
if let Some(purl) = build_pypi_purl_from_package_record(&record.package_record) {
purls.get_or_insert_with(Vec::new).push(purl);
}
if let Some(purl) = assume_conda_is_pypi(purls.as_ref(), record) {
purls.get_or_insert_with(Vec::new).push(purl);
}

// If we have found some purls we overwrite whatever was there before.
Expand All @@ -251,3 +247,17 @@ pub fn amend_pypi_purls_for_record(

Ok(())
}

/// Try to assume that the conda-forge package is a PyPI package and return a purl.
pub fn assume_conda_is_pypi(
purls: Option<&Vec<PackageUrl>>,
record: &RepoDataRecord,
) -> Option<PackageUrl> {
if purls.is_none() && is_conda_forge_record(record) {
// Convert the conda package names to pypi package names. If the conversion
// fails we just assume that its not a valid python package.
build_pypi_purl_from_package_record(&record.package_record)
} else {
None
}
}
4 changes: 2 additions & 2 deletions src/lock_file/update.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1569,7 +1569,7 @@ async fn spawn_solve_conda_environment_task(
if has_pypi_dependencies {
pb.set_message("extracting pypi packages");
pypi_mapping::amend_pypi_purls(
client,
client.into(),
&pypi_name_mapping_location,
&mut records,
Some(pb.purl_amend_reporter()),
Expand Down Expand Up @@ -1803,7 +1803,7 @@ async fn spawn_solve_pypi_task(
let locked_pypi_records = locked_pypi_packages.records.clone();

pypi_mapping::amend_pypi_purls(
environment.project().client().clone(),
environment.project().client().clone().into(),
pypi_name_mapping_location,
&mut conda_records,
None,
Expand Down
5 changes: 5 additions & 0 deletions src/project/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,11 @@ impl Project {
})
.collect::<miette::Result<HashMap<Channel, String>>>()?;

// User can disable the mapping by providing an empty map
if channel_to_location_map.is_empty() {
return Ok(MappingSource::Disabled);
}

let project_channels: HashSet<_> = manifest
.parsed
.project
Expand Down
Loading

0 comments on commit aca1bd4

Please sign in to comment.