From 549c8ce1aec26c433e85e7e39e160bf65a0a93bc Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 30 Oct 2024 17:19:01 +0100 Subject: [PATCH 1/2] start config for concurrency --- src/global/project/mod.rs | 2 +- src/project/repodata.rs | 2 +- src/repodata.rs | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/global/project/mod.rs b/src/global/project/mod.rs index 8909a595a..93bae3807 100644 --- a/src/global/project/mod.rs +++ b/src/global/project/mod.rs @@ -1025,7 +1025,7 @@ impl Repodata for Project { self.repodata_gateway.get_or_init(|| { Self::repodata_gateway_init( self.authenticated_client().clone(), - self.config().clone().into(), + self.config(), ) }) } diff --git a/src/project/repodata.rs b/src/project/repodata.rs index 472915642..ddef3f3a3 100644 --- a/src/project/repodata.rs +++ b/src/project/repodata.rs @@ -8,7 +8,7 @@ impl Repodata for Project { self.repodata_gateway.get_or_init(|| { Self::repodata_gateway_init( self.authenticated_client().clone(), - self.config().clone().into(), + self.config(), ) }) } diff --git a/src/repodata.rs b/src/repodata.rs index 23942560e..567835d04 100644 --- a/src/repodata.rs +++ b/src/repodata.rs @@ -6,7 +6,7 @@ pub(crate) trait Repodata { /// Initialized the [`Gateway`] fn repodata_gateway_init( authenticated_client: reqwest_middleware::ClientWithMiddleware, - channel_config: ChannelConfig, + channel_config: Config, ) -> Gateway { // Determine the cache directory and fall back to sane defaults otherwise. let cache_dir = pixi_config::get_cache_dir().unwrap_or_else(|e| { @@ -22,6 +22,7 @@ pub(crate) trait Repodata { .with_cache_dir(cache_dir.join(pixi_consts::consts::CONDA_REPODATA_CACHE_DIR)) .with_package_cache(package_cache) .with_channel_config(channel_config) + .with_max_concurrent_requests(10) .finish() } From 69ba8483d844c63263a3e85434c80e557110f96f Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 30 Oct 2024 17:32:43 +0100 Subject: [PATCH 2/2] wire up the concurrency config --- crates/pixi_config/src/lib.rs | 27 +++++++++++++++++++++++++++ src/global/project/mod.rs | 4 +++- src/project/repodata.rs | 3 ++- src/repodata.rs | 5 +++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/crates/pixi_config/src/lib.rs b/crates/pixi_config/src/lib.rs index 968b3f75f..2f3ab9e9d 100644 --- a/crates/pixi_config/src/lib.rs +++ b/crates/pixi_config/src/lib.rs @@ -179,6 +179,7 @@ pub enum DetachedEnvironments { Boolean(bool), Path(PathBuf), } + impl DetachedEnvironments { pub fn is_false(&self) -> bool { matches!(self, DetachedEnvironments::Boolean(false)) @@ -260,6 +261,7 @@ pub enum PinningStrategy { // Calling it no-pin to make it simple to type, as other option was pin-unconstrained. NoPin, } + impl FromStr for PinningStrategy { type Err = serde::de::value::Error; fn from_str(s: &str) -> Result { @@ -348,6 +350,22 @@ impl PinningStrategy { } } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct Concurrency { + /// The maximum number of concurrent HTTP requests to make. + #[serde(default)] + pub network_requests: usize, +} + +impl Default for Concurrency { + fn default() -> Self { + Self { + network_requests: 50, + } + } +} + #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub struct Config { @@ -409,6 +427,9 @@ pub struct Config { /// it back to the .pixi folder. #[serde(skip_serializing_if = "Option::is_none")] pub detached_environments: Option, + + /// Concurrency settings. + pub concurrency: Option, } impl Default for Config { @@ -425,6 +446,7 @@ impl Default for Config { pypi_config: PyPIConfig::default(), detached_environments: Some(DetachedEnvironments::default()), pinning_strategy: Default::default(), + concurrency: None, } } } @@ -688,6 +710,7 @@ impl Config { pypi_config: other.pypi_config.merge(self.pypi_config), detached_environments: other.detached_environments.or(self.detached_environments), pinning_strategy: other.pinning_strategy.or(self.pinning_strategy), + concurrency: other.concurrency.or(self.concurrency), } } @@ -745,6 +768,10 @@ impl Config { self.detached_environments.clone().unwrap_or_default() } + pub fn concurrency(&self) -> Concurrency { + self.concurrency.clone().unwrap_or_default() + } + /// Modify this config with the given key and value /// /// # Note diff --git a/src/global/project/mod.rs b/src/global/project/mod.rs index 93bae3807..37254aec6 100644 --- a/src/global/project/mod.rs +++ b/src/global/project/mod.rs @@ -1023,9 +1023,11 @@ impl Repodata for Project { /// Returns the [`Gateway`] used by this project. fn repodata_gateway(&self) -> &Gateway { self.repodata_gateway.get_or_init(|| { + tracing::info!("Concurrency is set to: {}", self.config().concurrency().network_requests); Self::repodata_gateway_init( self.authenticated_client().clone(), - self.config(), + self.config().into(), + self.config().concurrency().network_requests, ) }) } diff --git a/src/project/repodata.rs b/src/project/repodata.rs index ddef3f3a3..8f61dc228 100644 --- a/src/project/repodata.rs +++ b/src/project/repodata.rs @@ -8,7 +8,8 @@ impl Repodata for Project { self.repodata_gateway.get_or_init(|| { Self::repodata_gateway_init( self.authenticated_client().clone(), - self.config(), + self.config().into(), + self.config().concurrency().network_requests, ) }) } diff --git a/src/repodata.rs b/src/repodata.rs index 567835d04..5197dc9fc 100644 --- a/src/repodata.rs +++ b/src/repodata.rs @@ -6,7 +6,8 @@ pub(crate) trait Repodata { /// Initialized the [`Gateway`] fn repodata_gateway_init( authenticated_client: reqwest_middleware::ClientWithMiddleware, - channel_config: Config, + channel_config: ChannelConfig, + max_concurrent_requests: usize, ) -> Gateway { // Determine the cache directory and fall back to sane defaults otherwise. let cache_dir = pixi_config::get_cache_dir().unwrap_or_else(|e| { @@ -22,7 +23,7 @@ pub(crate) trait Repodata { .with_cache_dir(cache_dir.join(pixi_consts::consts::CONDA_REPODATA_CACHE_DIR)) .with_package_cache(package_cache) .with_channel_config(channel_config) - .with_max_concurrent_requests(10) + .with_max_concurrent_requests(max_concurrent_requests) .finish() }