diff --git a/Cargo.lock b/Cargo.lock index 27548bc2589..243e63bd25b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4021,9 +4021,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.28.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" dependencies = [ "memchr", "serde", @@ -4289,9 +4289,9 @@ checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "reqsign" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6cb65eb3405f9c2de5c18bfc37338d6bbdb2c35eb8eb0e946208cbb564e4833" +checksum = "f88838c22f5c35c2e8d5552842427a425b34d0693189091a3607d24949894414" dependencies = [ "anyhow", "async-trait", @@ -4306,7 +4306,7 @@ dependencies = [ "log", "once_cell", "percent-encoding", - "quick-xml 0.28.2", + "quick-xml 0.29.0", "rand 0.8.5", "reqwest", "rsa", diff --git a/core/Cargo.toml b/core/Cargo.toml index 33f2d742701..3e2fbfd1c7b 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -193,7 +193,7 @@ cacache = { version = "11.6", default-features = false, features = [ chrono = "0.4.26" dashmap = { version = "5.4", optional = true } dirs = { version = "5.0.1", optional = true } -etcd-client = { version = "0.11", optional = true, features = ["tls"]} +etcd-client = { version = "0.11", optional = true, features = ["tls"] } flagset = "0.4" futures = { version = "0.3", default-features = false, features = ["std"] } governor = { version = "0.5", optional = true, features = ["std"] } @@ -228,7 +228,7 @@ redis = { version = "0.23", features = [ "tokio-comp", "connection-manager", ], optional = true } -reqsign = { version = "0.13.0", default-features = false, optional = true } +reqsign = { version = "0.14.0", default-features = false, optional = true } reqwest = { version = "0.11.18", features = [ "stream", ], default-features = false } diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 50fa32f6aeb..2a12a2b29ba 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -31,9 +31,10 @@ use log::warn; use md5::Digest; use md5::Md5; use once_cell::sync::Lazy; +use reqsign::AwsAssumeRoleLoader; use reqsign::AwsConfig; use reqsign::AwsCredentialLoad; -use reqsign::AwsLoader; +use reqsign::AwsDefaultLoader; use reqsign::AwsV4Signer; use super::core::*; @@ -69,33 +70,34 @@ pub struct S3Builder { bucket: String, endpoint: Option, region: Option, - role_arn: Option, - external_id: Option, + + // Credentials related values. access_key_id: Option, secret_access_key: Option, + security_token: Option, + role_arn: Option, + external_id: Option, + disable_config_load: bool, + disable_ec2_metadata: bool, + allow_anonymous: bool, + customed_credential_load: Option>, + + // S3 features flags server_side_encryption: Option, server_side_encryption_aws_kms_key_id: Option, server_side_encryption_customer_algorithm: Option, server_side_encryption_customer_key: Option, server_side_encryption_customer_key_md5: Option, default_storage_class: Option, - - /// temporary credentials, check the official [doc](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html) for detail - security_token: Option, - - disable_config_load: bool, - disable_ec2_metadata: bool, - allow_anonymous: bool, enable_virtual_host_style: bool, - http_client: Option, - customed_credential_load: Option>, - /// the part size of s3 multipart upload, which should be 5 MiB to 5 GiB. /// There is no minimum size limit on the last part of your multipart upload write_min_size: Option, /// batch_max_operations batch_max_operations: Option, + + http_client: Option, } impl Debug for S3Builder { @@ -191,6 +193,9 @@ impl S3Builder { } /// Set role_arn for this backend. + /// + /// If `role_arn` is set, we will use already known config as source + /// credential to assume role with `role_arn`. pub fn role_arn(&mut self, v: &str) -> &mut Self { if !v.is_empty() { self.role_arn = Some(v.to_string()) @@ -432,6 +437,9 @@ impl S3Builder { } /// Adding a customed credential load for service. + /// + /// If customed_credential_load has been set, we will ignore all other + /// credential load methods. pub fn customed_credential_load(&mut self, cred: Box) -> &mut Self { self.customed_credential_load = Some(cred); self @@ -756,32 +764,16 @@ impl Builder for S3Builder { })? }; + // This is our current config. let mut cfg = AwsConfig::default(); if !self.disable_config_load { cfg = cfg.from_profile(); cfg = cfg.from_env(); } - // Setting all value from user input if available. if let Some(v) = self.region.take() { cfg.region = Some(v); } - if let Some(v) = self.access_key_id.take() { - cfg.access_key_id = Some(v) - } - if let Some(v) = self.secret_access_key.take() { - cfg.secret_access_key = Some(v) - } - if let Some(v) = self.security_token.take() { - cfg.session_token = Some(v) - } - if let Some(v) = self.role_arn.take() { - cfg.role_arn = Some(v) - } - if let Some(v) = self.external_id.take() { - cfg.external_id = Some(v) - } - if cfg.region.is_none() { // AWS S3 requires region to be set. if self.endpoint.is_none() @@ -805,15 +797,66 @@ impl Builder for S3Builder { let endpoint = self.build_endpoint(®ion); debug!("backend use endpoint: {endpoint}"); - let mut loader = AwsLoader::new(client.client(), cfg); - if self.disable_ec2_metadata { - loader = loader.with_disable_ec2_metadata(); + // Setting all value from user input if available. + if let Some(v) = self.access_key_id.take() { + cfg.access_key_id = Some(v) + } + if let Some(v) = self.secret_access_key.take() { + cfg.secret_access_key = Some(v) } + if let Some(v) = self.security_token.take() { + cfg.session_token = Some(v) + } + + let mut loader: Option> = None; + // If customed_credential_load is set, we will use it. if let Some(v) = self.customed_credential_load.take() { - loader = loader.with_customed_credential_loader(v); + loader = Some(v); + } + + // If role_arn is set, we must use AssumeRoleLoad. + if let Some(role_arn) = self.role_arn.take() { + // use current env as source credential loader. + let default_loader = AwsDefaultLoader::new(client.client(), cfg.clone()); + + // Build the config for assume role. + let assume_role_cfg = AwsConfig { + region: Some(region.clone()), + role_arn: Some(role_arn), + external_id: self.external_id.clone(), + sts_regional_endpoints: "regional".to_string(), + ..Default::default() + }; + let assume_role_loader = AwsAssumeRoleLoader::new( + client.client(), + assume_role_cfg, + Box::new(default_loader), + ) + .map_err(|err| { + Error::new( + ErrorKind::ConfigInvalid, + "The assume_role_loader is misconfigured", + ) + .with_context("service", Scheme::S3) + .set_source(err) + })?; + loader = Some(Box::new(assume_role_loader)); } + // If loader is not set, we will use default loader. + let loader = match loader { + Some(v) => v, + None => { + let mut default_loader = AwsDefaultLoader::new(client.client(), cfg); + if self.disable_ec2_metadata { + default_loader = default_loader.with_disable_ec2_metadata(); + } + + Box::new(default_loader) + } + }; let signer = AwsV4Signer::new("s3", ®ion); + let write_min_size = self.write_min_size.unwrap_or(DEFAULT_WRITE_MIN_SIZE); if write_min_size < 5 * 1024 * 1024 { return Err(Error::new( diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs index 18746399b45..027d6f36a86 100644 --- a/core/src/services/s3/core.rs +++ b/core/src/services/s3/core.rs @@ -33,7 +33,7 @@ use http::HeaderValue; use http::Request; use http::Response; use reqsign::AwsCredential; -use reqsign::AwsLoader; +use reqsign::AwsCredentialLoad; use reqsign::AwsV4Signer; use serde::Deserialize; use serde::Serialize; @@ -79,7 +79,7 @@ pub struct S3Core { pub allow_anonymous: bool, pub signer: AwsV4Signer, - pub loader: AwsLoader, + pub loader: Box, pub client: HttpClient, pub write_min_size: usize, pub batch_max_operations: usize, @@ -100,7 +100,7 @@ impl S3Core { async fn load_credential(&self) -> Result> { let cred = self .loader - .load() + .load_credential(self.client.client()) .await .map_err(new_request_credential_error)?; diff --git a/core/src/services/wasabi/backend.rs b/core/src/services/wasabi/backend.rs index c54eca1617c..954eec8f6b0 100644 --- a/core/src/services/wasabi/backend.rs +++ b/core/src/services/wasabi/backend.rs @@ -32,7 +32,7 @@ use md5::Md5; use once_cell::sync::Lazy; use reqsign::AwsConfig; use reqsign::AwsCredentialLoad; -use reqsign::AwsLoader; +use reqsign::AwsDefaultLoader; use reqsign::AwsV4Signer; use super::core::*; @@ -854,13 +854,10 @@ impl Builder for WasabiBuilder { let endpoint = self.build_endpoint(®ion); debug!("backend use endpoint: {endpoint}"); - let mut loader = AwsLoader::new(client.client(), cfg); + let mut loader = AwsDefaultLoader::new(client.client(), cfg); if self.disable_ec2_metadata { loader = loader.with_disable_ec2_metadata(); } - if let Some(v) = self.customed_credential_load.take() { - loader = loader.with_customed_credential_loader(v); - } let signer = AwsV4Signer::new("s3", ®ion); diff --git a/core/src/services/wasabi/core.rs b/core/src/services/wasabi/core.rs index 5f28bd51d3a..0a074d49212 100644 --- a/core/src/services/wasabi/core.rs +++ b/core/src/services/wasabi/core.rs @@ -31,7 +31,7 @@ use http::HeaderValue; use http::Request; use http::Response; use reqsign::AwsCredential; -use reqsign::AwsLoader; +use reqsign::AwsDefaultLoader; use reqsign::AwsV4Signer; use serde::Deserialize; use serde::Serialize; @@ -82,7 +82,7 @@ pub struct WasabiCore { pub default_storage_class: Option, pub signer: AwsV4Signer, - pub loader: AwsLoader, + pub loader: AwsDefaultLoader, pub client: HttpClient, }