Skip to content

Commit

Permalink
refactor(services/azblob): add AzblobConfig
Browse files Browse the repository at this point in the history
  • Loading branch information
@acehinnnqru
acehinnnqru committed Nov 10, 2023
1 parent 87a7ac1 commit 6c88f62
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 59 deletions.
140 changes: 81 additions & 59 deletions core/src/services/azblob/backend.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ use log::debug;
use reqsign::AzureStorageConfig;
use reqsign::AzureStorageLoader;
use reqsign::AzureStorageSigner;
use serde::Deserialize;
use sha2::Digest;
use sha2::Sha256;

Expand All @@ -53,25 +54,23 @@ const KNOWN_AZBLOB_ENDPOINT_SUFFIX: &[&str] = &[

const AZBLOB_BATCH_LIMIT: usize = 256;
/// Azure Storage Blob services support.
#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct AzblobBuilder {
root: Option<String>,
container: String,
endpoint: Option<String>,
account_name: Option<String>,
account_key: Option<String>,
encryption_key: Option<String>,
encryption_key_sha256: Option<String>,
encryption_algorithm: Option<String>,
sas_token: Option<String>,
http_client: Option<HttpClient>,
batch_max_operations: Option<usize>,
#[derive(Default, Deserialize, Clone)]
pub struct AzblobConfig {
pub root: Option<String>,
pub container: String,
pub endpoint: Option<String>,
pub account_name: Option<String>,
pub account_key: Option<String>,
pub encryption_key: Option<String>,
pub encryption_key_sha256: Option<String>,
pub encryption_algorithm: Option<String>,
pub sas_token: Option<String>,
pub batch_max_operations: Option<usize>,
}

impl Debug for AzblobBuilder {
impl Debug for AzblobConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("Builder");
let mut ds = f.debug_struct("Config");

ds.field("root", &self.root);
ds.field("container", &self.container);
Expand All @@ -91,26 +90,43 @@ impl Debug for AzblobBuilder {
}
}

#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct AzblobBuilder {
config: AzblobConfig,
http_client: Option<HttpClient>,
}

impl Debug for AzblobBuilder {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("Builder");

ds.field("config", &self.config);

ds.finish()
}
}

impl AzblobBuilder {
/// Set root of this backend.
///
/// All operations will happen under this root.
pub fn root(&mut self, root: &str) -> &mut Self {
if !root.is_empty() {
self.root = Some(root.to_string())
self.config.root = Some(root.to_string())
}

self
}

/// Set container name of this backend.
pub fn container(&mut self, container: &str) -> &mut Self {
self.container = container.to_string();
self.config.container = container.to_string();

self
}

/// Set endpoint of this backend.
/// Set endpoint of this backend
///
/// Endpoint must be full uri, e.g.
///
Expand All @@ -119,7 +135,7 @@ impl AzblobBuilder {
pub fn endpoint(&mut self, endpoint: &str) -> &mut Self {
if !endpoint.is_empty() {
// Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
self.endpoint = Some(endpoint.trim_end_matches('/').to_string());
self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string());
}

self
Expand All @@ -131,7 +147,7 @@ impl AzblobBuilder {
/// - If not, we will try to load it from environment.
pub fn account_name(&mut self, account_name: &str) -> &mut Self {
if !account_name.is_empty() {
self.account_name = Some(account_name.to_string());
self.config.account_name = Some(account_name.to_string());
}

self
Expand All @@ -143,7 +159,7 @@ impl AzblobBuilder {
/// - If not, we will try to load it from environment.
pub fn account_key(&mut self, account_key: &str) -> &mut Self {
if !account_key.is_empty() {
self.account_key = Some(account_key.to_string());
self.config.account_key = Some(account_key.to_string());
}

self
Expand All @@ -163,7 +179,7 @@ impl AzblobBuilder {
/// Please use `server_side_encryption_with_*` helpers if even possible.
pub fn encryption_key(&mut self, v: &str) -> &mut Self {
if !v.is_empty() {
self.encryption_key = Some(v.to_string());
self.config.encryption_key = Some(v.to_string());
}

self
Expand All @@ -183,7 +199,7 @@ impl AzblobBuilder {
/// Please use `server_side_encryption_with_*` helpers if even possible.
pub fn encryption_key_sha256(&mut self, v: &str) -> &mut Self {
if !v.is_empty() {
self.encryption_key_sha256 = Some(v.to_string());
self.config.encryption_key_sha256 = Some(v.to_string());
}

self
Expand All @@ -203,7 +219,7 @@ impl AzblobBuilder {
/// Please use `server_side_encryption_with_*` helpers if even possible.
pub fn encryption_algorithm(&mut self, v: &str) -> &mut Self {
if !v.is_empty() {
self.encryption_algorithm = Some(v.to_string());
self.config.encryption_algorithm = Some(v.to_string());
}

self
Expand All @@ -224,9 +240,10 @@ impl AzblobBuilder {
/// for more info.
pub fn server_side_encryption_with_customer_key(&mut self, key: &[u8]) -> &mut Self {
// Only AES256 is supported for now
self.encryption_algorithm = Some("AES256".to_string());
self.encryption_key = Some(BASE64_STANDARD.encode(key));
self.encryption_key_sha256 = Some(BASE64_STANDARD.encode(Sha256::digest(key).as_slice()));
self.config.encryption_algorithm = Some("AES256".to_string());
self.config.encryption_key = Some(BASE64_STANDARD.encode(key));
self.config.encryption_key_sha256 =
Some(BASE64_STANDARD.encode(Sha256::digest(key).as_slice()));
self
}

Expand All @@ -239,7 +256,7 @@ impl AzblobBuilder {
/// for more info.
pub fn sas_token(&mut self, sas_token: &str) -> &mut Self {
if !sas_token.is_empty() {
self.sas_token = Some(sas_token.to_string());
self.config.sas_token = Some(sas_token.to_string());
}

self
Expand All @@ -258,7 +275,7 @@ impl AzblobBuilder {

/// Set maximum batch operations of this backend.
pub fn batch_max_operations(&mut self, batch_max_operations: usize) -> &mut Self {
self.batch_max_operations = Some(batch_max_operations);
self.config.batch_max_operations = Some(batch_max_operations);

self
}
Expand Down Expand Up @@ -331,6 +348,7 @@ impl AzblobBuilder {
} else if let Some(v) = conn_map.get("EndpointSuffix") {
let protocol = conn_map.get("DefaultEndpointsProtocol").unwrap_or(&"https");
let account_name = builder
.config
.account_name
.as_ref()
.ok_or_else(|| {
Expand Down Expand Up @@ -375,19 +393,19 @@ impl Builder for AzblobBuilder {
fn build(&mut self) -> Result<Self::Accessor> {
debug!("backend build started: {:?}", &self);

let root = normalize_root(&self.root.take().unwrap_or_default());
let root = normalize_root(&self.config.root.take().unwrap_or_default());
debug!("backend use root {}", root);

// Handle endpoint, region and container name.
let container = match self.container.is_empty() {
false => Ok(&self.container),
let container = match self.config.container.is_empty() {
false => Ok(&self.config.container),
true => Err(Error::new(ErrorKind::ConfigInvalid, "container is empty")
.with_operation("Builder::build")
.with_context("service", Scheme::Azblob)),
}?;
debug!("backend use container {}", &container);

let endpoint = match &self.endpoint {
let endpoint = match &self.config.endpoint {
Some(endpoint) => Ok(endpoint.clone()),
None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
.with_operation("Builder::build")
Expand All @@ -406,30 +424,31 @@ impl Builder for AzblobBuilder {

let config_loader = AzureStorageConfig {
account_name: self
.config
.account_name
.clone()
.or_else(|| infer_storage_name_from_endpoint(endpoint.as_str())),
account_key: self.account_key.clone(),
sas_token: self.sas_token.clone(),
account_key: self.config.account_key.clone(),
sas_token: self.config.sas_token.clone(),
..Default::default()
};

let encryption_key =
match &self.encryption_key {
match &self.config.encryption_key {
None => None,
Some(v) => Some(build_header_value(v).map_err(|err| {
err.with_context("key", "server_side_encryption_customer_key")
})?),
};

let encryption_key_sha256 = match &self.encryption_key_sha256 {
let encryption_key_sha256 = match &self.config.encryption_key_sha256 {
None => None,
Some(v) => Some(build_header_value(v).map_err(|err| {
err.with_context("key", "server_side_encryption_customer_key_sha256")
})?),
};

let encryption_algorithm = match &self.encryption_algorithm {
let encryption_algorithm = match &self.config.encryption_algorithm {
None => None,
Some(v) => {
if v == "AES256" {
Expand All @@ -449,7 +468,10 @@ impl Builder for AzblobBuilder {

let signer = AzureStorageSigner::new();

let batch_max_operations = self.batch_max_operations.unwrap_or(AZBLOB_BATCH_LIMIT);
let batch_max_operations = self
.config
.batch_max_operations
.unwrap_or(AZBLOB_BATCH_LIMIT);

debug!("backend build finished: {:?}", &self);
Ok(AzblobBackend {
Expand All @@ -459,14 +481,14 @@ impl Builder for AzblobBuilder {
encryption_key,
encryption_key_sha256,
encryption_algorithm,
container: self.container.clone(),
container: self.config.container.clone(),

client,
loader: cred_loader,
signer,
batch_max_operations,
}),
has_sas_token: self.sas_token.is_some(),
has_sas_token: self.config.sas_token.is_some(),
})
}
}
Expand Down Expand Up @@ -793,7 +815,7 @@ mod tests {
assert_eq!(azblob.core.container, "container".to_string());

assert_eq!(
azblob_builder.account_key.unwrap(),
azblob_builder.config.account_key.unwrap(),
"account-key".to_string()
);
}
Expand All @@ -814,7 +836,7 @@ mod tests {

assert_eq!(azblob.core.container, "container".to_string());

assert_eq!(azblob_builder.account_key, None);
assert_eq!(azblob_builder.config.account_key, None);
}

#[test]
Expand All @@ -837,11 +859,11 @@ mod tests {
assert_eq!(azblob.core.container, "container".to_string());

assert_eq!(
azblob_builder.account_key.unwrap(),
azblob_builder.config.account_key.unwrap(),
"account-key".to_string()
);

assert_eq!(azblob_builder.sas_token.unwrap(), "sas".to_string());
assert_eq!(azblob_builder.config.sas_token.unwrap(), "sas".to_string());
}

#[test]
Expand All @@ -858,11 +880,11 @@ TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;
.expect("from connection string must succeed");

assert_eq!(
builder.endpoint.unwrap(),
builder.config.endpoint.unwrap(),
"http://127.0.0.1:10000/devstoreaccount1"
);
assert_eq!(builder.account_name.unwrap(), "devstoreaccount1");
assert_eq!(builder.account_key.unwrap(), "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==");
assert_eq!(builder.config.account_name.unwrap(), "devstoreaccount1");
assert_eq!(builder.config.account_key.unwrap(), "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==");

let builder = AzblobBuilder::from_connection_string(
r#"
Expand All @@ -875,11 +897,11 @@ EndpointSuffix=core.chinacloudapi.cn;
.expect("from connection string must succeed");

assert_eq!(
builder.endpoint.unwrap(),
builder.config.endpoint.unwrap(),
"https://storagesample.blob.core.chinacloudapi.cn"
);
assert_eq!(builder.account_name.unwrap(), "storagesample");
assert_eq!(builder.account_key.unwrap(), "account-key")
assert_eq!(builder.config.account_name.unwrap(), "storagesample");
assert_eq!(builder.config.account_key.unwrap(), "account-key")
}

#[test]
Expand All @@ -896,12 +918,12 @@ SharedAccessSignature=sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:0
.expect("from connection string must succeed");

assert_eq!(
builder.endpoint.unwrap(),
builder.config.endpoint.unwrap(),
"http://127.0.0.1:10000/devstoreaccount1"
);
assert_eq!(builder.sas_token.unwrap(), "sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D");
assert_eq!(builder.account_name, None);
assert_eq!(builder.account_key, None);
assert_eq!(builder.config.sas_token.unwrap(), "sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D");
assert_eq!(builder.config.account_name, None);
assert_eq!(builder.config.account_key, None);
}

#[test]
Expand All @@ -917,8 +939,8 @@ SharedAccessSignature=sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:0
.expect("from connection string must succeed");

// SAS should be preferred over shared key
assert_eq!(builder.sas_token.unwrap(), "sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D");
assert_eq!(builder.account_name, None);
assert_eq!(builder.account_key, None);
assert_eq!(builder.config.sas_token.unwrap(), "sv=2021-01-01&ss=b&srt=c&sp=rwdlaciytfx&se=2022-01-01T11:00:14Z&st=2022-01-02T03:00:14Z&spr=https&sig=KEllk4N8f7rJfLjQCmikL2fRVt%2B%2Bl73UBkbgH%2FK3VGE%3D");
assert_eq!(builder.config.account_name, None);
assert_eq!(builder.config.account_key, None);
}
}
1 change: 1 addition & 0 deletions core/src/services/azblob/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

mod backend;
pub use backend::AzblobBuilder as Azblob;
pub use backend::AzblobConfig;

mod core;
mod error;
Expand Down
2 changes: 2 additions & 0 deletions core/src/services/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
mod azblob;
#[cfg(feature = "services-azblob")]
pub use azblob::Azblob;
#[cfg(feature = "services-azblob")]
pub use azblob::AzblobConfig;

#[cfg(feature = "services-azdls")]
mod azdls;
Expand Down

0 comments on commit 6c88f62

Please sign in to comment.