Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Implement Config for part of services #4277

Merged
merged 9 commits into from
Feb 27, 2024
Merged

feat: Implement Config for part of services #4277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3757ca9
add config for azdls
Xuanwo Feb 26, 2024
636b2bc
Add config for azfile
Xuanwo Feb 26, 2024
958b076
Add cacache
Xuanwo Feb 26, 2024
33b17b6
Impl config for cloudflare kv
Xuanwo Feb 26, 2024
f1676bb
Migrate cos
Xuanwo Feb 26, 2024
d0c65bd
Impl config for dashmap
Xuanwo Feb 26, 2024
380f83a
Add config for dbfs
Xuanwo Feb 26, 2024
5ec0f5f
Fix build
Xuanwo Feb 26, 2024
7d7f21e
Fix cos
Xuanwo Feb 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion core/src/services/azblob/backend.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ const KNOWN_AZBLOB_ENDPOINT_SUFFIX: &[&str] = &[
];

const AZBLOB_BATCH_LIMIT: usize = 256;

/// Azure Storage Blob services support.
#[derive(Default, Deserialize, Clone)]
pub struct AzblobConfig {
Expand Down Expand Up @@ -124,7 +125,7 @@ pub struct AzblobBuilder {

impl Debug for AzblobBuilder {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("Builder");
let mut ds = f.debug_struct("AzblobBuilder");

ds.field("config", &self.config);

Expand Down
73 changes: 44 additions & 29 deletions core/src/services/azdls/backend.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ use log::debug;
use reqsign::AzureStorageConfig;
use reqsign::AzureStorageLoader;
use reqsign::AzureStorageSigner;
use serde::Deserialize;

use super::core::AzdlsCore;
use super::error::parse_error;
use super::lister::AzdlsLister;
use super::writer::AzdlsWriter;
use super::writer::{AzdlsWriter, AzdlsWriters};
use crate::raw::*;
use crate::services::azdls::writer::AzdlsWriters;
use crate::*;

/// Known endpoint suffix Azure Data Lake Storage Gen2 URI syntax.
Expand All @@ -46,20 +46,18 @@ const KNOWN_AZDLS_ENDPOINT_SUFFIX: &[&str] = &[
];

/// Azure Data Lake Storage Gen2 Support.
#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct AzdlsBuilder {
#[derive(Default, Deserialize, Clone)]
pub struct AzdlsConfig {
root: Option<String>,
filesystem: String,
endpoint: Option<String>,
account_name: Option<String>,
account_key: Option<String>,
http_client: Option<HttpClient>,
}

impl Debug for AzdlsBuilder {
impl Debug for AzdlsConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("Builder");
let mut ds = f.debug_struct("AzdlsConfig");

ds.field("root", &self.root);
ds.field("filesystem", &self.filesystem);
Expand All @@ -76,21 +74,39 @@ impl Debug for AzdlsBuilder {
}
}

/// Azure Data Lake Storage Gen2 Support.
#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct AzdlsBuilder {
config: AzdlsConfig,
http_client: Option<HttpClient>,
}

impl Debug for AzdlsBuilder {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("AzdlsBuilder");

ds.field("config", &self.config);

ds.finish()
}
}

impl AzdlsBuilder {
/// Set root of this backend.
///
/// All operations will happen under this root.
pub fn root(&mut self, root: &str) -> &mut Self {
if !root.is_empty() {
self.root = Some(root.to_string())
self.config.root = Some(root.to_string())
}

self
}

/// Set filesystem name of this backend.
pub fn filesystem(&mut self, filesystem: &str) -> &mut Self {
self.filesystem = filesystem.to_string();
self.config.filesystem = filesystem.to_string();

self
}
Expand All @@ -104,7 +120,7 @@ impl AzdlsBuilder {
pub fn endpoint(&mut self, endpoint: &str) -> &mut Self {
if !endpoint.is_empty() {
// Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
self.endpoint = Some(endpoint.trim_end_matches('/').to_string());
self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string());
}

self
Expand All @@ -116,7 +132,7 @@ impl AzdlsBuilder {
/// - If not, we will try to load it from environment.
pub fn account_name(&mut self, account_name: &str) -> &mut Self {
if !account_name.is_empty() {
self.account_name = Some(account_name.to_string());
self.config.account_name = Some(account_name.to_string());
}

self
Expand All @@ -128,7 +144,7 @@ impl AzdlsBuilder {
/// - If not, we will try to load it from environment.
pub fn account_key(&mut self, account_key: &str) -> &mut Self {
if !account_key.is_empty() {
self.account_key = Some(account_key.to_string());
self.config.account_key = Some(account_key.to_string());
}

self
Expand All @@ -153,19 +169,19 @@ impl Builder for AzdlsBuilder {
fn build(&mut self) -> Result<Self::Accessor> {
debug!("backend build started: {:?}", &self);

let root = normalize_root(&self.root.take().unwrap_or_default());
let root = normalize_root(&self.config.root.take().unwrap_or_default());
debug!("backend use root {}", root);

// Handle endpoint, region and container name.
let filesystem = match self.filesystem.is_empty() {
false => Ok(&self.filesystem),
let filesystem = match self.config.filesystem.is_empty() {
false => Ok(&self.config.filesystem),
true => Err(Error::new(ErrorKind::ConfigInvalid, "filesystem is empty")
.with_operation("Builder::build")
.with_context("service", Scheme::Azdls)),
}?;
debug!("backend use filesystem {}", &filesystem);

let endpoint = match &self.endpoint {
let endpoint = match &self.config.endpoint {
Some(endpoint) => Ok(endpoint.clone()),
None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
.with_operation("Builder::build")
Expand All @@ -184,10 +200,11 @@ impl Builder for AzdlsBuilder {

let config_loader = AzureStorageConfig {
account_name: self
.config
.account_name
.clone()
.or_else(|| infer_storage_name_from_endpoint(endpoint.as_str())),
account_key: self.account_key.clone(),
account_key: self.config.account_key.clone(),
sas_token: None,
..Default::default()
};
Expand All @@ -198,7 +215,7 @@ impl Builder for AzdlsBuilder {
debug!("backend build finished: {:?}", &self);
Ok(AzdlsBackend {
core: Arc::new(AzdlsCore {
filesystem: self.filesystem.clone(),
filesystem: self.config.filesystem.clone(),
root,
endpoint,
client,
Expand All @@ -209,15 +226,13 @@ impl Builder for AzdlsBuilder {
}

fn from_map(map: HashMap<String, String>) -> Self {
let mut builder = AzdlsBuilder::default();
let config = AzdlsConfig::deserialize(ConfigDeserializer::new(map))
.expect("config deserialize must succeed");

map.get("root").map(|v| builder.root(v));
map.get("filesystem").map(|v| builder.filesystem(v));
map.get("endpoint").map(|v| builder.endpoint(v));
map.get("account_name").map(|v| builder.account_name(v));
map.get("account_key").map(|v| builder.account_key(v));

builder
AzdlsBuilder {
config,
http_client: None,
}
}
}

Expand Down Expand Up @@ -467,7 +482,7 @@ mod tests {
assert_eq!(azdls.core.filesystem, "filesystem".to_string());

assert_eq!(
azdls_builder.account_key.unwrap(),
azdls_builder.config.account_key.unwrap(),
"account-key".to_string()
);
}
Expand All @@ -488,6 +503,6 @@ mod tests {

assert_eq!(azdls.core.filesystem, "filesystem".to_string());

assert_eq!(azdls_builder.account_key, None);
assert_eq!(azdls_builder.config.account_key, None);
}
}
74 changes: 47 additions & 27 deletions core/src/services/azfile/backend.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ use log::debug;
use reqsign::AzureStorageConfig;
use reqsign::AzureStorageLoader;
use reqsign::AzureStorageSigner;
use serde::Deserialize;

use super::core::AzfileCore;
use super::error::parse_error;
Expand All @@ -39,31 +40,51 @@ use crate::*;
const DEFAULT_AZFILE_ENDPOINT_SUFFIX: &str = "file.core.windows.net";

/// Azure File services support.
#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct AzfileBuilder {
#[derive(Default, Deserialize, Clone)]
pub struct AzfileConfig {
root: Option<String>,
endpoint: Option<String>,
account_name: Option<String>,
share_name: String,
account_name: Option<String>,
account_key: Option<String>,
sas_token: Option<String>,
http_client: Option<HttpClient>,
}

impl Debug for AzfileBuilder {
impl Debug for AzfileConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("Builder");
let mut ds = f.debug_struct("AzfileConfig");

ds.field("root", &self.root);
ds.field("endpoint", &self.endpoint);
ds.field("share_name", &self.share_name);
ds.field("endpoint", &self.endpoint);

if self.account_name.is_some() {
ds.field("account_name", &"<redacted>");
}
if self.account_key.is_some() {
ds.field("account_key", &"<redacted>");
}
if self.sas_token.is_some() {
ds.field("sas_token", &"<redacted>");
}

ds.finish()
}
}

/// Azure File services support.
#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct AzfileBuilder {
config: AzfileConfig,
http_client: Option<HttpClient>,
}

impl Debug for AzfileBuilder {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("AzfileBuilder");

ds.field("config", &self.config);

ds.finish()
}
Expand All @@ -75,7 +96,7 @@ impl AzfileBuilder {
/// All operations will happen under this root.
pub fn root(&mut self, root: &str) -> &mut Self {
if !root.is_empty() {
self.root = Some(root.to_string())
self.config.root = Some(root.to_string())
}

self
Expand All @@ -85,7 +106,7 @@ impl AzfileBuilder {
pub fn endpoint(&mut self, endpoint: &str) -> &mut Self {
if !endpoint.is_empty() {
// Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
self.endpoint = Some(endpoint.trim_end_matches('/').to_string());
self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string());
}

self
Expand All @@ -97,7 +118,7 @@ impl AzfileBuilder {
/// - If not, we will try to load it from environment.
pub fn account_name(&mut self, account_name: &str) -> &mut Self {
if !account_name.is_empty() {
self.account_name = Some(account_name.to_string());
self.config.account_name = Some(account_name.to_string());
}

self
Expand All @@ -109,7 +130,7 @@ impl AzfileBuilder {
/// - If not, we will try to load it from environment.
pub fn account_key(&mut self, account_key: &str) -> &mut Self {
if !account_key.is_empty() {
self.account_key = Some(account_key.to_string());
self.config.account_key = Some(account_key.to_string());
}

self
Expand All @@ -121,7 +142,7 @@ impl AzfileBuilder {
/// You can find more about from: <https://learn.microsoft.com/en-us/rest/api/storageservices/operations-on-shares--file-service>
pub fn share_name(&mut self, share_name: &str) -> &mut Self {
if !share_name.is_empty() {
self.share_name = share_name.to_string();
self.config.share_name = share_name.to_string();
}

self
Expand All @@ -144,24 +165,22 @@ impl Builder for AzfileBuilder {
type Accessor = AzfileBackend;

fn from_map(map: HashMap<String, String>) -> Self {
let mut builder = AzfileBuilder::default();

map.get("root").map(|v| builder.root(v));
map.get("endpoint").map(|v| builder.endpoint(v));
map.get("account_name").map(|v| builder.account_name(v));
map.get("account_key").map(|v| builder.account_key(v));
map.get("share_name").map(|v| builder.share_name(v));
let config = AzfileConfig::deserialize(ConfigDeserializer::new(map))
.expect("config deserialize must succeed");

builder
AzfileBuilder {
config,
http_client: None,
}
}

fn build(&mut self) -> Result<Self::Accessor> {
debug!("backend build started: {:?}", &self);

let root = normalize_root(&self.root.take().unwrap_or_default());
let root = normalize_root(&self.config.root.take().unwrap_or_default());
debug!("backend use root {}", root);

let endpoint = match &self.endpoint {
let endpoint = match &self.config.endpoint {
Some(endpoint) => Ok(endpoint.clone()),
None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
.with_operation("Builder::build")
Expand All @@ -179,6 +198,7 @@ impl Builder for AzfileBuilder {
};

let account_name_option = self
.config
.account_name
.clone()
.or_else(|| infer_account_name_from_endpoint(endpoint.as_str()));
Expand All @@ -194,8 +214,8 @@ impl Builder for AzfileBuilder {

let config_loader = AzureStorageConfig {
account_name: Some(account_name),
account_key: self.account_key.clone(),
sas_token: self.sas_token.clone(),
account_key: self.config.account_key.clone(),
sas_token: self.config.sas_token.clone(),
..Default::default()
};

Expand All @@ -211,7 +231,7 @@ impl Builder for AzfileBuilder {
loader: cred_loader,
client,
signer,
share_name: self.share_name.clone(),
share_name: self.config.share_name.clone(),
}),
})
}
Expand Down Expand Up @@ -435,7 +455,7 @@ mod tests {
);

assert_eq!(
azfile_builder.account_key.unwrap(),
azfile_builder.config.account_key.unwrap(),
"account-key".to_string()
);
}
Expand Down
Loading
Loading