factor configs into own files

src/common/io-config/src/azure.rs

@@ -0,0 +1,25 @@
+use std::fmt::Display;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub struct AzureConfig {
+    pub storage_account: Option<String>,
+    pub access_key: Option<String>,
+    pub anonymous: bool,
+}
+
+impl Display for AzureConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
+        write!(
+            f,
+            "AzureConfig
+    storage_account: {:?}
+    access_key: {:?}
+    anonymous: {:?}",
+            self.storage_account, self.access_key, self.anonymous
+        )
+    }
+}

src/common/io-config/src/config.rs

@@ -4,108 +4,7 @@ use std::fmt::Formatter;
 use serde::Deserialize;
 use serde::Serialize;
 
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
-pub struct S3Config {
-    pub region_name: Option<String>,
-    pub endpoint_url: Option<String>,
-    pub key_id: Option<String>,
-    pub session_token: Option<String>,
-    pub access_key: Option<String>,
-    pub retry_initial_backoff_ms: u64,
-    pub connect_timeout_ms: u64,
-    pub read_timeout_ms: u64,
-    pub num_tries: u32,
-    pub retry_mode: Option<String>,
-    pub anonymous: bool,
-}
-
-impl Default for S3Config {
-    fn default() -> Self {
-        S3Config {
-            region_name: None,
-            endpoint_url: None,
-            key_id: None,
-            session_token: None,
-            access_key: None,
-            retry_initial_backoff_ms: 1000,
-            connect_timeout_ms: 60_000,
-            read_timeout_ms: 60_000,
-            num_tries: 5,
-            retry_mode: Some("standard".to_string()),
-            anonymous: false,
-        }
-    }
-}
-
-impl Display for S3Config {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
-        write!(
-            f,
-            "S3Config
-    region_name: {:?}
-    endpoint_url: {:?}
-    key_id: {:?}
-    session_token: {:?},
-    access_key: {:?}
-    retry_initial_backoff_ms: {},
-    connect_timeout_ms: {},
-    read_timeout_ms: {},
-    num_tries: {:?},
-    retry_mode: {:?},
-    anonymous: {}",
-            self.region_name,
-            self.endpoint_url,
-            self.key_id,
-            self.session_token,
-            self.access_key,
-            self.retry_initial_backoff_ms,
-            self.connect_timeout_ms,
-            self.read_timeout_ms,
-            self.num_tries,
-            self.retry_mode,
-            self.anonymous
-        )
-    }
-}
-
-#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
-pub struct AzureConfig {
-    pub storage_account: Option<String>,
-    pub access_key: Option<String>,
-    pub anonymous: bool,
-}
-
-impl Display for AzureConfig {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
-        write!(
-            f,
-            "AzureConfig
-    storage_account: {:?}
-    access_key: {:?}
-    anonymous: {:?}",
-            self.storage_account, self.access_key, self.anonymous
-        )
-    }
-}
-
-#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
-pub struct GCSConfig {
-    pub project_id: Option<String>,
-    pub anonymous: bool,
-}
-
-impl Display for GCSConfig {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
-        write!(
-            f,
-            "GCSConfig
-    project_id: {:?}
-    anonymous: {:?}",
-            self.project_id, self.anonymous
-        )
-    }
-}
-
+use crate::{AzureConfig, GCSConfig, S3Config};
 #[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
 pub struct IOConfig {
     pub s3: S3Config,

src/common/io-config/src/gcs.rs

@@ -0,0 +1,23 @@
+use std::fmt::Display;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub struct GCSConfig {
+    pub project_id: Option<String>,
+    pub anonymous: bool,
+}
+
+impl Display for GCSConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
+        write!(
+            f,
+            "GCSConfig
+    project_id: {:?}
+    anonymous: {:?}",
+            self.project_id, self.anonymous
+        )
+    }
+}

src/common/io-config/src/lib.rs

@@ -1,6 +1,9 @@
 #[cfg(feature = "python")]
 pub mod python;
 
-pub mod config;
+mod azure;
+mod config;
+mod gcs;
+mod s3;
 
-pub use config::{AzureConfig, GCSConfig, IOConfig, S3Config};
+pub use crate::{azure::AzureConfig, config::IOConfig, gcs::GCSConfig, s3::S3Config};

src/common/io-config/src/python.rs

@@ -25,7 +25,7 @@ use crate::config;
 #[derive(Clone, Default)]
 #[pyclass]
 pub struct S3Config {
-    pub config: config::S3Config,
+    pub config: crate::S3Config,
 }
 /// Create configurations to be used when accessing Azure Blob Storage
 ///
@@ -40,7 +40,7 @@ pub struct S3Config {
 #[derive(Clone, Default)]
 #[pyclass]
 pub struct AzureConfig {
-    pub config: config::AzureConfig,
+    pub config: crate::AzureConfig,
 }
 
 /// Create configurations to be used when accessing Google Cloud Storage
@@ -55,7 +55,7 @@ pub struct AzureConfig {
 #[derive(Clone, Default)]
 #[pyclass]
 pub struct GCSConfig {
-    pub config: config::GCSConfig,
+    pub config: crate::GCSConfig,
 }
 
 /// Create configurations to be used when accessing storage
@@ -149,9 +149,9 @@ impl S3Config {
         retry_mode: Option<String>,
         anonymous: Option<bool>,
     ) -> Self {
-        let def = config::S3Config::default();
+        let def = crate::S3Config::default();
         S3Config {
-            config: config::S3Config {
+            config: crate::S3Config {
                 region_name: region_name.or(def.region_name),
                 endpoint_url: endpoint_url.or(def.endpoint_url),
                 key_id: key_id.or(def.key_id),
@@ -242,9 +242,9 @@ impl AzureConfig {
         access_key: Option<String>,
         anonymous: Option<bool>,
     ) -> Self {
-        let def = config::AzureConfig::default();
+        let def = crate::AzureConfig::default();
         AzureConfig {
-            config: config::AzureConfig {
+            config: crate::AzureConfig {
                 storage_account: storage_account.or(def.storage_account),
                 access_key: access_key.or(def.access_key),
                 anonymous: anonymous.unwrap_or(def.anonymous),
@@ -274,9 +274,9 @@ impl GCSConfig {
     #[allow(clippy::too_many_arguments)]
     #[new]
     pub fn new(project_id: Option<String>, anonymous: Option<bool>) -> Self {
-        let def = config::GCSConfig::default();
+        let def = crate::GCSConfig::default();
         GCSConfig {
-            config: config::GCSConfig {
+            config: crate::GCSConfig {
                 project_id: project_id.or(def.project_id),
                 anonymous: anonymous.unwrap_or(def.anonymous),
             },

src/common/io-config/src/s3.rs

@@ -0,0 +1,69 @@
+use std::fmt::Display;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub struct S3Config {
+    pub region_name: Option<String>,
+    pub endpoint_url: Option<String>,
+    pub key_id: Option<String>,
+    pub session_token: Option<String>,
+    pub access_key: Option<String>,
+    pub retry_initial_backoff_ms: u64,
+    pub connect_timeout_ms: u64,
+    pub read_timeout_ms: u64,
+    pub num_tries: u32,
+    pub retry_mode: Option<String>,
+    pub anonymous: bool,
+}
+
+impl Default for S3Config {
+    fn default() -> Self {
+        S3Config {
+            region_name: None,
+            endpoint_url: None,
+            key_id: None,
+            session_token: None,
+            access_key: None,
+            retry_initial_backoff_ms: 1000,
+            connect_timeout_ms: 60_000,
+            read_timeout_ms: 60_000,
+            num_tries: 5,
+            retry_mode: Some("standard".to_string()),
+            anonymous: false,
+        }
+    }
+}
+
+impl Display for S3Config {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
+        write!(
+            f,
+            "S3Config
+    region_name: {:?}
+    endpoint_url: {:?}
+    key_id: {:?}
+    session_token: {:?},
+    access_key: {:?}
+    retry_initial_backoff_ms: {},
+    connect_timeout_ms: {},
+    read_timeout_ms: {},
+    num_tries: {:?},
+    retry_mode: {:?},
+    anonymous: {}",
+            self.region_name,
+            self.endpoint_url,
+            self.key_id,
+            self.session_token,
+            self.access_key,
+            self.retry_initial_backoff_ms,
+            self.connect_timeout_ms,
+            self.read_timeout_ms,
+            self.num_tries,
+            self.retry_mode,
+            self.anonymous
+        )
+    }
+}