From ca53000efe8ac93d00e156548dc697e92649b5cb Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 23 Apr 2023 11:01:24 +0800 Subject: [PATCH] refactor: Allow reusing the same operator to speed up tests (#2068) * refactor: Allow reusing the same operator to speed up tests Signed-off-by: Xuanwo * Fix scan root Signed-off-by: Xuanwo * Fix debug for cap Signed-off-by: Xuanwo * avoid conflict Signed-off-by: Xuanwo * Fix fs Signed-off-by: Xuanwo * Allow retry content incomplete Signed-off-by: Xuanwo * Allow retry for copy and rename Signed-off-by: Xuanwo * FIx list tests Signed-off-by: Xuanwo * Fix ensure root Signed-off-by: Xuanwo * Allow retry for redis error Signed-off-by: Xuanwo * Refactor random root Signed-off-by: Xuanwo * Use better cap debug format Signed-off-by: Xuanwo * Better! Signed-off-by: Xuanwo * Fix webdav Signed-off-by: Xuanwo * ftp should ensure parent dir exists Signed-off-by: Xuanwo * Refactor tests Signed-off-by: Xuanwo * format code Signed-off-by: Xuanwo --------- Signed-off-by: Xuanwo --- .github/workflows/service_test_http.yml | 2 + .github/workflows/service_test_ipfs.yml | 2 + core/src/layers/retry.rs | 28 +++++++++ core/src/raw/http_util/body.rs | 6 +- core/src/services/ftp/backend.rs | 46 +++++++++----- core/src/services/redis/backend.rs | 4 +- core/src/services/webdav/backend.rs | 4 ++ core/src/services/webdav/pager.rs | 11 ++-- core/src/types/capability.rs | 49 ++++++++++++++- core/tests/behavior/blocking_copy.rs | 41 ++++++------- core/tests/behavior/blocking_list.rs | 60 +++++++++--------- core/tests/behavior/blocking_read.rs | 39 ++++++------ core/tests/behavior/blocking_rename.rs | 41 ++++++------- core/tests/behavior/blocking_write.rs | 39 ++++++------ core/tests/behavior/copy.rs | 39 ++++++------ core/tests/behavior/list.rs | 81 ++++++++++++++----------- core/tests/behavior/list_only.rs | 35 +++++------ core/tests/behavior/main.rs | 68 +++++++++++++-------- core/tests/behavior/presign.rs | 35 +++++------ core/tests/behavior/read_only.rs | 35 +++++------ core/tests/behavior/rename.rs | 39 ++++++------ core/tests/behavior/utils.rs | 6 +- core/tests/behavior/write.rs | 35 +++++------ 23 files changed, 425 insertions(+), 320 deletions(-) diff --git a/.github/workflows/service_test_http.yml b/.github/workflows/service_test_http.yml index 7e3a1ff6709..ce32448a985 100644 --- a/.github/workflows/service_test_http.yml +++ b/.github/workflows/service_test_http.yml @@ -66,6 +66,7 @@ jobs: RUST_LOG: debug OPENDAL_HTTP_TEST: on OPENDAL_HTTP_ENDPOINT: http://127.0.0.1:8080 + OPENDAL_DISABLE_RANDOM_ROOT: true caddy: runs-on: ${{ matrix.os }} @@ -102,3 +103,4 @@ jobs: RUST_LOG: debug OPENDAL_HTTP_TEST: on OPENDAL_HTTP_ENDPOINT: http://127.0.0.1:8080 + OPENDAL_DISABLE_RANDOM_ROOT: true diff --git a/.github/workflows/service_test_ipfs.yml b/.github/workflows/service_test_ipfs.yml index ad0edc53970..b7138586d7b 100644 --- a/.github/workflows/service_test_ipfs.yml +++ b/.github/workflows/service_test_ipfs.yml @@ -63,6 +63,8 @@ jobs: OPENDAL_IPFS_TEST: on OPENDAL_IPFS_ROOT: /ipfs/QmPpCt1aYGb9JWJRmXRUnmJtVgeFFTJGzWFYEEX7bo9zGJ/ OPENDAL_IPFS_ENDPOINT: "http://127.0.0.1:8080" + OPENDAL_DISABLE_RANDOM_ROOT: true + # # ipfs.io can't pass our test by now, we should address them in the future. # ipfs-io: # runs-on: ubuntu-latest diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs index 2193202a24c..341fd75ef18 100644 --- a/core/src/layers/retry.rs +++ b/core/src/layers/retry.rs @@ -247,6 +247,34 @@ impl LayeredAccessor for RetryAccessor { .await } + async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result { + { || self.inner.copy(from, to, args.clone()) } + .retry(&self.builder) + .when(|e| e.is_temporary()) + .notify(|err, dur| { + warn!( + target: "opendal::service", + "operation={} -> retry after {}s: error={:?}", + Operation::Copy, dur.as_secs_f64(), err) + }) + .map(|v| v.map_err(|e| e.set_persistent())) + .await + } + + async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result { + { || self.inner.rename(from, to, args.clone()) } + .retry(&self.builder) + .when(|e| e.is_temporary()) + .notify(|err, dur| { + warn!( + target: "opendal::service", + "operation={} -> retry after {}s: error={:?}", + Operation::Rename, dur.as_secs_f64(), err) + }) + .map(|v| v.map_err(|e| e.set_persistent())) + .await + } + async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> { { || self.inner.list(path, args.clone()) } .retry(&self.builder) diff --git a/core/src/raw/http_util/body.rs b/core/src/raw/http_util/body.rs index 4d8eec3f098..28905707ad8 100644 --- a/core/src/raw/http_util/body.rs +++ b/core/src/raw/http_util/body.rs @@ -146,11 +146,13 @@ impl IncomingAsyncBody { Ordering::Less => Err(Error::new( ErrorKind::ContentIncomplete, &format!("reader got too less data, expect: {expect}, actual: {actual}"), - )), + ) + .set_temporary()), Ordering::Greater => Err(Error::new( ErrorKind::ContentTruncated, &format!("reader got too much data, expect: {expect}, actual: {actual}"), - )), + ) + .set_temporary()), } } } diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs index 0b112c05853..32338980660 100644 --- a/core/src/services/ftp/backend.rs +++ b/core/src/services/ftp/backend.rs @@ -335,22 +335,16 @@ impl Accessor for FtpBackend { for path in paths { curr_path.push_str(path); - // try to create directory - if curr_path.ends_with('/') { - match ftp_stream.mkdir(&curr_path).await { - // Do nothing if status is FileUnavailable or OK(()) is return. - Err(FtpError::UnexpectedResponse(Response { - status: Status::FileUnavailable, - .. - })) - | Ok(()) => (), - Err(e) => { - return Err(e.into()); - } + match ftp_stream.mkdir(&curr_path).await { + // Do nothing if status is FileUnavailable or OK(()) is return. + Err(FtpError::UnexpectedResponse(Response { + status: Status::FileUnavailable, + .. + })) + | Ok(()) => (), + Err(e) => { + return Err(e.into()); } - } else { - // else, create file - ftp_stream.put_file(&curr_path, &mut "".as_bytes()).await?; } } @@ -398,6 +392,28 @@ impl Accessor for FtpBackend { )); } + // Ensure the parent dir exists. + let parent = get_parent(path); + let paths: Vec<&str> = parent.split('/').collect(); + + // TODO: we can optimize this by checking dir existence first. + let mut ftp_stream = self.ftp_connect(Operation::Write).await?; + let mut curr_path = String::new(); + for path in paths { + curr_path.push_str(path); + match ftp_stream.mkdir(&curr_path).await { + // Do nothing if status is FileUnavailable or OK(()) is return. + Err(FtpError::UnexpectedResponse(Response { + status: Status::FileUnavailable, + .. + })) + | Ok(()) => (), + Err(e) => { + return Err(e.into()); + } + } + } + Ok(( RpWrite::new(), FtpWriter::new(self.clone(), path.to_string()), diff --git a/core/src/services/redis/backend.rs b/core/src/services/redis/backend.rs index 7d8ac059d3c..3e484af0ea1 100644 --- a/core/src/services/redis/backend.rs +++ b/core/src/services/redis/backend.rs @@ -356,6 +356,8 @@ impl kv::Adapter for Adapter { impl From for Error { fn from(e: RedisError) -> Self { - Error::new(ErrorKind::Unexpected, e.category()).set_source(e) + Error::new(ErrorKind::Unexpected, e.category()) + .set_source(e) + .set_temporary() } } diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index bc1558e0b45..4a073d6e9fe 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -648,6 +648,10 @@ impl WebdavBackend { } async fn ensure_parent_path(&self, path: &str) -> Result<()> { + if path == "/" { + return Ok(()); + } + // create dir recursively, split path by `/` and create each dir except the last one let abs_path = build_abs_path(&self.root, path); let abs_path = abs_path.as_str(); diff --git a/core/src/services/webdav/pager.rs b/core/src/services/webdav/pager.rs index 32cb5e0f80c..3cd8ebb81e3 100644 --- a/core/src/services/webdav/pager.rs +++ b/core/src/services/webdav/pager.rs @@ -54,12 +54,13 @@ impl oio::Page for WebdavPager { .into_iter() .filter_map(|de| { let path = de.href; - let normalized_path = if self.root != path { - build_rel_path(&self.root, &path) - } else { - path - }; + // Ignore the root path itself. + if self.root == path { + return None; + } + + let normalized_path = build_rel_path(&self.root, &path); if normalized_path == self.path { // WebDav server may return the current path as an entry. return None; diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs index eb6c756ba07..bb7a872d456 100644 --- a/core/src/types/capability.rs +++ b/core/src/types/capability.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::fmt::Debug; + /// Capability is used to describe what operations are supported /// by current Operator. /// @@ -42,7 +44,7 @@ /// - Operation with variants should be named like `read_can_seek`. /// - Operation with arguments should be named like `read_with_range`. /// - Operation with limtations should be named like `batch_max_operations`. -#[derive(Copy, Clone, Debug, Default)] +#[derive(Copy, Clone, Default)] pub struct Capability { /// If operator supports stat natively, it will be true. pub stat: bool, @@ -127,3 +129,48 @@ pub struct Capability { /// If operator supports blocking natively, it will be true. pub blocking: bool, } + +impl Debug for Capability { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut s = vec![]; + + if self.read { + s.push("Read"); + } + if self.stat { + s.push("Stat"); + } + if self.write { + s.push("Write"); + } + if self.create_dir { + s.push("CreateDir"); + } + if self.delete { + s.push("Delete"); + } + if self.list { + s.push("List"); + } + if self.scan { + s.push("Scan"); + } + if self.copy { + s.push("Copy"); + } + if self.rename { + s.push("Rename"); + } + if self.presign { + s.push("Presign"); + } + if self.batch { + s.push("Batch"); + } + if self.blocking { + s.push("Blocking"); + } + + write!(f, "{{ {} }}", s.join(" | ")) + } +} diff --git a/core/tests/behavior/blocking_copy.rs b/core/tests/behavior/blocking_copy.rs index 8cf724c4e58..346b8b456a5 100644 --- a/core/tests/behavior/blocking_copy.rs +++ b/core/tests/behavior/blocking_copy.rs @@ -30,31 +30,28 @@ use super::utils::*; macro_rules! behavior_blocking_copy_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[test] - $( - #[$meta] - )* - fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(true); - match op { - Some(op) if op.info().can_read() - && op.info().can_write() - && op.info().can_copy() - && op.info().can_blocking() => $crate::blocking_copy::$test(op.blocking()), - Some(_) => { - log::warn!("service {} doesn't support blocking_copy, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() + && op.info().can_write() + && op.info().can_copy() + && op.info().can_blocking() => $crate::blocking_copy::$test(op.blocking()), + Some(_) => { + log::warn!("service {} doesn't support blocking_copy, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } diff --git a/core/tests/behavior/blocking_list.rs b/core/tests/behavior/blocking_list.rs index c84084d3d72..20f333e5940 100644 --- a/core/tests/behavior/blocking_list.rs +++ b/core/tests/behavior/blocking_list.rs @@ -34,30 +34,27 @@ use super::utils::*; macro_rules! behavior_blocking_list_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[test] - $( - #[$meta] - )* - fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(true); - match op { - Some(op) if op.info().can_read() - && op.info().can_write() - && op.info().can_blocking() && (op.info().can_list()||op.info().can_scan()) => $crate::blocking_list::$test(op.blocking()), - Some(_) => { - log::warn!("service {} doesn't support blocking_list, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() + && op.info().can_write() + && op.info().can_blocking() && (op.info().can_list()||op.info().can_scan()) => $crate::blocking_list::$test(op.blocking()), + Some(_) => { + log::warn!("service {} doesn't support blocking_list, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } @@ -79,13 +76,14 @@ macro_rules! behavior_blocking_list_tests { /// List dir should return newly created file. pub fn test_list_dir(op: BlockingOperator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); + let parent = uuid::Uuid::new_v4().to_string(); + let path = format!("{parent}/{}", uuid::Uuid::new_v4()); debug!("Generate a random file: {}", &path); let (content, size) = gen_bytes(); op.write(&path, content).expect("write must succeed"); - let obs = op.list("/")?; + let obs = op.list(&format!("{parent}/"))?; let mut found = false; for de in obs { let de = de?; @@ -122,22 +120,30 @@ pub fn test_list_non_exist_dir(op: BlockingOperator) -> Result<()> { // Walk top down should output as expected pub fn test_scan(op: BlockingOperator) -> Result<()> { + let parent = uuid::Uuid::new_v4().to_string(); + let expected = vec![ "x/", "x/y", "x/x/", "x/x/y", "x/x/x/", "x/x/x/y", "x/x/x/x/", ]; for path in expected.iter() { if path.ends_with('/') { - op.create_dir(path)?; + op.create_dir(&format!("{parent}/{path}"))?; } else { - op.write(path, "test_scan")?; + op.write(&format!("{parent}/{path}"), "test_scan")?; } } - let w = op.scan("x/")?; + let w = op.scan(&format!("{parent}/x/"))?; let actual = w .collect::>() .into_iter() - .map(|v| v.unwrap().path().to_string()) + .map(|v| { + v.unwrap() + .path() + .strip_prefix(&format!("{parent}/")) + .unwrap() + .to_string() + }) .collect::>(); debug!("walk top down: {:?}", actual); diff --git a/core/tests/behavior/blocking_read.rs b/core/tests/behavior/blocking_read.rs index 2f4e77a75dc..2033398caec 100644 --- a/core/tests/behavior/blocking_read.rs +++ b/core/tests/behavior/blocking_read.rs @@ -30,30 +30,27 @@ use sha2::Sha256; macro_rules! behavior_blocking_read_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[test] - $( - #[$meta] - )* - fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(true); - match op { - Some(op) if op.info().can_read() - && !op.info().can_write() - && op.info().can_blocking() => $crate::blocking_read::$test(op.blocking()), - Some(_) => { - log::warn!("service {} doesn't support blocking_read, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() + && !op.info().can_write() + && op.info().can_blocking() => $crate::blocking_read::$test(op.blocking()), + Some(_) => { + log::warn!("service {} doesn't support blocking_read, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } diff --git a/core/tests/behavior/blocking_rename.rs b/core/tests/behavior/blocking_rename.rs index 7222b230456..3973a95478c 100644 --- a/core/tests/behavior/blocking_rename.rs +++ b/core/tests/behavior/blocking_rename.rs @@ -30,31 +30,28 @@ use super::utils::*; macro_rules! behavior_blocking_rename_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[test] - $( - #[$meta] - )* - fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(true); - match op { - Some(op) if op.info().can_read() - && op.info().can_write() - && op.info().can_rename() - && op.info().can_blocking() => $crate::blocking_rename::$test(op.blocking()), - Some(_) => { - log::warn!("service {} doesn't support blocking_rename, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() + && op.info().can_write() + && op.info().can_rename() + && op.info().can_blocking() => $crate::blocking_rename::$test(op.blocking()), + Some(_) => { + log::warn!("service {} doesn't support blocking_rename, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } diff --git a/core/tests/behavior/blocking_write.rs b/core/tests/behavior/blocking_write.rs index a444b1ca933..d1bd392b21a 100644 --- a/core/tests/behavior/blocking_write.rs +++ b/core/tests/behavior/blocking_write.rs @@ -36,30 +36,27 @@ use super::utils::*; macro_rules! behavior_blocking_write_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[test] - $( - #[$meta] - )* - fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(true); - match op { - Some(op) if op.info().can_read() - && op.info().can_write() - && op.info().can_blocking() => $crate::blocking_write::$test(op.blocking()), - Some(_) => { - log::warn!("service {} doesn't support blocking_write, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() + && op.info().can_write() + && op.info().can_blocking() => $crate::blocking_write::$test(op.blocking()), + Some(_) => { + log::warn!("service {} doesn't support blocking_write, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } diff --git a/core/tests/behavior/copy.rs b/core/tests/behavior/copy.rs index e09f6b8d400..dfe0a7cc4d6 100644 --- a/core/tests/behavior/copy.rs +++ b/core/tests/behavior/copy.rs @@ -29,30 +29,27 @@ use super::utils::*; macro_rules! behavior_copy_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[tokio::test] - $( - #[$meta] - )* - async fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(true); - match op { - Some(op) if op.info().can_read() - && op.info().can_write() - && op.info().can_copy() => $crate::copy::$test(op).await, - Some(_) => { - log::warn!("service {} doesn't support copy, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() + && op.info().can_write() + && op.info().can_copy() => RUNTIME.block_on($crate::copy::$test(op.clone())), + Some(_) => { + log::warn!("service {} doesn't support copy, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/list.rs index 3c8d534705d..648818dd264 100644 --- a/core/tests/behavior/list.rs +++ b/core/tests/behavior/list.rs @@ -37,31 +37,28 @@ use super::utils::*; macro_rules! behavior_list_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[tokio::test] - $( - #[$meta] - )* - async fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(true); - match op { - Some(op) if op.info().can_read() - && op.info().can_write() - && (op.info().can_list() - || op.info().can_scan()) => $crate::list::$test(op).await, - Some(_) => { - log::warn!("service {} doesn't support list, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() + && op.info().can_write() + && (op.info().can_list() + || op.info().can_scan()) => RUNTIME.block_on($crate::list::$test(op.clone())), + Some(_) => { + log::warn!("service {} doesn't support list, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } @@ -98,13 +95,14 @@ pub async fn test_check(op: Operator) -> Result<()> { /// List dir should return newly created file. pub async fn test_list_dir(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); + let parent = uuid::Uuid::new_v4().to_string(); + let path = format!("{parent}/{}", uuid::Uuid::new_v4()); debug!("Generate a random file: {}", &path); let (content, size) = gen_bytes(); op.write(&path, content).await.expect("write must succeed"); - let mut obs = op.list("/").await?; + let mut obs = op.list(&format!("{parent}/")).await?; let mut found = false; while let Some(de) = obs.try_next().await? { let meta = op.stat(de.path()).await?; @@ -289,33 +287,37 @@ pub async fn test_scan_root(op: Operator) -> Result<()> { .map(|v| v.path().to_string()) .collect::>(); - assert!( - actual.is_empty(), - "empty root should return empty, but got {:?}", - actual - ); + assert!(!actual.contains("/"), "empty root should return itself"); + assert!(!actual.contains(""), "empty root should return empty"); Ok(()) } // Walk top down should output as expected pub async fn test_scan(op: Operator) -> Result<()> { + let parent = uuid::Uuid::new_v4().to_string(); + let expected = vec![ "x/", "x/y", "x/x/", "x/x/y", "x/x/x/", "x/x/x/y", "x/x/x/x/", ]; for path in expected.iter() { if path.ends_with('/') { - op.create_dir(path).await?; + op.create_dir(&format!("{parent}/{path}")).await?; } else { - op.write(path, "test_scan").await?; + op.write(&format!("{parent}/{path}"), "test_scan").await?; } } - let w = op.scan("x/").await?; + let w = op.scan(&format!("{parent}/x/")).await?; let actual = w .try_collect::>() .await? .into_iter() - .map(|v| v.path().to_string()) + .map(|v| { + v.path() + .strip_prefix(&format!("{parent}/")) + .unwrap() + .to_string() + }) .collect::>(); debug!("walk top down: {:?}", actual); @@ -328,24 +330,29 @@ pub async fn test_scan(op: Operator) -> Result<()> { // Remove all should remove all in this path. pub async fn test_remove_all(op: Operator) -> Result<()> { + let parent = uuid::Uuid::new_v4().to_string(); + let expected = vec![ "x/", "x/y", "x/x/", "x/x/y", "x/x/x/", "x/x/x/y", "x/x/x/x/", ]; for path in expected.iter() { if path.ends_with('/') { - op.create_dir(path).await?; + op.create_dir(&format!("{parent}/{path}")).await?; } else { - op.write(path, "test_remove_all").await?; + op.write(&format!("{parent}/{path}"), "test_scan").await?; } } - op.remove_all("x/").await?; + op.remove_all(&format!("{parent}/x/")).await?; for path in expected.iter() { if path.ends_with('/') { continue; } - assert!(!op.is_exist(path).await?, "{path} should be removed") + assert!( + !op.is_exist(&format!("{parent}/{path}")).await?, + "{parent}/{path} should be removed" + ) } Ok(()) } diff --git a/core/tests/behavior/list_only.rs b/core/tests/behavior/list_only.rs index a8a292a4bb7..a7e86a4583f 100644 --- a/core/tests/behavior/list_only.rs +++ b/core/tests/behavior/list_only.rs @@ -29,28 +29,25 @@ use opendal::Operator; macro_rules! behavior_list_only_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[tokio::test] - $( - #[$meta] - )* - async fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(false); - match op { - Some(op) if op.info().can_list() && !op.info().can_write() => $crate::list_only::$test(op).await, - Some(_) => { - log::warn!("service {} doesn't support list, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_list() && !op.info().can_write() => RUNTIME.block_on($crate::list_only::$test(op.clone())), + Some(_) => { + log::warn!("service {} doesn't support list, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs index 63d63b16f72..c9038d05e8f 100644 --- a/core/tests/behavior/main.rs +++ b/core/tests/behavior/main.rs @@ -46,32 +46,48 @@ mod utils; /// Update function list while changed. macro_rules! behavior_tests { ($($service:ident),*) => { - $( - // can_read && !can_write - behavior_read_tests!($service); - // can_read && !can_write && can_blocking - behavior_blocking_read_tests!($service); - // can_read && can_write - behavior_write_tests!($service); - // can_read && can_write && can_blocking - behavior_blocking_write_tests!($service); - // can_read && can_write && can_copy - behavior_copy_tests!($service); - // can read && can_write && can_blocking && can_copy - behavior_blocking_copy_tests!($service); - // can_read && can_write && can_move - behavior_rename_tests!($service); - // can_read && can_write && can_blocking && can_move - behavior_blocking_rename_tests!($service); - // can_read && can_write && can_list - behavior_list_tests!($service); - // can_read && can_write && can_presign - behavior_presign_tests!($service); - // can_read && can_write && can_blocking && can_list - behavior_blocking_list_tests!($service); - // can_list && !can_write - behavior_list_only_tests!($service); - )* + paste::item! { + $( + mod [] { + use once_cell::sync::Lazy; + + static RUNTIME: Lazy = Lazy::new(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + }); + static OPERATOR: Lazy> = Lazy::new(|| + $crate::utils::init_service::() + ); + + // can_read && !can_write + behavior_read_tests!($service); + // can_read && !can_write && can_blocking + behavior_blocking_read_tests!($service); + // can_read && can_write + behavior_write_tests!($service); + // can_read && can_write && can_blocking + behavior_blocking_write_tests!($service); + // can_read && can_write && can_copy + behavior_copy_tests!($service); + // can read && can_write && can_blocking && can_copy + behavior_blocking_copy_tests!($service); + // can_read && can_write && can_move + behavior_rename_tests!($service); + // can_read && can_write && can_blocking && can_move + behavior_blocking_rename_tests!($service); + // can_read && can_write && can_list + behavior_list_tests!($service); + // can_read && can_write && can_presign + behavior_presign_tests!($service); + // can_read && can_write && can_blocking && can_list + behavior_blocking_list_tests!($service); + // can_list && !can_write + behavior_list_only_tests!($service); + } + )* + } }; } diff --git a/core/tests/behavior/presign.rs b/core/tests/behavior/presign.rs index 8e42f902930..ae4ae039fa5 100644 --- a/core/tests/behavior/presign.rs +++ b/core/tests/behavior/presign.rs @@ -37,28 +37,25 @@ use super::utils::*; macro_rules! behavior_presign_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[tokio::test] - $( - #[$meta] - )* - async fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(true); - match op { - Some(op) if op.info().can_read() && op.info().can_write() && op.info().can_presign() => $crate::presign::$test(op).await, - Some(_) => { - log::warn!("service {} doesn't support presign, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() && op.info().can_write() && op.info().can_presign() => RUNTIME.block_on($crate::presign::$test(op.clone())), + Some(_) => { + log::warn!("service {} doesn't support presign, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } diff --git a/core/tests/behavior/read_only.rs b/core/tests/behavior/read_only.rs index 71054a74e0f..6d3194ccce4 100644 --- a/core/tests/behavior/read_only.rs +++ b/core/tests/behavior/read_only.rs @@ -30,28 +30,25 @@ use sha2::Sha256; macro_rules! behavior_read_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[tokio::test] - $( - #[$meta] - )* - async fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(false); - match op { - Some(op) if op.info().can_read() && !op.info().can_write() => $crate::read_only::$test(op).await, - Some(_) => { - log::warn!("service {} doesn't support read_only, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() && !op.info().can_write() => RUNTIME.block_on($crate::read_only::$test(op.clone())), + Some(_) => { + log::warn!("service {} doesn't support read_only, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } diff --git a/core/tests/behavior/rename.rs b/core/tests/behavior/rename.rs index 14744869bf4..cf4c0173063 100644 --- a/core/tests/behavior/rename.rs +++ b/core/tests/behavior/rename.rs @@ -29,30 +29,27 @@ use super::utils::*; macro_rules! behavior_rename_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[tokio::test] - $( - #[$meta] - )* - async fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(true); - match op { - Some(op) if op.info().can_read() - && op.info().can_write() - && op.info().can_rename() => $crate::rename::$test(op).await, - Some(_) => { - log::warn!("service {} doesn't support rename, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() + && op.info().can_write() + && op.info().can_rename() => RUNTIME.block_on($crate::rename::$test(op.clone())), + Some(_) => { + log::warn!("service {} doesn't support rename, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; } diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs index b909d43611f..5592ecfec56 100644 --- a/core/tests/behavior/utils.rs +++ b/core/tests/behavior/utils.rs @@ -34,7 +34,7 @@ use sha2::Sha256; /// /// - If `opendal_{schema}_test` is on, construct a new Operator with given root. /// - Else, returns a `None` to represent no valid config for operator. -pub fn init_service(random_root: bool) -> Option { +pub fn init_service() -> Option { let _ = env_logger::builder().is_test(true).try_init(); let _ = dotenvy::dotenv(); @@ -54,7 +54,9 @@ pub fn init_service(random_root: bool) -> Option { return None; } - if random_root { + // Use random root unless OPENDAL_DISABLE_RANDOM_ROOT is set to true. + let disable_random_root = env::var("OPENDAL_DISABLE_RANDOM_ROOT").unwrap_or_default() == "true"; + if !disable_random_root { let root = format!( "{}{}/", cfg.get("root").cloned().unwrap_or_else(|| "/".to_string()), diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs index 7dffcb24461..bf2d5c18202 100644 --- a/core/tests/behavior/write.rs +++ b/core/tests/behavior/write.rs @@ -40,28 +40,25 @@ use super::utils::*; macro_rules! behavior_write_test { ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { paste::item! { - mod [] { + $( + #[test] $( - #[tokio::test] - $( - #[$meta] - )* - async fn [< $test >]() -> anyhow::Result<()> { - let op = $crate::utils::init_service::(true); - match op { - Some(op) if op.info().can_read() && op.info().can_write() => $crate::write::$test(op).await, - Some(_) => { - log::warn!("service {} doesn't support write, ignored", opendal::Scheme::$service); - Ok(()) - }, - None => { - log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); - Ok(()) - } + #[$meta] + )* + fn []() -> anyhow::Result<()> { + match OPERATOR.as_ref() { + Some(op) if op.info().can_read() && op.info().can_write() => RUNTIME.block_on($crate::write::$test(op.clone())), + Some(_) => { + log::warn!("service {} doesn't support write, ignored", opendal::Scheme::$service); + Ok(()) + }, + None => { + log::warn!("service {} not initiated, ignored", opendal::Scheme::$service); + Ok(()) } } - )* - } + } + )* } }; }