From 98b5aa1e28398563342666797e4875d0870728fd Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 6 Dec 2023 14:04:35 +0800 Subject: [PATCH 1/9] ci: Add behavior test for b2 Signed-off-by: Xuanwo --- .github/services/b2/b2/action.yml | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/services/b2/b2/action.yml diff --git a/.github/services/b2/b2/action.yml b/.github/services/b2/b2/action.yml new file mode 100644 index 00000000000..9d3bd434e40 --- /dev/null +++ b/.github/services/b2/b2/action.yml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: b2 +description: 'Behavior test for B2' + +runs: + using: "composite" + steps: + - name: Setup + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OPENDAL_B2_BUCKET: op://services/b2/bucket + OPENDAL_B2_BUCKET_ID: op://services/b2/bucket_id + OPENDAL_B2_APPLICATION_KEY_ID: op://services/b2/application_key_id + OPENDAL_B2_APPLICATION_KEY: op://services/b2/application_key From 0b2e99fce96e73fc9500429ce3aaf33fd92c6a55 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 6 Dec 2023 14:08:39 +0800 Subject: [PATCH 2/9] Setup Signed-off-by: Xuanwo --- bindings/java/Cargo.toml | 2 ++ bindings/nodejs/Cargo.toml | 2 ++ bindings/python/Cargo.toml | 2 ++ core/Cargo.toml | 4 +++- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index e9d5ade68a5..36d5b328501 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -88,6 +88,7 @@ services-all = [ "services-libsql", "services-swift", "services-alluxio", + "services-b2", ] # Default services provided by opendal. @@ -109,6 +110,7 @@ services-webhdfs = ["opendal/services-webhdfs"] # Optional services provided by opendal. services-alluxio = ["opendal/services-alluxio"] services-azfile = ["opendal/services-azfile"] +services-b2 = ["opendal/services-b2"] services-cacache = ["opendal/services-cacache"] services-dashmap = ["opendal/services-dashmap"] services-dropbox = ["opendal/services-dropbox"] diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml index 39949f711da..62d45d3bb89 100644 --- a/bindings/nodejs/Cargo.toml +++ b/bindings/nodejs/Cargo.toml @@ -83,6 +83,7 @@ services-all = [ "services-sqlite", "services-libsql", "services-alluxio", + "services-b2", ] # Default services provided by opendal. @@ -104,6 +105,7 @@ services-webhdfs = ["opendal/services-webhdfs"] # Optional services provided by opendal. services-alluxio = ["opendal/services-alluxio"] services-azfile = ["opendal/services-azfile"] +services-b2 = ["opendal/services-b2"] services-cacache = ["opendal/services-cacache"] services-dashmap = ["opendal/services-dashmap"] services-dropbox = ["opendal/services-dropbox"] diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index f1e358d4808..329e75304f3 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -82,6 +82,7 @@ services-all = [ "services-sqlite", "services-libsql", "services-alluxio", + "services-b2", ] # Default services provided by opendal. @@ -103,6 +104,7 @@ services-webhdfs = ["opendal/services-webhdfs"] # Optional services provided by opendal. services-alluxio = ["opendal/services-alluxio"] services-azfile = ["opendal/services-azfile"] +services-b2 = ["opendal/services-b2"] services-cacache = ["opendal/services-cacache"] services-dashmap = ["opendal/services-dashmap"] services-dropbox = ["opendal/services-dropbox"] diff --git a/core/Cargo.toml b/core/Cargo.toml index 83a1c8d4206..d18a9984c61 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -257,7 +257,9 @@ mini-moka = { version = "0.10", optional = true } minitrace = { version = "0.6", optional = true } moka = { version = "0.12", optional = true, features = ["future", "sync"] } mongodb = { version = "2.7.0", optional = true, features = ["tokio-runtime"] } -mysql_async = { version = "0.32.2", default-features = false, features = ["default-rustls"], optional = true } +mysql_async = { version = "0.32.2", default-features = false, features = [ + "default-rustls", +], optional = true } once_cell = "1" openssh = { version = "0.10.0", optional = true } openssh-sftp-client = { version = "0.14.0", optional = true, features = [ From fbe848ed5be49ac7c11f4040228dbc7517823960 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 6 Dec 2023 14:35:09 +0800 Subject: [PATCH 3/9] Fix typo Signed-off-by: Xuanwo --- core/src/services/b2/core.rs | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/core/src/services/b2/core.rs b/core/src/services/b2/core.rs index f56eb33558f..0308560e4be 100644 --- a/core/src/services/b2/core.rs +++ b/core/src/services/b2/core.rs @@ -89,7 +89,7 @@ impl B2Core { { let mut signer = self.signer.write().await; - let req = Request::get("https://api.backblazeb2.com/b2api/v3/b2_authorize_account") + let req = Request::get("https://api.backblazeb2.com/b2api/v2/b2_authorize_account") .header( header::AUTHORIZATION, format_authorization_by_basic( @@ -110,8 +110,8 @@ impl B2Core { .map_err(new_json_deserialize_error)?; signer.auth_info = AuthInfo { authorization_token: token.authorization_token.clone(), - api_url: token.api_info.storage_api.api_url.clone(), - download_url: token.api_info.storage_api.download_url.clone(), + api_url: token.api_url.clone(), + download_url: token.download_url.clone(), // This authorization token is valid for at most 24 hours. expires_in: Utc::now() + chrono::Duration::hours(20), }; @@ -149,7 +149,7 @@ impl B2Core { let range = args.range(); if !range.is_full() { - req = req.header(http::header::RANGE, range.to_header()); + req = req.header(header::RANGE, range.to_header()); } let req = req @@ -210,7 +210,7 @@ impl B2Core { let range = args.range(); if !range.is_full() { - req = req.header(http::header::RANGE, range.to_header()); + req = req.header(header::RANGE, range.to_header()); } let body = GetDownloadAuthorizationRequest { bucket_id: self.bucket_id.clone(), @@ -586,18 +586,6 @@ pub struct AuthorizeAccountResponse { /// An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header. This authorization token is valid for at most 24 hours. /// So we should call b2_authorize_account every 24 hours. pub authorization_token: String, - pub api_info: ApiInfo, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ApiInfo { - pub storage_api: StorageApi, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct StorageApi { pub api_url: String, pub download_url: String, } From 637f8ea00b450f99bdcfd5a2c663a4b79c9f73e9 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 6 Dec 2023 16:41:17 +0800 Subject: [PATCH 4/9] Fix rate limited error Signed-off-by: Xuanwo --- core/src/services/b2/error.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/services/b2/error.rs b/core/src/services/b2/error.rs index 2a3ead79154..24b8bb1839f 100644 --- a/core/src/services/b2/error.rs +++ b/core/src/services/b2/error.rs @@ -44,6 +44,7 @@ pub async fn parse_error(resp: Response) -> Result { 304 | 412 => (ErrorKind::ConditionNotMatch, false), // Service b2 could return 403, show the authorization error 401 => (ErrorKind::PermissionDenied, true), + 429 => (ErrorKind::RateLimited, true), 500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true), _ => (ErrorKind::Unexpected, false), }; From f4d48a3c43e081f51cd006e0768325e6a5ad9894 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 6 Dec 2023 16:54:00 +0800 Subject: [PATCH 5/9] Fix b2 list Signed-off-by: Xuanwo --- core/src/services/b2/lister.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/services/b2/lister.rs b/core/src/services/b2/lister.rs index d0f250dfe05..78692d29c8a 100644 --- a/core/src/services/b2/lister.rs +++ b/core/src/services/b2/lister.rs @@ -80,7 +80,11 @@ impl oio::PageList for B2Lister { let output: ListFileNamesResponse = serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?; - ctx.done = output.next_file_name.is_none(); + if let Some(token) = output.next_file_name { + ctx.token = token; + } else { + ctx.done = true; + } for file in output.files { if let Some(start_after) = self.start_after.clone() { From 758c6fe354ca6fc16c20df32e8bade2cd16db2f1 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 6 Dec 2023 17:02:36 +0800 Subject: [PATCH 6/9] Fix b2 list Signed-off-by: Xuanwo --- core/src/services/b2/lister.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/services/b2/lister.rs b/core/src/services/b2/lister.rs index 78692d29c8a..476725dbaeb 100644 --- a/core/src/services/b2/lister.rs +++ b/core/src/services/b2/lister.rs @@ -67,7 +67,11 @@ impl oio::PageList for B2Lister { Some(&self.path), self.delimiter, self.limit, - self.start_after.clone(), + if ctx.token.is_empty() { + self.start_after.clone() + } else { + Some(ctx.token.clone()) + }, ) .await?; From 2e004209e7a6cd32808ca6eb75aff4e095821c10 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 6 Dec 2023 17:23:59 +0800 Subject: [PATCH 7/9] Fix abs Signed-off-by: Xuanwo --- core/src/services/b2/core.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/services/b2/core.rs b/core/src/services/b2/core.rs index 0308560e4be..20e24f11ac6 100644 --- a/core/src/services/b2/core.rs +++ b/core/src/services/b2/core.rs @@ -444,7 +444,6 @@ impl B2Core { } if let Some(start_after) = start_after { - let start_after = build_abs_path(&self.root, &start_after); url.push_str(&format!( "&startFileName={}", percent_encode_path(&start_after) From ce237d5ceceeefac18fe8100e4c43b177c56b932 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 6 Dec 2023 17:26:51 +0800 Subject: [PATCH 8/9] Fix build Signed-off-by: Xuanwo --- core/src/services/b2/lister.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/services/b2/lister.rs b/core/src/services/b2/lister.rs index 476725dbaeb..706a8b96086 100644 --- a/core/src/services/b2/lister.rs +++ b/core/src/services/b2/lister.rs @@ -68,7 +68,9 @@ impl oio::PageList for B2Lister { self.delimiter, self.limit, if ctx.token.is_empty() { - self.start_after.clone() + self.start_after + .map(|v| build_abs_path(&self.core.root, &v)) + .clone() } else { Some(ctx.token.clone()) }, From 1628078d8a5c98316754b418c3bbeb943ecf21cf Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 6 Dec 2023 17:32:32 +0800 Subject: [PATCH 9/9] Fix build Signed-off-by: Xuanwo --- core/src/services/b2/lister.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/services/b2/lister.rs b/core/src/services/b2/lister.rs index 706a8b96086..0dee20a9d14 100644 --- a/core/src/services/b2/lister.rs +++ b/core/src/services/b2/lister.rs @@ -69,8 +69,8 @@ impl oio::PageList for B2Lister { self.limit, if ctx.token.is_empty() { self.start_after - .map(|v| build_abs_path(&self.core.root, &v)) - .clone() + .as_ref() + .map(|v| build_abs_path(&self.core.root, v)) } else { Some(ctx.token.clone()) },