From 501db835d75aa765d80cb45dbe6f4d0547cf59a4 Mon Sep 17 00:00:00 2001 From: owl Date: Sun, 25 Jun 2023 01:15:48 +0800 Subject: [PATCH 1/9] feat(services/redb): support redb service Signed-off-by: owl --- Cargo.lock | 25 ++- core/Cargo.toml | 3 + core/src/services/mod.rs | 5 + core/src/services/redb/backend.rs | 234 +++++++++++++++++++++++++++++ core/src/services/redb/docs.md | 41 +++++ core/src/services/redb/mod.rs | 20 +++ core/src/types/operator/builder.rs | 2 + core/src/types/scheme.rs | 3 + core/tests/behavior/main.rs | 2 + 9 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 core/src/services/redb/backend.rs create mode 100644 core/src/services/redb/docs.md create mode 100644 core/src/services/redb/mod.rs diff --git a/Cargo.lock b/Cargo.lock index ad8d3adce5c..e687a795244 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2773,6 +2773,7 @@ dependencies = [ "prost", "quick-xml 0.27.1", "rand 0.8.5", + "redb", "redis", "reqsign", "reqwest", @@ -3538,7 +3539,7 @@ dependencies = [ "libc", "memoffset", "parking_lot 0.12.1", - "pyo3-build-config", + "pyo3-build-config 0.18.3", "pyo3-ffi", "pyo3-macros", "unindent", @@ -3567,6 +3568,16 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "pyo3-build-config" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713eccf888fb05f1a96eb78c0dbc51907fee42b3377272dc902eb38985f418d5" +dependencies = [ + "once_cell", + "target-lexicon", +] + [[package]] name = "pyo3-ffi" version = "0.18.3" @@ -3574,7 +3585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd4d7c5337821916ea2a1d21d1092e8443cf34879e53a0ac653fbb98f44ff65c" dependencies = [ "libc", - "pyo3-build-config", + "pyo3-build-config 0.18.3", ] [[package]] @@ -3808,6 +3819,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb" +[[package]] +name = "redb" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ccfb694235b9b5214d32756192abe324083e49363f29bc7c91a69854effe4c" +dependencies = [ + "libc", + "pyo3-build-config 0.19.0", +] + [[package]] name = "redis" version = "0.22.3" diff --git a/core/Cargo.toml b/core/Cargo.toml index 3169cbdab0d..dbdb4b8237c 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -50,6 +50,7 @@ default = [ "services-s3", "services-webdav", "services-webhdfs", + "services-redb", ] # Build docs or not. @@ -157,6 +158,7 @@ services-wasabi = [ ] services-webdav = [] services-webhdfs = [] +services-redb = ["dep:redb"] [lib] bench = false @@ -221,6 +223,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = { version = "0.10", optional = true } sled = { version = "0.34.7", optional = true } +redb = { version = "1.0.0", optional = true } suppaftp = { version = "4.5", default-features = false, features = [ "async-secure", "async-rustls", diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index d27ec1d41f0..ecf8e764c6d 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -168,3 +168,8 @@ pub use webhdfs::Webhdfs; mod vercel_artifacts; #[cfg(feature = "services-vercel-artifacts")] pub use vercel_artifacts::VercelArtifacts; + +#[cfg(feature = "services-redb")] +mod redb; +#[cfg(feature = "services-redb")] +pub use self::redb::Redb; \ No newline at end of file diff --git a/core/src/services/redb/backend.rs b/core/src/services/redb/backend.rs new file mode 100644 index 00000000000..3acdca60c2f --- /dev/null +++ b/core/src/services/redb/backend.rs @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt::Debug; +use std::fmt::Formatter; +use std::sync::Arc; + +use async_trait::async_trait; +use redb::ReadableTable; + +use crate::raw::adapters::kv; +use crate::Builder; +use crate::Error; +use crate::ErrorKind; +use crate::Scheme; +use crate::*; + +/// Redb service support. +#[doc = include_str!("docs.md")] +#[derive(Default)] +pub struct RedbBuilder { + /// That path to the redb data directory. + datadir: Option, + root: Option, + table: Option, +} + +impl RedbBuilder { + /// Set the path to the redb data directory. Will create if not exists. + pub fn datadir(&mut self, path: &str) -> &mut Self { + self.datadir = Some(path.into()); + self + } + + /// Set the table name for Redb. + pub fn table(&mut self, table: &str) -> &mut Self { + self.table = Some(table.into()); + self + } + + /// Set the root for Redb. + pub fn root(&mut self, path: &str) -> &mut Self { + self.root = Some(path.into()); + self + } +} + +impl Builder for RedbBuilder { + const SCHEME: Scheme = Scheme::Redb; + type Accessor = RedbBackend; + + fn from_map(map: HashMap) -> Self { + let mut builder = RedbBuilder::default(); + + map.get("datadir").map(|v| builder.datadir(v)); + map.get("root").map(|v| builder.root(v)); + + builder + } + + fn build(&mut self) -> Result { + let datadir_path = self.datadir.take().ok_or_else(|| { + Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set") + .with_context("service", Scheme::Redb) + })?; + + let table_name = self.table.take().ok_or_else(|| { + Error::new(ErrorKind::ConfigInvalid, "table is required but not set") + .with_context("service", Scheme::Redb) + })?; + + let db = redb::Database::create(&datadir_path).map_err(|e| { + Error::new(ErrorKind::ConfigInvalid, "open db") + .with_context("service", Scheme::Redb) + .with_context("datadir", datadir_path.clone()) + .set_source(e) + })?; + + let db = Arc::new(db); + + Ok(RedbBackend::new(Adapter { + datadir: datadir_path, + table: table_name, + db, + }) + .with_root(self.root.as_deref().unwrap_or_default())) + } +} + +/// Backend for Redb services. +pub type RedbBackend = kv::Backend; + +#[derive(Clone)] +pub struct Adapter { + datadir: String, + table: String, + db: Arc, +} + +impl Debug for Adapter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Adapter"); + ds.field("path", &self.datadir); + ds.finish() + } +} + +#[async_trait] +impl kv::Adapter for Adapter { + fn metadata(&self) -> kv::Metadata { + kv::Metadata::new( + Scheme::Redb, + &self.datadir, + Capability { + read: true, + write: true, + blocking: true, + ..Default::default() + }, + ) + } + + async fn get(&self, path: &str) -> Result>> { + self.blocking_get(path) + } + + fn blocking_get(&self, path: &str) -> Result>> { + let read_txn = match self.db.begin_read() { + Ok(txn) => txn, + Err(e) => { + return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); + } + }; + + let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(&self.table); + + let table = match read_txn.open_table(table_define) { + Ok(table) => table, + Err(e) => { + return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); + } + }; + + let result = match table.get(path) { + Ok(Some(v)) => Ok(Some(v.value().to_vec())), + Ok(None) => Ok(None), + Err(e) => Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)), + }; + result + } + + async fn set(&self, path: &str, value: &[u8]) -> Result<()> { + self.blocking_set(path, value) + } + + fn blocking_set(&self, path: &str, value: &[u8]) -> Result<()> { + let write_txn = match self.db.begin_write() { + Ok(txn) => txn, + Err(e) => { + return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); + } + }; + + let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(&self.table); + + { + let mut table = match write_txn.open_table(table_define) { + Ok(table) => table, + Err(e) => { + return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); + } + }; + + if let Err(e) = table.insert(path, value) { + return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); + }; + } + + let result = match write_txn.commit() { + Ok(()) => Ok(()), + Err(e) => Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)), + }; + result + } + + async fn delete(&self, path: &str) -> Result<()> { + self.blocking_delete(path) + } + + fn blocking_delete(&self, path: &str) -> Result<()> { + let write_txn = match self.db.begin_write() { + Ok(txn) => txn, + Err(e) => { + return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); + } + }; + + let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(&self.table); + + { + let mut table = match write_txn.open_table(table_define) { + Ok(table) => table, + Err(e) => { + return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); + } + }; + + if let Err(e) = table.remove(path) { + return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); + }; + } + + let result = match write_txn.commit() { + Ok(()) => Ok(()), + Err(e) => Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)), + }; + result + } +} \ No newline at end of file diff --git a/core/src/services/redb/docs.md b/core/src/services/redb/docs.md new file mode 100644 index 00000000000..767390a9187 --- /dev/null +++ b/core/src/services/redb/docs.md @@ -0,0 +1,41 @@ +## Capabilities + +This service can be used to: + +- [x] stat +- [x] read +- [x] write +- [x] create_dir +- [x] delete +- [x] copy +- [x] rename +- [ ] ~~list~~ +- [ ] scan +- [ ] ~~presign~~ +- [x] blocking + +## Configuration + +- `datadir`: Set the path to the redb data directory + +You can refer to [`RedbBuilder`]'s docs for more information + +## Example + +### Via Builder + +```rust +use anyhow::Result; +use opendal::services::Redb; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = Redb::default(); + builder.datadir("/tmp/opendal/redb"); + builder.table("opendal-redb"); + + let op: Operator = Operator::new(builder)?.finish(); + Ok(()) +} +``` diff --git a/core/src/services/redb/mod.rs b/core/src/services/redb/mod.rs new file mode 100644 index 00000000000..f53ba572207 --- /dev/null +++ b/core/src/services/redb/mod.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod backend; + +pub use backend::RedbBuilder as Redb; diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs index 4f91366123c..c4079c250ea 100644 --- a/core/src/types/operator/builder.rs +++ b/core/src/types/operator/builder.rs @@ -211,6 +211,8 @@ impl Operator { Scheme::Webdav => Self::from_map::(map)?.finish(), #[cfg(feature = "services-webhdfs")] Scheme::Webhdfs => Self::from_map::(map)?.finish(), + #[cfg(feature = "services-redb")] + Scheme::Redb => Self::from_map::(map)?.finish(), v => { return Err(Error::new( ErrorKind::Unsupported, diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs index 3555783133d..4ae707fd970 100644 --- a/core/src/types/scheme.rs +++ b/core/src/types/scheme.rs @@ -92,6 +92,8 @@ pub enum Scheme { Webdav, /// [webhdfs][crate::services::Webhdfs]: WebHDFS RESTful API Services Webhdfs, + /// [redb][crate::services::Redb]: Redb Services + Redb, /// Custom that allow users to implement services outside of OpenDAL. /// /// # NOTE @@ -190,6 +192,7 @@ impl From for &'static str { Scheme::Wasabi => "wasabi", Scheme::Webdav => "webdav", Scheme::Webhdfs => "webhdfs", + Scheme::Redb => "redb", Scheme::Custom(v) => v, } } diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs index 5d047cf8ecb..e1357de0025 100644 --- a/core/tests/behavior/main.rs +++ b/core/tests/behavior/main.rs @@ -154,6 +154,8 @@ fn main() -> anyhow::Result<()> { tests.extend(behavior_test::()); #[cfg(feature = "services-webhdfs")] tests.extend(behavior_test::()); + #[cfg(feature = "services-redb")] + tests.extend(behavior_test::()); // Don't init logging while building operator which may break cargo // nextest output From b47309bf2c59c1fa644e3049d6a519bced5ac1c8 Mon Sep 17 00:00:00 2001 From: owl Date: Sun, 25 Jun 2023 01:22:17 +0800 Subject: [PATCH 2/9] feat(services/redb): add redb workflow Signed-off-by: owl --- .github/workflows/service_test_redb.yml | 56 +++++++++++++++++++++++++ core/src/services/mod.rs | 2 +- 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/service_test_redb.yml diff --git a/.github/workflows/service_test_redb.yml b/.github/workflows/service_test_redb.yml new file mode 100644 index 00000000000..315cf745338 --- /dev/null +++ b/.github/workflows/service_test_redb.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Service Test Redb + +on: + push: + branches: + - main + pull_request: + branches: + - main + paths: + - "core/src/**" + - "core/tests/**" + - "!core/src/docs/**" + - "!core/src/services/**" + - "core/src/services/redb/**" + - ".github/workflows/redb.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + sled: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup Rust toolchain + uses: ./.github/actions/setup + - name: Test + shell: bash + working-directory: core + run: cargo test redb --features services-redb -j=1 + env: + RUST_BACKTRACE: full + RUST_LOG: debug + OPENDAL_REDB_TEST: on + OPENDAL_REDB_ROOT: / + OPENDAL_REDB_DATADIR: /tmp/opendal/redb/ + OPENDAL_REDB_TABLE: redb-table diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index ecf8e764c6d..9aa631beda3 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -172,4 +172,4 @@ pub use vercel_artifacts::VercelArtifacts; #[cfg(feature = "services-redb")] mod redb; #[cfg(feature = "services-redb")] -pub use self::redb::Redb; \ No newline at end of file +pub use self::redb::Redb; From 1a9ad44102a62ed3840765a7bad2cd2241101c6f Mon Sep 17 00:00:00 2001 From: owl Date: Sun, 25 Jun 2023 01:43:12 +0800 Subject: [PATCH 3/9] feat(services/redb): add redb workflow Signed-off-by: owl --- .env.example | 4 ++++ .github/workflows/service_test_redb.yml | 6 ++++++ core/src/services/redb/backend.rs | 28 +++++++++++++------------ 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index 259947f431a..1a0a02c6f50 100644 --- a/.env.example +++ b/.env.example @@ -114,3 +114,7 @@ OPENDAL_WASABI_BUCKET= OPENDAL_WASABI_ENDPOINT= OPENDAL_WASABI_ACCESS_KEY_ID= OPENDAL_WASABI_SECRET_ACCESS_KEY= +# redb +OPENDAL_REDB_TEST=false +OPENDAL_REDB_DATADIR=/path/to/database +OPENDAL_REDB_TABLE=redb-table diff --git a/.github/workflows/service_test_redb.yml b/.github/workflows/service_test_redb.yml index 315cf745338..910bc3482ea 100644 --- a/.github/workflows/service_test_redb.yml +++ b/.github/workflows/service_test_redb.yml @@ -43,6 +43,12 @@ jobs: - uses: actions/checkout@v3 - name: Setup Rust toolchain uses: ./.github/actions/setup + - name: Prepare for tests + shell: bash + working-directory: core + run: mkdir -p $OPENDAL_REDB_DATADIR + env: + OPENDAL_REDB_DATADIR: /tmp/opendal/redb/ - name: Test shell: bash working-directory: core diff --git a/core/src/services/redb/backend.rs b/core/src/services/redb/backend.rs index 3acdca60c2f..0241075f3bb 100644 --- a/core/src/services/redb/backend.rs +++ b/core/src/services/redb/backend.rs @@ -68,6 +68,7 @@ impl Builder for RedbBuilder { let mut builder = RedbBuilder::default(); map.get("datadir").map(|v| builder.datadir(v)); + map.get("table").map(|v| builder.table(v)); map.get("root").map(|v| builder.root(v)); builder @@ -146,8 +147,9 @@ impl kv::Adapter for Adapter { return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); } }; - - let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(&self.table); + + let table_define: redb::TableDefinition<&str, &[u8]> = + redb::TableDefinition::new(&self.table); let table = match read_txn.open_table(table_define) { Ok(table) => table, @@ -176,7 +178,8 @@ impl kv::Adapter for Adapter { } }; - let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(&self.table); + let table_define: redb::TableDefinition<&str, &[u8]> = + redb::TableDefinition::new(&self.table); { let mut table = match write_txn.open_table(table_define) { @@ -185,17 +188,16 @@ impl kv::Adapter for Adapter { return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); } }; - + if let Err(e) = table.insert(path, value) { return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); }; } - let result = match write_txn.commit() { + match write_txn.commit() { Ok(()) => Ok(()), Err(e) => Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)), - }; - result + } } async fn delete(&self, path: &str) -> Result<()> { @@ -210,7 +212,8 @@ impl kv::Adapter for Adapter { } }; - let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(&self.table); + let table_define: redb::TableDefinition<&str, &[u8]> = + redb::TableDefinition::new(&self.table); { let mut table = match write_txn.open_table(table_define) { @@ -219,16 +222,15 @@ impl kv::Adapter for Adapter { return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); } }; - + if let Err(e) = table.remove(path) { return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); }; } - let result = match write_txn.commit() { + match write_txn.commit() { Ok(()) => Ok(()), Err(e) => Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)), - }; - result + } } -} \ No newline at end of file +} From d4227ccfa22f3e29e403be6fd50f7fc3364e3073 Mon Sep 17 00:00:00 2001 From: owl Date: Sun, 25 Jun 2023 23:03:16 +0800 Subject: [PATCH 4/9] feat(services/redb): fix code Signed-off-by: owl --- .github/workflows/service_test_redb.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/service_test_redb.yml b/.github/workflows/service_test_redb.yml index 910bc3482ea..0bed7c21303 100644 --- a/.github/workflows/service_test_redb.yml +++ b/.github/workflows/service_test_redb.yml @@ -58,5 +58,5 @@ jobs: RUST_LOG: debug OPENDAL_REDB_TEST: on OPENDAL_REDB_ROOT: / - OPENDAL_REDB_DATADIR: /tmp/opendal/redb/ + OPENDAL_REDB_DATADIR: /tmp/opendal/ OPENDAL_REDB_TABLE: redb-table From 9fa5659a2763edd1cc300f8bbbb0f94133b30c34 Mon Sep 17 00:00:00 2001 From: owl Date: Sun, 25 Jun 2023 23:14:58 +0800 Subject: [PATCH 5/9] feat(services/redb): fix code Signed-off-by: owl --- .github/workflows/service_test_redb.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/service_test_redb.yml b/.github/workflows/service_test_redb.yml index 0bed7c21303..7fba12b6712 100644 --- a/.github/workflows/service_test_redb.yml +++ b/.github/workflows/service_test_redb.yml @@ -37,7 +37,7 @@ concurrency: cancel-in-progress: true jobs: - sled: + redb: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -48,7 +48,7 @@ jobs: working-directory: core run: mkdir -p $OPENDAL_REDB_DATADIR env: - OPENDAL_REDB_DATADIR: /tmp/opendal/redb/ + OPENDAL_REDB_DATADIR: /tmp/opendal/ - name: Test shell: bash working-directory: core @@ -58,5 +58,5 @@ jobs: RUST_LOG: debug OPENDAL_REDB_TEST: on OPENDAL_REDB_ROOT: / - OPENDAL_REDB_DATADIR: /tmp/opendal/ + OPENDAL_REDB_DATADIR: /tmp/opendal/redb OPENDAL_REDB_TABLE: redb-table From 3ff9cfb54a3f6c6100942071e037fbe402410315 Mon Sep 17 00:00:00 2001 From: owl Date: Sun, 25 Jun 2023 23:59:05 +0800 Subject: [PATCH 6/9] feat(services/redb): fix code Signed-off-by: owl --- core/src/services/redb/backend.rs | 106 ++++++++++++++---------------- 1 file changed, 48 insertions(+), 58 deletions(-) diff --git a/core/src/services/redb/backend.rs b/core/src/services/redb/backend.rs index 0241075f3bb..fb8227c9993 100644 --- a/core/src/services/redb/backend.rs +++ b/core/src/services/redb/backend.rs @@ -85,12 +85,7 @@ impl Builder for RedbBuilder { .with_context("service", Scheme::Redb) })?; - let db = redb::Database::create(&datadir_path).map_err(|e| { - Error::new(ErrorKind::ConfigInvalid, "open db") - .with_context("service", Scheme::Redb) - .with_context("datadir", datadir_path.clone()) - .set_source(e) - })?; + let db = redb::Database::create(&datadir_path).map_err(parse_database_error)?; let db = Arc::new(db); @@ -141,27 +136,19 @@ impl kv::Adapter for Adapter { } fn blocking_get(&self, path: &str) -> Result>> { - let read_txn = match self.db.begin_read() { - Ok(txn) => txn, - Err(e) => { - return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); - } - }; + let read_txn = self.db.begin_read().map_err(parse_transaction_error)?; let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(&self.table); - let table = match read_txn.open_table(table_define) { - Ok(table) => table, - Err(e) => { - return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); - } - }; + let table = read_txn + .open_table(table_define) + .map_err(parse_table_error)?; let result = match table.get(path) { Ok(Some(v)) => Ok(Some(v.value().to_vec())), Ok(None) => Ok(None), - Err(e) => Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)), + Err(e) => Err(parse_storage_error(e)), }; result } @@ -171,33 +158,21 @@ impl kv::Adapter for Adapter { } fn blocking_set(&self, path: &str, value: &[u8]) -> Result<()> { - let write_txn = match self.db.begin_write() { - Ok(txn) => txn, - Err(e) => { - return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); - } - }; + let write_txn = self.db.begin_write().map_err(parse_transaction_error)?; let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(&self.table); { - let mut table = match write_txn.open_table(table_define) { - Ok(table) => table, - Err(e) => { - return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); - } - }; - - if let Err(e) = table.insert(path, value) { - return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); - }; - } + let mut table = write_txn + .open_table(table_define) + .map_err(parse_table_error)?; - match write_txn.commit() { - Ok(()) => Ok(()), - Err(e) => Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)), + table.insert(path, value).map_err(parse_storage_error)?; } + + write_txn.commit().map_err(parse_commit_error)?; + Ok(()) } async fn delete(&self, path: &str) -> Result<()> { @@ -205,32 +180,47 @@ impl kv::Adapter for Adapter { } fn blocking_delete(&self, path: &str) -> Result<()> { - let write_txn = match self.db.begin_write() { - Ok(txn) => txn, - Err(e) => { - return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); - } - }; + let write_txn = self.db.begin_write().map_err(parse_transaction_error)?; let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(&self.table); { - let mut table = match write_txn.open_table(table_define) { - Ok(table) => table, - Err(e) => { - return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); - } - }; - - if let Err(e) = table.remove(path) { - return Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)); - }; + let mut table = write_txn + .open_table(table_define) + .map_err(parse_table_error)?; + + table.remove(path).map_err(parse_storage_error)?; } - match write_txn.commit() { - Ok(()) => Ok(()), - Err(e) => Err(Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)), + write_txn.commit().map_err(parse_commit_error)?; + Ok(()) + } +} + +fn parse_transaction_error(e: redb::TransactionError) -> Error { + match e { + _ => Error::new(ErrorKind::Unexpected, "error from redb").set_source(e), + } +} + +fn parse_table_error(e: redb::TableError) -> Error { + match e { + redb::TableError::TableDoesNotExist(_) => { + Error::new(ErrorKind::NotFound, "error from redb").set_source(e) } + _ => Error::new(ErrorKind::Unexpected, "error from redb").set_source(e), } } + +fn parse_storage_error(e: redb::StorageError) -> Error { + Error::new(ErrorKind::Unexpected, "error from redb").set_source(e) +} + +fn parse_database_error(e: redb::DatabaseError) -> Error { + Error::new(ErrorKind::Unexpected, "error from redb").set_source(e) +} + +fn parse_commit_error(e: redb::CommitError) -> Error { + Error::new(ErrorKind::Unexpected, "error from redb").set_source(e) +} From 56d7db7b586c574371b44cb6e94a67d053307993 Mon Sep 17 00:00:00 2001 From: owl Date: Mon, 26 Jun 2023 00:10:00 +0800 Subject: [PATCH 7/9] feat(services/redb): fix code Signed-off-by: owl --- core/src/services/redb/backend.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/services/redb/backend.rs b/core/src/services/redb/backend.rs index fb8227c9993..2d721800ef2 100644 --- a/core/src/services/redb/backend.rs +++ b/core/src/services/redb/backend.rs @@ -199,9 +199,7 @@ impl kv::Adapter for Adapter { } fn parse_transaction_error(e: redb::TransactionError) -> Error { - match e { - _ => Error::new(ErrorKind::Unexpected, "error from redb").set_source(e), - } + Error::new(ErrorKind::Unexpected, "error from redb").set_source(e) } fn parse_table_error(e: redb::TableError) -> Error { From 7e804617ea1332e09125ecc6198cf8514a50d992 Mon Sep 17 00:00:00 2001 From: owl Date: Mon, 26 Jun 2023 00:18:36 +0800 Subject: [PATCH 8/9] feat(services/redb): fix code Signed-off-by: owl --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 1a0a02c6f50..cb3161c2b7b 100644 --- a/.env.example +++ b/.env.example @@ -116,5 +116,5 @@ OPENDAL_WASABI_ACCESS_KEY_ID= OPENDAL_WASABI_SECRET_ACCESS_KEY= # redb OPENDAL_REDB_TEST=false -OPENDAL_REDB_DATADIR=/path/to/database +OPENDAL_REDB_DATADIR=/tmp/redb OPENDAL_REDB_TABLE=redb-table From 1be527aaa78a465ffb0e0eefab67019f9db09ff9 Mon Sep 17 00:00:00 2001 From: owl Date: Mon, 26 Jun 2023 00:39:25 +0800 Subject: [PATCH 9/9] feat(services/redb): fix code Signed-off-by: owl --- core/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index dbdb4b8237c..154bf69335a 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -50,7 +50,6 @@ default = [ "services-s3", "services-webdav", "services-webhdfs", - "services-redb", ] # Build docs or not.