refactor: Remove the concept of Object (#1496)

* check clippy first Signed-off-by: Xuanwo <[email protected]> * Build passed Signed-off-by: Xuanwo <[email protected]> * Fix test Signed-off-by: Xuanwo <[email protected]> * cleanup error code Signed-off-by: Xuanwo <[email protected]> * Fix doc test Signed-off-by: Xuanwo <[email protected]> * format code Signed-off-by: Xuanwo <[email protected]> * Fix docs Signed-off-by: Xuanwo <[email protected]> * Fix topic Signed-off-by: Xuanwo <[email protected]> * Fix nodejs test Signed-off-by: Xuanwo <[email protected]> * FIx typo Signed-off-by: Xuanwo <[email protected]> * Don't build python bindings Signed-off-by: Xuanwo <[email protected]> --------- Signed-off-by: Xuanwo <[email protected]>
apache · Mar 7, 2023 · a3c7b33 · a3c7b33 · github-actions · Mar 7, 2023
1 parent 046acca
commit a3c7b33
Show file tree

Hide file tree

Showing 101 changed files with 2,512 additions and 3,472 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -57,10 +57,10 @@ jobs:
         uses: korandoru/hawkeye@v1
       - name: Cargo format
         run: cargo fmt --all -- --check
-      - name: Cargo doc
-        run: cargo doc --no-deps --all-features
       - name: Cargo clippy
         run: cargo clippy --all-targets --all-features --workspace -- -D warnings
+      - name: Cargo doc
+        run: cargo doc --lib --no-deps --all-features
 
   msrv_check:
     runs-on: ubuntu-latest
@@ -103,7 +103,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
       - name: Build
-        run: cargo build --workspace
+        run: cargo build -p opendal -p oli -p object_store_opendal
 
   build_all_features:
     runs-on: ${{ matrix.os }}
@@ -129,7 +129,7 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
       - name: Build
-        run: cargo build --all-features --workspace
+        run: cargo build --all-features
 
   unit:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -36,7 +36,7 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: doc
-          args: --no-deps --all-features -p opendal
+          args: --lib --no-deps --all-features -p opendal
         env:
           LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }}
 

diff --git a/benches/ops/read.rs b/benches/ops/read.rs
@@ -55,8 +55,7 @@ fn bench_read_full(c: &mut Criterion, name: &str, op: Operator) {
         group.bench_with_input(size.to_string(), &(op.clone(), &path), |b, (op, path)| {
             b.to_async(&*TOKIO).iter(|| async {
                 let r = op
-                    .object(path)
-                    .range_reader(0..=size.bytes() as u64)
+                    .range_reader(path, 0..=size.bytes() as u64)
                     .await
                     .unwrap();
                 io::copy(r, &mut io::sink()).await.unwrap();
@@ -89,7 +88,7 @@ fn bench_read_part(c: &mut Criterion, name: &str, op: Operator) {
         group.throughput(criterion::Throughput::Bytes(size.bytes() as u64));
         group.bench_with_input(size.to_string(), &(op.clone(), &path), |b, (op, path)| {
             b.to_async(&*TOKIO).iter(|| async {
-                let r = op.object(path).range_reader(offset..).await.unwrap();
+                let r = op.range_reader(path, offset..).await.unwrap();
                 io::copy(r, &mut io::sink()).await.unwrap();
             })
         });
@@ -128,8 +127,7 @@ fn bench_read_parallel(c: &mut Criterion, name: &str, op: Operator) {
                             .map(|_| async {
                                 let mut buf = buf.clone();
                                 let mut r = op
-                                    .object(path)
-                                    .range_reader(offset..=offset + size.bytes() as u64)
+                                    .range_reader(path, offset..=offset + size.bytes() as u64)
                                     .await
                                     .unwrap();
                                 r.read_exact(&mut buf).await.unwrap();

diff --git a/benches/ops/utils.rs b/benches/ops/utils.rs
@@ -66,12 +66,7 @@ impl TempData {
     }
 
     pub fn generate(op: Operator, path: &str, content: Bytes) -> Self {
-        TOKIO.block_on(async {
-            op.object(path)
-                .write(content)
-                .await
-                .expect("create test data")
-        });
+        TOKIO.block_on(async { op.write(path, content).await.expect("create test data") });
 
         Self {
             op,
@@ -83,11 +78,7 @@ impl TempData {
 impl Drop for TempData {
     fn drop(&mut self) {
         TOKIO.block_on(async {
-            self.op
-                .object(&self.path)
-                .delete()
-                .await
-                .expect("cleanup test data");
+            self.op.delete(&self.path).await.expect("cleanup test data");
         })
     }
 }
diff --git a/benches/ops/write.rs b/benches/ops/write.rs
@@ -53,7 +53,7 @@ fn bench_write_once(c: &mut Criterion, name: &str, op: Operator) {
             &(op.clone(), &path, content.clone()),
             |b, (op, path, content)| {
                 b.to_async(&*TOKIO).iter(|| async {
-                    op.object(path).write(content.clone()).await.unwrap();
+                    op.write(path, content.clone()).await.unwrap();
                 })
             },
         );

diff --git a/binaries/oli/src/commands/cp.rs b/binaries/oli/src/commands/cp.rs
@@ -27,15 +27,14 @@ pub async fn main(args: Option<ArgMatches>) -> Result<()> {
         .get_one::<String>("source")
         .ok_or_else(|| anyhow!("missing source"))?;
     let (src_op, src_path) = parse_location(src)?;
-    let src_o = src_op.object(src_path);
 
     let dst = args
         .get_one::<String>("destination")
         .ok_or_else(|| anyhow!("missing target"))?;
     let (dst_op, dst_path) = parse_location(dst)?;
-    let mut dst_w = dst_op.object(dst_path).writer().await?;
+    let mut dst_w = dst_op.writer(dst_path).await?;
 
-    let reader = src_o.reader().await?;
+    let reader = src_op.reader(src_path).await?;
     let buf_reader = futures::io::BufReader::with_capacity(8 * 1024 * 1024, reader);
     futures::io::copy_buf(buf_reader, &mut dst_w).await?;
     Ok(())

diff --git a/binaries/oli/src/utils/location.rs b/binaries/oli/src/utils/location.rs
@@ -35,7 +35,7 @@ pub fn parse_location(s: &str) -> Result<(Operator, &str)> {
             None => s,
         };
 
-        return Ok((Operator::create(fs)?.finish(), filename));
+        return Ok((Operator::new(fs)?.finish(), filename));
     }
 
     let s = s.splitn(2, "://").collect::<Vec<_>>();
@@ -48,7 +48,7 @@ pub fn parse_location(s: &str) -> Result<(Operator, &str)> {
                 let (bucket, location) = parse_s3_uri(s[1]);
                 let mut builder = services::S3::default();
                 builder.bucket(bucket);
-                Ok((Operator::create(builder)?.finish(), location))
+                Ok((Operator::new(builder)?.finish(), location))
             }
             _ => todo!(),
         }

diff --git a/bindings/nodejs/.gitignore b/bindings/nodejs/.gitignore
@@ -121,7 +121,7 @@ dist
 .AppleDouble
 .LSOverride
 
-# Icon must end with two 
+# Icon must end with two
 Icon
 
 
@@ -190,3 +190,5 @@ Cargo.lock
 .yarn
 *.node
 
+generated.js
+index.d.ts
diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml
@@ -22,7 +22,6 @@ version = "0.0.0"
 crate-type = ["cdylib"]
 
 [dependencies]
-# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
 futures = "0.3.26"
 napi = { version = "2.11.2", default-features = false, features = [
   "napi6",

diff --git a/bindings/nodejs/__test__/index.spec.mjs b/bindings/nodejs/__test__/index.spec.mjs
@@ -23,18 +23,16 @@ test('test memory write & read', async (t) => {
   let content = "hello world"
   let path = 'test'
 
-  let o = op.object(path)
+  await op.write(path, new TextEncoder().encode(content))
 
-  await o.write(new TextEncoder().encode(content))
-
-  let meta = await o.stat()
+  let meta = await op.stat(path)
   t.is(meta.mode, 0)
   t.is(meta.contentLength, BigInt(content.length))
 
-  let res = await o.read()
+  let res = await op.read(path)
   t.is(content, new TextDecoder().decode(res))
 
-  await o.delete()
+  await op.delete(path)
 })
 
 
@@ -44,47 +42,43 @@ test('test memory write & read synchronously', (t) => {
   let content = "hello world"
   let path = 'test'
 
-  let o = op.object(path)
-
-  o.writeSync(new TextEncoder().encode(content))
+  op.writeSync(path, new TextEncoder().encode(content))
 
-  let meta = o.statSync()
+  let meta = op.statSync(path)
   t.is(meta.mode, 0)
   t.is(meta.contentLength, BigInt(content.length))
 
-  let res = o.readSync()
+  let res = op.readSync(path)
   t.is(content, new TextDecoder().decode(res))
 
-  o.deleteSync()
+  op.deleteSync(path)
 })
 
 test('test scan', async (t) => {
   let op = new Operator(Scheme.Memory)
   let content = "hello world"
   let pathPrefix = 'test'
   let paths = new Array(10).fill(0).map((_, index) => pathPrefix + index)
-  let objects = paths.map(p => op.object(p))
 
-  let writeTasks = objects.map((o) => new Promise(async (resolve, reject) => {
-    await o.write(new TextEncoder().encode(content))
+  let writeTasks = paths.map((path) => new Promise(async (resolve, reject) => {
+    await op.write(path, new TextEncoder().encode(content))
     resolve()
   }))
 
   await Promise.all(writeTasks)
 
-  let dir = op.object("")
-  let objList = await dir.scan()
-  let objectCount = 0
+  let objList = await op.scan("")
+  let entryCount = 0
   while (true) {
-    let o = await objList.next()
-    if (o === null) break
-    objectCount++
-    t.is(new TextDecoder().decode(await o.read()), content)
+    let entry = await objList.next()
+    if (entry === null) break
+    entryCount++
+    t.is(new TextDecoder().decode(await op.read(entry.path())), content)
   }
 
-  t.is(objectCount, paths.length)
+  t.is(entryCount, paths.length)
 
-  objects.forEach(async (o) => {
-    await o.delete()
+  paths.forEach(async (path) => {
+    await op.delete(path)
   })
 })