refactor: improve the python binding implementation

bindings/python/opendal.pyi

@@ -1,3 +1,19 @@
+# Copyright 2022 Datafuse Labs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class Error(Exception): ...
+
 class Operator:
     def __init__(scheme: str, **kwargs): ...
     def read(self, path: str) -> bytes: ...
@@ -11,4 +27,5 @@ class AsyncOperator:
     async def stat(self, path: str) -> Metadata: ...
 
 class Metadata:
+    @property
     def content_length(self) -> int: ...

bindings/python/src/lib.rs

@@ -16,11 +16,14 @@ use std::collections::HashMap;
 use std::str::FromStr;
 
 use ::opendal as od;
-use pyo3::exceptions::{PyFileNotFoundError, PyRuntimeError};
+use pyo3::create_exception;
+use pyo3::exceptions::{PyException, PyFileNotFoundError};
 use pyo3::prelude::*;
 use pyo3::types::{PyBytes, PyDict};
 use pyo3_asyncio::tokio::future_into_py;
 
+create_exception!(opendal, Error, PyException);
+
 fn build_operator(scheme: od::Scheme, map: HashMap<String, String>) -> PyResult<od::Operator> {
     use od::services::*;
 
@@ -64,9 +67,9 @@ impl AsyncOperator {
         let this = self.0.clone();
         let path = path.to_string();
         future_into_py(py, async move {
-            let res = this.read(&path).await.map_err(format_pyerr)?;
-            let bytes = Python::with_gil(|py| PyBytes::new(py, &res).to_object(py));
-            Ok(bytes)
+            let res: Vec<u8> = this.read(&path).await.map_err(format_pyerr)?;
+            let pybytes: PyObject = Python::with_gil(|py| PyBytes::new(py, &res).into());
+            Ok(pybytes)
         })
     }
 
@@ -112,10 +115,11 @@ impl Operator {
         Ok(Operator(build_operator(scheme, map)?.blocking()))
     }
 
-    pub fn read<'p>(&'p self, py: Python<'p>, path: &str) -> PyResult<&'p PyBytes> {
-        let res = self.0.read(path).map_err(format_pyerr)?;
-        let bytes = PyBytes::new(py, &res);
-        Ok(bytes)
+    pub fn read<'p>(&'p self, py: Python<'p>, path: &str) -> PyResult<&'p PyAny> {
+        self.0
+            .read(path)
+            .map_err(format_pyerr)
+            .map(|res| PyBytes::new(py, &res).into())
     }
 
     pub fn write(&self, path: &str, bs: Vec<u8>) -> PyResult<()> {
@@ -132,6 +136,7 @@ struct Metadata(od::Metadata);
 
 #[pymethods]
 impl Metadata {
+    #[getter]
     pub fn content_length(&self) -> u64 {
         self.0.content_length()
     }
@@ -141,13 +146,14 @@ fn format_pyerr(err: od::Error) -> PyErr {
     use od::ErrorKind::*;
     match err.kind() {
         NotFound => PyFileNotFoundError::new_err(err.to_string()),
-        _ => PyRuntimeError::new_err(err.to_string()),
+        _ => Error::new_err(err.to_string()),
     }
 }
 
 #[pymodule]
-fn opendal(_py: Python, m: &PyModule) -> PyResult<()> {
+fn opendal(py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<Operator>()?;
     m.add_class::<AsyncOperator>()?;
+    m.add("Error", py.get_type::<Error>())?;
     Ok(())
 }

bindings/python/test_requirements.txt

@@ -0,0 +1,2 @@
+pytest
+pytest-asyncio

bindings/python/tests/test_core.py

@@ -0,0 +1,61 @@
+# Copyright 2022 Datafuse Labs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import opendal
+import pytest
+
+
+def test_blocking():
+    op = opendal.Operator("memory")
+    op.write("test", b"Hello, World!")
+    bs = op.read("test")
+    assert bs == b"Hello, World!", bs
+    meta = op.stat("test")
+    assert meta.content_length == 13, meta.content_length
+
+
+@pytest.mark.asyncio
+async def test_async():
+    op = opendal.AsyncOperator("memory")
+    await op.write("test", b"Hello, World!")
+    bs = await op.read("test")
+    assert bs == b"Hello, World!", bs
+    meta = await op.stat("test")
+    assert meta.content_length == 13, meta.content_length
+
+
+def test_blocking_fs(tmp_path):
+    op = opendal.Operator("fs", root=str(tmp_path))
+    op.write("test.txt", b"Hello, World!")
+    bs = op.read("test.txt")
+    assert bs == b"Hello, World!", bs
+    meta = op.stat("test.txt")
+    assert meta.content_length == 13, meta.content_length
+
+
+@pytest.mark.asyncio
+async def test_async_fs(tmp_path):
+    op = opendal.AsyncOperator("fs", root=str(tmp_path))
+    await op.write("test.txt", b"Hello, World!")
+    bs = await op.read("test.txt")
+    assert bs == b"Hello, World!", bs
+    meta = await op.stat("test.txt")
+    assert meta.content_length == 13, meta.content_length
+
+
+def test_error():
+    op = opendal.Operator("memory")
+    with pytest.raises(FileNotFoundError):
+        op.read("test")

licenserc.toml

@@ -26,6 +26,7 @@ excludes = [
   "bindings/nodejs/.npmignore",
   "bindings/nodejs/generated.js",
   "bindings/nodejs/index.d.ts",
+  "bindings/python/test_requirements.txt",
   ".env.example",
 ]