diff --git a/.github/workflows/test_examples.yml b/.github/workflows/test_examples.yml deleted file mode 100644 index 7a6d5759290..00000000000 --- a/.github/workflows/test_examples.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Examples - -on: - push: - branches: - - main - pull_request: - branches: - - main - paths: - - "examples/**" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true - -jobs: - rust-00-setup: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - name: Test - shell: bash - working-directory: examples/rust/00-setup - run: cargo run - - rust-01-init-operator: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - name: Test - shell: bash - working-directory: examples/rust/01-init-operator - run: cargo run - - rust-02-async-io: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - name: Test - shell: bash - working-directory: examples/rust/02-async-io - run: cargo run diff --git a/core/Cargo.lock b/core/Cargo.lock index 250a5a1f114..367d4d1b0fc 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2421,7 +2421,7 @@ dependencies = [ [[package]] name = "edge_test_aws_s3_assume_role_with_web_identity" -version = "0.0.0" +version = "0.47.0" dependencies = [ "opendal", "tokio", @@ -2430,7 +2430,7 @@ dependencies = [ [[package]] name = "edge_test_file_write_on_full_disk" -version = "0.0.0" +version = "0.47.0" dependencies = [ "futures", "opendal", @@ -2440,7 +2440,7 @@ dependencies = [ [[package]] name = "edge_test_s3_read_on_wasm" -version = "0.0.0" +version = "0.47.0" dependencies = [ "opendal", "wasm-bindgen", @@ -4854,6 +4854,33 @@ dependencies = [ "uuid", ] +[[package]] +name = "opendal-examples-basic" +version = "0.47.0" +dependencies = [ + "futures", + "opendal", + "tokio", +] + +[[package]] +name = "opendal-examples-concurrent-upload" +version = "0.47.0" +dependencies = [ + "futures", + "opendal", + "tokio", +] + +[[package]] +name = "opendal-examples-multipart-upload" +version = "0.47.0" +dependencies = [ + "futures", + "opendal", + "tokio", +] + [[package]] name = "opendal-fuzz" version = "0.0.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index d49b9de04e0..48655977289 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -36,7 +36,13 @@ rustdoc-args = ["--cfg", "docs"] [workspace] default-members = ["."] -members = [".", "fuzz", "edge/*", "benches/vs_*"] +members = [".", "examples/*", "fuzz", "edge/*", "benches/vs_*"] + +[workspace.package] +edition = "2021" +license = "Apache-2.0" +rust-version = "1.75" +version = "0.47.0" [features] default = ["reqwest/rustls-tls", "executors-tokio", "services-memory"] @@ -95,6 +101,7 @@ layers-async-backtrace = ["dep:async-backtrace"] layers-blocking = ["internal-tokio-rt"] layers-dtrace = ["dep:probe"] +services-aliyun-drive = [] services-alluxio = [] services-atomicserver = ["dep:atomic_lib"] services-azblob = [ @@ -193,7 +200,6 @@ services-vercel-blob = [] services-webdav = [] services-webhdfs = [] services-yandex-disk = [] -services-aliyun-drive = [] [lib] bench = false @@ -233,7 +239,10 @@ chrono = { version = "0.4.28", default-features = false, features = [ "std", ] } flagset = "0.4" -futures = { version = "0.3", default-features = false, features = ["std", "async-await"] } +futures = { version = "0.3", default-features = false, features = [ + "std", + "async-await", +] } http = "1.1" log = "0.4" md-5 = "0.10" @@ -335,7 +344,11 @@ hdfs-native = { version = "0.9.4", optional = true } # for services-surrealdb surrealdb = { version = "1.3.0", optional = true, features = ["protocol-http"] } # for services-compfs -compio = { version = "0.10.0", optional = true, features = ["runtime", "bytes", "polling"] } +compio = { version = "0.10.0", optional = true, features = [ + "runtime", + "bytes", + "polling", +] } # for services-s3 crc32c = { version = "0.6.6", optional = true } diff --git a/core/README.md b/core/README.md index 94b3cc2ae35..eb31e363a16 100644 --- a/core/README.md +++ b/core/README.md @@ -17,6 +17,7 @@ OpenDAL offers a unified data access layer, empowering users to seamlessly and e ## Useful Links - Documentation: [release](https://docs.rs/opendal/) | [dev](https://opendal.apache.org/docs/rust/opendal/) +- [Examples](./examples) - [Release Notes](https://docs.rs/opendal/latest/opendal/docs/changelog/index.html) - [Upgrade Guide](https://docs.rs/opendal/latest/opendal/docs/upgrade/index.html) - [RFC List](https://docs.rs/opendal/latest/opendal/docs/rfcs/index.html) @@ -194,7 +195,15 @@ async fn main() -> Result<()> { ## Examples -The examples are available at [here](../examples/rust). +| Name | Description | +|---------------------|---------------------------------------------------------------| +| [Basic] | Show how to use opendal to operate storage service. | +| [Concurrent Upload] | Show how to perform upload concurrently to a storage service. | +| [Multipart Upload] | Show how to perform a multipart upload to a storage service. | + +[Basic]: ./examples/basic +[Concurrent Upload]: ./examples/concurrent-upload +[Multipart Upload]: ./examples/multipart-upload ## Contributing diff --git a/core/edge/file_write_on_full_disk/Cargo.toml b/core/edge/file_write_on_full_disk/Cargo.toml index 339c73a3d72..57bfb10a3d9 100644 --- a/core/edge/file_write_on_full_disk/Cargo.toml +++ b/core/edge/file_write_on_full_disk/Cargo.toml @@ -16,12 +16,13 @@ # under the License. [package] -edition = "2021" -license = "Apache-2.0" name = "edge_test_file_write_on_full_disk" + +edition.workspace = true +license.workspace = true publish = false -rust-version = "1.75" -version = "0.0.0" +rust-version.workspace = true +version.workspace = true [dependencies] futures = "0.3" diff --git a/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml b/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml index f503ef8e6f3..0e898209e50 100644 --- a/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml +++ b/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml @@ -16,12 +16,13 @@ # under the License. [package] -edition = "2021" -license = "Apache-2.0" name = "edge_test_aws_s3_assume_role_with_web_identity" + +edition.workspace = true +license.workspace = true publish = false -rust-version = "1.75" -version = "0.0.0" +rust-version.workspace = true +version.workspace = true [dependencies] opendal = { path = "../..", features = ["tests"] } diff --git a/core/edge/s3_read_on_wasm/Cargo.toml b/core/edge/s3_read_on_wasm/Cargo.toml index 4c14b7678a4..df57849cb6f 100644 --- a/core/edge/s3_read_on_wasm/Cargo.toml +++ b/core/edge/s3_read_on_wasm/Cargo.toml @@ -16,12 +16,13 @@ # under the License. [package] -edition = "2021" -license = "Apache-2.0" name = "edge_test_s3_read_on_wasm" + +edition.workspace = true +license.workspace = true publish = false -rust-version = "1.75" -version = "0.0.0" +rust-version.workspace = true +version.workspace = true [lib] crate-type = ["cdylib"] diff --git a/core/examples/README.md b/core/examples/README.md new file mode 100644 index 00000000000..6c761c5f9d3 --- /dev/null +++ b/core/examples/README.md @@ -0,0 +1,23 @@ +# Apache OpenDAL™ Rust Core Examples + +Thank you for using OpenDAL! + +Those examples are designed to help you to understand how to use OpenDAL Rust Core. + +## Setup + +All examples following the same setup steps: + +To run this example, please copy the `.env.example`, which is at project root, to `.env` and change the values on need. + +Take `fs` for example, we need to change to enable behavior test on `fs` on `/tmp`. + +```dotenv +OPENDAL_FS_ROOT=/path/to/dir +``` + +into + +```dotenv +OPENDAL_FS_ROOT=/tmp +``` diff --git a/core/examples/basic/Cargo.toml b/core/examples/basic/Cargo.toml new file mode 100644 index 00000000000..185d4b8e51c --- /dev/null +++ b/core/examples/basic/Cargo.toml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "opendal-examples-basic" +publish = false + +version.workspace = true +edition.workspace = true +rust-version.workspace = true + +[dependencies] +futures = "0.3" +opendal = { path = "../..", features = ["tests"] } +tokio = { version = "1.38.0", features = ["macros", "rt-multi-thread"] } diff --git a/core/examples/basic/README.md b/core/examples/basic/README.md new file mode 100644 index 00000000000..65621a18c9e --- /dev/null +++ b/core/examples/basic/README.md @@ -0,0 +1,15 @@ +# Basic + +This example will show how to use opendal to operate storage service. + +## Setup + +Refer [Setup](../README.md) to setup all examples. + +## Run + +Use `OPENDAL_TEST` to control which service to run example: + +```shell +OPENDAL_TEST=fs cargo run +``` \ No newline at end of file diff --git a/core/examples/basic/src/main.rs b/core/examples/basic/src/main.rs new file mode 100644 index 00000000000..8a7c3ba88d8 --- /dev/null +++ b/core/examples/basic/src/main.rs @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use opendal::{Operator, Result}; + +async fn example(op: Operator) -> Result<()> { + // Write data to S3. + op.write("test.txt", "Hello, World!").await?; + println!("write succeeded"); + + // Read data from s3. + let bs = op.read("test.txt").await?; + println!("read: {}", String::from_utf8(bs.to_vec()).unwrap()); + + // Fetch metadata of s3. + let meta = op.stat("test.txt").await?; + println!("stat: {:?}", meta); + + // Delete data from s3. + op.delete("test.txt").await?; + println!("delete succeeded"); + + Ok(()) +} + +// --- The following data is not part of this example --- + +#[tokio::main] +async fn main() -> Result<()> { + use opendal::raw::tests::init_test_service; + let op = init_test_service()?.expect("OPENDAL_TEST must be set"); + + println!("service {:?} has been initialized", op.info()); + example(op).await +} diff --git a/core/examples/concurrent-upload/Cargo.toml b/core/examples/concurrent-upload/Cargo.toml new file mode 100644 index 00000000000..8a9ba0963fb --- /dev/null +++ b/core/examples/concurrent-upload/Cargo.toml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "opendal-examples-concurrent-upload" +publish = false + +version.workspace = true +edition.workspace = true +rust-version.workspace = true + +[dependencies] +futures = "0.3" +opendal = { path = "../..", features = ["tests"] } +tokio = { version = "1.38.0", features = ["macros", "rt-multi-thread"] } diff --git a/core/examples/concurrent-upload/README.md b/core/examples/concurrent-upload/README.md new file mode 100644 index 00000000000..d1e735924de --- /dev/null +++ b/core/examples/concurrent-upload/README.md @@ -0,0 +1,15 @@ +# Concurrent Upload + +This example will show how to perform upload concurrently to a storage service. + +## Setup + +Refer [Setup](../README.md) to setup all examples. + +## Run + +Use `OPENDAL_TEST` to control which service to run example: + +```shell +OPENDAL_TEST=s3 cargo run +``` \ No newline at end of file diff --git a/core/examples/concurrent-upload/src/main.rs b/core/examples/concurrent-upload/src/main.rs new file mode 100644 index 00000000000..d5350b67154 --- /dev/null +++ b/core/examples/concurrent-upload/src/main.rs @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use opendal::{Operator, Result}; +use std::time::SystemTime; + +async fn example(op: Operator) -> Result<()> { + let mut w = op.writer_with("test.txt").concurrent(4).await?; + + let now = SystemTime::now(); + // Write a 10MiB chunk. + w.write(vec![0; 10 * 1024 * 1024]).await?; + println!( + "vec![0; 10 * 1024 * 1024] uploaded at {:03}", + now.elapsed().unwrap().as_secs_f64() + ); + // Write another 10MiB chunk. + w.write(vec![1; 10 * 1024 * 1024]).await?; + println!( + "vec![1; 10 * 1024 * 1024] uploaded at {:03}", + now.elapsed().unwrap().as_secs_f64() + ); + // Finish the upload. + w.close().await?; + println!( + "upload finished at {:03}", + now.elapsed().unwrap().as_secs_f64() + ); + + let bs = op.read("test.txt").await?; + println!("read: {} bytes", bs.len()); + assert_eq!( + bs.to_vec(), + vec![0; 10 * 1024 * 1024] + .into_iter() + .chain(vec![1; 10 * 1024 * 1024]) + .collect::>() + ); + + Ok(()) +} + +// --- The following data is not part of this example --- + +#[tokio::main] +async fn main() -> Result<()> { + use opendal::raw::tests::init_test_service; + let op = init_test_service()?.expect("OPENDAL_TEST must be set"); + + println!("service {:?} has been initialized", op.info()); + example(op).await +} diff --git a/core/examples/multipart-upload/Cargo.toml b/core/examples/multipart-upload/Cargo.toml new file mode 100644 index 00000000000..575f37aa33d --- /dev/null +++ b/core/examples/multipart-upload/Cargo.toml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "opendal-examples-multipart-upload" +publish = false + +version.workspace = true +edition.workspace = true +rust-version.workspace = true + +[dependencies] +futures = "0.3" +opendal = { path = "../..", features = ["tests"] } +tokio = { version = "1.38.0", features = ["macros", "rt-multi-thread"] } diff --git a/core/examples/multipart-upload/README.md b/core/examples/multipart-upload/README.md new file mode 100644 index 00000000000..defd177f482 --- /dev/null +++ b/core/examples/multipart-upload/README.md @@ -0,0 +1,15 @@ +# Multipart Upload + +This example will show how to perform a multipart upload to a storage service. + +## Setup + +Refer [Setup](../README.md) to setup all examples. + +## Run + +Use `OPENDAL_TEST` to control which service to run example: + +```shell +OPENDAL_TEST=s3 cargo run +``` diff --git a/core/examples/multipart-upload/src/main.rs b/core/examples/multipart-upload/src/main.rs new file mode 100644 index 00000000000..7f9e4a1a27d --- /dev/null +++ b/core/examples/multipart-upload/src/main.rs @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use opendal::{Operator, Result}; + +async fn example(op: Operator) -> Result<()> { + let mut w = op.writer("test.txt").await?; + + // Write a 10MiB chunk. + w.write(vec![0; 10 * 1024 * 1024]).await?; + println!("vec![0; 10 * 1024 * 1024] uploaded"); + // Write another 10MiB chunk. + w.write(vec![1; 10 * 1024 * 1024]).await?; + println!("vec![1; 10 * 1024 * 1024] uploaded"); + // Finish the upload. + w.close().await?; + println!("upload finished"); + + let bs = op.read("test.txt").await?; + println!("read: {} bytes", bs.len()); + assert_eq!( + bs.to_vec(), + vec![0; 10 * 1024 * 1024] + .into_iter() + .chain(vec![1; 10 * 1024 * 1024]) + .collect::>() + ); + + Ok(()) +} + +// --- The following data is not part of this example --- + +#[tokio::main] +async fn main() -> Result<()> { + use opendal::raw::tests::init_test_service; + let op = init_test_service()?.expect("OPENDAL_TEST must be set"); + + println!("service {:?} has been initialized", op.info()); + example(op).await +} diff --git a/examples/README.md b/examples/README.md index a072e882b92..3cf11479db5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -6,5 +6,5 @@ The goal of these documents is to provide you with everything you need to start ## Examples -- [Rust](rust/README.md) +- [Rust](../core/examples/README.md) - [C++](cpp/README.md) \ No newline at end of file diff --git a/examples/rust/00-setup/.gitignore b/examples/rust/00-setup/.gitignore deleted file mode 100644 index 2c96eb1b651..00000000000 --- a/examples/rust/00-setup/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -target/ -Cargo.lock diff --git a/examples/rust/00-setup/Cargo.toml b/examples/rust/00-setup/Cargo.toml deleted file mode 100644 index 2c28f6f6eea..00000000000 --- a/examples/rust/00-setup/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -# name for this package -name = "setup" -# edition is the rust edition that used to compile this package. -# We will not discuss about this a lot, use `2021` is a good choice. -# -# For more information: -edition = "2021" -# version is the semver for this package. -# We will not publish this package publicly, so the version is leave AS-IS. -version = "0.1.0" - -# In this example, we are depending on two crates (packages): `futures` and `opendal` -[dependencies] -futures = "0.3" -# The `0.45` is the semver version of opendal. -# -# We use `0.45` to indicate all versions that `>=0.45.0 && <0.46.0`. -opendal = "0.45" diff --git a/examples/rust/00-setup/README.md b/examples/rust/00-setup/README.md deleted file mode 100644 index e8e9aded442..00000000000 --- a/examples/rust/00-setup/README.md +++ /dev/null @@ -1,98 +0,0 @@ -# Chapter-00: Set up Your First Rust Project - -Welcome to our first chapter. In this example we will set up our first Rust project with OpenDAL. - -## Install Rust - -First of all, we need to install Rust. Refer [here](../../../CONTRIBUTING.md#bring-your-own-toolbox) to figure it out. - -After Rust has been installed, we will have the following components: - -- `rustup`: [rustup](https://rust-lang.github.io/rustup/) is a command-line toolchain manager for the Rust programming language. It allows you to easily install, manage, and update different versions of the Rust compiler and associated tools on your system. -- `cargo`: [cargo](https://doc.rust-lang.org/cargo/index.html) is the package manager and build system for the Rust programming language. It is designed to make it easy to develop, build, and manage Rust projects. -- `rustc`: [rustc](https://rustc-dev-guide.rust-lang.org/) is the official compiler for the Rust programming language. It takes Rust source code as input and translates it into machine-readable instructions that can be executed by the computer's hardware. Most time, `rustc` will be called by `cargo` and we don't need to use it directly. - -We can use `cargo --version` to check if it works as expected: - -```shell -> cargo --version -cargo 1.70.0 (ec8a8a0ca 2023-04-25) -``` - -## Rust Project Structure - -A typical Rust project follows a specific directory structure. The most simple project structure will contain the following files: - -- `Cargo.toml`: The Cargo.toml file is the manifest file for the project. It specifies metadata about the project, including its name, version, dependencies, and build configuration. It is also where you declare the project's dependencies using the Cargo package manager. -- `src/`: The `src` directory is where the main source code files of the project reside. It typically contains the Rust source files with the code implementation for the project's functionality. - -Let's go into more depth by our example code. - -### `Cargo.toml` - -Let's take a look over [`Cargo.toml`](Cargo.toml) first. - -The most simple `Cargo.toml` will contains two parts: - -- `package`: The metadata of this package like `name`, `version`. -- `dependencies`: The dependencies that this package will depend on. `cargo` will download them from and compile them. - -### `src/` - -Then, let's read [`main.rs`](./src/main.rs). - -The most simple `main.rs` will contain only one function: `fn main()`, this is the entry of an application. - -In our example, we did the following things: - -```rust -use opendal::Scheme; -``` - -Import the `opendal::Scheme` into current namespace, so that we can use `Scheme` instead of the full naming `opendal::Scheme` everywhere. - -``` rust -fn main() { - ... -} -``` - -Declare our main function, this is the most simple function that not take any input and not return any output. - -```rust -println!("Hello, {}", Scheme::S3) -``` - -- `println!()` is a built macro in rust to prints to the standard output, with a newline. Macro will be expanded to real code during compilation. -- `"Hello, {}", Scheme::S3` is the format string in rust. It will convert `Scheme::S3` to string, and construct a new string in this format. - -## Build our first rust project! - -Now, it's time to build our first rust project! - -Make sure we are under folder `examples/rust/00-setup`, running: - -```shell -cargo run -``` - -We will see output like: - -```shell -:) cargo run - Compiling setup v0.1.0 (/home/xuanwo/Code/apache/opendal/examples/rust/00-setup) - Finished dev [unoptimized + debuginfo] target(s) in 0.32s - Running `target/debug/setup` -Hello, s3 -``` - -Congrats! Our first Rust project is built and running with success! - -After built, we will find that there are some new files created: - -- `Cargo.lock`: Cargo.lock is a file generated by the cargo package manager when you build or run a Rust project. It serves as a lock file and records the exact versions of dependencies that were used during the previous successful build or run of the project. It's always a good idea to commit `Cargo.lock` to your repo, so developers can reproduce the same build result with you. -- `target`: `target` folder is a directory automatically generated by `rustc` and managed cargo. It contains the compiled artifacts and build output for specific target platforms and architectures. - -## Conclusion - -In this chapter we have learnt the Rust project structure and run our first Rust example. We will use this knowledge to build our first Rust application in the next chapter! diff --git a/examples/rust/00-setup/src/main.rs b/examples/rust/00-setup/src/main.rs deleted file mode 100644 index 626a518c373..00000000000 --- a/examples/rust/00-setup/src/main.rs +++ /dev/null @@ -1,5 +0,0 @@ -use opendal::Scheme; - -fn main() { - println!("Hello, {}", Scheme::S3) -} diff --git a/examples/rust/01-init-operator/.gitignore b/examples/rust/01-init-operator/.gitignore deleted file mode 100644 index 2c96eb1b651..00000000000 --- a/examples/rust/01-init-operator/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -target/ -Cargo.lock diff --git a/examples/rust/01-init-operator/Cargo.toml b/examples/rust/01-init-operator/Cargo.toml deleted file mode 100644 index 546c7e1fcbd..00000000000 --- a/examples/rust/01-init-operator/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "init-operator" - -edition = "2021" -version = "0.1.0" - -[dependencies] -futures = "0.3" -opendal = "0.45" diff --git a/examples/rust/01-init-operator/README.md b/examples/rust/01-init-operator/README.md deleted file mode 100644 index 3387b7df2b9..00000000000 --- a/examples/rust/01-init-operator/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Chapter-01: Initiate OpenDAL Operator - -The core concept in OpenDAL is the `Operator`, which is used to handle operations on different storage services. This chapter will discuss how to initiate OpenDAL operators. - -## Warmup - -Before delving into the OpenDAL API, let's take a look at some new Rust features. - -### Enum and Set Returning Type for Function - -First of all, our main functions now returns `Result<()>`: - -```rust -use opendal::Result; - -fn main() -> Result<()> { - Ok(()) -} -``` - -This is one of the coolest part of Rust: [`enum`](https://doc.rust-lang.org/book/ch06-01-defining-an-enum.html). In rust, an `enum` (short for enumeration) is a data type that represents a value that can be one of several possible variants. It allows you to define a set of distinct options or choices, making it useful for scenarios where a variable can have a limited number of predefined states. - -For example, Rust provide a built-in enum called `Option`: - -```rust -enum Option { - Some(T), - None, -} -``` - -The `T` represents the type of the value that may or may not be present. `Some` variant holds the actual value, while `None` indicates the absence of a value. - -Rust also provides a `Result` enum for returning and propagating errors: - -```rust -enum Result { - Ok(T), - Err(E), -} -``` - -OpenDAL defines it own `Result` type for it easier to use: - -```rust -/// Result that is a wrapper of `Result` -pub type Result = std::result::Result; -``` - -So let's go back into our examples: - -```rust -use opendal::Result; - -fn main() -> Result<()> { - Ok(()) -} -``` - -> `use opendal::Result;` - -imports opendal's `Result` which will shadows Rust built-in result. - -> `fn main() -> Result<()>` - -The way that Rust to declare returning types of a function. This case means `main` will return a `Result` which holds `()`. - -> `()` - -The `()` type has exactly one value `()`, and is used when there is no other meaningful value that could be returned. In fact, the example in `00-setup` could is exactly `fn main() -> () {}`. - -> `Ok(())` - -The variants of `Result` are preluded, allowing us to use `Ok` and `Err` directly instead of having to write out `Result::Ok` and `Result::Err`. In this context, using `Ok(())` means creating a new value for the type `Result<()>`. - -### Call Functions - -Let's go next line: - -```rust -let op = init_operator_via_builder()?; -``` - -- `let a = b;` is the syntax in Rust for declaring an immutable variable. -- `xxx()` means call function that named `xxx`. -- the `?` is a syntax sugar in Rust that checks if the returned `Result` is an `Err`. If it is, then it immediately returns that error. Otherwise, it returns the value of `Ok`. - -So this line will call `init_operator_via_builder` function first and set to variable `op` if it returns `Ok`. - -## Initiate Via Builder - -Now, we can go deeper into the OpenDAL side. - -Let's take look over `init_operator_via_builder` function first. - -```rust -fn init_operator_via_builder() -> Result { - let mut builder = S3::default(); - builder.bucket("example"); - builder.access_key_id("access_key_id"); - builder.secret_access_key("secret_access_key"); - builder.region("us-east-1"); - - let op = Operator::new(builder)?.finish(); - Ok(op) -} -``` - -We have a new concept here: - -> let mut builder = xxx; - -The `mut` here means `mutable`, allowing its value to be changed later. - -In Rust, all variables are declared as `immutable` by default, meaning that users cannot change their values. However, by declaring them as `mut`, we can call mutable functions such as `bucket` and `access_key_id`. - -In this example, we initiate a new builder and call its functions to set different arguments. Finally, we use `Operator::new(builder)?.finish()` to build it. - -Each service provides its own builder, which can be accessed at [opendal::services](https://opendal.apache.org/docs/rust/opendal/services/index.html). - -## Initiate Via Map - -Calling builder's API is simple and directly, but sometimes developer will read config from files or env which will need to building operator from a map. - -```rust -fn init_operator_via_map() -> Result { - let mut map = HashMap::default(); - map.insert("bucket".to_string(), "example".to_string()); - map.insert("access_key_id".to_string(), "access_key_id".to_string()); - map.insert( - "secret_access_key".to_string(), - "secret_access_key".to_string(), - ); - map.insert("region".to_string(), "us-east-1".to_string()); - - let op = Operator::via_map(Scheme::S3, map)?; - Ok(op) -} -``` - -In this example, we use `Operator::via_map()` API the init a new operator. - -## Let's Go! - -We will see the output of this example in this way: - -```shell -:) cargo run - Compiling init-operator v0.1.0 (/home/xuanwo/Code/apache/opendal/examples/rust/01-init-operator) - Finished dev [unoptimized + debuginfo] target(s) in 0.38s - Running `target/debug/init-operator` -operator from builder: Operator { accessor: S3Backend { core: S3Core { bucket: "example", endpoint: "https://s3.us-east-1.amazonaws.com/example", root: "/", .. } }, limit: 1000 } -operator from map: Operator { accessor: S3Backend { core: S3Core { bucket: "example", endpoint: "https://s3.us-east-1.amazonaws.com/example", root: "/", .. } }, limit: 1000 } -``` - -Please feel free to try initiating other services or configuring new settings for experimentation purposes! - -## Conclusion - -In this chapter we learnt a lot of basic concepts in Rust! Now we know: - -- How to define a function that returns something. -- How to declare an immutable variable. -- How to call functions in Rust. -- How to define and use enums. -- How to handle results. - -With our newly acquired Rust skills, we can initiate the OpenDAL Operator in two main ways. This knowledge will be used to read and write data to storage in the next chapter! diff --git a/examples/rust/01-init-operator/src/main.rs b/examples/rust/01-init-operator/src/main.rs deleted file mode 100644 index 2110dad9560..00000000000 --- a/examples/rust/01-init-operator/src/main.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::collections::HashMap; - -use opendal::services::S3; -use opendal::Operator; -use opendal::Result; -use opendal::Scheme; - -fn main() -> Result<()> { - let op = init_operator_via_builder()?; - println!("operator from builder: {:?}", op); - - let op = init_operator_via_map()?; - println!("operator from map: {:?}", op); - - Ok(()) -} - -fn init_operator_via_builder() -> Result { - let mut builder = S3::default(); - builder.bucket("example"); - builder.region("us-east-1"); - builder.access_key_id("access_key_id"); - builder.secret_access_key("secret_access_key"); - - let op = Operator::new(builder)?.finish(); - Ok(op) -} - -fn init_operator_via_map() -> Result { - let mut map = HashMap::default(); - map.insert("bucket".to_string(), "example".to_string()); - map.insert("region".to_string(), "us-east-1".to_string()); - map.insert("access_key_id".to_string(), "access_key_id".to_string()); - map.insert( - "secret_access_key".to_string(), - "secret_access_key".to_string(), - ); - - let op = Operator::via_map(Scheme::S3, map)?; - Ok(op) -} diff --git a/examples/rust/02-async-io/.gitignore b/examples/rust/02-async-io/.gitignore deleted file mode 100644 index 2c96eb1b651..00000000000 --- a/examples/rust/02-async-io/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -target/ -Cargo.lock diff --git a/examples/rust/02-async-io/Cargo.toml b/examples/rust/02-async-io/Cargo.toml deleted file mode 100644 index bc120649e9e..00000000000 --- a/examples/rust/02-async-io/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "async-io" - -edition = "2021" -version = "0.1.0" - -[dependencies] -futures = "0.3" -opendal = "0.45" -tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/examples/rust/02-async-io/README.md b/examples/rust/02-async-io/README.md deleted file mode 100644 index f86606fd61c..00000000000 --- a/examples/rust/02-async-io/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Chapter-02: Async IO - -In [Chapter-01](../01-init-operator/README.md), we learnt how to initiate an `Operator`. All operations in OpenDAL are IO bound which means most of our users will use OpenDAL in an async runtime. This chapter will discuss how to call `Operator` in async runtime and how to use OpenDAL to read and write data! - -## Warmup - -Before delving into the OpenDAL API, let's take a look at some new Rust features. - -### Crate Features - -In this chapter's `Cargo.toml`, we add a new dependence `tokio`: - -```toml -tokio = { version = "1", features = ["full"] } -``` - -The syntax is different from what we used before: - -```diff -- tokio = "1" -+ tokio = { version = "1", features = ["full"] } -``` - -We use this syntax to specify additional options for this dependency. Essentially, `tokio = "1"` can be considered a shorthand for `tokio = { version = "1" }`. - -The most commonly used option is `features`, which works in conjunction with conditional compilation and the crate's configuration system. These features allow you to enable or disable specific pieces of code within the crate based on different use cases. - -For instance, since the `tokio` project has many components, not all of them are necessary for every project. By enabling only the required features, we can reduce build time. - -In our examples, we will just enable the `macros` and `"rt-multi-thread` feature which allow use to use `#[tokio::main]` in code. - -### Attributes - -Let's back to read our `main.rs`. We will find a new thing here: `#[tokio::main]`: the attribute macros denoted by the `#[...]` syntax and are used to add additional metadata or behavior to the annotated item. - -`[tokio::main]` here will expand the following code: - -```rust -#[tokio::main] -async fn main() { - println!("Hello world"); -} -``` - -into: - -```rust -fn main() { - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap() - .block_on(async { - println!("Hello world"); - }) -} -``` - -There are many different kinds of attributes in rust, we can refer to [The Rust Reference - Attributes](https://doc.rust-lang.org/reference/attributes.html#attributes) for more information. - -## Async Function - -Another new thing here is `async fn`. By prefix `async` before `fn`, we can declare an async function: - -```rust -async fn write_data(op: Operator) -> Result<()> { - ... -} -``` - -Every async function in Rust is essentially a state machine generator. When we call an async function, it creates a new state machine that can be polled to determine if it has finished running or not during runtime. We described this state machine as a [`Future`](https://doc.rust-lang.org/std/future/trait.Future.html). - -If you call an async function without using `await`, it will not do anything because we only create a `Future` but do not poll it. We will need to call async function like the following: - -```rust -async fn write_data(op: Operator) -> Result<()> { - op.write("test", "Hello, World!").await?; - - Ok(()) -} -``` - -When using `.await`, we submit the `Future` created by `op.write("test", "Hello, World!")` to the async runtime and continuously poll it until it finishes. - -The async runtime will have multiple workers running different `Future` tasks, allowing us to avoid blocking on IO and improving the performance of the entire application. - -## Write Data - -Now we can go deeper into OpenDAL's API. Writing data via OpenDAL is simple: - -```rust -async fn write_data(op: Operator) -> Result<()> { - op.write("test", "Hello, World!").await?; - - Ok(()) -} -``` - -The `write` API provided by `Operator` is: - -```rust -impl Operator { - pub async fn write(&self, path: &str, bs: impl Into) -> Result<()>; -} -``` - -`impl Into` here is a syntax sugar of rust, we can expand it like the following: - -```rust -impl Operator { - pub async fn write(&self, path: &str, bs: T) -> Result<()> - where T: Into; -} -``` - -This API means it accepts any types that implements `Into`. `Bytes` is provided by [`bytes::Bytes`](https://docs.rs/bytes/latest/bytes/struct.Bytes.html), we can find all types that implements `Into` here. - -So we can also call `write` on: - -- `Vec` -- `String` -- `&'static str` - -For example: - -```rust -async fn write_data(op: Operator) -> Result<()> { - let s = "Hello, World!".to_string(); - op.write("test", s).await?; - - Ok(()) -} -``` - -The easiest way to write data into OpenDAL is by calling the "write" function. However, OpenDAL also offers additional extensions for adding metadata to files and writing data in a streaming manner, which eliminates the need to hold all of the data in memory. We will cover these topics in upcoming chapters. - -## Read Data - -Reading data via OpenDAL is simple too: - -```rust -async fn read_data(op: Operator) -> Result<()> { - let bs = op.read("test").await?; - println!("read data: {}", String::from_utf8_lossy(&bs)); - - Ok(()) -} -``` - -The `read` API provided by `Operator` is: - -```rust -pub async fn read(&self, path: &str) -> Result>; -``` - -This API will read all data from `path` and return as a `Vec`. - -## Conclusion - -In this chapter we learnt a lot basic concepts in async rust! Now we have known that: - -- How to setup tokio async runtime. -- How to define and call an async function. -- How to write and read data via OpenDAL. - -## Challenge Time - -Now that we have mastered all the knowledge required to write and read data via OpenDAL. Let's take on a challenge! - -Our challenge is to - -- Write and read data from AWS S3. -- Explore the API related to `read_with` and `write_with`. - -See you in the next chapter! diff --git a/examples/rust/02-async-io/src/main.rs b/examples/rust/02-async-io/src/main.rs deleted file mode 100644 index 1ea8533056b..00000000000 --- a/examples/rust/02-async-io/src/main.rs +++ /dev/null @@ -1,25 +0,0 @@ -use opendal::services::Memory; -use opendal::Operator; -use opendal::Result; - -#[tokio::main] -async fn main() -> Result<()> { - let op = Operator::new(Memory::default())?.finish(); - - write_data(op.clone()).await?; - read_data(op.clone()).await?; - Ok(()) -} - -async fn write_data(op: Operator) -> Result<()> { - op.write("test", "Hello, World!").await?; - - Ok(()) -} - -async fn read_data(op: Operator) -> Result<()> { - let bs = op.read("test").await?; - println!("data: {}", String::from_utf8_lossy(&bs)); - - Ok(()) -} diff --git a/examples/rust/03-add-layers/.gitignore b/examples/rust/03-add-layers/.gitignore deleted file mode 100644 index 2c96eb1b651..00000000000 --- a/examples/rust/03-add-layers/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -target/ -Cargo.lock diff --git a/examples/rust/03-add-layers/Cargo.toml b/examples/rust/03-add-layers/Cargo.toml deleted file mode 100644 index 8ff929e41b9..00000000000 --- a/examples/rust/03-add-layers/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "add-layers" - -edition = "2021" -version = "0.1.0" - -[dependencies] -env_logger = "0.11" -futures = "0.3" -opendal = "0.45" -tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/examples/rust/03-add-layers/README.md b/examples/rust/03-add-layers/README.md deleted file mode 100644 index 7470a157566..00000000000 --- a/examples/rust/03-add-layers/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# Chapter-03: How To Add Layers - -In the preceding chapters, we have effectively utilized OpenDAL to access the storage service and execute basic data read and write operations. This chapter takes a step forward by introducing additional layers to manage data operations, allowing for controlled access behavior and the acquisition of observable data throughout the process. - -## What are Layers in OpenDAL - -OpenDAL offers native layer support, enabling users to implement middleware or intercept for all operations. By using layers, we can retry failed requests and resume from the point of failure with Retry Layer, provide native observability with Tracing Layer, and so on. - -## Simple Examples - -In this section, we will incorporate practical Layers into our OpenDAL data reading and writing operations, allowing us to experience the simplicity and convenience they offer. You can find the complete code [here](./src/main.rs). - -### Adding Logging Capability - -Logging is an invaluable feature in daily development. OpenDAL offers comprehensive logging support throughout various data access operations. Specifically, OpenDAL utilizes [log](https://docs.rs/log/latest/log/) crate to record structured logs. At the onset of each operation, an initial log entry is made. Upon completion of each operation, a log entry is recorded to indicate its success or failure along with the specific reasons for any failures encountered. - -```diff -+ env_logger = "0.11" -``` - -To utilize logging functionality, we need introduce the [env_logger](https://docs.rs/env_logger/latest/env_logger/) crate first. This is necessary because the [log](https://docs.rs/log/latest/log/) crate in Rust solely offers a unified API abstraction for logging. Users can select the appropriate crate and specific logging implementation based on their actual requirements. Here, we opt for the [env_logger](https://docs.rs/env_logger/latest/env_logger/) crate. - -```rust -#[tokio::main] -async fn main() -> Result<()> { - env_logger::init(); - let op = Operator::new(Memory::default()) - .expect("must init") - .layer(LoggingLayer::default()) - .finish(); - ... - Ok(()) -} -``` - -We simply need to add a layer to the creation process of the `Operator` in a chain call to apply the functions of the corresponding layers to the `Operator`. Here, we add a default Logger Layer for the `Operator`. Through log outputs, we can gain a clearer understanding of the operations performed by OpenDAL throughout the entire process, as well as the execution status of each operation. - -```shell -$ RUST_LOG=debug cargo run -[DEBUG opendal::services] service=memory operation=metadata -> started -[DEBUG opendal::services] service=memory operation=metadata -> finished: AccessorInfo { scheme: Memory, root: "/", name: "0x5565075f1ad0", native_capability: { Stat | Read | Write | Delete | List | Blocking }, full_capability: { Stat | Read | Write | CreateDir | Delete | List | Blocking } } -[DEBUG opendal::services] service=memory operation=write path=test -> started -[DEBUG opendal::services] service=memory operation=write path=test -> start writing -[DEBUG opendal::services] service=memory operation=write path=test written=13B -> data written finished -[DEBUG opendal::services] service=memory operation=stat path=test -> started -[DEBUG opendal::services] service=memory operation=stat path=test -> finished: RpStat { meta: Metadata { metakey: FlagSet(Complete | Mode | ContentLength), mode: FILE, cache_control: None, content_disposition: None, content_length: Some(13), content_md5: None, content_range: None, content_type: None, etag: None, last_modified: None, version: None } } -[DEBUG opendal::services] service=memory operation=read path=test range=0-12 -> started -[DEBUG opendal::services] service=memory operation=read path=test range=0-12 -> got reader -[DEBUG opendal::services] service=memory operation=read path=test read=13 -> data read finished -data: Hello, World! -``` - -Next let's execute the code. It's important to note that when running the code, you'll need to include RUST_LOG and specify the log output level. After running the sample code, you'll see output similar to the above on your terminal. By enabling Logging Layer, OpenDAL no longer appears as a black box! By controlling the location and level of log output, you can access all the details you wish to know. - -### Stacking Timeout Layer - -OpenDAL not only supports adding a layer to the `Operator`, but also enables the addition of all necessary layers to the `Operator` through chain calls. Let's continue by stacking the Timeout Layer on top of the Logger Layer. - -```Rust -let op = Operator::new(Memory::default()) - .expect("must init") - .layer(LoggingLayer::default()) - .layer( - TimeoutLayer::default() - .with_timeout(Duration::from_secs(5)) - .with_io_timeout(Duration::from_secs(3)) - ) - .finish(); -``` - -We can directly add the Timeout Layer in a unified manner based on the previous code. Each operation in the `Operator` can be categorized into IO operations and non-IO operations. Users can set different timeouts for each operation according to their actual needs. In this example, we set a 5-second timeout for non-IO operations and a 3-second timeout for IO operations in the `Operator`. - -Through the Timeout Layer, we can prevent any operation in OpenDAL from waiting excessively long or getting stuck indefinitely due to unforeseen circumstances. For instance, a dead connection might cause a SQL query to hang indefinitely. The Timeout Layer will terminate such connections and return an error, enabling users to handle them by retrying or directly informing the users. - -## More Useful Layers - -### Retry Layer - -The Retry Layer provides the ability for OpenDAL to automatically retry related operations when an error occurs in a data access operation based on OpenDAL. - -OpenDAL uses an exponential backoff algorithm to retry erroneous requests to avoid request current limiting and conflicts to the greatest extent, and provides relevant settings to further control retry behavior. Specifically, OpenDAL supports setting the maximum number of retries, the minimum retry interval, the maximum retry interval, the jitter and exponential coefficients in the exponential backoff algorithm, and setting the callback function for each retry. - -In addition, OpenDAL divides errors in requests into three types: `Permanent`, `Temporary` and `Persistent`: - -- `Permanent` means that the error cannot be solved by retrying, such as `Not Found` error and permission authentication error. At this time, OpenDAL will not retry the error request to avoid adding additional unnecessary request traffic; -- `Temporary` means that the error is temporary and may be resolved by retrying. For example, the storage service returns a traffic restriction error. After adding the Retry Layer, OpenDAL will automatically retry this type of request; -- `Persistent` means that the request error cannot be resolved after retrying, such as continuing network error. `Persistent` is transformed from `Temporary`. - -Users can further judge the service running status based on the error type returned by the OpenDAL request. - -### Concurrent Limit Layer - -Through the Concurrency Limit Layer, users can control the maximum number of concurrent connections between OpenDAL and the underlying storage service. This control can effectively manage system resources and avoid the risk of excessive resource usage leading to performance degradation or system crash. - -### Blocking Layer - -As introduced in the previous chapters, OpenDAL uses the Rust asynchronous runtime to ensure efficient processing of IO requests. Blocking Layer provides a synchronous semantic wrapper for operations in OpenDAL, allowing users to bridge the use of many underlying asynchronous services provided by OpenDAL with synchronous semantics. - -In addition, since synchronization bridging involving asynchronous functions involves knowledge about the [Tokio](https://docs.rs/tokio/latest/tokio/) runtime, users need additional processing when using this type of interface. This is beyond the scope of this tutorial, more examples can be found in https://docs.rs/opendal/latest/opendal/layers/struct.BlockingLayer.html. - -### Built-in Layers in OpenDAL - -An interesting fact is that OpenDAL applies some layers by default for every operation, such as Error Context Layer and Complete Layer. These built-in layers are usually very thin, light and in a zero-cost way. So there is no need to worry about its impact on data access performance, but they can greatly enhance the user experience. - -Error Context Layer injects context information, such as service (the `Scheme` of underlying service), operation (the `Operation` of this operation), path (the path of this operation), into all returned errors of operations. - -Complete Layer adds necessary capabilities to services, complete underlying services features. OpenDAL exposes a unified interface implementation through Complete Layer, so that users can use them in the same way, and OpenDAL will always choose the optimal way to initiate the request. - -For example, S3 cannot support `seek`, but OpenDAL will implement it at zero cost manner according to existing methods provided by S3. So users don't have to worry about whether the underlying service supports, they only need to pay attention to what methods OpenDAL provides. And the good news is OpenDAL always provides interfaces in a unified way. ^_^ - -### Others - -In addition to the many practical layers mentioned above, OpenDAL also provides support for corresponding observability and fine-grained control layers of data access for various scenarios, such as: - -- [Metrics Layer](https://docs.rs/opendal/latest/opendal/layers/struct.MetricsLayer.html): add [metrics](https://docs.rs/metrics/) for every operations; -- [Tracing Layer](https://docs.rs/opendal/latest/opendal/layers/struct.TracingLayer.html): add [tracing](https://docs.rs/tracing/) for every operations; -- [Prometheus Layer](https://docs.rs/opendal/latest/opendal/layers/struct.PrometheusLayer.html): add [prometheus](https://docs.rs/prometheus) for every operations; -- [Minitrace Layer](https://docs.rs/opendal/latest/opendal/layers/struct.MinitraceLayer.html): add [minitrace](https://docs.rs/minitrace/) for every operations; -- [Immutable Index Layer](https://docs.rs/opendal/latest/opendal/layers/struct.ImmutableIndexLayer.html): add an immutable in-memory index for underlying storage services; -- [Throttle Layer](https://docs.rs/opendal/latest/opendal/layers/struct.ThrottleLayer.html): add a bandwidth rate limiter to the underlying services; -- [Await Tree Layer](https://docs.rs/opendal/latest/opendal/layers/struct.AwaitTreeLayer.html): add a instrument await-tree for actor-based applications to the underlying services; -- [Async Backtrace Layer](https://docs.rs/opendal/latest/opendal/layers/struct.AsyncBacktraceLayer.html): add efficient, logical stack traces of async functions for the underlying services; -- [Madsim Layer](https://docs.rs/opendal/latest/opendal/layers/struct.MadsimServer.html): add deterministic simulation for async operations; -- [Chaos Layer](https://docs.rs/opendal/latest/opendal/layers/struct.ChaosLayer.html): inject chaos into underlying services for robustness test. - -In the end, this article serves as an introductory tutorial to introduce the techniques of using various layers in OpenDAL. You can refer to https://docs.rs/opendal/latest/opendal/layers/index.html to learn more about the layers provided in OpenDAL. Maybe there is an integrated solution you are looking for! diff --git a/examples/rust/03-add-layers/src/main.rs b/examples/rust/03-add-layers/src/main.rs deleted file mode 100644 index 20b88c77929..00000000000 --- a/examples/rust/03-add-layers/src/main.rs +++ /dev/null @@ -1,40 +0,0 @@ -use std::time::Duration; - -use env_logger; -use opendal::layers::LoggingLayer; -use opendal::layers::TimeoutLayer; -use opendal::services::Memory; -use opendal::Operator; -use opendal::Result; - -#[tokio::main] -async fn main() -> Result<()> { - env_logger::init(); - - let op = Operator::new(Memory::default()) - .expect("must init") - .layer(LoggingLayer::default()) - .layer( - TimeoutLayer::default() - .with_timeout(Duration::from_secs(5)) - .with_io_timeout(Duration::from_secs(3)), - ) - .finish(); - - write_data(op.clone()).await?; - read_data(op.clone()).await?; - Ok(()) -} - -async fn write_data(op: Operator) -> Result<()> { - op.write("test", "Hello, World!").await?; - - Ok(()) -} - -async fn read_data(op: Operator) -> Result<()> { - let bs = op.read("test").await?; - println!("data: {}", String::from_utf8_lossy(&bs)); - - Ok(()) -} diff --git a/examples/rust/README.md b/examples/rust/README.md deleted file mode 100644 index 5c179c3b306..00000000000 --- a/examples/rust/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# OpenDAL Rust Examples - -Thank you for using OpenDAL Rust Core! - -The goal of these documents is to give you everything you need to start using OpenDAL Rust Core. We will start with the Rust basics and move on to more OpenDAL features. After reading and using all these examples, we hope you will become a master of OpenDAL and be empowered to build your bravo applications! - -## Layout - -Our examples are organised like a book with different chapters. - -It's a good idea to start reading from the first chapter. It's also ok to pick the topics that interest you the most. Each chapter will be a separate project that can be run and referenced. - -## Contents - -- [Chapter-00: Set up Your First Rust Project](./00-setup/README.md) -- [Chapter-01: Initiate OpenDAL Operator](./01-init-operator/README.md) -- [Chapter-02: Async IO](./02-async-io/README.md) -- [Chapter-03: How To Add Layers](./03-add-layers/README.md)