From 4f17a120cc2980b5c9e71ad004b585b4475b4fc0 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 13 Oct 2023 16:44:03 +0800 Subject: [PATCH 1/4] feat: Add edge test cases for OpenDAL Core Signed-off-by: Xuanwo --- .../{fs_write_full_disk.yml => edge_test.yml} | 30 ++++++++----- Cargo.lock | 10 +++++ Cargo.toml | 1 + core/edge/README.md | 3 ++ core/edge/file_write_on_full_disk/Cargo.toml | 11 +++++ core/edge/file_write_on_full_disk/README.md | 14 +++++++ core/edge/file_write_on_full_disk/src/main.rs | 42 +++++++++++++++++++ 7 files changed, 100 insertions(+), 11 deletions(-) rename .github/workflows/{fs_write_full_disk.yml => edge_test.yml} (67%) create mode 100644 core/edge/README.md create mode 100644 core/edge/file_write_on_full_disk/Cargo.toml create mode 100644 core/edge/file_write_on_full_disk/README.md create mode 100644 core/edge/file_write_on_full_disk/src/main.rs diff --git a/.github/workflows/fs_write_full_disk.yml b/.github/workflows/edge_test.yml similarity index 67% rename from .github/workflows/fs_write_full_disk.yml rename to .github/workflows/edge_test.yml index 862657d497d..7ea59a600ac 100644 --- a/.github/workflows/fs_write_full_disk.yml +++ b/.github/workflows/edge_test.yml @@ -15,15 +15,24 @@ # specific language governing permissions and limitations # under the License. -name: Fs write full disk +name: Edge Test on: push: branches: - main + pull_request: + branches: + - main + paths: + - "core/src/**" + - "!core/src/docs/**" + - "!core/src/services/**" + - "core/src/services/fs/**" + - ".github/workflows/edge_test.yml" jobs: - test: + test_file_write_on_full_disk: runs-on: ubuntu-latest steps: @@ -37,16 +46,15 @@ jobs: - name: Mount disk image run: | - mkdir ./td - sudo mount -o loop disk.img ./td + mkdir /tmp/test_dir + sudo mount -o loop disk.img /tmp/test_dir - name: Set permissions - run: chmod a+wr ./td + run: sudo chmod a+wr /tmp/test_dir - # Add more steps for testing as needed + - name: Test + working-directory: core/edge/file_write_on_full_disk + run: cargo run + env: + OPENDAL_FS_ROOT: /tmp/test_dir - - name: Clean up - run: | - sudo umount ./td - rm -rf ./td - rm disk.img diff --git a/Cargo.lock b/Cargo.lock index d5f269a7b17..d1a0bcc01ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1771,6 +1771,16 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" +[[package]] +name = "edge_test_file_write_on_full_disk" +version = "0.0.0" +dependencies = [ + "futures", + "opendal", + "rand 0.8.5", + "tokio", +] + [[package]] name = "either" version = "1.8.1" diff --git a/Cargo.toml b/Cargo.toml index 06212688038..1d83e191b70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ exclude = ["examples"] members = [ "core", "core/fuzz", + "core/edge/*", "bindings/c", "bindings/nodejs", diff --git a/core/edge/README.md b/core/edge/README.md new file mode 100644 index 00000000000..44c59f06ef2 --- /dev/null +++ b/core/edge/README.md @@ -0,0 +1,3 @@ +# OpenDAL Edge Tests + +OpenDAL edge tests served as edge tests for the OpenDAL project. They will have pre-set data and will test the functionality of the OpenDAL project. diff --git a/core/edge/file_write_on_full_disk/Cargo.toml b/core/edge/file_write_on_full_disk/Cargo.toml new file mode 100644 index 00000000000..efb29d36c5f --- /dev/null +++ b/core/edge/file_write_on_full_disk/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "edge_test_file_write_on_full_disk" +edition = "2021" +version = "0.0.0" +publish = false + +[dependencies] +futures = "0.3" +opendal = { workspace = true } +tokio = { version = "1", features = ["full"] } +rand = "0.8" diff --git a/core/edge/file_write_on_full_disk/README.md b/core/edge/file_write_on_full_disk/README.md new file mode 100644 index 00000000000..682057da61b --- /dev/null +++ b/core/edge/file_write_on_full_disk/README.md @@ -0,0 +1,14 @@ +# File Write on Fill Disk + +Reported by [The `FsBackend` only writes partial bytes and doesn't raise any errors](https://github.com/apache/incubator-opendal/issues/3052). + +Setup: + +```shell +fallocate -l 512K disk.img +mkfs disk.img +mkdir ./td +sudo mount -o loop td.img ./td +chmod a+wr ./td +``` + diff --git a/core/edge/file_write_on_full_disk/src/main.rs b/core/edge/file_write_on_full_disk/src/main.rs new file mode 100644 index 00000000000..b5f19ae035f --- /dev/null +++ b/core/edge/file_write_on_full_disk/src/main.rs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use opendal::services::Fs; +use opendal::Operator; +use opendal::Result; +use rand::prelude::*; +use std::env; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = Fs::default(); + builder.root(&env::var("OPENDAL_FS_ROOT").expect("root must be set for this test")); + let op = Operator::new(builder)?.finish(); + + let size = thread_rng().gen_range(512 * 1024 + 1..4 * 1024 * 1024); + let mut bs = vec![0; size]; + thread_rng().fill_bytes(&mut bs); + + let result = op.write("/test", bs).await; + // Write file with size > 512KB should fail + assert!( + result.is_err(), + "wirte file on full disk should return error" + ); + + Ok(()) +} From e0537b0292754b086c8755419d1598b9ee49968b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 13 Oct 2023 16:45:18 +0800 Subject: [PATCH 2/4] Fix typo Signed-off-by: Xuanwo --- core/edge/file_write_on_full_disk/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/edge/file_write_on_full_disk/src/main.rs b/core/edge/file_write_on_full_disk/src/main.rs index b5f19ae035f..63cfe6cf050 100644 --- a/core/edge/file_write_on_full_disk/src/main.rs +++ b/core/edge/file_write_on_full_disk/src/main.rs @@ -35,7 +35,7 @@ async fn main() -> Result<()> { // Write file with size > 512KB should fail assert!( result.is_err(), - "wirte file on full disk should return error" + "write file on full disk should return error" ); Ok(()) From 58657bbbca1e760c6f5ab814b970992425008e47 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 13 Oct 2023 16:46:00 +0800 Subject: [PATCH 3/4] Fix license Signed-off-by: Xuanwo --- core/edge/file_write_on_full_disk/Cargo.toml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/core/edge/file_write_on_full_disk/Cargo.toml b/core/edge/file_write_on_full_disk/Cargo.toml index efb29d36c5f..aefee22d47b 100644 --- a/core/edge/file_write_on_full_disk/Cargo.toml +++ b/core/edge/file_write_on_full_disk/Cargo.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [package] name = "edge_test_file_write_on_full_disk" edition = "2021" From 47749126dd4994dc73f464d87c6e71e9e633ec3a Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 13 Oct 2023 16:48:08 +0800 Subject: [PATCH 4/4] Use checkout v4 Signed-off-by: Xuanwo --- .github/workflows/edge_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/edge_test.yml b/.github/workflows/edge_test.yml index 7ea59a600ac..1d89afccb0e 100644 --- a/.github/workflows/edge_test.yml +++ b/.github/workflows/edge_test.yml @@ -37,7 +37,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Create disk image run: |