diff --git a/.github/workflows/service_test_webhdfs.yml b/.github/workflows/service_test_webhdfs.yml index 2e000d637531..ba33b7a6e317 100644 --- a/.github/workflows/service_test_webhdfs.yml +++ b/.github/workflows/service_test_webhdfs.yml @@ -39,28 +39,41 @@ concurrency: jobs: hdfs: runs-on: ubuntu-latest - strategy: - matrix: - hdfs-version: ["2.10.1", "3.2.3", "3.3.2"] steps: - uses: actions/checkout@v3 - - name: Setup-hdfs env - uses: beyondstorage/setup-hdfs@master - with: - hdfs-version: ${{ matrix.hdfs-version }} + - name: Configure Webhdfs + # namenode will use ports: 9870 and 9000 + # datanode will use ports: 9864 + run: | + docker run -d \ + --name namenode \ + --network host \ + -e CLUSTER_NAME=test \ + -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \ + -e CORE_CONF_hadoop_http_staticuser_user=root \ + bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 + + docker run -d \ + --name datanode \ + --network host \ + -e CLUSTER_NAME=test \ + -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \ + -e CORE_CONF_hadoop_http_staticuser_user=root \ + bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 + + curl --retry 30 --retry-delay 1 --retry-connrefused http://localhost:9870 - name: Setup Rust toolchain uses: ./.github/actions/setup + - name: Test shell: bash working-directory: core - continue-on-error: true run: cargo test webhdfs -- --show-output env: RUST_BACKTRACE: full RUST_LOG: debug OPENDAL_WEBHDFS_TEST: on - OPENDAL_WEBHDFS_ROOT: /tmp/opendal/ - OPENDAL_WEBHDFS_USER: runner - OPENDAL_WEBHDFS_ENDPOINT: http://${{ env.HDFS_NAMENODE_HTTP_ADDR }} + OPENDAL_WEBHDFS_ROOT: / + OPENDAL_WEBHDFS_ENDPOINT: http://127.0.0.1:9870 diff --git a/core/src/services/webhdfs/pager.rs b/core/src/services/webhdfs/pager.rs index fb997ec4ed70..17fc626a198e 100644 --- a/core/src/services/webhdfs/pager.rs +++ b/core/src/services/webhdfs/pager.rs @@ -45,7 +45,11 @@ impl oio::Page for WebhdfsPager { let mut entries = Vec::with_capacity(self.statuses.len()); while let Some(status) = self.statuses.pop() { - let mut path = format!("{}/{}", &self.path, status.path_suffix); + let mut path = if self.path.is_empty() { + status.path_suffix.to_string() + } else { + format!("{}/{}", self.path, status.path_suffix) + }; let meta: Metadata = status.try_into()?; if meta.mode().is_dir() {