ci: Re-enable webhdfs test (apache#1925)

* ci: Re-enable webhdfs test Signed-off-by: Xuanwo <[email protected]> * trigger Signed-off-by: Xuanwo <[email protected]> * Add health check Signed-off-by: Xuanwo <[email protected]> * FIx docker Signed-off-by: Xuanwo <[email protected]> * Wait docker Signed-off-by: Xuanwo <[email protected]> * Let's wait Signed-off-by: Xuanwo <[email protected]> * Fix retry Signed-off-by: Xuanwo <[email protected]> --------- Signed-off-by: Xuanwo <[email protected]>
wcy-fdu · Apr 12, 2023 · c101078 · c101078
1 parent 96f650b
commit c101078
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 12 deletions.
diff --git a/.github/workflows/service_test_webhdfs.yml b/.github/workflows/service_test_webhdfs.yml
@@ -39,28 +39,41 @@ concurrency:
 jobs:
   hdfs:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        hdfs-version: ["2.10.1", "3.2.3", "3.3.2"]
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup-hdfs env
-        uses: beyondstorage/setup-hdfs@master
-        with:
-          hdfs-version: ${{ matrix.hdfs-version }}
+      - name: Configure Webhdfs
+        # namenode will use ports: 9870 and 9000
+        # datanode will use ports: 9864
+        run: |
+          docker run -d \
+            --name namenode \
+            --network host \
+            -e CLUSTER_NAME=test \
+            -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \
+            -e CORE_CONF_hadoop_http_staticuser_user=root \
+            bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
+
+          docker run -d \
+            --name datanode \
+            --network host \
+            -e CLUSTER_NAME=test \
+            -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \
+            -e CORE_CONF_hadoop_http_staticuser_user=root \
+            bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
+
+          curl --retry 30 --retry-delay 1 --retry-connrefused http://localhost:9870
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup
+
       - name: Test
         shell: bash
         working-directory: core
-        continue-on-error: true
         run: cargo test webhdfs -- --show-output
         env:
           RUST_BACKTRACE: full
           RUST_LOG: debug
           OPENDAL_WEBHDFS_TEST: on
-          OPENDAL_WEBHDFS_ROOT: /tmp/opendal/
-          OPENDAL_WEBHDFS_USER: runner
-          OPENDAL_WEBHDFS_ENDPOINT: http://${{ env.HDFS_NAMENODE_HTTP_ADDR }}
+          OPENDAL_WEBHDFS_ROOT: /
+          OPENDAL_WEBHDFS_ENDPOINT: http://127.0.0.1:9870
diff --git a/core/src/services/webhdfs/pager.rs b/core/src/services/webhdfs/pager.rs
@@ -45,7 +45,11 @@ impl oio::Page for WebhdfsPager {
         let mut entries = Vec::with_capacity(self.statuses.len());
 
         while let Some(status) = self.statuses.pop() {
-            let mut path = format!("{}/{}", &self.path, status.path_suffix);
+            let mut path = if self.path.is_empty() {
+                status.path_suffix.to_string()
+            } else {
+                format!("{}/{}", self.path, status.path_suffix)
+            };
 
             let meta: Metadata = status.try_into()?;
             if meta.mode().is_dir() {