fix hdfs datasource storage path empty bug

WeBankBlockchain · Nov 7, 2024 · e699a8f · e699a8f
1 parent 6378d0d
commit e699a8f
Showing 1 changed file with 14 additions and 0 deletions.
diff --git a/...ava/com/webank/wedpr/components/dataset/datasource/processor/HdfsDataSourceProcessor.java b/...ava/com/webank/wedpr/components/dataset/datasource/processor/HdfsDataSourceProcessor.java
@@ -2,6 +2,7 @@
 
 import com.webank.wedpr.common.protocol.StorageType;
 import com.webank.wedpr.common.utils.Common;
+import com.webank.wedpr.common.utils.ObjectMapperFactory;
 import com.webank.wedpr.components.dataset.config.DatasetConfig;
 import com.webank.wedpr.components.dataset.datasource.DataSourceMeta;
 import com.webank.wedpr.components.dataset.datasource.category.HdfsDataSource;
@@ -15,6 +16,7 @@
 import java.io.File;
 import java.util.Arrays;
 import java.util.List;
+import lombok.SneakyThrows;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -101,10 +103,13 @@ public void prepareData() throws DatasetException {
                 endTimeMillis - startTimeMillis);
     }
 
+    @SneakyThrows
     @Override
     public void analyzeData() throws DatasetException {
         String cvsFilePath = dataSourceProcessorContext.getCvsFilePath();
         Dataset dataset = dataSourceProcessorContext.getDataset();
+        HdfsDataSource hdfsDataSource =
+                (HdfsDataSource) dataSourceProcessorContext.getDataSourceMeta();
 
         long startTimeMillis = System.currentTimeMillis();
 
@@ -132,6 +137,15 @@ public void analyzeData() throws DatasetException {
         this.dataSourceProcessorContext.getDataset().setDatasetVersionHash(md5Hash);
         this.dataSourceProcessorContext.getDataset().setDatasetSize(fileSize);
 
+        HDFSStoragePath storagePath = new HDFSStoragePath(hdfsDataSource.getFilePath());
+        String storagePathStr =
+                ObjectMapperFactory.getObjectMapper().writeValueAsString(storagePath);
+        this.dataSourceProcessorContext
+                .getDataset()
+                .setDatasetStorageType(storagePath.getStorageType());
+        this.dataSourceProcessorContext.getDataset().setDatasetStoragePath(storagePathStr);
+        this.dataSourceProcessorContext.setStoragePath(storagePath);
+
         String datasetId = dataset.getDatasetId();
 
         long endTimeMillis = System.currentTimeMillis();