diff --git a/wedpr-components/dataset/src/main/java/com/webank/wedpr/components/dataset/datasource/processor/HdfsDataSourceProcessor.java b/wedpr-components/dataset/src/main/java/com/webank/wedpr/components/dataset/datasource/processor/HdfsDataSourceProcessor.java index 44e4ee89..166535f9 100644 --- a/wedpr-components/dataset/src/main/java/com/webank/wedpr/components/dataset/datasource/processor/HdfsDataSourceProcessor.java +++ b/wedpr-components/dataset/src/main/java/com/webank/wedpr/components/dataset/datasource/processor/HdfsDataSourceProcessor.java @@ -2,6 +2,7 @@ import com.webank.wedpr.common.protocol.StorageType; import com.webank.wedpr.common.utils.Common; +import com.webank.wedpr.common.utils.ObjectMapperFactory; import com.webank.wedpr.components.dataset.config.DatasetConfig; import com.webank.wedpr.components.dataset.datasource.DataSourceMeta; import com.webank.wedpr.components.dataset.datasource.category.HdfsDataSource; @@ -15,6 +16,7 @@ import java.io.File; import java.util.Arrays; import java.util.List; +import lombok.SneakyThrows; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,10 +103,13 @@ public void prepareData() throws DatasetException { endTimeMillis - startTimeMillis); } + @SneakyThrows @Override public void analyzeData() throws DatasetException { String cvsFilePath = dataSourceProcessorContext.getCvsFilePath(); Dataset dataset = dataSourceProcessorContext.getDataset(); + HdfsDataSource hdfsDataSource = + (HdfsDataSource) dataSourceProcessorContext.getDataSourceMeta(); long startTimeMillis = System.currentTimeMillis(); @@ -132,6 +137,15 @@ public void analyzeData() throws DatasetException { this.dataSourceProcessorContext.getDataset().setDatasetVersionHash(md5Hash); this.dataSourceProcessorContext.getDataset().setDatasetSize(fileSize); + HDFSStoragePath storagePath = new HDFSStoragePath(hdfsDataSource.getFilePath()); + String storagePathStr = + ObjectMapperFactory.getObjectMapper().writeValueAsString(storagePath); + this.dataSourceProcessorContext + .getDataset() + .setDatasetStorageType(storagePath.getStorageType()); + this.dataSourceProcessorContext.getDataset().setDatasetStoragePath(storagePathStr); + this.dataSourceProcessorContext.setStoragePath(storagePath); + String datasetId = dataset.getDatasetId(); long endTimeMillis = System.currentTimeMillis();