Skip to content

Commit

Permalink
Fix the bug that HDFS iterator doesn't work
Browse files Browse the repository at this point in the history
Fix the bug that HDFS iterator doesn't work.
			pr-link: Alluxio#18429
			change-id: cid-dbb5f1a246bce706f7b3b5d179ad158218dccf83
  • Loading branch information
JiamingMai authored and ssz1997 committed Dec 15, 2023
1 parent 9a0fe3e commit 99222b2
Showing 1 changed file with 5 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public class HdfsUfsStatusIterator implements Iterator<UfsStatus> {

private final String mUfsSchemaUri;

private final String mPathToList;
private final AlluxioURI mPathToList;

/**
* Each element is a pair of (full path, UfsStatus).
Expand All @@ -58,7 +58,7 @@ public class HdfsUfsStatusIterator implements Iterator<UfsStatus> {
public HdfsUfsStatusIterator(String pathToList, FileSystem fs) {
mFs = fs;
mUfsSchemaUri = mFs.getUri().toString();
mPathToList = pathToList;
mPathToList = new AlluxioURI(pathToList);
initQueue(pathToList);
}

Expand Down Expand Up @@ -107,13 +107,12 @@ public UfsStatus next() {
UfsStatus ufsStatus;
Path path = fileStatus.getPath();

AlluxioURI rootUri = new AlluxioURI(mUfsSchemaUri);
if (fileStatus.isDirectory()) {
String relativePath = extractRelativePath(path.toUri().getPath());
ufsStatus = new UfsDirectoryStatus(relativePath, fileStatus.getOwner(),
fileStatus.getGroup(), fileStatus.getPermission().toShort(),
fileStatus.getModificationTime());
ufsStatus.setUfsFullPath(rootUri.join(ufsStatus.getName()));
ufsStatus.setUfsFullPath(mPathToList.join(ufsStatus.getName()));
mDirPathsToProcess.addLast(new Pair<>(path.toString(), ufsStatus));
} else {
String contentHash =
Expand All @@ -123,7 +122,7 @@ public UfsStatus next() {
ufsStatus = new UfsFileStatus(relativePath, contentHash, fileStatus.getLen(),
fileStatus.getModificationTime(), fileStatus.getOwner(), fileStatus.getGroup(),
fileStatus.getPermission().toShort(), fileStatus.getBlockSize());
ufsStatus.setUfsFullPath(rootUri.join(ufsStatus.getName()));
ufsStatus.setUfsFullPath(mPathToList.join(ufsStatus.getName()));
}
return ufsStatus;
} catch (IOException e) {
Expand All @@ -133,7 +132,7 @@ public UfsStatus next() {

private String extractRelativePath(String fullPath) {
String fullPathWithoutSchema = trimPathPrefix(mUfsSchemaUri, fullPath);
String pathToListWithoutSchema = trimPathPrefix(mUfsSchemaUri, mPathToList);
String pathToListWithoutSchema = trimPathPrefix(mUfsSchemaUri, mPathToList.toString());
if (!pathToListWithoutSchema.startsWith("/")) {
pathToListWithoutSchema = "/" + pathToListWithoutSchema;
}
Expand Down

0 comments on commit 99222b2

Please sign in to comment.