Skip to content

Commit

Permalink
Load ufs path directly
Browse files Browse the repository at this point in the history 
### What changes are proposed in this pull request?
Load ufs path directly

### Why are the changes needed?
Previously we concatenate root with relative path, now we directly pass ufs path So we can have multiple ufs support

### Does this PR introduce any user facing changes?
load have to input full ufs path

			pr-link: Alluxio#18289
			change-id: cid-3d9c4bad7b68af142ca7985954418bfef0f34b0b
  • Loading branch information
@jja725 @ssz1997
jja725 authored and ssz1997 committed Dec 15, 2023
1 parent 893a111 commit 90891b8
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 18 deletions.
5 changes: 1 addition & 4 deletions dora/shell/src/main/java/alluxio/cli/fs/command/LoadCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
import alluxio.annotation.PublicApi;
import alluxio.cli.CommandUtils;
import alluxio.client.file.FileSystemContext;
import alluxio.conf.Configuration;
import alluxio.conf.PropertyKey;
import alluxio.exception.AlluxioException;
import alluxio.exception.status.InvalidArgumentException;
import alluxio.grpc.JobProgressReportFormat;
Expand Down Expand Up @@ -150,8 +148,7 @@ public Options getOptions() {
@Override
public int run(CommandLine cl) throws AlluxioException, IOException {
String[] args = cl.getArgs();
AlluxioURI path =
new AlluxioURI(Configuration.getString(PropertyKey.DORA_CLIENT_UFS_ROOT)).join(args[0]);
AlluxioURI path = new AlluxioURI(args[0]);
if (path.containsWildcard()) {
throw new UnsupportedOperationException("Load does not support wildcard path");
}
Expand Down
16 changes: 8 additions & 8 deletions ...tegration/src/test/java/alluxio/client/cli/fs/command/DoraLoadCommandIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public void before() throws Exception {
public void testCommand() throws Exception {
File testRoot = mTestFolder.newFolder("testRoot");
mTestFolder.newFolder("testRoot/testDirectory");

String path = testRoot.getAbsolutePath();
createByteFileInUfs("/testRoot/testFileA", Constants.MB);
createByteFileInUfs("/testRoot/testFileB", Constants.MB);
createByteFileInUfs("/testRoot/testDirectory/testFileC", Constants.MB);
Expand All @@ -56,10 +56,10 @@ public void testCommand() throws Exception {
assertEquals(0, mFileSystem.getStatus(uriA).getInAlluxioPercentage());
assertEquals(0, mFileSystem.getStatus(uriB).getInAlluxioPercentage());
assertEquals(0, mFileSystem.getStatus(uriC).getInAlluxioPercentage());

// Testing loading of a directory
assertEquals(0, mFsShell.run("load", "/testRoot", "--submit", "--verify"));
assertEquals(0, mFsShell.run("load", "/testRoot", "--progress"));

assertEquals(0, mFsShell.run("load", path, "--submit", "--verify"));
assertEquals(0, mFsShell.run("load", path, "--progress"));

FileSystemUtils.waitForAlluxioPercentage(mFileSystem, uriA, 100);
FileSystemUtils.waitForAlluxioPercentage(mFileSystem, uriB, 100);
Expand All @@ -69,18 +69,18 @@ public void testCommand() throws Exception {
fileInStream.positionedRead(0, buffer, 0, Constants.MB);
assertTrue(BufferUtils.equalIncreasingByteArray(Constants.MB, buffer));
while (!mOutput.toString().contains("SUCCEEDED")) {
assertEquals(0, mFsShell.run("load", "/testRoot", "--progress"));
assertEquals(0, mFsShell.run("load", path, "--progress"));
Thread.sleep(1000);
}
assertTrue(mOutput.toString().contains("Inodes Processed: 4"));
assertTrue(mOutput.toString().contains("Bytes Loaded: 3072.00KB out of 3072.00KB"));
assertTrue(mOutput.toString().contains("Files Failed: 0"));
assertEquals(0, mFsShell.run("load", "/testRoot", "--stop"));
assertEquals(0, mFsShell.run("load", path, "--stop"));
assertEquals(-2, mFsShell.run("load", "/testRootNotExists", "--progress"));
assertTrue(mOutput.toString().contains("cannot be found."));
mFsShell.run("load", "/testRoot", "--progress", "--format", "JSON");
mFsShell.run("load", path, "--progress", "--format", "JSON");
assertTrue(mOutput.toString().contains("\"mJobState\":\"SUCCEEDED\""));
mFsShell.run("load", "/testRoot", "--progress", "--format", "JSON", "--verbose");
mFsShell.run("load", path, "--progress", "--format", "JSON", "--verbose");
assertTrue(mOutput.toString().contains("\"mVerbose\":true"));
}
}
13 changes: 7 additions & 6 deletions ...st/java/alluxio/client/cli/fs/command/DoraLoadCommandWithVirtualBlockIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ public void before() throws Exception {
@Test
public void testCommand() throws Exception {
File testRoot = mTestFolder.newFolder("testRoot");
String path = testRoot.getAbsolutePath();
mTestFolder.newFolder("testRoot/testDirectory");

int lengthA = 16 * Constants.MB;
Expand All @@ -63,8 +64,8 @@ public void testCommand() throws Exception {
assertEquals(0, mFileSystem.getStatus(uriC).getInAlluxioPercentage());

// Testing loading of a directory
assertEquals(0, mFsShell.run("load", "/testRoot", "--submit", "--verify"));
assertEquals(0, mFsShell.run("load", "/testRoot", "--progress"));
assertEquals(0, mFsShell.run("load", path, "--submit", "--verify"));
assertEquals(0, mFsShell.run("load", path, "--progress"));

FileSystemUtils.waitForAlluxioPercentage(mFileSystem, uriA, 100);
FileSystemUtils.waitForAlluxioPercentage(mFileSystem, uriB, 100);
Expand All @@ -82,20 +83,20 @@ public void testCommand() throws Exception {
fileInStream.positionedRead(0, buffer, 0, lengthC);
assertTrue(BufferUtils.equalIncreasingByteArray(lengthC, buffer));
while (!mOutput.toString().contains("SUCCEEDED")) {
assertEquals(0, mFsShell.run("load", "/testRoot", "--progress"));
assertEquals(0, mFsShell.run("load", path, "--progress"));
Thread.sleep(1000);
}
assertTrue(mOutput.toString().contains("Inodes Processed: 4"));
int bytes = (lengthA + lengthB + lengthC) / Constants.MB;
assertTrue(mOutput.toString().contains(
String.format("Bytes Loaded: %s.00MB out of %s.00MB", bytes, bytes)));
assertTrue(mOutput.toString().contains("Files Failed: 0"));
assertEquals(0, mFsShell.run("load", "/testRoot", "--stop"));
assertEquals(0, mFsShell.run("load", path, "--stop"));
assertEquals(-2, mFsShell.run("load", "/testRootNotExists", "--progress"));
assertTrue(mOutput.toString().contains("cannot be found."));
mFsShell.run("load", "/testRoot", "--progress", "--format", "JSON");
mFsShell.run("load", path, "--progress", "--format", "JSON");
assertTrue(mOutput.toString().contains("\"mJobState\":\"SUCCEEDED\""));
mFsShell.run("load", "/testRoot", "--progress", "--format", "JSON", "--verbose");
mFsShell.run("load", path, "--progress", "--format", "JSON", "--verbose");
assertTrue(mOutput.toString().contains("\"mVerbose\":true"));
}
}

0 comments on commit 90891b8

Please sign in to comment.