Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix and re-enable UfsFileIn/OutStreamTest #18170

Merged
merged 3 commits into from
Sep 23, 2023
Merged

Fix and re-enable UfsFileIn/OutStreamTest #18170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5ecb6e7
trying to fix UfsFileIn/OutStreamTest
voddle Sep 19, 2023
a1d27d5
fix
voddle Sep 20, 2023
ba27d2c
removed redundent
voddle Sep 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 4 additions & 24 deletions dora/core/client/fs/src/test/java/alluxio/client/file/ufs/AbstractUfsStreamTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,59 +14,38 @@
import alluxio.AlluxioTestDirectory;
import alluxio.AlluxioURI;
import alluxio.ClientContext;
import alluxio.annotation.dora.DoraTestTodoItem;
import alluxio.client.file.FileSystem;
import alluxio.client.file.FileSystemContext;
import alluxio.client.file.URIStatus;
import alluxio.client.file.options.FileSystemOptions;
import alluxio.client.file.options.UfsFileSystemOptions;
import alluxio.conf.Configuration;
import alluxio.conf.InstancedConfiguration;
import alluxio.conf.PropertyKey;
import alluxio.conf.Source;
import alluxio.exception.AlluxioException;
import alluxio.grpc.DeletePOptions;
import alluxio.underfs.UnderFileSystemFactoryRegistry;
import alluxio.underfs.local.LocalUnderFileSystemFactory;

import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.UUID;

/**
* Add unit tests for streams of {@link UfsBaseFileSystem}.
*/
@RunWith(Parameterized.class)
@DoraTestTodoItem(action = DoraTestTodoItem.Action.FIX, owner = "LuQQiu",
comment = "fix the tests here because those APIs are still relevant")
@Ignore
public abstract class AbstractUfsStreamTest {
protected static final int CHUNK_SIZE = 100;
protected InstancedConfiguration mConf;
protected AlluxioURI mRootUfs;
protected FileSystem mFileSystem;

@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {{false}, {true}});
voddle marked this conversation as resolved.
Show resolved Hide resolved
}

/**
* Runs {@link AbstractUfsStreamTest} with different configuration combinations.
*
* @param localDataCacheEnabled whether local data cache is enabled
*/
public AbstractUfsStreamTest(boolean localDataCacheEnabled) {
public AbstractUfsStreamTest() {
mConf = Configuration.copyGlobal();
mConf.set(PropertyKey.USER_CLIENT_CACHE_ENABLED,
PropertyKey.USER_CLIENT_CACHE_ENABLED.formatValue(localDataCacheEnabled), Source.RUNTIME);
}

/**
Expand All @@ -78,9 +57,10 @@ public void before() {
mRootUfs = new AlluxioURI(ufs);
UnderFileSystemFactoryRegistry.register(new LocalUnderFileSystemFactory());
final FileSystemOptions fileSystemOptions =
FileSystemOptions.Builder
.fromConf(mConf)
new FileSystemOptions.Builder()
.setUfsFileSystemOptions(new UfsFileSystemOptions(ufs))
.setDoraCacheEnabled(false)
.setDataCacheEnabled(false)
Comment on lines +62 to +63
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fyi @dbw9580 @JiamingMai @LuQQiu In the test for UfsFileStream we are manually disabling meta/data client cache. Let us know if this is incorrect. Thanks!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looks good to me

.build();
mFileSystem = FileSystem.Factory.create(FileSystemContext.create(
ClientContext.create(mConf)), fileSystemOptions);
Expand Down
20 changes: 2 additions & 18 deletions dora/core/client/fs/src/test/java/alluxio/client/file/ufs/UfsFileInStreamTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,14 @@
import static org.junit.Assert.assertTrue;

import alluxio.AlluxioURI;
import alluxio.annotation.dora.DoraTestTodoItem;
import alluxio.client.file.FileInStream;
import alluxio.conf.PropertyKey;
import alluxio.exception.AlluxioException;
import alluxio.util.io.BufferUtils;

import org.junit.Assert;
import org.junit.Assume;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import java.io.IOException;
import java.io.OutputStream;
Expand All @@ -37,18 +33,12 @@
/**
* Add unit tests for {@link UfsFileInStream}.
*/
@RunWith(Parameterized.class)
@DoraTestTodoItem(action = DoraTestTodoItem.Action.FIX, owner = "LuQQiu",
comment = "check if UfsBaseFS is still relevant and fix the tests if so")
@Ignore
public class UfsFileInStreamTest extends AbstractUfsStreamTest {
/**
* Runs {@link UfsFileInStreamTest} with different configuration combinations.
*
* @param localDataCacheEnabled whether local data cache is enabled
*/
public UfsFileInStreamTest(boolean localDataCacheEnabled) {
super(localDataCacheEnabled);
public UfsFileInStreamTest() {
super();
}

@Test
Expand Down Expand Up @@ -95,7 +85,6 @@ public void manyBytesRead() throws IOException, AlluxioException {
}

@Test
@Ignore
public void manyBytesReadByteBuffer() throws IOException, AlluxioException {
AlluxioURI ufsPath = getUfsPath();
createFile(ufsPath, CHUNK_SIZE);
Expand All @@ -107,7 +96,6 @@ public void manyBytesReadByteBuffer() throws IOException, AlluxioException {
}

@Test
@Ignore
public void readAll() throws IOException, AlluxioException {
int len = CHUNK_SIZE * 5;
int start = 0;
Expand All @@ -124,7 +112,6 @@ public void readAll() throws IOException, AlluxioException {
}

@Test
@Ignore
public void readAllByteBuffer() throws IOException, AlluxioException {
int len = CHUNK_SIZE * 5;
int start = 0;
Expand Down Expand Up @@ -242,7 +229,6 @@ public void readNullBuffer() throws IOException, AlluxioException {
}

@Test
@Ignore
public void readNullArrayOffset() throws IOException, AlluxioException {
AlluxioURI ufsPath = getUfsPath();
createFile(ufsPath, CHUNK_SIZE);
Expand Down Expand Up @@ -353,7 +339,6 @@ public void seekToBeginning() throws IOException, AlluxioException {
}

@Test
@Ignore
public void seekForwardAndBackward() throws IOException, AlluxioException {
AlluxioURI ufsPath = getUfsPath();
createFile(ufsPath, CHUNK_SIZE);
Expand Down Expand Up @@ -423,7 +408,6 @@ public void skipToEnd() throws IOException, AlluxioException {
}

@Test
@Ignore
public void skipPassEnd() throws IOException, AlluxioException {
AlluxioURI ufsPath = getUfsPath();
createFile(ufsPath, CHUNK_SIZE);
Expand Down
14 changes: 2 additions & 12 deletions dora/core/client/fs/src/test/java/alluxio/client/file/ufs/UfsFileOutStreamTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,35 +16,25 @@
import static org.junit.Assert.assertThrows;

import alluxio.AlluxioURI;
import alluxio.annotation.dora.DoraTestTodoItem;
import alluxio.client.file.FileOutStream;
import alluxio.exception.AlluxioException;
import alluxio.util.io.BufferUtils;

import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import java.io.IOException;
import java.io.InputStream;

/**
* Add unit tests for {@link UfsFileOutStream}.
*/
@RunWith(Parameterized.class)
@DoraTestTodoItem(action = DoraTestTodoItem.Action.FIX, owner = "LuQQiu",
comment = "check if UfsBaseFS is still relevant and fix the tests if so")
@Ignore
public class UfsFileOutStreamTest extends AbstractUfsStreamTest {
/**
* Runs {@link UfsFileInStreamTest} with different configuration combinations.
*
* @param localDataCacheEnabled whether local data cache is enabled
*/
public UfsFileOutStreamTest(boolean localDataCacheEnabled) {
super(localDataCacheEnabled);
public UfsFileOutStreamTest() {
super();
}

@Test
Expand Down
Loading