Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(S3BasicFileAttributes): Do IO in methods that throw IOException #342

Merged
merged 2 commits into from
Dec 28, 2023
Merged

chore(S3BasicFileAttributes): Do IO in methods that throw IOException #342

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3ce472a
chore(S3BasicFileAttributes): Do IO on methods that throw IOExceptions
guicamest Dec 25, 2023
5ad4374
chore(S3BasicFileAttributes): update javadocs
guicamest Dec 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/main/java/software/amazon/nio/spi/s3/S3BasicFileAttributeView.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package software.amazon.nio.spi.s3;

import java.io.IOException;
import java.nio.file.attribute.BasicFileAttributeView;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
Expand Down Expand Up @@ -37,8 +38,8 @@ public String name() {
* @return the file attributes
*/
@Override
public BasicFileAttributes readAttributes() {
return new S3BasicFileAttributes(path, Duration.ofMinutes(TimeOutUtils.TIMEOUT_TIME_LENGTH_1));
public BasicFileAttributes readAttributes() throws IOException {
return S3BasicFileAttributes.get(path, Duration.ofMinutes(TimeOutUtils.TIMEOUT_TIME_LENGTH_1));
}

/**
Expand Down
144 changes: 87 additions & 57 deletions src/main/java/software/amazon/nio/spi/s3/S3BasicFileAttributes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@

import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static software.amazon.nio.spi.s3.util.TimeOutUtils.createAndLogTimeOutMessage;

import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Files;
Expand All @@ -24,36 +26,49 @@
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.nio.spi.s3.util.TimeOutUtils;

/**
* Representation of {@link BasicFileAttributes} for an S3 object
*/
class S3BasicFileAttributes implements BasicFileAttributes {

private static final FileTime EPOCH_FILE_TIME = FileTime.from(Instant.EPOCH);

private static final S3BasicFileAttributes DIRECTORY_ATTRIBUTES = new S3BasicFileAttributes(
EPOCH_FILE_TIME,
0L,
null,
true,
false
);

private static final Set<String> METHOD_NAMES_TO_FILTER_OUT =
Set.of("wait", "toString", "hashCode", "getClass", "notify", "notifyAll");

private static final Logger logger = LoggerFactory.getLogger(S3BasicFileAttributes.class.getName());

private final S3Path path;
private final S3AsyncClient client;
private final String bucketName;
private final Duration timeout;

private final FileTime lastModifiedTime;
private final Long size;
private final Object eTag;
private final boolean isDirectory;
private final boolean isRegularFile;

/**
* Constructor for the attributes of a path
*
* @param path the path to represent the attributes of
* @param timeout timeout for requests to get attributes
*/
S3BasicFileAttributes(S3Path path, Duration timeout) {
this.path = path;
this.client = path.getFileSystem().client();
this.bucketName = path.bucketName();
this.timeout = timeout;
private S3BasicFileAttributes(FileTime lastModifiedTime,
Long size,
Object eTag,
boolean isDirectory,
boolean isRegularFile
) {
this.lastModifiedTime = lastModifiedTime;
this.size = size;
this.eTag = eTag;
this.isDirectory = isDirectory;
this.isRegularFile = isRegularFile;
}

/**
Expand All @@ -65,15 +80,10 @@ class S3BasicFileAttributes implements BasicFileAttributes {
*
* @return a {@code FileTime} representing the time the file was last
* modified.
* @throws RuntimeException if the S3Clients {@code RetryConditions} configuration was not able to handle the exception.
*/
@Override
public FileTime lastModifiedTime() {
if (path.isDirectory()) {
return FileTime.from(Instant.EPOCH);
}

return FileTime.from(getObjectMetadata("lastModifiedTime()").lastModified());
return lastModifiedTime;
}

/**
Expand Down Expand Up @@ -109,7 +119,7 @@ public FileTime creationTime() {
*/
@Override
public boolean isRegularFile() {
return !path.isDirectory();
return isRegularFile;
}

/**
Expand All @@ -119,7 +129,7 @@ public boolean isRegularFile() {
*/
@Override
public boolean isDirectory() {
return path.isDirectory();
return isDirectory;
}

/**
Expand Down Expand Up @@ -151,50 +161,21 @@ public boolean isOther() {
* therefore unspecified.
*
* @return the file size, in bytes
* @throws RuntimeException if the S3Clients {@code RetryConditions} configuration was not able to handle the exception.
*/
@Override
public long size() {
if (isDirectory()) {
return 0;
}
return getObjectMetadata("size()").contentLength();
return size;
}

/**
* Returns the S3 etag for the object
*
* @return the etag for an object, or {@code null} for a "directory"
* @throws RuntimeException if the S3Clients {@code RetryConditions} configuration was not able to handle the exception.
* @see Files#walkFileTree
*/
@Override
public Object fileKey() {
if (path.isDirectory()) {
return null;
}
return getObjectMetadata("fileKey()").eTag();
}

private HeadObjectResponse getObjectMetadata(String forOperation) {
try {
return client.headObject(req -> req
.bucket(bucketName)
.key(path.getKey())
).get(timeout.toMillis(), MILLISECONDS);
} catch (ExecutionException e) {
var errMsg = format(
"an '%s' error occurred while obtaining the metadata (for operation %s) of '%s'" +
"that was not handled successfully by the S3Client's configured RetryConditions",
e.getCause().toString(), forOperation, path.toUri());
logger.error(errMsg);
throw new RuntimeException(errMsg, e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(e);
} catch (TimeoutException e) {
throw TimeOutUtils.logAndGenerateExceptionOnTimeOut(logger, forOperation, timeout.toMillis(), MILLISECONDS);
}
return eTag;
}

/**
Expand All @@ -214,13 +195,62 @@ protected Map<String, Object> asMap(Predicate<String> attributeFilter) {
return method.invoke(this);
} catch (IllegalAccessException | InvocationTargetException e) {
// should not ever happen as these are all public no arg methods
var errorMsg = format(
"an exception has occurred during a reflection operation on the methods of the file attributes" +
"of '%s', check if your Java SecurityManager is configured to allow reflection.",
path.toUri());
var errorMsg =
"an exception has occurred during a reflection operation on the methods of file attributes," +
"check if your Java SecurityManager is configured to allow reflection."
;
logger.error("{}, caused by {}", errorMsg, e.getCause().getMessage());
throw new RuntimeException(errorMsg, e);
}
})));
}

/**
* @param path the path to represent the attributes of
* @param readTimeout timeout for requests to get attributes
* @return path BasicFileAttributes
* @throws IOException Errors getting the metadata of the object represented by the path are wrapped in IOException
*/
static S3BasicFileAttributes get(S3Path path, Duration readTimeout) throws IOException {
if (path.isDirectory()) {
return DIRECTORY_ATTRIBUTES;
}

var headResponse = getObjectMetadata(path, readTimeout);
return new S3BasicFileAttributes(
FileTime.from(headResponse.lastModified()),
headResponse.contentLength(),
headResponse.eTag(),
false,
true
);
}

private static HeadObjectResponse getObjectMetadata(
S3Path path,
Duration timeout
) throws IOException {
var client = path.getFileSystem().client();
var bucketName = path.bucketName();
try {
return client.headObject(req -> req
.bucket(bucketName)
.key(path.getKey())
).get(timeout.toMillis(), MILLISECONDS);
} catch (ExecutionException e) {
var errMsg = format(
"an '%s' error occurred while obtaining the metadata (for operation getFileAttributes) of '%s'" +
"that was not handled successfully by the S3Client's configured RetryConditions",
e.getCause().toString(), path.toUri());
logger.error(errMsg);
throw new IOException(errMsg, e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException(e);
} catch (TimeoutException e) {
var msg = createAndLogTimeOutMessage(logger, "getFileAttributes", timeout.toMillis(), MILLISECONDS);
throw new IOException(msg, e);
}
}

}
8 changes: 4 additions & 4 deletions src/main/java/software/amazon/nio/spi/s3/S3FileSystemProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -626,13 +626,13 @@ public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V>
* @return the file attributes or {@code null} if {@code path} is inferred to be a directory.
*/
@Override
public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) {
public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException {
Objects.requireNonNull(type);
var s3Path = checkPath(path);

if (type.equals(BasicFileAttributes.class)) {
@SuppressWarnings("unchecked")
var a = (A) new S3BasicFileAttributes(s3Path, Duration.ofMinutes(TimeOutUtils.TIMEOUT_TIME_LENGTH_1));
var a = (A) S3BasicFileAttributes.get(s3Path, Duration.ofMinutes(TimeOutUtils.TIMEOUT_TIME_LENGTH_1));
return a;
} else {
throw new UnsupportedOperationException("cannot read attributes of type: " + type);
Expand Down Expand Up @@ -660,7 +660,7 @@ public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type
* may be invoked to check for additional permissions.
*/
@Override
public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) {
public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException {
Objects.requireNonNull(attributes);
var s3Path = checkPath(path);

Expand All @@ -669,7 +669,7 @@ public Map<String, Object> readAttributes(Path path, String attributes, LinkOpti
}

var attributesFilter = attributesFilterFor(attributes);
return new S3BasicFileAttributes(s3Path, Duration.ofMinutes(TimeOutUtils.TIMEOUT_TIME_LENGTH_1)).asMap(attributesFilter);
return S3BasicFileAttributes.get(s3Path, Duration.ofMinutes(TimeOutUtils.TIMEOUT_TIME_LENGTH_1)).asMap(attributesFilter);
}

/**
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/software/amazon/nio/spi/s3/S3SeekableByteChannel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,9 +201,9 @@ public long size() throws IOException {
return this.size;
}

private void fetchSize() {
private void fetchSize() throws IOException {
synchronized (this) {
this.size = new S3BasicFileAttributes(path, Duration.ofMinutes(TimeOutUtils.TIMEOUT_TIME_LENGTH_1)).size();
this.size = S3BasicFileAttributes.get(path, Duration.ofMinutes(TimeOutUtils.TIMEOUT_TIME_LENGTH_1)).size();
LOGGER.debug("size of '{}' is '{}'", path.toUri(), this.size);
}
}
Expand Down
Loading