Skip to content

Commit

Permalink
fix: use environment configuration when creating a byte channel (#405)
Browse files Browse the repository at this point in the history
  • Loading branch information
markjschreiber committed Mar 11, 2024
1 parent 9797b89 commit 49be81e
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -239,4 +239,12 @@ private CompletableFuture<ByteBuffer> computeFragmentFuture(int fragmentIndex) {
Integer fragmentIndexForByteNumber(long byteNumber) {
return Math.toIntExact(Math.floorDiv(byteNumber, (long) maxFragmentSize));
}

public int getMaxFragmentSize() {
return maxFragmentSize;
}

public int getMaxNumberFragments() {
return maxNumberFragments;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.nio.spi.s3.config.S3NioSpiConfiguration;
import software.amazon.nio.spi.s3.util.TimeOutUtils;

class S3SeekableByteChannel implements SeekableByteChannel {
Expand Down Expand Up @@ -54,9 +53,7 @@ private S3SeekableByteChannel(S3Path s3Path, S3AsyncClient s3Client, long startA
throw new IOException("The SYNC/DSYNC options is not supported");
}

// later we will add a constructor that allows providing delegates for composition

var config = new S3NioSpiConfiguration();
var config = s3Path.getFileSystem().configuration();
if (options.contains(StandardOpenOption.WRITE)) {
LOGGER.debug("using S3WritableByteChannel as write delegate for path '{}'", s3Path.toUri());
readDelegate = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,38 +5,46 @@

package software.amazon.nio.spi.s3;

import java.time.Instant;
import org.mockito.Mock;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.async.AsyncRequestBody;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.*;
import static java.nio.file.StandardOpenOption.CREATE;
import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
import static software.amazon.nio.spi.s3.S3Matchers.anyConsumer;

import java.io.IOException;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.OpenOption;
import java.time.Instant;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import org.junit.jupiter.api.AfterEach;

import static java.nio.file.StandardOpenOption.CREATE;
import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;
import static org.junit.jupiter.api.Assertions.*;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;

import static org.mockito.Mockito.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
import static software.amazon.nio.spi.s3.S3Matchers.anyConsumer;

import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.async.AsyncRequestBody;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;

@ExtendWith(MockitoExtension.class)
@SuppressWarnings("unchecked")
Expand Down Expand Up @@ -149,4 +157,15 @@ private S3SeekableByteChannel seekableByteChannelForRead() throws IOException {
return new S3SeekableByteChannel(path, mockClient, Collections.singleton(READ));
}

// test that the S3SeekableByteChannel uses the buffer size from the configuration set for the FileSystem
@Test
public void testBufferSize() throws IOException {
fs.configuration().withMaxFragmentSize(10000);
fs.configuration().withMaxFragmentNumber(10);
try(var channel = (S3SeekableByteChannel) fs.provider().newByteChannel(path, Set.of(READ))) {
assertEquals(10000, ((S3ReadAheadByteChannel) channel.getReadDelegate()).getMaxFragmentSize());
assertEquals(10, ((S3ReadAheadByteChannel) channel.getReadDelegate()).getMaxNumberFragments());
}
}

}

0 comments on commit 49be81e

Please sign in to comment.