diff --git a/AZURE.md b/AZURE.md index de5b07ef..50edb6e7 100644 --- a/AZURE.md +++ b/AZURE.md @@ -38,7 +38,9 @@ Start the server with the following run profiles enabled: `prod,secure,azure` ## Client Setup Follow the regular client setup of access token and storage and metadata urls. -However when running the client use azure pofile option: +However when running the client use azure profile option: ```bash $ bin/score-client --profile azure ... -``` \ No newline at end of file +``` + +Upload block size for Azure can be configured with the `azure.blockSize` configuration property. diff --git a/score-client/README.md b/score-client/README.md index e40cb862..3e5d1590 100644 --- a/score-client/README.md +++ b/score-client/README.md @@ -77,3 +77,6 @@ To enable logging of request bodies and headers, append the following to the com `--logging.level.org.apache.http=DEBUG` +## Azure +For azure support, use the `azure` profile. Block sizes for uploads +can be configured with the `azure.blockSize` property. \ No newline at end of file diff --git a/score-client/src/main/java/bio/overture/score/client/upload/azure/AzureUploadService.java b/score-client/src/main/java/bio/overture/score/client/upload/azure/AzureUploadService.java index 83978b44..d62b367b 100644 --- a/score-client/src/main/java/bio/overture/score/client/upload/azure/AzureUploadService.java +++ b/score-client/src/main/java/bio/overture/score/client/upload/azure/AzureUploadService.java @@ -49,8 +49,8 @@ public class AzureUploadService implements UploadService { @Value("${transport.parallel}") private int parallelUploads; - // Azure used to have a 4MiB maximum part size, it has been since increased. Setting to 100MiB for now. - private final static int BLOCK_SIZE = 100 * 1024 * 1024; + @Value("${azure.blockSize:104857600") + private int blockSize = 100 * 1024 * 1024; @Override public void upload(File file, String objectId, String md5, boolean redo) throws IOException { @@ -58,7 +58,7 @@ public void upload(File file, String objectId, String md5, boolean redo) throws val spec = storageService.initiateUpload(objectId, file.length(), redo, md5); // Calculate expected number of parts to track progress against - val partInfo = calculateNumBlocks(file.length(), BLOCK_SIZE); + val partInfo = calculateNumBlocks(file.length(), blockSize); // Will contain one Part as far as Storage Service is concerned. There is no need for more than one SAS to be // generated for an entire file; each Part does not require its own pre-signed URL. @@ -83,7 +83,7 @@ public void upload(File file, String objectId, String md5, boolean redo) throws @Override public void eventOccurred(ResponseReceivedEvent eventArg) { - long bytesSent = BLOCK_SIZE; + long bytesSent = blockSize; int partCount = completedParts.incrementAndGet(); if (partCount <= partInfo.getLeft()) { @@ -99,7 +99,7 @@ public void eventOccurred(ResponseReceivedEvent eventArg) { @Override public void eventOccurred(ResponseReceivedEvent eventArg) { - long bytesSent = BLOCK_SIZE; + long bytesSent = blockSize; int partCount = completedParts.incrementAndGet(); if (partCount <= partInfo.getLeft()) { @@ -122,7 +122,7 @@ public void eventOccurred(RetryingEvent eventArg) { val options = new BlobRequestOptions(); options.setConcurrentRequestCount(parallelUploads); - blob.setStreamWriteSizeInBytes(BLOCK_SIZE); + blob.setStreamWriteSizeInBytes(blockSize); progress.start(); progress.startTransfer(); diff --git a/score-client/src/main/resources/application.yml b/score-client/src/main/resources/application.yml index 40d955ba..5f515f96 100644 --- a/score-client/src/main/resources/application.yml +++ b/score-client/src/main/resources/application.yml @@ -98,6 +98,17 @@ client: --- +############################################################################### +# Profile - "azure" +############################################################################### + +spring.profiles: azure + +azure: + blockSize: 104857600 + +--- + ############################################################################### # Profile - "kf" ############################################################################### @@ -130,7 +141,7 @@ metadata: client: ssl: - custom: false + custom: false ---