From 128a5bb0fa3b1e094a8714f4658e54b5e485c532 Mon Sep 17 00:00:00 2001 From: Eric Davies Date: Fri, 16 Dec 2022 18:54:46 -0600 Subject: [PATCH 1/4] Properly order Project.toml --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index c808fd4e48..f148911026 100644 --- a/Project.toml +++ b/Project.toml @@ -38,8 +38,8 @@ julia = "1.6" [extras] Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" +Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" From 5b3860f09d8e770229cf07c347494e47301d05b9 Mon Sep 17 00:00:00 2001 From: Eric Davies Date: Fri, 16 Dec 2022 18:55:42 -0600 Subject: [PATCH 2/4] Add SlowDown as a throttling error (used in S3) --- src/utilities/request.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utilities/request.jl b/src/utilities/request.jl index d22198cd6b..62036f5589 100644 --- a/src/utilities/request.jl +++ b/src/utilities/request.jl @@ -114,6 +114,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers "LimitExceededException", "RequestThrottled", "PriorRequestNotComplete", + "SlowDown", ] request.headers["User-Agent"] = user_agent[] From 6bf345269ce4ef9e76cdfdd0d284e683d24044cc Mon Sep 17 00:00:00 2001 From: Eric Davies Date: Fri, 16 Dec 2022 18:56:28 -0600 Subject: [PATCH 3/4] Make the number of retries configurable --- src/AWSConfig.jl | 29 +++++++++++++++++-- src/utilities/request.jl | 2 +- src/utilities/utilities.jl | 2 +- test/AWS.jl | 59 ++++++++++++++++++++++++++++++++++++++ test/patch.jl | 21 ++++++++++++++ 5 files changed, 109 insertions(+), 4 deletions(-) diff --git a/src/AWSConfig.jl b/src/AWSConfig.jl index f63d7ad494..9097aa49bf 100644 --- a/src/AWSConfig.jl +++ b/src/AWSConfig.jl @@ -1,18 +1,43 @@ abstract type AbstractAWSConfig end + +# https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html +const AWS_MAX_RETRY_ATTEMPTS = 3 + +""" + max_attempts(::AbstractAWSConfig) -> Int + +The number of AWS request retry attempts to make. +Each backend may perform an additional layer backend-specific retries. + +`AbstractAWSConfig` subtypes should allow users to override the default. +The default is 3, per the recommendations at +https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html +""" +max_attempts(::AbstractAWSConfig) = AWS_MAX_RETRY_ATTEMPTS + mutable struct AWSConfig <: AbstractAWSConfig credentials::Union{AWSCredentials,Nothing} region::String output::String + max_attempts::Int end +# provide a default for the new field since people were using the default constructor +AWSConfig(creds, region, output) = AWSConfig(creds, region, output, AWS_MAX_RETRY_ATTEMPTS) + credentials(aws::AWSConfig) = aws.credentials region(aws::AWSConfig) = aws.region +max_attempts(aws::AWSConfig) = aws.max_attempts function AWSConfig(; - profile=nothing, creds=AWSCredentials(; profile=profile), region=nothing, output="json" + profile=nothing, + creds=AWSCredentials(; profile=profile), + region=nothing, + output="json", + max_attempts=AWS_MAX_RETRY_ATTEMPTS, ) region = @something region aws_get_region(; profile=profile) - return AWSConfig(creds, region, output) + return AWSConfig(creds, region, output, max_attempts) end """ diff --git a/src/utilities/request.jl b/src/utilities/request.jl index 62036f5589..a214eec436 100644 --- a/src/utilities/request.jl +++ b/src/utilities/request.jl @@ -191,7 +191,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers return false end - delays = AWSExponentialBackoff(; max_attempts=3) + delays = AWSExponentialBackoff(; max_attempts=max_attempts(aws)) retry(upgrade_error(get_response); check=check, delays=delays)() diff --git a/src/utilities/utilities.jl b/src/utilities/utilities.jl index 52eaca2518..8464f941ed 100644 --- a/src/utilities/utilities.jl +++ b/src/utilities/utilities.jl @@ -143,7 +143,7 @@ end # https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html # Default values for AWS's `standard` retry mode. Note: these can be overridden elsewhere. Base.@kwdef struct AWSExponentialBackoff - max_attempts::Int = 3 + max_attempts::Int = AWS_MAX_RETRY_ATTEMPTS max_backoff::Float64 = 20.0 rng::AbstractRNG = Random.GLOBAL_RNG end diff --git a/test/AWS.jl b/test/AWS.jl index 624d4f64f2..a6bc9e1e2e 100644 --- a/test/AWS.jl +++ b/test/AWS.jl @@ -193,6 +193,65 @@ end @test String(response.body) == Patches.body end + @testset "Default throttling" begin + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + retries = Ref{Int}(0) + exception = apply(Patches._throttling_patch(retries)) do + try + AWS.submit_request(aws, request) + return nothing + catch e + if e isa AWSException + return e + else + rethrow() + end + end + end + + @test exception isa AWSException + @test exception.code == "SlowDown" + @test retries[] == AWS.max_attempts(aws) + end + + @testset "Custom throttling" begin + aws = AWS.AWSConfig(max_attempts=1) + @test AWS.max_attempts(aws) == 1 + + request = Request(; + service="s3", + api_version="api_version", + request_method="GET", + url="https://s3.us-east-1.amazonaws.com/sample-bucket", + use_response_type=true, + ) + + retries = Ref{Int}(0) + exception = apply(Patches._throttling_patch(retries)) do + try + AWS.submit_request(aws, request) + return nothing + catch e + if e isa AWSException + return e + else + rethrow() + end + end + end + + @test exception isa AWSException + @test exception.code == "SlowDown" + @test retries[] == AWS.max_attempts(aws) + end + @testset "Not authorized" begin request = Request(; service="s3", diff --git a/test/patch.jl b/test/patch.jl index 19ec42a936..d57723b5f7 100644 --- a/test/patch.jl +++ b/test/patch.jl @@ -66,6 +66,27 @@ function _aws_http_request_patch(response::AWS.Response=_response()) return p end +function _throttling_patch(retries::Ref{Int}) + status = 503 + body = """ + \n + + SlowDown + Please reduce your request rate + /mybucket/myfoto.jpg + 4442587FB7D0A2F9 + + """ + + p = @patch function AWS._http_request(::AWS.AbstractBackend, request::Request, ::IO) + retries[] += 1 + resp = _response(status=status, body=body).response + err = HTTP.StatusError(status, request.request_method, request.resource, resp) + throw(AWS.AWSException(err, body)) + end + return p +end + _cred_file_patch = @patch function dot_aws_credentials_file() return "" end From a078000f98e4fbae58b5b91cc6937d500b31a91b Mon Sep 17 00:00:00 2001 From: Eric Davies Date: Fri, 16 Dec 2022 19:01:39 -0600 Subject: [PATCH 4/4] Apply autoformat suggestions Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/AWS.jl | 2 +- test/patch.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/AWS.jl b/test/AWS.jl index a6bc9e1e2e..7dfc2fbde5 100644 --- a/test/AWS.jl +++ b/test/AWS.jl @@ -222,7 +222,7 @@ end end @testset "Custom throttling" begin - aws = AWS.AWSConfig(max_attempts=1) + aws = AWS.AWSConfig(; max_attempts=1) @test AWS.max_attempts(aws) == 1 request = Request(; diff --git a/test/patch.jl b/test/patch.jl index d57723b5f7..7315533f91 100644 --- a/test/patch.jl +++ b/test/patch.jl @@ -80,7 +80,7 @@ function _throttling_patch(retries::Ref{Int}) p = @patch function AWS._http_request(::AWS.AbstractBackend, request::Request, ::IO) retries[] += 1 - resp = _response(status=status, body=body).response + resp = _response(; status=status, body=body).response err = HTTP.StatusError(status, request.request_method, request.resource, resp) throw(AWS.AWSException(err, body)) end