-
Notifications
You must be signed in to change notification settings - Fork 787
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add retry manager to reduce RateLimitExceeded errors
- Loading branch information
1 parent
4e6b82b
commit 9e8c545
Showing
3 changed files
with
101 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| package cloud | ||
|
|
||
| import ( | ||
| "github.com/aws/aws-sdk-go-v2/aws/retry" | ||
| ) | ||
|
|
||
| const ( | ||
| // retryMaxAttempt sets max number of EC2 API call attempts. | ||
| // Set high enough to ensure default sidecar timeout will cancel context long before we stop retrying. | ||
| retryMaxAttempt = 50 | ||
| ) | ||
|
|
||
| // newAdaptiveRetryer restricts attempts of API calls that recently hit throttle errors. | ||
| func newAdaptiveRetryer() *retry.AdaptiveMode { | ||
| return retry.NewAdaptiveMode(func(ao *retry.AdaptiveModeOptions) { | ||
| ao.StandardOptions = append(ao.StandardOptions, func(so *retry.StandardOptions) { | ||
| so.MaxAttempts = retryMaxAttempt | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| // retryManager dictates the retry strategies of EC2 API calls. | ||
| // Each mutating EC2 API has its own retryer because the AWS SDK throttles on a retryer object level, not by API name. | ||
| // While default AWS accounts share request tokens between mutating APIs, users can raise limits for individual APIs. | ||
| // Separate retryers ensures that throttling one API doesn't unintentionally throttle others with separate token buckets. | ||
| type retryManager struct { | ||
| createVolumeRetryer *retry.AdaptiveMode | ||
| deleteVolumeRetryer *retry.AdaptiveMode | ||
| attachVolumeRetryer *retry.AdaptiveMode | ||
| detachVolumeRetryer *retry.AdaptiveMode | ||
| modifyVolumeRetryer *retry.AdaptiveMode | ||
| createSnapshotRetryer *retry.AdaptiveMode | ||
| deleteSnapshotRetryer *retry.AdaptiveMode | ||
| enableFastSnapshotRestoresRetryer *retry.AdaptiveMode | ||
| unbatchableDescribeVolumesModificationsRetryer *retry.AdaptiveMode | ||
| } | ||
|
|
||
| func newRetryManager() *retryManager { | ||
| return &retryManager{ | ||
| createVolumeRetryer: newAdaptiveRetryer(), | ||
| attachVolumeRetryer: newAdaptiveRetryer(), | ||
| deleteVolumeRetryer: newAdaptiveRetryer(), | ||
| detachVolumeRetryer: newAdaptiveRetryer(), | ||
| modifyVolumeRetryer: newAdaptiveRetryer(), | ||
| createSnapshotRetryer: newAdaptiveRetryer(), | ||
| deleteSnapshotRetryer: newAdaptiveRetryer(), | ||
| enableFastSnapshotRestoresRetryer: newAdaptiveRetryer(), | ||
| unbatchableDescribeVolumesModificationsRetryer: newAdaptiveRetryer(), | ||
| } | ||
| } |