service/s3: Add support for regional S3 us-east-1 endpoint (#2938)

Adds support for S3 configuring an SDK Amazon S3 client for the regional
us-east-1 endpoint instead of the default global s3 endpoint.

Adds a new configuration option, `S3UsEast1RegionalEndpoint` which when
set to RegionalS3UsEast1Endpoint, and region is `us-east-1` the S3
client will resolve the `us-east-1` regional endpoint,
`s3.us-east-1.amazonaws.com` instead of the global S3 endpoint,
`s3.amazonaws.com`. The SDK defaults to the current global S3 endpoint
resolution for backwards compatibility.

Opt-in to the `us-east-1` regional endpoint via the SDK's Config,
environment variable, `AWS_S3_US_EAST_1_REGIONAL_ENDPOINT=regional`, or
shared config option, `s3_us_east_1_regional_endpoint=regional`.

Note the SDK does not support the shared configuration file by default.
You must opt-in to that behavior via Session Option `SharedConfigState`,
or `AWS_SDK_LOAD_CONFIG=true` environment variable.

CHANGELOG_PENDING.md

@@ -1,5 +1,10 @@
 ### SDK Features
 
 ### SDK Enhancements
+* `aws/endpoints`: Add support for regional S3 us-east-1 endpoint
+  * Adds support for S3 configuring an SDK Amazon S3 client for the regional us-east-1 endpoint instead of the default global S3 endpoint.
+  * Adds a new configuration option, `S3UsEast1RegionalEndpoint` which when set to `RegionalS3UsEast1Endpoint`, and region is `us-east-1` the S3 client will resolve the `us-east-1` regional endpoint, `s3.us-east-1.amazonaws.com` instead of the global S3 endpoint, `s3.amazonaws.com`. The SDK defaults to the current global S3 endpoint resolution for backwards compatibility.
+  * Opt-in to the `us-east-1` regional endpoint via the SDK's Config, environment variable, `AWS_S3_US_EAST_1_REGIONAL_ENDPOINT=regional`, or shared config option, `s3_us_east_1_regional_endpoint=regional`.
+  * Note the SDK does not support the shared configuration file by default.  You must opt-in to that behavior via Session Option `SharedConfigState`, or `AWS_SDK_LOAD_CONFIG=true` environment variable.
 
 ### SDK Bugs

aws/config.go

@@ -249,6 +249,9 @@ type Config struct {
 
 	// STSRegionalEndpoint will enable regional or legacy endpoint resolving
 	STSRegionalEndpoint endpoints.STSRegionalEndpoint
+
+	// S3UsEast1RegionalEndpoint will enable regional or legacy endpoint resolving
+	S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint
 }
 
 // NewConfig returns a new Config pointer that can be chained with builder
@@ -430,6 +433,13 @@ func (c *Config) WithSTSRegionalEndpoint(sre endpoints.STSRegionalEndpoint) *Con
 	return c
 }
 
+// WithS3UsEast1RegionalEndpoint will set whether or not to use regional endpoint flag
+// when resolving the endpoint for a service
+func (c *Config) WithS3UsEast1RegionalEndpoint(sre endpoints.S3UsEast1RegionalEndpoint) *Config {
+	c.S3UsEast1RegionalEndpoint = sre
+	return c
+}
+
 func mergeInConfig(dst *Config, other *Config) {
 	if other == nil {
 		return
@@ -534,6 +544,10 @@ func mergeInConfig(dst *Config, other *Config) {
 	if other.STSRegionalEndpoint != endpoints.UnsetSTSEndpoint {
 		dst.STSRegionalEndpoint = other.STSRegionalEndpoint
 	}
+
+	if other.S3UsEast1RegionalEndpoint != endpoints.UnsetS3UsEast1Endpoint {
+		dst.S3UsEast1RegionalEndpoint = other.S3UsEast1RegionalEndpoint
+	}
 }
 
 // Copy will return a shallow copy of the Config object. If any additional

aws/endpoints/decode.go

@@ -83,6 +83,7 @@ func decodeV3Endpoints(modelDef modelDefinition, opts DecodeModelOptions) (Resol
 		p := &ps[i]
 		custAddEC2Metadata(p)
 		custAddS3DualStack(p)
+		custRegionalS3(p)
 		custRmIotDataService(p)
 		custFixAppAutoscalingChina(p)
 		custFixAppAutoscalingUsGov(p)
@@ -100,6 +101,33 @@ func custAddS3DualStack(p *partition) {
 	custAddDualstack(p, "s3-control")
 }
 
+func custRegionalS3(p *partition) {
+	if p.ID != "aws" {
+		return
+	}
+
+	service, ok := p.Services["s3"]
+	if !ok {
+		return
+	}
+
+	// If global endpoint already exists no customization needed.
+	if _, ok := service.Endpoints["aws-global"]; ok {
+		return
+	}
+
+	service.PartitionEndpoint = "aws-global"
+	service.Endpoints["us-east-1"] = endpoint{}
+	service.Endpoints["aws-global"] = endpoint{
+		Hostname: "s3.amazonaws.com",
+		CredentialScope: credentialScope{
+			Region: "us-east-1",
+		},
+	}
+
+	p.Services["s3"] = service
+}
+
 func custAddDualstack(p *partition, svcName string) {
 	s, ok := p.Services[svcName]
 	if !ok {

aws/endpoints/endpoints.go

@@ -50,12 +50,28 @@ type Options struct {
 
 	// STS Regional Endpoint flag helps with resolving the STS endpoint
 	STSRegionalEndpoint STSRegionalEndpoint
+
+	// S3 Regional Endpoint flag helps with resolving the S3 endpoint
+	S3UsEast1RegionalEndpoint S3UsEast1RegionalEndpoint
 }
 
-// STSRegionalEndpoint is an enum type alias for int
-// It is used internally by the core sdk as STS Regional Endpoint flag value
+// STSRegionalEndpoint is an enum for the states of the STS Regional Endpoint
+// options.
 type STSRegionalEndpoint int
 
+func (e STSRegionalEndpoint) String() string {
+	switch e {
+	case LegacySTSEndpoint:
+		return "legacy"
+	case RegionalSTSEndpoint:
+		return "regional"
+	case UnsetSTSEndpoint:
+		return ""
+	default:
+		return "unknown"
+	}
+}
+
 const (
 
 	// UnsetSTSEndpoint represents that STS Regional Endpoint flag is not specified.
@@ -86,6 +102,55 @@ func GetSTSRegionalEndpoint(s string) (STSRegionalEndpoint, error) {
 	}
 }
 
+// S3UsEast1RegionalEndpoint is an enum for the states of the S3 us-east-1
+// Regional Endpoint options.
+type S3UsEast1RegionalEndpoint int
+
+func (e S3UsEast1RegionalEndpoint) String() string {
+	switch e {
+	case LegacyS3UsEast1Endpoint:
+		return "legacy"
+	case RegionalS3UsEast1Endpoint:
+		return "regional"
+	case UnsetS3UsEast1Endpoint:
+		return ""
+	default:
+		return "unknown"
+	}
+}
+
+const (
+
+	// UnsetS3UsEast1Endpoint represents that S3 Regional Endpoint flag is not
+	// specified.
+	UnsetS3UsEast1Endpoint S3UsEast1RegionalEndpoint = iota
+
+	// LegacyS3UsEast1Endpoint represents when S3 Regional Endpoint flag is
+	// specified to use legacy endpoints.
+	LegacyS3UsEast1Endpoint
+
+	// RegionalS3UsEast1Endpoint represents when S3 Regional Endpoint flag is
+	// specified to use regional endpoints.
+	RegionalS3UsEast1Endpoint
+)
+
+// GetS3UsEast1RegionalEndpoint function returns the S3UsEast1RegionalEndpointFlag based
+// on the input string provided in env config or shared config by the user.
+//
+// `legacy`, `regional` are the only case-insensitive valid strings for
+// resolving the S3 regional Endpoint flag.
+func GetS3UsEast1RegionalEndpoint(s string) (S3UsEast1RegionalEndpoint, error) {
+	switch {
+	case strings.EqualFold(s, "legacy"):
+		return LegacyS3UsEast1Endpoint, nil
+	case strings.EqualFold(s, "regional"):
+		return RegionalS3UsEast1Endpoint, nil
+	default:
+		return UnsetS3UsEast1Endpoint,
+			fmt.Errorf("unable to resolve the value of S3UsEast1RegionalEndpoint for %v", s)
+	}
+}
+
 // Set combines all of the option functions together.
 func (o *Options) Set(optFns ...func(*Options)) {
 	for _, fn := range optFns {

aws/endpoints/legacy_regions.go

@@ -0,0 +1,24 @@
+package endpoints
+
+var legacyGlobalRegions = map[string]map[string]struct{}{
+	"sts": {
+		"ap-northeast-1": {},
+		"ap-south-1":     {},
+		"ap-southeast-1": {},
+		"ap-southeast-2": {},
+		"ca-central-1":   {},
+		"eu-central-1":   {},
+		"eu-north-1":     {},
+		"eu-west-1":      {},
+		"eu-west-2":      {},
+		"eu-west-3":      {},
+		"sa-east-1":      {},
+		"us-east-1":      {},
+		"us-east-2":      {},
+		"us-west-1":      {},
+		"us-west-2":      {},
+	},
+	"s3": {
+		"us-east-1": {},
+	},
+}

aws/endpoints/v3model.go

@@ -110,8 +110,9 @@ func (p partition) EndpointFor(service, region string, opts ...func(*Options)) (
 		region = s.PartitionEndpoint
 	}
 
-	if service == "sts" && opt.STSRegionalEndpoint != RegionalSTSEndpoint {
-		if _, ok := stsLegacyGlobalRegions[region]; ok {
+	if (service == "sts" && opt.STSRegionalEndpoint != RegionalSTSEndpoint) ||
+		(service == "s3" && opt.S3UsEast1RegionalEndpoint != RegionalS3UsEast1Endpoint) {
+		if _, ok := legacyGlobalRegions[service][region]; ok {
 			region = "aws-global"
 		}
 	}
@@ -240,11 +241,23 @@ func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs [
 	merged.mergeIn(e)
 	e = merged
 
-	hostname := e.Hostname
+	signingRegion := e.CredentialScope.Region
+	if len(signingRegion) == 0 {
+		signingRegion = region
+	}
 
+	signingName := e.CredentialScope.Service
+	var signingNameDerived bool
+	if len(signingName) == 0 {
+		signingName = service
+		signingNameDerived = true
+	}
+
+	hostname := e.Hostname
 	// Offset the hostname for dualstack if enabled
 	if opts.UseDualStack && e.HasDualStack == boxedTrue {
 		hostname = e.DualStackHostname
+		region = signingRegion
 	}
 
 	u := strings.Replace(hostname, "{service}", service, 1)
@@ -254,18 +267,6 @@ func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs [
 	scheme := getEndpointScheme(e.Protocols, opts.DisableSSL)
 	u = fmt.Sprintf("%s://%s", scheme, u)
 
-	signingRegion := e.CredentialScope.Region
-	if len(signingRegion) == 0 {
-		signingRegion = region
-	}
-
-	signingName := e.CredentialScope.Service
-	var signingNameDerived bool
-	if len(signingName) == 0 {
-		signingName = service
-		signingNameDerived = true
-	}
-
 	return ResolvedEndpoint{
 		URL:                u,
 		PartitionID:        partitionID,