Fix using global region in regional services

.gitignore

@@ -20,3 +20,4 @@ cq-provider-aws
 # .tfstate files
 *.tfstate
 *.tfstate.*
+.swp

.golangci.yml

@@ -28,7 +28,6 @@ linters:
     - gosimple
     - govet
     - ineffassign
-    - interfacer
     - misspell
     - nakedret
     - prealloc

client/client.go

@@ -9,6 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/autoscaling"
 	"github.com/aws/aws-sdk-go-v2/service/cloudtrail"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
@@ -85,6 +86,7 @@ type Services struct {
 	Redshift         RedshiftClient
 	RDS              RdsClient
 	S3               S3Client
+	S3Manager        S3ManagerClient
 }
 
 type Client struct {
@@ -105,6 +107,24 @@ type Client struct {
 	ReportUsers interface{}
 }
 
+// This is needed because https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/feature/s3/manager
+// has different structure then all other services (i.e no service but just a function) and we need
+// the ability to mock it.
+// Also we need to use s3 manager to be able to query the bucket-region https://github.com/aws/aws-sdk-go-v2/pull/1027#issuecomment-759818990
+type S3Manager struct {
+	s3Client *s3.Client
+}
+
+func newS3ManagerFromConfig(cfg aws.Config) S3Manager {
+	return S3Manager{
+		s3Client: s3.NewFromConfig(cfg),
+	}
+}
+
+func (s3Manager S3Manager) GetBucketRegion(ctx context.Context, bucket string, optFns ...func(*s3.Options)) (string, error) {
+	return manager.GetBucketRegion(ctx, s3Manager.s3Client, bucket, optFns...)
+}
+
 func NewAwsClient(logger hclog.Logger, regions []string) Client {
 	return Client{
 		services: map[string]*Services{},
@@ -195,17 +215,21 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe
 		if awsConfig.AWSDebug {
 			awsCfg.ClientLogMode = aws.LogRequest | aws.LogResponse | aws.LogRetries
 		}
-		awsCfg.Retryer = newRetryer(awsConfig.MaxRetries, awsConfig.MaxRetries)
+		awsCfg.Retryer = newRetryer(awsConfig.MaxRetries, awsConfig.MaxBackoff)
 		svc := sts.NewFromConfig(awsCfg)
 		output, err := svc.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}, func(o *sts.Options) {
-			o.Region = "us-east-1"
+			o.Region = "aws-global"
 		})
 		if err != nil {
 			return nil, err
 		}
 		// This is a work-around to skip disabled regions
 		// https://github.com/aws/aws-sdk-go-v2/issues/1068
-		res, err := ec2.NewFromConfig(awsCfg).DescribeRegions(ctx, nil)
+		res, err := ec2.NewFromConfig(awsCfg).DescribeRegions(ctx,
+			&ec2.DescribeRegionsInput{AllRegions: false},
+			func(o *ec2.Options) {
+				o.Region = "us-east-1"
+			})
 		if err != nil {
 			return nil, err
 		}
@@ -245,6 +269,7 @@ func initServices(awsCfg aws.Config) Services {
 		Organizations:    organizations.NewFromConfig(awsCfg),
 		RDS:              rds.NewFromConfig(awsCfg),
 		Redshift:         redshift.NewFromConfig(awsCfg),
+		S3Manager:        newS3ManagerFromConfig(awsCfg),
 	}
 }
 

client/mocks/builders_test.go

@@ -907,6 +907,7 @@ func buildOrganizationsAccounts(t *testing.T, ctrl *gomock.Controller) client.Se
 }
 
 func buildS3Buckets(t *testing.T, ctrl *gomock.Controller) client.Services {
+	mgr := mocks.NewMockS3ManagerClient(ctrl)
 	m := mocks.NewMockS3Client(ctrl)
 	b := s3Types.Bucket{}
 	err := faker.FakeData(&b)
@@ -955,8 +956,6 @@ func buildS3Buckets(t *testing.T, ctrl *gomock.Controller) client.Services {
 		&s3.ListBucketsOutput{
 			Buckets: []s3Types.Bucket{b},
 		}, nil)
-	m.EXPECT().GetBucketLocation(gomock.Any(), gomock.Any(), gomock.Any()).Return(
-		&bloc, nil)
 	m.EXPECT().GetBucketLogging(gomock.Any(), gomock.Any(), gomock.Any()).Return(
 		&blog, nil)
 	m.EXPECT().GetBucketPolicy(gomock.Any(), gomock.Any(), gomock.Any()).Return(
@@ -973,7 +972,10 @@ func buildS3Buckets(t *testing.T, ctrl *gomock.Controller) client.Services {
 		}, nil)
 	m.EXPECT().GetBucketEncryption(gomock.Any(), gomock.Any(), gomock.Any()).Return(
 		&bencryption, nil)
+	mgr.EXPECT().GetBucketRegion(gomock.Any(), gomock.Any(), gomock.Any()).Return(
+		"us-east-1", nil)
 	return client.Services{
-		S3: m,
+		S3:        m,
+		S3Manager: mgr,
 	}
 }

client/services.go

@@ -1,7 +1,7 @@
 // We define interfaces so we can easily mock AWS calls
 package client
 
-//go:generate mockgen -destination=./mocks/services.go -package=mocks . AutoscalingClient,CloudtrailClient,CloudwatchClient,CloudwatchLogsClient,DirectconnectClient,Ec2Client,EcrClient,EfsClient,ElasticbeanstalkClient,ElbV2Client,EmrClient,FsxClient,IamClient,KmsClient,OrganizationsClient,RdsClient,S3Client,SnsClient,EksClient,RedshiftClient,EcsClient
+//go:generate mockgen -destination=./mocks/services.go -package=mocks . AutoscalingClient,CloudtrailClient,CloudwatchClient,CloudwatchLogsClient,DirectconnectClient,Ec2Client,EcrClient,EfsClient,ElasticbeanstalkClient,ElbV2Client,EmrClient,FsxClient,IamClient,KmsClient,OrganizationsClient,RdsClient,S3Client,SnsClient,EksClient,RedshiftClient,EcsClient,S3ManagerClient
 
 import (
 	"context"
@@ -161,3 +161,7 @@ type RedshiftClient interface {
 	DescribeClusters(ctx context.Context, params *redshift.DescribeClustersInput, optFns ...func(*redshift.Options)) (*redshift.DescribeClustersOutput, error)
 	DescribeClusterSubnetGroups(ctx context.Context, params *redshift.DescribeClusterSubnetGroupsInput, optFns ...func(*redshift.Options)) (*redshift.DescribeClusterSubnetGroupsOutput, error)
 }
+
+type S3ManagerClient interface {
+	GetBucketRegion(ctx context.Context, bucket string, optFns ...func(*s3.Options)) (string, error)
+}

go.mod

@@ -3,9 +3,10 @@ module github.com/cloudquery/cq-provider-aws
 go 1.15
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.3.1
-	github.com/aws/aws-sdk-go-v2/config v1.1.2
-	github.com/aws/aws-sdk-go-v2/credentials v1.1.2
+	github.com/aws/aws-sdk-go-v2 v1.3.2
+	github.com/aws/aws-sdk-go-v2/config v1.1.6
+	github.com/aws/aws-sdk-go-v2/credentials v1.1.6
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.1.3
 	github.com/aws/aws-sdk-go-v2/service/autoscaling v1.2.0
 	github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.1.2
 	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.1.2
@@ -25,10 +26,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/organizations v1.2.1
 	github.com/aws/aws-sdk-go-v2/service/rds v1.2.1
 	github.com/aws/aws-sdk-go-v2/service/redshift v1.3.0
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.4.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.5.0
 	github.com/aws/aws-sdk-go-v2/service/sns v1.1.2
-	github.com/aws/aws-sdk-go-v2/service/sts v1.1.2
-	github.com/aws/smithy-go v1.3.0
+	github.com/aws/aws-sdk-go-v2/service/sts v1.3.0
+	github.com/aws/smithy-go v1.3.1
 	github.com/cloudquery/cq-provider-sdk v0.1.6
 	github.com/cloudquery/faker/v3 v3.7.4
 	github.com/golang/mock v1.5.0

go.sum

@@ -6,12 +6,22 @@ github.com/aws/aws-sdk-go-v2 v1.2.1/go.mod h1:hTQc/9pYq5bfFACIUY9tc/2SYWd9Vnmw+t
 github.com/aws/aws-sdk-go-v2 v1.3.0/go.mod h1:hTQc/9pYq5bfFACIUY9tc/2SYWd9Vnmw+testmuQeRY=
 github.com/aws/aws-sdk-go-v2 v1.3.1 h1:KKstwh6zsuUhQH3GvSor7M3am/+imPqydFOZHzlkTKc=
 github.com/aws/aws-sdk-go-v2 v1.3.1/go.mod h1:5SmWRTjN6uTRFNCc7rR69xHsdcUJnthmaRHGDsYhpTE=
+github.com/aws/aws-sdk-go-v2 v1.3.2 h1:RQj8l98yKUm0UV2Wd3w/Ms+TXV9Rs1E6Kr5tRRMfyU4=
+github.com/aws/aws-sdk-go-v2 v1.3.2/go.mod h1:7OaACgj2SX3XGWnrIjGlJM22h6yD6MEWKvm7levnnM8=
 github.com/aws/aws-sdk-go-v2/config v1.1.2 h1:H2r6cwMvvINFpEC55Y7jcNaR/oc7zYIChrG2497wmBI=
 github.com/aws/aws-sdk-go-v2/config v1.1.2/go.mod h1:77yIk+qmCS/94JlxbwV1d+YEyu6Z8FBlCGcSz3TdM6A=
+github.com/aws/aws-sdk-go-v2/config v1.1.6 h1:tg8KyxrxDt1CrYmZXWs9lc6IFE1yxtk9kn6eS/v2fdA=
+github.com/aws/aws-sdk-go-v2/config v1.1.6/go.mod h1:Kx90DDOgkMpRfSkzGbF13AVXHHfBNct1liO+95KxXsU=
 github.com/aws/aws-sdk-go-v2/credentials v1.1.2 h1:YoNqfhxAJGZI+lStIbqgx30UcCqQ86fr7FjTLUvrFOc=
 github.com/aws/aws-sdk-go-v2/credentials v1.1.2/go.mod h1:hofjw//lM0XLplgvzPPMA7oD0doQU1QpaIK1nweEEWg=
+github.com/aws/aws-sdk-go-v2/credentials v1.1.6 h1:efaeh6FsO/jzyJ+U4ZxduKC6rRJDrUpu+Z0k5+guqHo=
+github.com/aws/aws-sdk-go-v2/credentials v1.1.6/go.mod h1:q1wQ5jHdFNhc4wnNcOEpnovs4keJA5Ds+qESCnfEsgU=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3 h1:d3bKAGy4XdJyK8hz3Nx3WJJ4TCmYp2498G4mFY5wly0=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.3/go.mod h1:Zr1Mj+KUMGVQ+WJvTT68EZJxqhjiie2PWSPGEUPaNY0=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.6 h1:zoOz5V56jO/rGixsCDnrQtAzYRYM2hGA/43U6jVMFbo=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.6/go.mod h1:0+fWMitrmIpENiY8/1DyhdYPUCAPvd9UNz9mtCsEoLQ=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.1.3 h1:uJ7ZQbDAzCb0lQ/gmk6lzMkZVHPaZdoLii6awFBHxFI=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.1.3/go.mod h1:m3OfJqvCXS+jU83bUDtBiPQODurln7id6zL3nNRQSHA=
 github.com/aws/aws-sdk-go-v2/service/autoscaling v1.2.0 h1:7dqTi+BP80MdTWZdnYDJwkl7t/4fD51+uuTPXZ0LhE0=
 github.com/aws/aws-sdk-go-v2/service/autoscaling v1.2.0/go.mod h1:F9sOgLk8RCWpkJ9DgH5LQCDqHnJzW1e28ShymxCM7Cw=
 github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.1.2 h1:TQG/DLTpnn2tOPqQ/eYA2MsDR+ALiRTEDdc02fpgTrc=
@@ -44,12 +54,18 @@ github.com/aws/aws-sdk-go-v2/service/iam v1.3.0 h1:V95YLxbxLGlTcFR0KMMSZEaudIxYC
 github.com/aws/aws-sdk-go-v2/service/iam v1.3.0/go.mod h1:gPUYT7MBEb30j9eAsJ17LN9KbXtD1uqKOOKesCC4tjc=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.3 h1:iLFz4nrWkXMTFeVn0n99wRyc4Xib4SlDbtAM3h2z8P8=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.3/go.mod h1:g3Xw4tO/W+ae4EMzkxB6nGnJ48cLM4i1Z61WmD+IKtY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.4 h1:8yeByqOL6UWBsOOXsHnW93/ukwL66O008tRfxXxnTwA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.4/go.mod h1:BCfU3Uo2fhKcMZFp9zU5QQGQxqWCOYmZ/27Dju3S/do=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.3/go.mod h1:C50Z41fJaJ7WgaeeCulOGAU3q4+4se4B3uOPFdhBi2I=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.4/go.mod h1:DGOKKGeqXdIWX3xD5DKr4otrgNw5cstwUCJYwSKxbp0=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.5 h1:GbW4bbc1iED64aIL203xcGSfLzWOWuIdnKV0guMcJvg=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.5/go.mod h1:MW0O/RpmVpS6MWKn6W03XEJmqXlG7+d3iaYLzkd2fAc=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.6 h1:ldYIsOP4WyjdzW8t6RC/aSieajrlx+3UN3UCZy1KM5Y=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.6/go.mod h1:L0KWr0ASo83PRZu9NaZaDsw3koS6PspKv137DMDZjHo=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.2.1 h1:wCzfVBrF1QRQFacZn1ywE/o2p92FzfpDNI2aCpIv+sY=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.2.1/go.mod h1:6A0VfJAnYwhcXzt7KsixOdFlITEH5NFl4QeYxlZ5TtQ=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.2.2 h1:aU8H58DoYxNo8R1TaSPTofkuxfQNnoqZmWL+G3+k/vA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.2.2/go.mod h1:nnutjMLuna0s3GVY/MAkpLX03thyNER06gXvnMAPj5g=
 github.com/aws/aws-sdk-go-v2/service/kms v1.2.1 h1:X77wgZdglt2hU9zZS3DufyJiR4ZGW4F5x72OauDMSsM=
 github.com/aws/aws-sdk-go-v2/service/kms v1.2.1/go.mod h1:VJL8/fcYPX11A7hdOPAXtzU6+yDifhKW5MgwaA6HIwY=
 github.com/aws/aws-sdk-go-v2/service/organizations v1.2.1 h1:TvDVD1mBXP60NIHrqbP8uuzTf4vu48HlOm5jtoQQcW0=
@@ -60,15 +76,23 @@ github.com/aws/aws-sdk-go-v2/service/redshift v1.3.0 h1:Eiv449szSm77R2AFB8Up0oW3
 github.com/aws/aws-sdk-go-v2/service/redshift v1.3.0/go.mod h1:FSg9SNvu62Ac+jjWjTS7R/mmtXj9zl6nXwacwDGxK0s=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.4.0 h1:045tK3IL+TxOSWWQyG199A0BYJ/Yhgk8XV9xo+nQkLQ=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.4.0/go.mod h1:zFD4go1gW0I/WxeGfCNSsz/BnZSJyu5arLPMPnw0gvQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.5.0 h1:VbwXUI3L0hyhVmrFxbDxrs6cBX8TNFX0YxCpooMNjvY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.5.0/go.mod h1:uwA7gs93Qcss43astPUb1eq4RyceNmYWAQjZFDOAMLo=
 github.com/aws/aws-sdk-go-v2/service/sns v1.1.2 h1:1U/FujyBEkNwrvANUcZFuVnAQqy0EAUEGToso5Dcijs=
 github.com/aws/aws-sdk-go-v2/service/sns v1.1.2/go.mod h1:/vvAGyo3/TG5CSrJQarIlwzjE6O/DjBIvJTRkpYkvwA=
 github.com/aws/aws-sdk-go-v2/service/sso v1.1.2 h1:9BnjX/ALn5uLo2DbgkwMpUkPL1VLQVBXcjZxqJBhf44=
 github.com/aws/aws-sdk-go-v2/service/sso v1.1.2/go.mod h1:5yU1oE3+CVYYLUsaHt2AVU3CJJZ6ER4pwsrRD1L2KSc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.1.5 h1:B7ec5wE4+3Ldkurmq0C4gfQFtElGTG+/iTpi/YPMzi4=
+github.com/aws/aws-sdk-go-v2/service/sso v1.1.5/go.mod h1:bpGz0tidC4y39sZkQSkpO/J0tzWCMXHbw6FZ0j1GkWM=
 github.com/aws/aws-sdk-go-v2/service/sts v1.1.2 h1:7Kxqov7uQeP8WUEO0iHz3j9Bh0E1rJrn6cf/OGfcDds=
 github.com/aws/aws-sdk-go-v2/service/sts v1.1.2/go.mod h1:zu7rotIY9P4Aoc6ytqLP9jeYrECDHUODB5Gbp+BSHl8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.3.0 h1:4o69U9waE25xhRbsnXa4jjQac03BFJcNfcZkSedk3e4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.3.0/go.mod h1:ssRzzJ2RZOVuKj2Vx1YE7ypfil/BIlgmQnCSW4DistU=
 github.com/aws/smithy-go v1.2.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
 github.com/aws/smithy-go v1.3.0 h1:awbB2OJBZ/Txj+c4q+qhDQs3Ob0sRhBuIIkOD4Aq8yc=
 github.com/aws/smithy-go v1.3.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
+github.com/aws/smithy-go v1.3.1 h1:xJFO4pK0y9J8fCl34uGsSJX5KNnGbdARDlA5BPhXnwE=
+github.com/aws/smithy-go v1.3.1/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudquery/cq-provider-sdk v0.1.6 h1:CCmSItk6XRT7I4KZs5ZuQWLoHUw9uLEEddMwpinCZaQ=

resources/cloudtrail_trails.go

@@ -212,9 +212,13 @@ func fetchCloudtrailTrails(ctx context.Context, meta schema.ClientMeta, parent *
 	return nil
 }
 func postCloudtrailTrailResolver(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource) error {
-	svc := meta.(*client.Client).Services().Cloudtrail
+	c := meta.(*client.Client)
+	svc := c.Services().Cloudtrail
 	r := resource.Item.(types.Trail)
-	response, err := svc.GetTrailStatus(ctx, &cloudtrail.GetTrailStatusInput{Name: r.TrailARN})
+	response, err := svc.GetTrailStatus(ctx,
+		&cloudtrail.GetTrailStatusInput{Name: r.TrailARN}, func(o *cloudtrail.Options) {
+			o.Region = c.Region
+		})
 	if err != nil {
 		return err
 	}

resources/s3_buckets.go

@@ -176,10 +176,9 @@ func resolveS3BucketsAttributes(ctx context.Context, meta schema.ClientMeta, res
 	var ae smithy.APIError
 	log := meta.Logger()
 	r := resource.Item.(types.Bucket)
-	svc := meta.(*client.Client).Services().S3
-	output, err := svc.GetBucketLocation(ctx, &s3.GetBucketLocationInput{
-		Bucket: r.Name,
-	})
+	log.Info("bucket name", r.Name)
+	mgr := meta.(*client.Client).Services().S3Manager
+	output, err := mgr.GetBucketRegion(ctx, *r.Name)
 	if err != nil {
 		if errors.As(err, &ae) && ae.ErrorCode() == "NoSuchBucket" {
 			// https://aws.amazon.com/premiumsupport/knowledge-center/s3-listing-deleted-bucket/
@@ -189,10 +188,11 @@ func resolveS3BucketsAttributes(ctx context.Context, meta schema.ClientMeta, res
 		}
 		return err
 	}
+	svc := meta.(*client.Client).Services().S3
 	bucketRegion := "us-east-1"
-	if output.LocationConstraint != "" {
+	if output != "" {
 		// This is a weird corner case by AWS API https://github.com/aws/aws-sdk-net/issues/323#issuecomment-196584538
-		bucketRegion = string(output.LocationConstraint)
+		bucketRegion = output
 	}
 	resource.Set("region", bucketRegion)