From b38e1d2c3ccbda6a06abc91480e6539f7f9d9317 Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 22 Nov 2021 14:35:02 -0500 Subject: [PATCH 01/13] intital --- client/client.go | 2 + go.mod | 10 +- go.sum | 16 +- resources/provider.go | 1 + resources/s3_accounts.go | 443 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 462 insertions(+), 10 deletions(-) create mode 100644 resources/s3_accounts.go diff --git a/client/client.go b/client/client.go index 3bf58ffd1..54d4baab3 100644 --- a/client/client.go +++ b/client/client.go @@ -46,6 +46,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/route53domains" "github.com/aws/aws-sdk-go-v2/service/s3" + s3control "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sts" @@ -377,6 +378,7 @@ func initServices(region string, c aws.Config) Services { Route53: route53.NewFromConfig(awsCfg), Route53Domains: route53domains.NewFromConfig(awsCfg), S3: s3.NewFromConfig(awsCfg), + S3Control: s3control.NewFromConfig(awsCfg), S3Manager: newS3ManagerFromConfig(awsCfg), SNS: sns.NewFromConfig(awsCfg), SQS: sqs.NewFromConfig(awsCfg), diff --git a/go.mod b/go.mod index 338cb7f30..97bf8151f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/Masterminds/squirrel v1.5.0 - github.com/aws/aws-sdk-go-v2 v1.11.0 + github.com/aws/aws-sdk-go-v2 v1.11.1 github.com/aws/aws-sdk-go-v2/config v1.3.0 github.com/aws/aws-sdk-go-v2/credentials v1.2.1 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.2.1 @@ -64,7 +64,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.3.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.1 // indirect github.com/aws/aws-sdk-go-v2/service/route53domains v1.6.0 github.com/aws/aws-sdk-go-v2/service/sso v1.2.1 // indirect github.com/creasty/defaults v1.5.2 // indirect @@ -131,7 +131,9 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) +require github.com/aws/aws-sdk-go-v2/service/s3control v1.14.1 + require ( - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6b4c61a31..cb9581bd3 100644 --- a/go.sum +++ b/go.sum @@ -135,10 +135,10 @@ github.com/aws/aws-sdk-go-v2 v1.3.2/go.mod h1:7OaACgj2SX3XGWnrIjGlJM22h6yD6MEWKv github.com/aws/aws-sdk-go-v2 v1.5.0/go.mod h1:tI4KhsR5VkzlUa2DZAdwx7wCAYGwkZZ1H31PYrBFx1w= github.com/aws/aws-sdk-go-v2 v1.6.0/go.mod h1:tI4KhsR5VkzlUa2DZAdwx7wCAYGwkZZ1H31PYrBFx1w= github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.9.1 h1:ZbovGV/qo40nrOJ4q8G33AGICzaPI45FHQWJ9650pF4= github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= -github.com/aws/aws-sdk-go-v2 v1.11.0 h1:HxyD62DyNhCfiFGUHqJ/xITD6rAjJ7Dm/2nLxLmO4Ag= github.com/aws/aws-sdk-go-v2 v1.11.0/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= +github.com/aws/aws-sdk-go-v2 v1.11.1 h1:GzvOVAdTbWxhEMRK4FfiblkGverOkAT0UodDxC1jHQM= +github.com/aws/aws-sdk-go-v2 v1.11.1/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= github.com/aws/aws-sdk-go-v2/config v1.1.5/go.mod h1:P3F1hku7qzC81txjwXnwOM6Ex6ezkU6+/557Teyb64E= github.com/aws/aws-sdk-go-v2/config v1.3.0 h1:0JAnp0WcsgKilFLiZEScUTKIvTKa2LkicadZADza+u0= github.com/aws/aws-sdk-go-v2/config v1.3.0/go.mod h1:lOxzHWDt/k7MMidA/K8DgXL4+ynnZYsDq65Qhs/l3dg= @@ -151,10 +151,12 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.1.1/go.mod h1:GTXAhrxHQOj9N+J5t github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.1.2/go.mod h1:Azf567f5wBUfUbwpyJJnLM/geFFIzEulGR30L+nQZOE= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.2.1 h1:ZZs6209e+yocx7jnT+TySOjt6/jk1LKdAPtT1fAPuio= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.2.1/go.mod h1:2JOqaBP3I6TEm27NLb11UiD9j4HZsJ+EW4N7vCf8WGQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.0 h1:zY8cNmbBXt3pzjgWgdIbzpQ6qxoCwt+Nx9JbrAf2mbY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.0/go.mod h1:NO3Q5ZTTQtO2xIg2+xTXYDiT7knSejfeDm7WGDaOo0U= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.0 h1:Z3aR/OXBnkYK9zXkNkfitHX6SmUBzSsx8VMHbH4Lvhw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.1 h1:LZwqhOyqQ2w64PZk04V0Om9AEExtW8WMkCRoE1h9/94= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.1/go.mod h1:22SEiBSQm5AyKEjoPcG1hzpeTI+m9CXfE6yt1h49wBE= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.0/go.mod h1:anlUzBoEWglcUxUQwZA7HQOEVEnQALVZsizAapB2hq8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.1 h1:ObMfGNk0xjOWduPxsrRWVwZZia3e9fOcO6zlKCkt38s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.1/go.mod h1:1xvCD+I5BcDuQUc+psZr7LI1a9pclAWZs3S3Gce5+lg= github.com/aws/aws-sdk-go-v2/internal/ini v1.0.0 h1:k7I9E6tyVWBo7H9ffpnxDWudtjau6Qt9rnOYgV+ciEQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.0.0/go.mod h1:g3XMXuxvqSMUjnsXXp/960152w0wFS4CXVYgQaSVOHE= github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.4.1 h1:rGY4jUqM06SztzHgdU56MQr2gq2w3n1ByxLH0+caXR4= @@ -214,8 +216,9 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.1.1/go.mod h1:2+e github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 h1:VNJ5NLBteVXEwE2F1zEXVmyIH58mZ6kIQGJoC7C+vkg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.2.2/go.mod h1:nnutjMLuna0s3GVY/MAkpLX03thyNER06gXvnMAPj5g= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.3.1 h1:VH1Y4k+IZ5kcRVqSNw7eAkXyfS7k2/ibKjrNtbhYhV4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.3.1/go.mod h1:IpjxfORBAFfkMM0VEx5gPPnEy6WV4Hk0F/+zb/SUWyw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.1 h1:ACJBfyfa2TxVBzwiKOdzLVdRymu6XKDXLLkfAC6rNBM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.1/go.mod h1:wnxXx7N+DjBf8mDy1qAzoSqWmpOOzCHW6hRqIUxPQEw= github.com/aws/aws-sdk-go-v2/service/kms v1.2.1 h1:X77wgZdglt2hU9zZS3DufyJiR4ZGW4F5x72OauDMSsM= github.com/aws/aws-sdk-go-v2/service/kms v1.2.1/go.mod h1:VJL8/fcYPX11A7hdOPAXtzU6+yDifhKW5MgwaA6HIwY= github.com/aws/aws-sdk-go-v2/service/lambda v1.3.0 h1:ChVmaOi+4HHvbmVrYlWzoj+4EmrrPCR8zvz2PRujlWM= @@ -235,6 +238,8 @@ github.com/aws/aws-sdk-go-v2/service/route53domains v1.6.0/go.mod h1:qPnejxOymP2 github.com/aws/aws-sdk-go-v2/service/s3 v1.5.0/go.mod h1:uwA7gs93Qcss43astPUb1eq4RyceNmYWAQjZFDOAMLo= github.com/aws/aws-sdk-go-v2/service/s3 v1.8.0 h1:rljno3viFN46b59CbjkIqYwxEAzk4naLe+djOb/exLs= github.com/aws/aws-sdk-go-v2/service/s3 v1.8.0/go.mod h1:zHCjYoODbYRLz/iFicYswq1gRoxBnHvpY5h2Vg3/tJ4= +github.com/aws/aws-sdk-go-v2/service/s3control v1.14.1 h1:Nmcb6pxJtjJof+mmF9TJvyWuSbzv7sCn5YoK3MAsPek= +github.com/aws/aws-sdk-go-v2/service/s3control v1.14.1/go.mod h1:dTnxIRqR69JUZobQDUh47rlbYe8PzTd0k4o+gDkHeV4= github.com/aws/aws-sdk-go-v2/service/sns v1.1.2 h1:1U/FujyBEkNwrvANUcZFuVnAQqy0EAUEGToso5Dcijs= github.com/aws/aws-sdk-go-v2/service/sns v1.1.2/go.mod h1:/vvAGyo3/TG5CSrJQarIlwzjE6O/DjBIvJTRkpYkvwA= github.com/aws/aws-sdk-go-v2/service/sqs v1.9.1 h1:8m+6iuSldxMrVQbjHRcWPnUxdpD3RCPtacmFFNkR4Vw= @@ -253,7 +258,6 @@ github.com/aws/smithy-go v1.2.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAm github.com/aws/smithy-go v1.3.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.3.1/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.4.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.8.0 h1:AEwwwXQZtUwP5Mz506FeXXrKBe0jA8gVM+1gEcSRooc= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.9.0 h1:c7FUdEqrQA1/UVKKCNDFQPNKGp4FQg3YW4Ck5SLTG58= github.com/aws/smithy-go v1.9.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= diff --git a/resources/provider.go b/resources/provider.go index ec12c5c2d..c7c746d18 100644 --- a/resources/provider.go +++ b/resources/provider.go @@ -102,6 +102,7 @@ func Provider() *provider.Provider { "route53.reusable_delegation_sets": Route53ReusableDelegationSets(), "route53.traffic_policies": Route53TrafficPolicies(), "s3.buckets": S3Buckets(), + "s3.accounts": S3Accounts(), "sns.subscriptions": SnsSubscriptions(), "sns.topics": SnsTopics(), "sqs.queues": SQSQueues(), diff --git a/resources/s3_accounts.go b/resources/s3_accounts.go new file mode 100644 index 000000000..1d5805ce2 --- /dev/null +++ b/resources/s3_accounts.go @@ -0,0 +1,443 @@ +package resources + +import ( + "context" + + "github.com/cloudquery/cq-provider-aws/client" + "github.com/cloudquery/cq-provider-sdk/provider/schema" +) + +func S3Accounts() *schema.Table { + return &schema.Table{ + Name: "aws_s3_accounts", + Description: "An Amazon S3 bucket is a public cloud storage resource available in Amazon Web Services' (AWS) Simple Storage Service (S3)", + Resolver: fetchS3Account, + Multiplex: client.AccountMultiplex, + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + DeleteFilter: client.DeleteAccountFilter, + PostResourceResolver: resolveS3BucketsAttributes, + Options: schema.TableCreationOptions{PrimaryKeys: []string{"account_id", "name"}}, + Columns: []schema.Column{ + { + Name: "account_id", + Description: "The AWS Account ID of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSAccount, + }, + { + Name: "region", + Description: "The AWS Region of the resource.", + Type: schema.TypeString, + }, + { + Name: "logging_target_prefix", + Type: schema.TypeString, + }, + { + Name: "logging_target_bucket", + Type: schema.TypeString, + }, + { + Name: "versioning_status", + Type: schema.TypeString, + }, + { + Name: "versioning_mfa_delete", + Type: schema.TypeString, + }, + { + Name: "policy", + Type: schema.TypeJSON, + }, + { + Name: "tags", + Type: schema.TypeJSON, + }, + { + Name: "creation_date", + Description: "Date the bucket was created", + Type: schema.TypeTimestamp, + }, + { + Name: "name", + Description: "The name of the bucket.", + Type: schema.TypeString, + }, + { + Name: "block_public_acls", + Description: "Specifies whether Amazon S3 should block public access control lists (ACLs) for this bucket and objects in this bucket", + Type: schema.TypeBool, + }, + { + Name: "block_public_policy", + Description: "Specifies whether Amazon S3 should block public bucket policies for this bucket. Setting this element to TRUE causes Amazon S3 to reject calls to PUT Bucket policy if the specified bucket policy allows public access", + Type: schema.TypeBool, + }, + { + Name: "ignore_public_acls", + Description: "Specifies whether Amazon S3 should ignore public ACLs for this bucket and objects in this bucket", + Type: schema.TypeBool, + }, + { + Name: "restrict_public_buckets", + Description: "Specifies whether Amazon S3 should restrict public bucket policies for this bucket", + Type: schema.TypeBool, + }, + { + Name: "replication_role", + Description: "The Amazon Resource Name (ARN) of the AWS Identity and Access Management (IAM) role that Amazon S3 assumes when replicating objects", + Type: schema.TypeString, + Resolver: schema.PathResolver("Role"), + }, + { + Name: "arn", + Description: "The Amazon Resource Name (ARN) for the s3 bucket", + Type: schema.TypeString, + Resolver: resolveS3BucketsArn, + }, + }, + Relations: []*schema.Table{ + { + Name: "aws_s3_bucket_grants", + Description: "Container for grant information.", + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + Resolver: fetchS3BucketGrants, + Columns: []schema.Column{ + { + Name: "bucket_cq_id", + Description: "Unique CloudQuery ID of aws_s3_buckets table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "type", + Description: "Type of grantee", + Type: schema.TypeString, + Resolver: schema.PathResolver("Grantee.Type"), + }, + { + Name: "display_name", + Description: "Screen name of the grantee.", + Type: schema.TypeString, + Resolver: schema.PathResolver("Grantee.DisplayName"), + }, + { + Name: "email_address", + Description: "Email address of the grantee", + Type: schema.TypeString, + Resolver: schema.PathResolver("Grantee.EmailAddress"), + }, + { + Name: "grantee_id", + Description: "The canonical user ID of the grantee.", + Type: schema.TypeString, + Resolver: schema.PathResolver("Grantee.ID"), + }, + { + Name: "uri", + Description: "URI of the grantee group.", + Type: schema.TypeString, + Resolver: schema.PathResolver("Grantee.URI"), + }, + { + Name: "permission", + Description: "Specifies the permission given to the grantee.", + Type: schema.TypeString, + }, + }, + }, + { + Name: "aws_s3_bucket_cors_rules", + Description: "Specifies a cross-origin access rule for an Amazon S3 bucket.", + Resolver: fetchS3BucketCorsRules, + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + Columns: []schema.Column{ + { + Name: "bucket_cq_id", + Description: "Unique CloudQuery ID of aws_s3_buckets table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "allowed_methods", + Description: "An HTTP method that you allow the origin to execute", + Type: schema.TypeStringArray, + }, + { + Name: "allowed_origins", + Description: "One or more origins you want customers to be able to access the bucket from.", + Type: schema.TypeStringArray, + }, + { + Name: "allowed_headers", + Description: "Headers that are specified in the Access-Control-Request-Headers header", + Type: schema.TypeStringArray, + }, + { + Name: "expose_headers", + Description: "One or more headers in the response that you want customers to be able to access from their applications (for example, from a JavaScript XMLHttpRequest object).", + Type: schema.TypeStringArray, + }, + { + Name: "id", + Description: "Unique identifier for the rule", + Type: schema.TypeString, + Resolver: schema.PathResolver("ID"), + }, + { + Name: "max_age_seconds", + Description: "The time in seconds that your browser is to cache the preflight response for the specified resource.", + Type: schema.TypeInt, + }, + }, + }, + { + Name: "aws_s3_bucket_encryption_rules", + Description: "Specifies the default server-side encryption configuration.", + Resolver: fetchS3BucketEncryptionRules, + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + Columns: []schema.Column{ + { + Name: "bucket_cq_id", + Description: "Unique CloudQuery ID of aws_s3_buckets table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "sse_algorithm", + Description: "Server-side encryption algorithm to use for the default encryption.", + Type: schema.TypeString, + Resolver: schema.PathResolver("ApplyServerSideEncryptionByDefault.SSEAlgorithm"), + }, + { + Name: "kms_master_key_id", + Description: "AWS Key Management Service (KMS) customer master key ID to use for the default encryption", + Type: schema.TypeString, + Resolver: schema.PathResolver("ApplyServerSideEncryptionByDefault.KMSMasterKeyID"), + }, + { + Name: "bucket_key_enabled", + Description: "Specifies whether Amazon S3 should use an S3 Bucket Key with server-side encryption using KMS (SSE-KMS) for new objects in the bucket", + Type: schema.TypeBool, + }, + }, + }, + { + Name: "aws_s3_bucket_replication_rules", + Description: "Specifies which Amazon S3 objects to replicate and where to store the replicas.", + Resolver: fetchS3BucketReplicationRules, + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + Options: schema.TableCreationOptions{PrimaryKeys: []string{"bucket_cq_id", "id"}}, + Columns: []schema.Column{ + { + Name: "bucket_cq_id", + Description: "Unique CloudQuery ID of aws_s3_buckets table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "destination_bucket", + Description: "The Amazon Resource Name (ARN) of the bucket where you want Amazon S3 to store the results.", + Type: schema.TypeString, + Resolver: schema.PathResolver("Destination.Bucket"), + }, + { + Name: "destination_access_control_translation_owner", + Description: "Specifies the replica ownership", + Type: schema.TypeString, + Resolver: schema.PathResolver("Destination.AccessControlTranslation.Owner"), + }, + { + Name: "destination_account", + Description: "Destination bucket owner account ID", + Type: schema.TypeString, + Resolver: schema.PathResolver("Destination.Account"), + }, + { + Name: "destination_encryption_configuration_replica_kms_key_id", + Description: "Specifies the ID (Key ARN or Alias ARN) of the customer managed customer master key (CMK) stored in AWS Key Management Service (KMS) for the destination bucket. Amazon S3 uses this key to encrypt replica objects", + Type: schema.TypeString, + Resolver: schema.PathResolver("Destination.EncryptionConfiguration.ReplicaKmsKeyID"), + }, + { + Name: "destination_metrics_status", + Description: "Specifies whether the replication metrics are enabled.", + Type: schema.TypeString, + Resolver: schema.PathResolver("Destination.Metrics.Status"), + }, + { + Name: "destination_metrics_event_threshold_minutes", + Description: "Contains an integer specifying time in minutes", + Type: schema.TypeInt, + Resolver: schema.PathResolver("Destination.Metrics.EventThreshold.Minutes"), + }, + { + Name: "destination_replication_time_status", + Description: "Specifies whether the replication time is enabled.", + Type: schema.TypeString, + Resolver: schema.PathResolver("Destination.ReplicationTime.Status"), + }, + { + Name: "destination_replication_time_minutes", + Description: "Contains an integer specifying time in minutes", + Type: schema.TypeInt, + Resolver: schema.PathResolver("Destination.ReplicationTime.Time.Minutes"), + }, + { + Name: "destination_storage_class", + Description: "The storage class to use when replicating objects, such as S3 Standard or reduced redundancy", + Type: schema.TypeString, + Resolver: schema.PathResolver("Destination.StorageClass"), + }, + { + Name: "status", + Description: "Specifies whether the rule is enabled.", + Type: schema.TypeString, + }, + { + Name: "delete_marker_replication_status", + Description: "Indicates whether to replicate delete markers", + Type: schema.TypeString, + Resolver: schema.PathResolver("DeleteMarkerReplication.Status"), + }, + { + Name: "existing_object_replication_status", + Type: schema.TypeString, + Resolver: schema.PathResolver("ExistingObjectReplication.Status"), + }, + { + Name: "filter", + Description: "A filter that identifies the subset of objects to which the replication rule applies", + Type: schema.TypeJSON, + Resolver: resolveS3BucketReplicationRuleFilter, + }, + { + Name: "id", + Description: "A unique identifier for the rule", + Type: schema.TypeString, + Resolver: schema.PathResolver("ID"), + }, + { + Name: "prefix", + Description: "An object key name prefix that identifies the object or objects to which the rule applies", + Type: schema.TypeString, + }, + { + Name: "priority", + Description: "The priority indicates which rule has precedence whenever two or more replication rules conflict", + Type: schema.TypeInt, + }, + { + Name: "source_replica_modifications_status", + Description: "Specifies whether Amazon S3 replicates modifications on replicas.", + Type: schema.TypeString, + Resolver: schema.PathResolver("SourceSelectionCriteria.ReplicaModifications.Status"), + }, + { + Name: "source_sse_kms_encrypted_objects_status", + Description: "Specifies whether Amazon S3 replicates objects created with server-side encryption using a customer master key (CMK) stored in AWS Key Management Service.", + Type: schema.TypeString, + Resolver: schema.PathResolver("SourceSelectionCriteria.SseKmsEncryptedObjects.Status"), + }, + }, + }, + { + Name: "aws_s3_bucket_lifecycles", + Description: "A lifecycle rule for individual objects in an Amazon S3 bucket.", + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + Resolver: fetchS3BucketLifecycles, + Options: schema.TableCreationOptions{PrimaryKeys: []string{"bucket_cq_id", "id"}}, + Columns: []schema.Column{ + { + Name: "bucket_cq_id", + Description: "Unique CloudQuery ID of aws_s3_buckets table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "status", + Description: "If 'Enabled', the rule is currently being applied", + Type: schema.TypeString, + }, + { + Name: "abort_incomplete_multipart_upload_days_after_initiation", + Description: "Specifies the number of days after which Amazon S3 aborts an incomplete multipart upload.", + Type: schema.TypeInt, + Resolver: schema.PathResolver("AbortIncompleteMultipartUpload.DaysAfterInitiation"), + }, + { + Name: "expiration_date", + Description: "Indicates at what date the object is to be moved or deleted", + Type: schema.TypeTimestamp, + Resolver: schema.PathResolver("Expiration.Date"), + }, + { + Name: "expiration_days", + Description: "Indicates the lifetime, in days, of the objects that are subject to the rule. The value must be a non-zero positive integer.", + Type: schema.TypeInt, + Resolver: schema.PathResolver("Expiration.Days"), + }, + { + Name: "expiration_expired_object_delete_marker", + Description: "Indicates whether Amazon S3 will remove a delete marker with no noncurrent versions", + Type: schema.TypeBool, + Resolver: schema.PathResolver("Expiration.ExpiredObjectDeleteMarker"), + }, + { + Name: "filter", + Description: "The Filter is used to identify objects that a Lifecycle Rule applies to", + Type: schema.TypeJSON, + Resolver: resolveS3BucketLifecycleFilter, + }, + { + Name: "id", + Description: "Unique identifier for the rule", + Type: schema.TypeString, + Resolver: schema.PathResolver("ID"), + }, + { + Name: "noncurrent_version_expiration_days", + Description: "Specifies the number of days an object is noncurrent before Amazon S3 can perform the associated action", + Type: schema.TypeInt, + Resolver: schema.PathResolver("NoncurrentVersionExpiration.NoncurrentDays"), + }, + { + Name: "noncurrent_version_transitions", + Description: "Specifies the transition rule for the lifecycle rule that describes when noncurrent objects transition to a specific storage class", + Type: schema.TypeJSON, + Resolver: resolveS3BucketLifecycleNoncurrentVersionTransitions, + }, + { + Name: "prefix", + Description: "Prefix identifying one or more objects to which the rule applies", + Type: schema.TypeString, + }, + { + Name: "transitions", + Description: "Specifies when an Amazon S3 object transitions to a specified storage class.", + Type: schema.TypeJSON, + Resolver: resolveS3BucketLifecycleTransitions, + }, + }, + }, + }, + } +} + +// ==================================================================================================================== +// Table Resolver Functions +// ==================================================================================================================== +func fetchS3Account(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan interface{}) error { + svc := meta.(*client.Client).Services().S3 + response, err := svc.ListBuckets(ctx, nil) + if err != nil { + return err + } + wb := make([]*WrappedBucket, len(response.Buckets)) + for i, b := range response.Buckets { + wb[i] = &WrappedBucket{b, nil, nil} + } + + res <- wb + return nil +} From 47a552d16e40384860826d989c47c3cb78695d93 Mon Sep 17 00:00:00 2001 From: bbernays Date: Mon, 22 Nov 2021 18:23:14 -0500 Subject: [PATCH 02/13] Add Support for S3 Account Settings --- client/client.go | 2 +- client/services.go | 6 + go.mod | 7 +- go.sum | 11 +- resources/s3_accounts.go | 457 +++++---------------------------------- 5 files changed, 77 insertions(+), 406 deletions(-) diff --git a/client/client.go b/client/client.go index 54d4baab3..8566eb96a 100644 --- a/client/client.go +++ b/client/client.go @@ -120,6 +120,7 @@ type Services struct { Route53Domains Route53DomainsClient RDS RdsClient S3 S3Client + S3Control S3ControlClient S3Manager S3ManagerClient SQS SQSClient Apigateway ApigatewayClient @@ -129,7 +130,6 @@ type Services struct { Waf WafClient WafV2 WafV2Client } - type ServicesAccountRegionMap map[string]map[string]*Services // ServicesManager will hold the entire map of (account X region) services diff --git a/client/services.go b/client/services.go index 16e45481e..d4240a38f 100644 --- a/client/services.go +++ b/client/services.go @@ -36,6 +36,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/route53" "github.com/aws/aws-sdk-go-v2/service/route53domains" "github.com/aws/aws-sdk-go-v2/service/s3" + s3control "github.com/aws/aws-sdk-go-v2/service/s3control" "github.com/aws/aws-sdk-go-v2/service/sns" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/waf" @@ -282,6 +283,11 @@ type RdsClient interface { DescribeCertificates(ctx context.Context, params *rds.DescribeCertificatesInput, optFns ...func(*rds.Options)) (*rds.DescribeCertificatesOutput, error) } +//go:generate mockgen -package=mocks -destination=./mocks/mock_s3Control.go . S3ControlClient +type S3ControlClient interface { + GetPublicAccessBlock(ctx context.Context, params *s3control.GetPublicAccessBlockInput, optFns ...func(*s3control.Options)) (*s3control.GetPublicAccessBlockOutput, error) +} + //go:generate mockgen -package=mocks -destination=./mocks/mock_s3.go . S3Client type S3Client interface { ListBuckets(ctx context.Context, params *s3.ListBucketsInput, optFns ...func(*s3.Options)) (*s3.ListBucketsOutput, error) diff --git a/go.mod b/go.mod index 97bf8151f..914ee2309 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/rds v1.2.1 github.com/aws/aws-sdk-go-v2/service/redshift v1.3.0 github.com/aws/aws-sdk-go-v2/service/route53 v1.4.0 - github.com/aws/aws-sdk-go-v2/service/s3 v1.8.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.19.1 github.com/aws/aws-sdk-go-v2/service/sns v1.1.2 github.com/aws/aws-sdk-go-v2/service/sqs v1.9.1 github.com/aws/aws-sdk-go-v2/service/sts v1.4.1 @@ -62,8 +62,8 @@ require ( github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.1.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.1 // indirect github.com/aws/aws-sdk-go-v2/service/route53domains v1.6.0 github.com/aws/aws-sdk-go-v2/service/sso v1.2.1 // indirect @@ -134,6 +134,7 @@ require ( require github.com/aws/aws-sdk-go-v2/service/s3control v1.14.1 require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.1 // indirect ) diff --git a/go.sum b/go.sum index cb9581bd3..d3f135708 100644 --- a/go.sum +++ b/go.sum @@ -139,6 +139,8 @@ github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVj github.com/aws/aws-sdk-go-v2 v1.11.0/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= github.com/aws/aws-sdk-go-v2 v1.11.1 h1:GzvOVAdTbWxhEMRK4FfiblkGverOkAT0UodDxC1jHQM= github.com/aws/aws-sdk-go-v2 v1.11.1/go.mod h1:SQfA+m2ltnu1cA0soUkj4dRSsmITiVQUJvBIZjzfPyQ= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 h1:yVUAwvJC/0WNPbyl0nA3j1L6CW1CN8wBubCRqtG7JLI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0/go.mod h1:Xn6sxgRuIDflLRJFj5Ev7UxABIkNbccFPV/p8itDReM= github.com/aws/aws-sdk-go-v2/config v1.1.5/go.mod h1:P3F1hku7qzC81txjwXnwOM6Ex6ezkU6+/557Teyb64E= github.com/aws/aws-sdk-go-v2/config v1.3.0 h1:0JAnp0WcsgKilFLiZEScUTKIvTKa2LkicadZADza+u0= github.com/aws/aws-sdk-go-v2/config v1.3.0/go.mod h1:lOxzHWDt/k7MMidA/K8DgXL4+ynnZYsDq65Qhs/l3dg= @@ -208,13 +210,15 @@ github.com/aws/aws-sdk-go-v2/service/fsx v1.2.0/go.mod h1:BFqIiyAdn9j1EaiCN418uB github.com/aws/aws-sdk-go-v2/service/iam v1.3.0 h1:V95YLxbxLGlTcFR0KMMSZEaudIxYCAhycSGcO7/Favs= github.com/aws/aws-sdk-go-v2/service/iam v1.3.0/go.mod h1:gPUYT7MBEb30j9eAsJ17LN9KbXtD1uqKOOKesCC4tjc= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.0.4/go.mod h1:BCfU3Uo2fhKcMZFp9zU5QQGQxqWCOYmZ/27Dju3S/do= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.1.0 h1:XwqxIO9LtNXznBbEMNGumtLN60k4nVqDpVwVWx3XU/o= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.1.0/go.mod h1:zdjOOy0ojUn3iNELo6ycIHSMCp4xUbycSHfb8PnbbyM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.5.0 h1:lPLbw4Gn59uoKqvOfSnkJr54XWk5Ak1NK20ZEiSWb3U= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.5.0/go.mod h1:80NaCIH9YU3rzTTs/J/ECATjXuRqzo/wB6ukO6MZ0XY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.5/go.mod h1:MW0O/RpmVpS6MWKn6W03XEJmqXlG7+d3iaYLzkd2fAc= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.6/go.mod h1:L0KWr0ASo83PRZu9NaZaDsw3koS6PspKv137DMDZjHo= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.1.1/go.mod h1:2+ehJPkdIdl46VCj67Emz/EH2hpebHZtaLdzqg+sWOI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 h1:VNJ5NLBteVXEwE2F1zEXVmyIH58mZ6kIQGJoC7C+vkg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.1 h1:ZFSfgetO5kf4WXy+a2B8zug6DXGUYjsWacyvwx5cgXU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.1/go.mod h1:fEaHB2bi+wVZw4uKMHEXTL9LwtT4EL//DOhTeflqIVo= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.2.2/go.mod h1:nnutjMLuna0s3GVY/MAkpLX03thyNER06gXvnMAPj5g= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.3.1/go.mod h1:IpjxfORBAFfkMM0VEx5gPPnEy6WV4Hk0F/+zb/SUWyw= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.1 h1:ACJBfyfa2TxVBzwiKOdzLVdRymu6XKDXLLkfAC6rNBM= @@ -236,8 +240,9 @@ github.com/aws/aws-sdk-go-v2/service/route53 v1.4.0/go.mod h1:BNPIchdQA/UcqNmWkP github.com/aws/aws-sdk-go-v2/service/route53domains v1.6.0 h1:weG1nlBo27e/jz3urXdLFJ4z1mkqa/K+eAfRy8Z9kRw= github.com/aws/aws-sdk-go-v2/service/route53domains v1.6.0/go.mod h1:qPnejxOymP2/tcqFuYAWJyaeCgSuEjahjXT5s/2bteI= github.com/aws/aws-sdk-go-v2/service/s3 v1.5.0/go.mod h1:uwA7gs93Qcss43astPUb1eq4RyceNmYWAQjZFDOAMLo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.8.0 h1:rljno3viFN46b59CbjkIqYwxEAzk4naLe+djOb/exLs= github.com/aws/aws-sdk-go-v2/service/s3 v1.8.0/go.mod h1:zHCjYoODbYRLz/iFicYswq1gRoxBnHvpY5h2Vg3/tJ4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.19.1 h1:v7n7a2v9fN+We4Jna/u7+35Fhch5YDgtxjglRBNjYh4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.19.1/go.mod h1:wcAYHjbvrLxDNWJmwCgwxudlHIkSLyU2m4Q1tWO6QZw= github.com/aws/aws-sdk-go-v2/service/s3control v1.14.1 h1:Nmcb6pxJtjJof+mmF9TJvyWuSbzv7sCn5YoK3MAsPek= github.com/aws/aws-sdk-go-v2/service/s3control v1.14.1/go.mod h1:dTnxIRqR69JUZobQDUh47rlbYe8PzTd0k4o+gDkHeV4= github.com/aws/aws-sdk-go-v2/service/sns v1.1.2 h1:1U/FujyBEkNwrvANUcZFuVnAQqy0EAUEGToso5Dcijs= diff --git a/resources/s3_accounts.go b/resources/s3_accounts.go index 1d5805ce2..4405a71cf 100644 --- a/resources/s3_accounts.go +++ b/resources/s3_accounts.go @@ -2,442 +2,101 @@ package resources import ( "context" + "errors" + aws "github.com/aws/aws-sdk-go-v2/aws" + s3control "github.com/aws/aws-sdk-go-v2/service/s3control" + s3controlTypes "github.com/aws/aws-sdk-go-v2/service/s3control/types" + "github.com/aws/smithy-go" "github.com/cloudquery/cq-provider-aws/client" "github.com/cloudquery/cq-provider-sdk/provider/schema" ) func S3Accounts() *schema.Table { return &schema.Table{ - Name: "aws_s3_accounts", - Description: "An Amazon S3 bucket is a public cloud storage resource available in Amazon Web Services' (AWS) Simple Storage Service (S3)", - Resolver: fetchS3Account, - Multiplex: client.AccountMultiplex, - IgnoreError: client.IgnoreAccessDeniedServiceDisabled, - DeleteFilter: client.DeleteAccountFilter, - PostResourceResolver: resolveS3BucketsAttributes, - Options: schema.TableCreationOptions{PrimaryKeys: []string{"account_id", "name"}}, + Name: "aws_s3_accounts", + Description: "TODO-FILL-THIS-IN", + Resolver: fetchS3AccountConfig, + Multiplex: client.AccountMultiplex, + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + DeleteFilter: client.DeleteAccountFilter, + Options: schema.TableCreationOptions{PrimaryKeys: []string{"account_id"}}, Columns: []schema.Column{ { - Name: "account_id", - Description: "The AWS Account ID of the resource.", - Type: schema.TypeString, - Resolver: client.ResolveAWSAccount, + Name: "account_id", + Type: schema.TypeString, + Resolver: client.ResolveAWSAccount, }, { - Name: "region", - Description: "The AWS Region of the resource.", - Type: schema.TypeString, + Name: "region", + Type: schema.TypeString, + Resolver: client.ResolveAWSRegion, }, { - Name: "logging_target_prefix", - Type: schema.TypeString, - }, - { - Name: "logging_target_bucket", - Type: schema.TypeString, - }, - { - Name: "versioning_status", - Type: schema.TypeString, - }, - { - Name: "versioning_mfa_delete", - Type: schema.TypeString, - }, - { - Name: "policy", - Type: schema.TypeJSON, - }, - { - Name: "tags", - Type: schema.TypeJSON, - }, - { - Name: "creation_date", - Description: "Date the bucket was created", - Type: schema.TypeTimestamp, - }, - { - Name: "name", - Description: "The name of the bucket.", - Type: schema.TypeString, + Name: "config_exists", + Type: schema.TypeBool, + Description: "Specifies whether Amazon S3 access control exists", }, { Name: "block_public_acls", - Description: "Specifies whether Amazon S3 should block public access control lists (ACLs) for this bucket and objects in this bucket", Type: schema.TypeBool, + Description: "Specifies whether Amazon S3 should block public access control lists (ACLs) for buckets in this account", }, { Name: "block_public_policy", - Description: "Specifies whether Amazon S3 should block public bucket policies for this bucket. Setting this element to TRUE causes Amazon S3 to reject calls to PUT Bucket policy if the specified bucket policy allows public access", Type: schema.TypeBool, + Description: "Specifies whether Amazon S3 should block public bucket policies for buckets in this account.", }, + { Name: "ignore_public_acls", - Description: "Specifies whether Amazon S3 should ignore public ACLs for this bucket and objects in this bucket", Type: schema.TypeBool, + Description: "Specifies whether Amazon S3 should ignore public ACLs for buckets in this account", }, { Name: "restrict_public_buckets", - Description: "Specifies whether Amazon S3 should restrict public bucket policies for this bucket", Type: schema.TypeBool, - }, - { - Name: "replication_role", - Description: "The Amazon Resource Name (ARN) of the AWS Identity and Access Management (IAM) role that Amazon S3 assumes when replicating objects", - Type: schema.TypeString, - Resolver: schema.PathResolver("Role"), - }, - { - Name: "arn", - Description: "The Amazon Resource Name (ARN) for the s3 bucket", - Type: schema.TypeString, - Resolver: resolveS3BucketsArn, - }, - }, - Relations: []*schema.Table{ - { - Name: "aws_s3_bucket_grants", - Description: "Container for grant information.", - IgnoreError: client.IgnoreAccessDeniedServiceDisabled, - Resolver: fetchS3BucketGrants, - Columns: []schema.Column{ - { - Name: "bucket_cq_id", - Description: "Unique CloudQuery ID of aws_s3_buckets table (FK)", - Type: schema.TypeUUID, - Resolver: schema.ParentIdResolver, - }, - { - Name: "type", - Description: "Type of grantee", - Type: schema.TypeString, - Resolver: schema.PathResolver("Grantee.Type"), - }, - { - Name: "display_name", - Description: "Screen name of the grantee.", - Type: schema.TypeString, - Resolver: schema.PathResolver("Grantee.DisplayName"), - }, - { - Name: "email_address", - Description: "Email address of the grantee", - Type: schema.TypeString, - Resolver: schema.PathResolver("Grantee.EmailAddress"), - }, - { - Name: "grantee_id", - Description: "The canonical user ID of the grantee.", - Type: schema.TypeString, - Resolver: schema.PathResolver("Grantee.ID"), - }, - { - Name: "uri", - Description: "URI of the grantee group.", - Type: schema.TypeString, - Resolver: schema.PathResolver("Grantee.URI"), - }, - { - Name: "permission", - Description: "Specifies the permission given to the grantee.", - Type: schema.TypeString, - }, - }, - }, - { - Name: "aws_s3_bucket_cors_rules", - Description: "Specifies a cross-origin access rule for an Amazon S3 bucket.", - Resolver: fetchS3BucketCorsRules, - IgnoreError: client.IgnoreAccessDeniedServiceDisabled, - Columns: []schema.Column{ - { - Name: "bucket_cq_id", - Description: "Unique CloudQuery ID of aws_s3_buckets table (FK)", - Type: schema.TypeUUID, - Resolver: schema.ParentIdResolver, - }, - { - Name: "allowed_methods", - Description: "An HTTP method that you allow the origin to execute", - Type: schema.TypeStringArray, - }, - { - Name: "allowed_origins", - Description: "One or more origins you want customers to be able to access the bucket from.", - Type: schema.TypeStringArray, - }, - { - Name: "allowed_headers", - Description: "Headers that are specified in the Access-Control-Request-Headers header", - Type: schema.TypeStringArray, - }, - { - Name: "expose_headers", - Description: "One or more headers in the response that you want customers to be able to access from their applications (for example, from a JavaScript XMLHttpRequest object).", - Type: schema.TypeStringArray, - }, - { - Name: "id", - Description: "Unique identifier for the rule", - Type: schema.TypeString, - Resolver: schema.PathResolver("ID"), - }, - { - Name: "max_age_seconds", - Description: "The time in seconds that your browser is to cache the preflight response for the specified resource.", - Type: schema.TypeInt, - }, - }, - }, - { - Name: "aws_s3_bucket_encryption_rules", - Description: "Specifies the default server-side encryption configuration.", - Resolver: fetchS3BucketEncryptionRules, - IgnoreError: client.IgnoreAccessDeniedServiceDisabled, - Columns: []schema.Column{ - { - Name: "bucket_cq_id", - Description: "Unique CloudQuery ID of aws_s3_buckets table (FK)", - Type: schema.TypeUUID, - Resolver: schema.ParentIdResolver, - }, - { - Name: "sse_algorithm", - Description: "Server-side encryption algorithm to use for the default encryption.", - Type: schema.TypeString, - Resolver: schema.PathResolver("ApplyServerSideEncryptionByDefault.SSEAlgorithm"), - }, - { - Name: "kms_master_key_id", - Description: "AWS Key Management Service (KMS) customer master key ID to use for the default encryption", - Type: schema.TypeString, - Resolver: schema.PathResolver("ApplyServerSideEncryptionByDefault.KMSMasterKeyID"), - }, - { - Name: "bucket_key_enabled", - Description: "Specifies whether Amazon S3 should use an S3 Bucket Key with server-side encryption using KMS (SSE-KMS) for new objects in the bucket", - Type: schema.TypeBool, - }, - }, - }, - { - Name: "aws_s3_bucket_replication_rules", - Description: "Specifies which Amazon S3 objects to replicate and where to store the replicas.", - Resolver: fetchS3BucketReplicationRules, - IgnoreError: client.IgnoreAccessDeniedServiceDisabled, - Options: schema.TableCreationOptions{PrimaryKeys: []string{"bucket_cq_id", "id"}}, - Columns: []schema.Column{ - { - Name: "bucket_cq_id", - Description: "Unique CloudQuery ID of aws_s3_buckets table (FK)", - Type: schema.TypeUUID, - Resolver: schema.ParentIdResolver, - }, - { - Name: "destination_bucket", - Description: "The Amazon Resource Name (ARN) of the bucket where you want Amazon S3 to store the results.", - Type: schema.TypeString, - Resolver: schema.PathResolver("Destination.Bucket"), - }, - { - Name: "destination_access_control_translation_owner", - Description: "Specifies the replica ownership", - Type: schema.TypeString, - Resolver: schema.PathResolver("Destination.AccessControlTranslation.Owner"), - }, - { - Name: "destination_account", - Description: "Destination bucket owner account ID", - Type: schema.TypeString, - Resolver: schema.PathResolver("Destination.Account"), - }, - { - Name: "destination_encryption_configuration_replica_kms_key_id", - Description: "Specifies the ID (Key ARN or Alias ARN) of the customer managed customer master key (CMK) stored in AWS Key Management Service (KMS) for the destination bucket. Amazon S3 uses this key to encrypt replica objects", - Type: schema.TypeString, - Resolver: schema.PathResolver("Destination.EncryptionConfiguration.ReplicaKmsKeyID"), - }, - { - Name: "destination_metrics_status", - Description: "Specifies whether the replication metrics are enabled.", - Type: schema.TypeString, - Resolver: schema.PathResolver("Destination.Metrics.Status"), - }, - { - Name: "destination_metrics_event_threshold_minutes", - Description: "Contains an integer specifying time in minutes", - Type: schema.TypeInt, - Resolver: schema.PathResolver("Destination.Metrics.EventThreshold.Minutes"), - }, - { - Name: "destination_replication_time_status", - Description: "Specifies whether the replication time is enabled.", - Type: schema.TypeString, - Resolver: schema.PathResolver("Destination.ReplicationTime.Status"), - }, - { - Name: "destination_replication_time_minutes", - Description: "Contains an integer specifying time in minutes", - Type: schema.TypeInt, - Resolver: schema.PathResolver("Destination.ReplicationTime.Time.Minutes"), - }, - { - Name: "destination_storage_class", - Description: "The storage class to use when replicating objects, such as S3 Standard or reduced redundancy", - Type: schema.TypeString, - Resolver: schema.PathResolver("Destination.StorageClass"), - }, - { - Name: "status", - Description: "Specifies whether the rule is enabled.", - Type: schema.TypeString, - }, - { - Name: "delete_marker_replication_status", - Description: "Indicates whether to replicate delete markers", - Type: schema.TypeString, - Resolver: schema.PathResolver("DeleteMarkerReplication.Status"), - }, - { - Name: "existing_object_replication_status", - Type: schema.TypeString, - Resolver: schema.PathResolver("ExistingObjectReplication.Status"), - }, - { - Name: "filter", - Description: "A filter that identifies the subset of objects to which the replication rule applies", - Type: schema.TypeJSON, - Resolver: resolveS3BucketReplicationRuleFilter, - }, - { - Name: "id", - Description: "A unique identifier for the rule", - Type: schema.TypeString, - Resolver: schema.PathResolver("ID"), - }, - { - Name: "prefix", - Description: "An object key name prefix that identifies the object or objects to which the rule applies", - Type: schema.TypeString, - }, - { - Name: "priority", - Description: "The priority indicates which rule has precedence whenever two or more replication rules conflict", - Type: schema.TypeInt, - }, - { - Name: "source_replica_modifications_status", - Description: "Specifies whether Amazon S3 replicates modifications on replicas.", - Type: schema.TypeString, - Resolver: schema.PathResolver("SourceSelectionCriteria.ReplicaModifications.Status"), - }, - { - Name: "source_sse_kms_encrypted_objects_status", - Description: "Specifies whether Amazon S3 replicates objects created with server-side encryption using a customer master key (CMK) stored in AWS Key Management Service.", - Type: schema.TypeString, - Resolver: schema.PathResolver("SourceSelectionCriteria.SseKmsEncryptedObjects.Status"), - }, - }, - }, - { - Name: "aws_s3_bucket_lifecycles", - Description: "A lifecycle rule for individual objects in an Amazon S3 bucket.", - IgnoreError: client.IgnoreAccessDeniedServiceDisabled, - Resolver: fetchS3BucketLifecycles, - Options: schema.TableCreationOptions{PrimaryKeys: []string{"bucket_cq_id", "id"}}, - Columns: []schema.Column{ - { - Name: "bucket_cq_id", - Description: "Unique CloudQuery ID of aws_s3_buckets table (FK)", - Type: schema.TypeUUID, - Resolver: schema.ParentIdResolver, - }, - { - Name: "status", - Description: "If 'Enabled', the rule is currently being applied", - Type: schema.TypeString, - }, - { - Name: "abort_incomplete_multipart_upload_days_after_initiation", - Description: "Specifies the number of days after which Amazon S3 aborts an incomplete multipart upload.", - Type: schema.TypeInt, - Resolver: schema.PathResolver("AbortIncompleteMultipartUpload.DaysAfterInitiation"), - }, - { - Name: "expiration_date", - Description: "Indicates at what date the object is to be moved or deleted", - Type: schema.TypeTimestamp, - Resolver: schema.PathResolver("Expiration.Date"), - }, - { - Name: "expiration_days", - Description: "Indicates the lifetime, in days, of the objects that are subject to the rule. The value must be a non-zero positive integer.", - Type: schema.TypeInt, - Resolver: schema.PathResolver("Expiration.Days"), - }, - { - Name: "expiration_expired_object_delete_marker", - Description: "Indicates whether Amazon S3 will remove a delete marker with no noncurrent versions", - Type: schema.TypeBool, - Resolver: schema.PathResolver("Expiration.ExpiredObjectDeleteMarker"), - }, - { - Name: "filter", - Description: "The Filter is used to identify objects that a Lifecycle Rule applies to", - Type: schema.TypeJSON, - Resolver: resolveS3BucketLifecycleFilter, - }, - { - Name: "id", - Description: "Unique identifier for the rule", - Type: schema.TypeString, - Resolver: schema.PathResolver("ID"), - }, - { - Name: "noncurrent_version_expiration_days", - Description: "Specifies the number of days an object is noncurrent before Amazon S3 can perform the associated action", - Type: schema.TypeInt, - Resolver: schema.PathResolver("NoncurrentVersionExpiration.NoncurrentDays"), - }, - { - Name: "noncurrent_version_transitions", - Description: "Specifies the transition rule for the lifecycle rule that describes when noncurrent objects transition to a specific storage class", - Type: schema.TypeJSON, - Resolver: resolveS3BucketLifecycleNoncurrentVersionTransitions, - }, - { - Name: "prefix", - Description: "Prefix identifying one or more objects to which the rule applies", - Type: schema.TypeString, - }, - { - Name: "transitions", - Description: "Specifies when an Amazon S3 object transitions to a specified storage class.", - Type: schema.TypeJSON, - Resolver: resolveS3BucketLifecycleTransitions, - }, - }, + Description: "Specifies whether Amazon S3 should restrict public bucket policies for buckets in this account.", }, }, } } -// ==================================================================================================================== -// Table Resolver Functions -// ==================================================================================================================== -func fetchS3Account(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan interface{}) error { - svc := meta.(*client.Client).Services().S3 - response, err := svc.ListBuckets(ctx, nil) +func fetchS3AccountConfig(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan interface{}) error { + var s3AccountConfig S3AccountConfig + c := meta.(*client.Client) + + svc := c.Services().S3Control + var accountConfig s3control.GetPublicAccessBlockInput + accountConfig.AccountId = aws.String(c.AccountID) + s3AccountConfig.ConfigExists = false + resp, err := svc.GetPublicAccessBlock(ctx, &accountConfig, func(options *s3control.Options) { + options.Region = c.Region + }) if err != nil { - return err + // If we received any error other than NoSuchPublicAccessBlockConfiguration, we return and error + var ae smithy.APIError + if errors.As(err, &ae) && ae.ErrorCode() != "NoSuchPublicAccessBlockConfiguration" { + return err + } } - wb := make([]*WrappedBucket, len(response.Buckets)) - for i, b := range response.Buckets { - wb[i] = &WrappedBucket{b, nil, nil} + if resp != nil { + s3AccountConfig.BlockPublicAcls = resp.PublicAccessBlockConfiguration.BlockPublicAcls + s3AccountConfig.BlockPublicPolicy = resp.PublicAccessBlockConfiguration.BlockPublicPolicy + s3AccountConfig.IgnorePublicAcls = resp.PublicAccessBlockConfiguration.IgnorePublicAcls + s3AccountConfig.RestrictPublicBuckets = resp.PublicAccessBlockConfiguration.RestrictPublicBuckets + } else { + s3AccountConfig.BlockPublicAcls = false + s3AccountConfig.BlockPublicPolicy = false + s3AccountConfig.IgnorePublicAcls = false + s3AccountConfig.RestrictPublicBuckets = false } - res <- wb + res <- s3AccountConfig return nil } + +type S3AccountConfig struct { + s3controlTypes.PublicAccessBlockConfiguration + ConfigExists bool +} From 23292efeeb3d3a8278dd70ea71e53ca24aea881b Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 06:04:20 -0500 Subject: [PATCH 03/13] Update resources/s3_accounts.go Co-authored-by: Ron <38083777+roneli@users.noreply.github.com> --- resources/s3_accounts.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/resources/s3_accounts.go b/resources/s3_accounts.go index 4405a71cf..2a1299448 100644 --- a/resources/s3_accounts.go +++ b/resources/s3_accounts.go @@ -81,16 +81,8 @@ func fetchS3AccountConfig(ctx context.Context, meta schema.ClientMeta, _ *schema } } if resp != nil { - s3AccountConfig.BlockPublicAcls = resp.PublicAccessBlockConfiguration.BlockPublicAcls - s3AccountConfig.BlockPublicPolicy = resp.PublicAccessBlockConfiguration.BlockPublicPolicy - s3AccountConfig.IgnorePublicAcls = resp.PublicAccessBlockConfiguration.IgnorePublicAcls - s3AccountConfig.RestrictPublicBuckets = resp.PublicAccessBlockConfiguration.RestrictPublicBuckets - } else { - s3AccountConfig.BlockPublicAcls = false - s3AccountConfig.BlockPublicPolicy = false - s3AccountConfig.IgnorePublicAcls = false - s3AccountConfig.RestrictPublicBuckets = false - } + res <- s3AccountConfig{resp.PublicAccessBlockConfiguration, true} + } res <- s3AccountConfig return nil From 6cce91f6942cab2046e3442152e969cf50fd94d6 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 06:21:48 -0500 Subject: [PATCH 04/13] Update s3_accounts.go --- resources/s3_accounts.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/resources/s3_accounts.go b/resources/s3_accounts.go index 2a1299448..220023c37 100644 --- a/resources/s3_accounts.go +++ b/resources/s3_accounts.go @@ -81,10 +81,11 @@ func fetchS3AccountConfig(ctx context.Context, meta schema.ClientMeta, _ *schema } } if resp != nil { - res <- s3AccountConfig{resp.PublicAccessBlockConfiguration, true} - } + res <- S3AccountConfig{*resp.PublicAccessBlockConfiguration, true} + } else { + res <- s3AccountConfig + } - res <- s3AccountConfig return nil } From 0283a05c60ce8afd733a50845b92bc7dca8d28a0 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 06:22:39 -0500 Subject: [PATCH 05/13] Stub out docs --- docs/tables/aws_s3_accounts.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/tables/aws_s3_accounts.md diff --git a/docs/tables/aws_s3_accounts.md b/docs/tables/aws_s3_accounts.md new file mode 100644 index 000000000..41d374b68 --- /dev/null +++ b/docs/tables/aws_s3_accounts.md @@ -0,0 +1,13 @@ + +# Table: aws_s3_accounts +TODO-FILL-THIS-IN +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|account_id|text|| +|region|text|| +|config_exists|boolean|Specifies whether Amazon S3 access control exists| +|block_public_acls|boolean|Specifies whether Amazon S3 should block public access control lists (ACLs) for buckets in this account| +|block_public_policy|boolean|Specifies whether Amazon S3 should block public bucket policies for buckets in this account.| +|ignore_public_acls|boolean|Specifies whether Amazon S3 should ignore public ACLs for buckets in this account| +|restrict_public_buckets|boolean|Specifies whether Amazon S3 should restrict public bucket policies for buckets in this account.| From 017e8f20dc15e91e3668223cde8020de641ac668 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 06:31:34 -0500 Subject: [PATCH 06/13] Update s3_accounts.go --- resources/s3_accounts.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/resources/s3_accounts.go b/resources/s3_accounts.go index 220023c37..f91dc45d4 100644 --- a/resources/s3_accounts.go +++ b/resources/s3_accounts.go @@ -79,11 +79,9 @@ func fetchS3AccountConfig(ctx context.Context, meta schema.ClientMeta, _ *schema if errors.As(err, &ae) && ae.ErrorCode() != "NoSuchPublicAccessBlockConfiguration" { return err } - } - if resp != nil { - res <- S3AccountConfig{*resp.PublicAccessBlockConfiguration, true} - } else { res <- s3AccountConfig + } else { + res <- S3AccountConfig{*resp.PublicAccessBlockConfiguration, true} } return nil From f598426e24706494c6270cd8d81f80ecd17b6f09 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 06:53:20 -0500 Subject: [PATCH 07/13] Update s3_accounts.go --- resources/s3_accounts.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/resources/s3_accounts.go b/resources/s3_accounts.go index f91dc45d4..093d480cc 100644 --- a/resources/s3_accounts.go +++ b/resources/s3_accounts.go @@ -3,11 +3,11 @@ package resources import ( "context" "errors" + "log" aws "github.com/aws/aws-sdk-go-v2/aws" s3control "github.com/aws/aws-sdk-go-v2/service/s3control" s3controlTypes "github.com/aws/aws-sdk-go-v2/service/s3control/types" - "github.com/aws/smithy-go" "github.com/cloudquery/cq-provider-aws/client" "github.com/cloudquery/cq-provider-sdk/provider/schema" ) @@ -73,12 +73,20 @@ func fetchS3AccountConfig(ctx context.Context, meta schema.ClientMeta, _ *schema resp, err := svc.GetPublicAccessBlock(ctx, &accountConfig, func(options *s3control.Options) { options.Region = c.Region }) + + var re *awshttp.ResponseError + if errors.As(err, &re) { + log.Printf("requestID: %s, error: %v", re.ServiceRequestID(), re.Unwrap()) + } + if err != nil { // If we received any error other than NoSuchPublicAccessBlockConfiguration, we return and error - var ae smithy.APIError - if errors.As(err, &ae) && ae.ErrorCode() != "NoSuchPublicAccessBlockConfiguration" { + + var nspabc *s3controlTypes.NoSuchPublicAccessBlockConfiguration + if !errors.As(err, &nspabc) { return err } + res <- s3AccountConfig } else { res <- S3AccountConfig{*resp.PublicAccessBlockConfiguration, true} From 3953e9536663641c88d5705d6ce78337e0d1ae9b Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 06:53:41 -0500 Subject: [PATCH 08/13] Update s3_accounts.go --- resources/s3_accounts.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/resources/s3_accounts.go b/resources/s3_accounts.go index 093d480cc..bcca483aa 100644 --- a/resources/s3_accounts.go +++ b/resources/s3_accounts.go @@ -3,7 +3,6 @@ package resources import ( "context" "errors" - "log" aws "github.com/aws/aws-sdk-go-v2/aws" s3control "github.com/aws/aws-sdk-go-v2/service/s3control" @@ -74,19 +73,12 @@ func fetchS3AccountConfig(ctx context.Context, meta schema.ClientMeta, _ *schema options.Region = c.Region }) - var re *awshttp.ResponseError - if errors.As(err, &re) { - log.Printf("requestID: %s, error: %v", re.ServiceRequestID(), re.Unwrap()) - } - if err != nil { // If we received any error other than NoSuchPublicAccessBlockConfiguration, we return and error - var nspabc *s3controlTypes.NoSuchPublicAccessBlockConfiguration if !errors.As(err, &nspabc) { return err } - res <- s3AccountConfig } else { res <- S3AccountConfig{*resp.PublicAccessBlockConfiguration, true} From e845b2b6e95caf34911cff280eb8c109c037e739 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 07:03:00 -0500 Subject: [PATCH 09/13] clean up --- docs/tables/aws_s3_account_config.md | 12 ++++++++++++ resources/s3_accounts.go | 11 +++-------- 2 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 docs/tables/aws_s3_account_config.md diff --git a/docs/tables/aws_s3_account_config.md b/docs/tables/aws_s3_account_config.md new file mode 100644 index 000000000..7604b182c --- /dev/null +++ b/docs/tables/aws_s3_account_config.md @@ -0,0 +1,12 @@ + +# Table: aws_s3_account_config +Account configurations for S3 +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|account_id|text|| +|config_exists|boolean|Specifies whether Amazon S3 public access control config exists| +|block_public_acls|boolean|Specifies whether Amazon S3 should block public access control lists (ACLs) for buckets in this account| +|block_public_policy|boolean|Specifies whether Amazon S3 should block public bucket policies for buckets in this account.| +|ignore_public_acls|boolean|Specifies whether Amazon S3 should ignore public ACLs for buckets in this account| +|restrict_public_buckets|boolean|Specifies whether Amazon S3 should restrict public bucket policies for buckets in this account.| diff --git a/resources/s3_accounts.go b/resources/s3_accounts.go index bcca483aa..18c137fa7 100644 --- a/resources/s3_accounts.go +++ b/resources/s3_accounts.go @@ -13,8 +13,8 @@ import ( func S3Accounts() *schema.Table { return &schema.Table{ - Name: "aws_s3_accounts", - Description: "TODO-FILL-THIS-IN", + Name: "aws_s3_account_config", + Description: "Account configurations for S3", Resolver: fetchS3AccountConfig, Multiplex: client.AccountMultiplex, IgnoreError: client.IgnoreAccessDeniedServiceDisabled, @@ -26,15 +26,10 @@ func S3Accounts() *schema.Table { Type: schema.TypeString, Resolver: client.ResolveAWSAccount, }, - { - Name: "region", - Type: schema.TypeString, - Resolver: client.ResolveAWSRegion, - }, { Name: "config_exists", Type: schema.TypeBool, - Description: "Specifies whether Amazon S3 access control exists", + Description: "Specifies whether Amazon S3 public access control config exists", }, { Name: "block_public_acls", From 3fd2a14a907048689cc4a88011ef6b9d5cd8630f Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 07:48:54 -0500 Subject: [PATCH 10/13] Update s3_accounts.go --- resources/s3_accounts.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/resources/s3_accounts.go b/resources/s3_accounts.go index 18c137fa7..d057d6371 100644 --- a/resources/s3_accounts.go +++ b/resources/s3_accounts.go @@ -57,13 +57,11 @@ func S3Accounts() *schema.Table { } func fetchS3AccountConfig(ctx context.Context, meta schema.ClientMeta, _ *schema.Resource, res chan interface{}) error { - var s3AccountConfig S3AccountConfig c := meta.(*client.Client) svc := c.Services().S3Control var accountConfig s3control.GetPublicAccessBlockInput accountConfig.AccountId = aws.String(c.AccountID) - s3AccountConfig.ConfigExists = false resp, err := svc.GetPublicAccessBlock(ctx, &accountConfig, func(options *s3control.Options) { options.Region = c.Region }) @@ -74,7 +72,7 @@ func fetchS3AccountConfig(ctx context.Context, meta schema.ClientMeta, _ *schema if !errors.As(err, &nspabc) { return err } - res <- s3AccountConfig + res <- S3AccountConfig{s3controlTypes.PublicAccessBlockConfiguration{}, false} } else { res <- S3AccountConfig{*resp.PublicAccessBlockConfiguration, true} } From b5f5ea9622a5f607fde0d73ecd8ecfe5bb1e7e41 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 07:56:43 -0500 Subject: [PATCH 11/13] Delete aws_s3_accounts.md --- docs/tables/aws_s3_accounts.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 docs/tables/aws_s3_accounts.md diff --git a/docs/tables/aws_s3_accounts.md b/docs/tables/aws_s3_accounts.md deleted file mode 100644 index 41d374b68..000000000 --- a/docs/tables/aws_s3_accounts.md +++ /dev/null @@ -1,13 +0,0 @@ - -# Table: aws_s3_accounts -TODO-FILL-THIS-IN -## Columns -| Name | Type | Description | -| ------------- | ------------- | ----- | -|account_id|text|| -|region|text|| -|config_exists|boolean|Specifies whether Amazon S3 access control exists| -|block_public_acls|boolean|Specifies whether Amazon S3 should block public access control lists (ACLs) for buckets in this account| -|block_public_policy|boolean|Specifies whether Amazon S3 should block public bucket policies for buckets in this account.| -|ignore_public_acls|boolean|Specifies whether Amazon S3 should ignore public ACLs for buckets in this account| -|restrict_public_buckets|boolean|Specifies whether Amazon S3 should restrict public bucket policies for buckets in this account.| From 5f1b0f2eb9f77171856b371dcc29216360663281 Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 08:45:01 -0500 Subject: [PATCH 12/13] add mocks --- client/mocks/mock_s3Control.go | 56 ++++++++++++++++++++++++++++++++++ go.mod | 3 ++ 2 files changed, 59 insertions(+) create mode 100644 client/mocks/mock_s3Control.go diff --git a/client/mocks/mock_s3Control.go b/client/mocks/mock_s3Control.go new file mode 100644 index 000000000..29f2a32dc --- /dev/null +++ b/client/mocks/mock_s3Control.go @@ -0,0 +1,56 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/cloudquery/cq-provider-aws/client (interfaces: S3ControlClient) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + s3control "github.com/aws/aws-sdk-go-v2/service/s3control" + gomock "github.com/golang/mock/gomock" +) + +// MockS3ControlClient is a mock of S3ControlClient interface. +type MockS3ControlClient struct { + ctrl *gomock.Controller + recorder *MockS3ControlClientMockRecorder +} + +// MockS3ControlClientMockRecorder is the mock recorder for MockS3ControlClient. +type MockS3ControlClientMockRecorder struct { + mock *MockS3ControlClient +} + +// NewMockS3ControlClient creates a new mock instance. +func NewMockS3ControlClient(ctrl *gomock.Controller) *MockS3ControlClient { + mock := &MockS3ControlClient{ctrl: ctrl} + mock.recorder = &MockS3ControlClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockS3ControlClient) EXPECT() *MockS3ControlClientMockRecorder { + return m.recorder +} + +// GetPublicAccessBlock mocks base method. +func (m *MockS3ControlClient) GetPublicAccessBlock(arg0 context.Context, arg1 *s3control.GetPublicAccessBlockInput, arg2 ...func(*s3control.Options)) (*s3control.GetPublicAccessBlockOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetPublicAccessBlock", varargs...) + ret0, _ := ret[0].(*s3control.GetPublicAccessBlockOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPublicAccessBlock indicates an expected call of GetPublicAccessBlock. +func (mr *MockS3ControlClientMockRecorder) GetPublicAccessBlock(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPublicAccessBlock", reflect.TypeOf((*MockS3ControlClient)(nil).GetPublicAccessBlock), varargs...) +} diff --git a/go.mod b/go.mod index 914ee2309..07b861006 100644 --- a/go.mod +++ b/go.mod @@ -137,4 +137,7 @@ require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.1 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/tools v0.1.5 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) From 7ed112a8df16af18ea38650920c4c906ed15ceab Mon Sep 17 00:00:00 2001 From: bbernays Date: Tue, 23 Nov 2021 10:29:23 -0500 Subject: [PATCH 13/13] Create aws_s3_account_test.go --- .../integration_tests/aws_s3_account_test.go | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 resources/integration_tests/aws_s3_account_test.go diff --git a/resources/integration_tests/aws_s3_account_test.go b/resources/integration_tests/aws_s3_account_test.go new file mode 100644 index 000000000..441e71ed7 --- /dev/null +++ b/resources/integration_tests/aws_s3_account_test.go @@ -0,0 +1,32 @@ +package integration_tests + +import ( + "testing" + + "github.com/Masterminds/squirrel" + "github.com/cloudquery/cq-provider-aws/resources" + providertest "github.com/cloudquery/cq-provider-sdk/provider/testing" +) + +func TestIntegrationS3Account(t *testing.T) { + awsTestIntegrationHelper(t, resources.S3Accounts(), nil, func(res *providertest.ResourceIntegrationTestData) providertest.ResourceIntegrationVerification { + return providertest.ResourceIntegrationVerification{ + Name: "aws_s3_account_config", + Filter: func(sq squirrel.SelectBuilder, res *providertest.ResourceIntegrationTestData) squirrel.SelectBuilder { + return sq.Where(squirrel.Eq{"config_exists": false}) + }, + ExpectedValues: []providertest.ExpectedValue{ + { + Count: 1, + Data: map[string]interface{}{ + "config_exists": false, + "block_public_acls": false, + "block_public_policy": false, + "ignore_public_acls": false, + "restrict_public_buckets": false, + }, + }, + }, + } + }) +}