From 24d27225fc6cf8494865ca06eb46be37a2b0c23f Mon Sep 17 00:00:00 2001 From: Kemal <223029+disq@users.noreply.github.com> Date: Tue, 21 Jun 2022 10:03:17 +0100 Subject: [PATCH] feat: YAML config support (#1067) Co-authored-by: Kemal Hadimli --- client/config.go | 84 ++++++++++++++++++++++++---------- client/testing.go | 5 +- go.mod | 2 +- go.sum | 4 +- resources/provider/provider.go | 5 +- 5 files changed, 70 insertions(+), 30 deletions(-) diff --git a/client/config.go b/client/config.go index 50a96cd07..72ed2aee9 100644 --- a/client/config.go +++ b/client/config.go @@ -1,39 +1,51 @@ package client +import "github.com/cloudquery/cq-provider-sdk/cqproto" + type Account struct { - ID string `hcl:",label"` + ID string `yaml:"id" hcl:",label"` AccountID string - AccountName string `hcl:"account_name,optional"` - LocalProfile string `hcl:"local_profile,optional"` - RoleARN string `hcl:"role_arn,optional"` - RoleSessionName string `hcl:"role_session_name,optional"` - ExternalID string `hcl:"external_id,optional"` - Regions []string `hcl:"regions,optional"` + AccountName string `yaml:"account_name,omitempty" hcl:"account_name,optional"` + LocalProfile string `yaml:"local_profile,omitempty" hcl:"local_profile,optional"` + RoleARN string `yaml:"role_arn,omitempty" hcl:"role_arn,optional"` + RoleSessionName string `yaml:"role_session_name,omitempty" hcl:"role_session_name,optional"` + ExternalID string `yaml:"external_id,omitempty" hcl:"external_id,optional"` + Regions []string `yaml:"regions,omitempty" hcl:"regions,optional"` source string } type AwsOrg struct { - OrganizationUnits []string `hcl:"organization_units,optional"` - AdminAccount *Account `hcl:"admin_account,block"` - MemberCredentials *Account `hcl:"member_trusted_principal,block"` - ChildAccountRoleName string `hcl:"member_role_name,optional"` - ChildAccountRoleSessionName string `hcl:"member_role_session_name,optional"` - ChildAccountExternalID string `hcl:"member_external_id,optional"` - ChildAccountRegions []string `hcl:"member_regions,optional"` + OrganizationUnits []string `yaml:"organization_units,omitempty" hcl:"organization_units,optional"` + AdminAccount *Account `yaml:"admin_account" hcl:"admin_account,block"` + MemberCredentials *Account `yaml:"member_trusted_principal" hcl:"member_trusted_principal,block"` + ChildAccountRoleName string `yaml:"member_role_name,omitempty" hcl:"member_role_name,optional"` + ChildAccountRoleSessionName string `yaml:"member_role_session_name,omitempty" hcl:"member_role_session_name,optional"` + ChildAccountExternalID string `yaml:"member_external_id,omitempty" hcl:"member_external_id,optional"` + ChildAccountRegions []string `yaml:"member_regions,omitempty" hcl:"member_regions,optional"` } type Config struct { - Regions []string `hcl:"regions,optional"` - Accounts []Account `hcl:"accounts,block"` - Organization *AwsOrg `hcl:"org,block"` - AWSDebug bool `hcl:"aws_debug,optional"` - MaxRetries int `hcl:"max_retries,optional" default:"10"` - MaxBackoff int `hcl:"max_backoff,optional" default:"30"` - GlobalRegion string `hcl:"global_region,optional" default:"us-east-1"` + Regions []string `yaml:"regions,omitempty" hcl:"regions,optional"` + Accounts []Account `yaml:"accounts" hcl:"accounts,block"` + Organization *AwsOrg `yaml:"org" hcl:"org,block"` + AWSDebug bool `yaml:"aws_debug,omitempty" hcl:"aws_debug,optional"` + MaxRetries int `yaml:"max_retries,omitempty" hcl:"max_retries,optional" default:"10"` + MaxBackoff int `yaml:"max_backoff,omitempty" hcl:"max_backoff,optional" default:"30"` + GlobalRegion string `yaml:"global_region,omitempty" hcl:"global_region,optional" default:"us-east-1"` + + requestedFormat cqproto.ConfigFormat +} + +func NewConfig(f cqproto.ConfigFormat) *Config { + return &Config{ + requestedFormat: f, + } } -func (Config) Example() string { - return ` configuration { +func (c Config) Example() string { + switch c.requestedFormat { + case cqproto.ConfigHCL: + return ` configuration { // Optional, Repeated. Add an 'accounts' block for every account you want to assume-role into and fetch data from. // accounts "" { // Optional. Role ARN we want to assume when accessing this account @@ -51,4 +63,30 @@ func (Config) Example() string { // max_backoff = 30 } ` + default: + return ` +Optional, Repeated. Add an accounts block for every account you want to assume-role into and fetch data from. +accounts: + - id: +Optional. Role ARN we want to assume when accessing this account + role_arn: < YOUR_ROLE_ARN > +Optional. Named profile in config or credential file from where CQ should grab credentials + local_profile = < PROFILE_NAME > + +Optional. by default assumes all regions +regions: + - us-east-1 + us-west-2 +Optional. Enable AWS SDK debug logging. + aws_debug: false +The maximum number of times that a request will be retried for failures. Defaults to 10 retry attempts. +max_retries: 10 +The maximum back off delay between attempts. The backoff delays exponentially with a jitter based on the number of attempts. Defaults to 30 seconds. +max_backoff: 30 +` + } +} + +func (c Config) Format() cqproto.ConfigFormat { + return c.requestedFormat } diff --git a/client/testing.go b/client/testing.go index 970bad1a5..6b767932c 100644 --- a/client/testing.go +++ b/client/testing.go @@ -3,6 +3,7 @@ package client import ( "testing" + "github.com/cloudquery/cq-provider-sdk/cqproto" "github.com/cloudquery/cq-provider-sdk/logging" "github.com/cloudquery/cq-provider-sdk/provider" "github.com/cloudquery/cq-provider-sdk/provider/diag" @@ -47,8 +48,8 @@ func AwsMockTestHelper(t *testing.T, table *schema.Table, builder func(*testing. ResourceMap: map[string]*schema.Table{ "test_resource": table, }, - Config: func() provider.Config { - return &Config{} + Config: func(f cqproto.ConfigFormat) provider.Config { + return NewConfig(f) }, }, Config: cfg, diff --git a/go.mod b/go.mod index 63f836605..4c1c8e17a 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( github.com/basgys/goxml2json v1.1.0 github.com/bxcodec/faker v2.0.1+incompatible github.com/cloudquery/cq-gen v0.0.5 - github.com/cloudquery/cq-provider-sdk v0.11.4 + github.com/cloudquery/cq-provider-sdk v0.12.0 github.com/cloudquery/faker/v3 v3.7.5 github.com/gocarina/gocsv v0.0.0-20210516172204-ca9e8a8ddea8 github.com/golang/mock v1.6.0 diff --git a/go.sum b/go.sum index da9e8275b..bd3bec714 100644 --- a/go.sum +++ b/go.sum @@ -363,8 +363,8 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h github.com/cloudquery/cq-gen v0.0.5 h1:yhDhM4RCqqGLZulDzfA51VMy0b7TIdtifoEiZXnFbUc= github.com/cloudquery/cq-gen v0.0.5/go.mod h1:zrjBcuCGtED9P4RzA4gK+P3loxn0Ij1wEcBZX97JTnI= github.com/cloudquery/cq-provider-sdk v0.8.2/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ= -github.com/cloudquery/cq-provider-sdk v0.11.4 h1:/y/AB+/mKRGFoc0rLin3KRtfqc5eAooYjqdltrsGi8I= -github.com/cloudquery/cq-provider-sdk v0.11.4/go.mod h1:q6hYy9S+XtKG8cyOiLG5gqSIkXEJ72MHQ316zW6DMiA= +github.com/cloudquery/cq-provider-sdk v0.12.0 h1:S3AAkxmXhoGwvn7E77GmcOzkIFAxOjXEhPr+F0jW+Pg= +github.com/cloudquery/cq-provider-sdk v0.12.0/go.mod h1:o5czsmX3MeP8cY/KtabkqavkS7asF8HwFAO1n5+CvoQ= github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= github.com/cloudquery/faker/v3 v3.7.5 h1:G7ANdEEcm8TvAAjIwNWSLrYK36CFCiSlrCqOTGCccL0= github.com/cloudquery/faker/v3 v3.7.5/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo= diff --git a/resources/provider/provider.go b/resources/provider/provider.go index 981bc31b1..c2a70fa60 100644 --- a/resources/provider/provider.go +++ b/resources/provider/provider.go @@ -59,6 +59,7 @@ import ( "github.com/cloudquery/cq-provider-aws/resources/services/wafv2" "github.com/cloudquery/cq-provider-aws/resources/services/workspaces" "github.com/cloudquery/cq-provider-aws/resources/services/xray" + "github.com/cloudquery/cq-provider-sdk/cqproto" "github.com/cloudquery/cq-provider-sdk/provider" "github.com/cloudquery/cq-provider-sdk/provider/module" "github.com/cloudquery/cq-provider-sdk/provider/schema" @@ -244,8 +245,8 @@ func Provider() *provider.Provider { "xray.sampling_rules": xray.SamplingRules(), //"iot.security_profiles": iot.IotSecurityProfiles(), //TODO disabled because of api error NotFoundException: No method found matching route security-profiles for http method GET. }, - Config: func() provider.Config { - return &client.Config{} + Config: func(f cqproto.ConfigFormat) provider.Config { + return client.NewConfig(f) }, } }