feat: YAML config support (#1067)

Co-authored-by: Kemal Hadimli <[email protected]>

client/config.go

@@ -1,39 +1,51 @@
 package client
 
+import "github.com/cloudquery/cq-provider-sdk/cqproto"
+
 type Account struct {
-	ID              string `hcl:",label"`
+	ID              string `yaml:"id" hcl:",label"`
 	AccountID       string
-	AccountName     string   `hcl:"account_name,optional"`
-	LocalProfile    string   `hcl:"local_profile,optional"`
-	RoleARN         string   `hcl:"role_arn,optional"`
-	RoleSessionName string   `hcl:"role_session_name,optional"`
-	ExternalID      string   `hcl:"external_id,optional"`
-	Regions         []string `hcl:"regions,optional"`
+	AccountName     string   `yaml:"account_name,omitempty" hcl:"account_name,optional"`
+	LocalProfile    string   `yaml:"local_profile,omitempty" hcl:"local_profile,optional"`
+	RoleARN         string   `yaml:"role_arn,omitempty" hcl:"role_arn,optional"`
+	RoleSessionName string   `yaml:"role_session_name,omitempty" hcl:"role_session_name,optional"`
+	ExternalID      string   `yaml:"external_id,omitempty" hcl:"external_id,optional"`
+	Regions         []string `yaml:"regions,omitempty" hcl:"regions,optional"`
 	source          string
 }
 
 type AwsOrg struct {
-	OrganizationUnits           []string `hcl:"organization_units,optional"`
-	AdminAccount                *Account `hcl:"admin_account,block"`
-	MemberCredentials           *Account `hcl:"member_trusted_principal,block"`
-	ChildAccountRoleName        string   `hcl:"member_role_name,optional"`
-	ChildAccountRoleSessionName string   `hcl:"member_role_session_name,optional"`
-	ChildAccountExternalID      string   `hcl:"member_external_id,optional"`
-	ChildAccountRegions         []string `hcl:"member_regions,optional"`
+	OrganizationUnits           []string `yaml:"organization_units,omitempty" hcl:"organization_units,optional"`
+	AdminAccount                *Account `yaml:"admin_account" hcl:"admin_account,block"`
+	MemberCredentials           *Account `yaml:"member_trusted_principal" hcl:"member_trusted_principal,block"`
+	ChildAccountRoleName        string   `yaml:"member_role_name,omitempty" hcl:"member_role_name,optional"`
+	ChildAccountRoleSessionName string   `yaml:"member_role_session_name,omitempty" hcl:"member_role_session_name,optional"`
+	ChildAccountExternalID      string   `yaml:"member_external_id,omitempty" hcl:"member_external_id,optional"`
+	ChildAccountRegions         []string `yaml:"member_regions,omitempty" hcl:"member_regions,optional"`
 }
 
 type Config struct {
-	Regions      []string  `hcl:"regions,optional"`
-	Accounts     []Account `hcl:"accounts,block"`
-	Organization *AwsOrg   `hcl:"org,block"`
-	AWSDebug     bool      `hcl:"aws_debug,optional"`
-	MaxRetries   int       `hcl:"max_retries,optional" default:"10"`
-	MaxBackoff   int       `hcl:"max_backoff,optional" default:"30"`
-	GlobalRegion string    `hcl:"global_region,optional" default:"us-east-1"`
+	Regions      []string  `yaml:"regions,omitempty" hcl:"regions,optional"`
+	Accounts     []Account `yaml:"accounts" hcl:"accounts,block"`
+	Organization *AwsOrg   `yaml:"org" hcl:"org,block"`
+	AWSDebug     bool      `yaml:"aws_debug,omitempty" hcl:"aws_debug,optional"`
+	MaxRetries   int       `yaml:"max_retries,omitempty" hcl:"max_retries,optional" default:"10"`
+	MaxBackoff   int       `yaml:"max_backoff,omitempty" hcl:"max_backoff,optional" default:"30"`
+	GlobalRegion string    `yaml:"global_region,omitempty" hcl:"global_region,optional" default:"us-east-1"`
+
+	requestedFormat cqproto.ConfigFormat
+}
+
+func NewConfig(f cqproto.ConfigFormat) *Config {
+	return &Config{
+		requestedFormat: f,
+	}
 }
 
-func (Config) Example() string {
-	return ` configuration {
+func (c Config) Example() string {
+	switch c.requestedFormat {
+	case cqproto.ConfigHCL:
+		return ` configuration {
   // Optional, Repeated. Add an 'accounts' block for every account you want to assume-role into and fetch data from.
   // accounts "<UNIQUE ACCOUNT IDENTIFIER>" {
     // Optional. Role ARN we want to assume when accessing this account
@@ -51,4 +63,30 @@ func (Config) Example() string {
   // max_backoff = 30
 }
 `
+	default:
+		return `
+Optional, Repeated. Add an accounts block for every account you want to assume-role into and fetch data from.
+accounts:
+  - id: <UNIQUE ACCOUNT IDENTIFIER>
+Optional. Role ARN we want to assume when accessing this account
+    role_arn: < YOUR_ROLE_ARN >
+Optional. Named profile in config or credential file from where CQ should grab credentials
+    local_profile = < PROFILE_NAME >
+
+Optional. by default assumes all regions
+regions:
+  - us-east-1
+    us-west-2
+Optional. Enable AWS SDK debug logging.
+  aws_debug: false
+The maximum number of times that a request will be retried for failures. Defaults to 10 retry attempts.
+max_retries: 10
+The maximum back off delay between attempts. The backoff delays exponentially with a jitter based on the number of attempts. Defaults to 30 seconds.
+max_backoff: 30
+`
+	}
+}
+
+func (c Config) Format() cqproto.ConfigFormat {
+	return c.requestedFormat
 }

client/testing.go

@@ -3,6 +3,7 @@ package client
 import (
 	"testing"
 
+	"github.com/cloudquery/cq-provider-sdk/cqproto"
 	"github.com/cloudquery/cq-provider-sdk/logging"
 	"github.com/cloudquery/cq-provider-sdk/provider"
 	"github.com/cloudquery/cq-provider-sdk/provider/diag"
@@ -47,8 +48,8 @@ func AwsMockTestHelper(t *testing.T, table *schema.Table, builder func(*testing.
 			ResourceMap: map[string]*schema.Table{
 				"test_resource": table,
 			},
-			Config: func() provider.Config {
-				return &Config{}
+			Config: func(f cqproto.ConfigFormat) provider.Config {
+				return NewConfig(f)
 			},
 		},
 		Config:           cfg,

go.mod

@@ -66,7 +66,7 @@ require (
 	github.com/basgys/goxml2json v1.1.0
 	github.com/bxcodec/faker v2.0.1+incompatible
 	github.com/cloudquery/cq-gen v0.0.5
-	github.com/cloudquery/cq-provider-sdk v0.11.4
+	github.com/cloudquery/cq-provider-sdk v0.12.0
 	github.com/cloudquery/faker/v3 v3.7.5
 	github.com/gocarina/gocsv v0.0.0-20210516172204-ca9e8a8ddea8
 	github.com/golang/mock v1.6.0

go.sum

@@ -363,8 +363,8 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h
 github.com/cloudquery/cq-gen v0.0.5 h1:yhDhM4RCqqGLZulDzfA51VMy0b7TIdtifoEiZXnFbUc=
 github.com/cloudquery/cq-gen v0.0.5/go.mod h1:zrjBcuCGtED9P4RzA4gK+P3loxn0Ij1wEcBZX97JTnI=
 github.com/cloudquery/cq-provider-sdk v0.8.2/go.mod h1:IHxqY7TOttWhNQhMRqYl1vBo2JS2szLAf5Mhg78MwTQ=
-github.com/cloudquery/cq-provider-sdk v0.11.4 h1:/y/AB+/mKRGFoc0rLin3KRtfqc5eAooYjqdltrsGi8I=
-github.com/cloudquery/cq-provider-sdk v0.11.4/go.mod h1:q6hYy9S+XtKG8cyOiLG5gqSIkXEJ72MHQ316zW6DMiA=
+github.com/cloudquery/cq-provider-sdk v0.12.0 h1:S3AAkxmXhoGwvn7E77GmcOzkIFAxOjXEhPr+F0jW+Pg=
+github.com/cloudquery/cq-provider-sdk v0.12.0/go.mod h1:o5czsmX3MeP8cY/KtabkqavkS7asF8HwFAO1n5+CvoQ=
 github.com/cloudquery/faker/v3 v3.7.4/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo=
 github.com/cloudquery/faker/v3 v3.7.5 h1:G7ANdEEcm8TvAAjIwNWSLrYK36CFCiSlrCqOTGCccL0=
 github.com/cloudquery/faker/v3 v3.7.5/go.mod h1:1b8WVG9Gh0T2hVo1a8dWeXfu0AhqSB6J/mmJaesqOeo=

resources/provider/provider.go

@@ -59,6 +59,7 @@ import (
 	"github.com/cloudquery/cq-provider-aws/resources/services/wafv2"
 	"github.com/cloudquery/cq-provider-aws/resources/services/workspaces"
 	"github.com/cloudquery/cq-provider-aws/resources/services/xray"
+	"github.com/cloudquery/cq-provider-sdk/cqproto"
 	"github.com/cloudquery/cq-provider-sdk/provider"
 	"github.com/cloudquery/cq-provider-sdk/provider/module"
 	"github.com/cloudquery/cq-provider-sdk/provider/schema"
@@ -244,8 +245,8 @@ func Provider() *provider.Provider {
 			"xray.sampling_rules":                     xray.SamplingRules(),
 			//"iot.security_profiles": 				 iot.IotSecurityProfiles(), //TODO disabled because of api error NotFoundException: No method found matching route security-profiles for http method GET.
 		},
-		Config: func() provider.Config {
-			return &client.Config{}
+		Config: func(f cqproto.ConfigFormat) provider.Config {
+			return client.NewConfig(f)
 		},
 	}
 }