diff --git a/.gitignore b/.gitignore index 0f41531c1..6cadda860 100644 --- a/.gitignore +++ b/.gitignore @@ -40,5 +40,6 @@ __debug_bin .env terraform/downloaded* config.hcl +cloudquery.yml terraform/tfplan.binary cq-provider* diff --git a/client/client.go b/client/client.go index 24c400088..b96d3c93c 100644 --- a/client/client.go +++ b/client/client.go @@ -508,7 +508,7 @@ func Configure(logger hclog.Logger, providerConfig interface{}) (schema.ClientMe } if isAllRegions(localRegions) { - logger.Info("All regions specified in config.yml. Assuming all regions") + logger.Info("All regions specified in `cloudquery.yml`. Assuming all regions") } awsCfg, err := configureAwsClient(ctx, logger, awsConfig, account, adminAccountSts) diff --git a/client/config.go b/client/config.go index 67b75d92e..c324b3c0b 100644 --- a/client/config.go +++ b/client/config.go @@ -75,7 +75,7 @@ Optional. Named profile in config or credential file from where CQ should grab c Optional. by default assumes all regions regions: - us-east-1 - us-west-2 + - us-west-2 Optional. Enable AWS SDK debug logging. aws_debug: false The maximum number of times that a request will be retried for failures. Defaults to 10 retry attempts. diff --git a/docs/index.md b/docs/index.md index e8e4407b4..29c85bb44 100644 --- a/docs/index.md +++ b/docs/index.md @@ -72,18 +72,16 @@ Then, you can either export the `AWS_PROFILE` environment variable (On Linux/Mac export AWS_PROFILE=myprofile ``` -or, configure your desired profile in the `local_profile` field of your CloudQuery `config.hcl`: - -```hcl title="config.hcl" -provider "aws" { - configuration { - accounts "" { - local_profile = "myprofile" - } - ... - } - ... -} +or, configure your desired profile in the `local_profile` field of your CloudQuery `cloudquery.yml`: + +```yml title="cloudquery.yml" +providers: + # provider configurations + - name: aws + configuration: + accounts: + id: + local_profile: myprofile ``` ### IAM Roles for AWS Compute Resources @@ -96,27 +94,32 @@ credentials manually. For more information on configuring IAM, see the AWS docs The following configuration section can be automatically generated by `cloudquery init aws`: -```hcl -provider "aws" { - configuration { - // Optional. if you want to assume role to multiple account and fetch data from them - // Optional. by default assumes all regions or explicitly state all regions by including the `*` character as the only argument in the array - // regions = ["us-east-1", "us-west-2"] - // accounts "" { - // Optional. Role ARN we want to assume when accessing this account - // role_arn = < YOUR_ROLE_ARN > - // Optional. Override provider configs for a specific account - // regions = ["us-east-1", "us-east-2"] - } - // Optional. Enable AWS SDK debug logging. - // aws_debug = false - // The maximum number of times that a request will be retried for failures. Defaults to 20 retry attempts. - // max_retries = 10 - // The maximum back off delay between attempts. The backoff delays exponentially with a jitter based on the number of attempts. Defaults to 90 seconds. - // max_backoff = 30 - } - - resources = ["*"] +```yml +providers: + # provider configurations + - name: aws + configuration: + # Optional, Repeated. Add an accounts block for every account you want to assume-role into and fetch data from. + # accounts: + # - id: + # Optional. Role ARN we want to assume when accessing this account + # role_arn: < YOUR_ROLE_ARN > + # Optional. Named profile in config or credential file from where CQ should grab credentials + # local_profile = < PROFILE_NAME > + # Optional. by default assumes all regions + # regions: + # - us-east-1 + # - us-west-2 + # Optional. Enable AWS SDK debug logging. + # aws_debug: false + # The maximum number of times that a request will be retried for failures. Defaults to 10 retry attempts. + # max_retries: 10 + # The maximum back off delay between attempts. The backoff delays exponentially with a jitter based on the number of attempts. Defaults to 30 seconds. + # max_backoff: 30 + # + # list of resources to fetch + resources: + - "*" } ``` @@ -150,78 +153,68 @@ Prerequisites for using AWS Org functionality: Using AWS Organization: 1. Specify member role name: -```hcl - org { - member_role_name = "OrganizationAccountAccessRole" - } +```yml + org: + member_role_name: OrganizationAccountAccessRole ``` 2. Getting credentials that have the necessary `organizations` permissions: 1. Sourcing Credentials from the default credential tool chain: - ```hcl - org { - member_role_name = "OrganizationAccountAccessRole" - } + ```yml + org: + member_role_name: OrganizationAccountAccessRole ``` 2. Sourcing credentials from a named profile in the shared configuration or credentials file - ```hcl - org { - member_role_name = "OrganizationAccountAccessRole" - admin_account "admin" { - local_profile = "" - } - } + ```yml + org: + member_role_name: OrganizationAccountAccessRole + admin_account: + local_profile: ``` 3. Assuming a role in admin account using credentials in the shared configuration or credentials file: - ```hcl - org { - member_role_name = "OrganizationAccountAccessRole" - admin_account "admin" { - local_profile = "" - - role_arn = "arn:aws:iam:::role/" + ```yml + org: + member_role_name: OrganizationAccountAccessRole + admin_account: + local_profile: + role_arn: arn:aws:iam:::role/ // Optional. Specify the name of the session - // role_session_name = "" + // role_session_name: "" // Optional. Specify the ExternalID if required for trust policy - // external_id = " - - } - } + // external_id: "" ``` 3. Optional. If the trust policy configured for the member accounts requires different credentials than you configured in the previous step, then you can specify the credentials to use in the `member_trusted_principal` block -```hcl - org { - member_role_name = "OrganizationAccountAccessRole" - admin_account "admin" { - local_profile = "" - } - member_trusted_principal "trusted" { - - } - - organization_units = ["ou-","ou-"] - } +```yml + org: + member_role_name: OrganizationAccountAccessRole + admin_account: + local_profile: + member_trusted_principal: + local_profile: + organization_units: + - ou- + - ou- ``` 4. Optional. If you want to specify specific Organizational Units to fetch from you can add them to the `organization_units` list. -```hcl - org { - member_role_name = "OrganizationAccountAccessRole" - admin_account "admin" { - local_profile = "" - } - organization_units = ["ou-","ou-"] - } +```yml + org: + member_role_name: OrganizationAccountAccessRole + admin_account: + local_profile: + organization_units: + - ou- + - ou- ``` @@ -250,26 +243,23 @@ Using AWS Organization: CloudQuery can fetch from multiple accounts in parallel by using AssumeRole (You will need to use credentials that can AssumeRole to all other specified account. Following is an example configuration: -```hcl -provider "aws" { - configuration { - // Optional. if you want to assume role to multiple account and fetch data from them - accounts "" { - // Optional. Role ARN we want to assume when accessing this account - role_arn = "" - // Optional. Local Profile is the named profile in your shared configuration file (usually `~/.aws/config`) that you want to use for this specific account - local_profile = "" - // Optional. Specify the Role Session name - role_session_name = "" - } - accounts "" { - // Optional. Role ARN we want to assume when accessing this account - role_arn = "" - } - } - - resources = ["*"] -} +```yml +providers: + - name: aws + resources: + - "*" + configuration: + accounts: + - id: + role_arn: + // Optional. Local Profile is the named profile in your shared configuration file (usually `~/.aws/config`) that you want to use for this specific account + local_profile: + // Optional. Specify the Role Session name + role_session_name: "" + - id: + local_profile: provider + // Optional. Role ARN we want to assume when accessing this account + role_arn: ``` #### Arguments for Accounts block: @@ -339,7 +329,7 @@ Running Provider locally: ```bash make pg-start ``` -3. [Optional] Configure the `config.hcl` +3. [Optional] Configure the `cloudquery.yml` ```bash make os=Linux arch=arm64 install ./cloudquery init aws