From 0fb2662dadba8098c9476f0ca01a06cc035f3883 Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Mon, 7 Feb 2022 14:36:58 +0000 Subject: [PATCH 1/4] add terraformer for azure --- Dockerfile | 4 +- Dockerfile.apispec | 4 +- Dockerfile.apispec.debian | 4 +- Dockerfile.debian | 4 +- Dockerfile.ubi7 | 4 +- pkg/terraformer/azure/azure_cloud_provider.go | 53 +++++++++++++++++++ pkg/terraformer/terraformer.go | 3 ++ 7 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 pkg/terraformer/azure/azure_cloud_provider.go diff --git a/Dockerfile b/Dockerfile index 9bd85f9eef4..8fc7931926f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,9 +39,11 @@ FROM alpine:3.14.3 RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip \ && unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip \ && mv terraform /usr/bin/terraform \ + && wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip \ && wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip \ + && unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip\ && unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip \ - && mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 ~/.terraform.d/plugins/linux_amd64 \ + && mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64 \ && apk add --no-cache \ git=2.32.0-r0 diff --git a/Dockerfile.apispec b/Dockerfile.apispec index da5b6dda2c5..4d376e20ea1 100644 --- a/Dockerfile.apispec +++ b/Dockerfile.apispec @@ -38,9 +38,11 @@ RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_am RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip RUN mv terraform /usr/bin/terraform +RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip +RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip -RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 ~/.terraform.d/plugins/linux_amd64 +RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64 # Install Git diff --git a/Dockerfile.apispec.debian b/Dockerfile.apispec.debian index 9b71c11b0e9..7aa860d8b22 100644 --- a/Dockerfile.apispec.debian +++ b/Dockerfile.apispec.debian @@ -56,9 +56,11 @@ RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_am RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip RUN mv terraform /usr/bin/terraform +RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip +RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip -RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 ~/.terraform.d/plugins/linux_amd64 +RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64 COPY --from=build_env /app/bin/kics /app/bin/kics diff --git a/Dockerfile.debian b/Dockerfile.debian index 3bfdbf3b1b0..d0664a379d6 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -57,9 +57,11 @@ RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_am RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip RUN mv terraform /usr/bin/terraform +RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip +RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip -RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 ~/.terraform.d/plugins/linux_amd64 +RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64 COPY --from=build_env /app/bin/kics /app/bin/kics diff --git a/Dockerfile.ubi7 b/Dockerfile.ubi7 index 502d0a69012..f188e7165a9 100644 --- a/Dockerfile.ubi7 +++ b/Dockerfile.ubi7 @@ -78,9 +78,11 @@ RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_am RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip RUN mv terraform /usr/bin/terraform +RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip +RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip -RUN mkdir /app/bin/.terraform.d && mkdir /app/bin/.terraform.d/plugins && mkdir /app/bin/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 /app/bin/.terraform.d/plugins/linux_amd64 +RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64 USER ${KUSER} diff --git a/pkg/terraformer/azure/azure_cloud_provider.go b/pkg/terraformer/azure/azure_cloud_provider.go new file mode 100644 index 00000000000..2470f37a67b --- /dev/null +++ b/pkg/terraformer/azure/azure_cloud_provider.go @@ -0,0 +1,53 @@ +//go:build !dev +// +build !dev + +package azure + +import ( + "context" + "errors" + "sync" + "time" + + importer "github.com/GoogleCloudPlatform/terraformer/cmd" + azureterraformer "github.com/GoogleCloudPlatform/terraformer/providers/azure" + "github.com/GoogleCloudPlatform/terraformer/terraformutils" +) + +var terraformerTimeout = time.Minute * 3 + +// ImporterFunc is the function kics uses to import resources (for testing porpuses) +var ImporterFunc func(provider terraformutils.ProviderGenerator, options importer.ImportOptions, args []string) error = importer.Import + +// CloudProvider is the AZURE Cloud Provider +type CloudProvider struct{} + +var provider = &azureterraformer.AzureProvider{} + +// Import imports the terraformer resources into the destination using terraformer +func (a CloudProvider) Import(ctx context.Context, options *importer.ImportOptions, destination string) error { + ctxT, cancel := context.WithTimeout(ctx, terraformerTimeout) + defer cancel() + wg := sync.WaitGroup{} + done := make(chan error, 1) + + for _, region := range options.Regions { + wg.Add(1) + go func(region string) { + defer wg.Done() + done <- ImporterFunc(provider, *options, []string{options.ResourceGroup}) + }(region) + } + + go func() { + defer close(done) + wg.Wait() + }() + + select { + case err := <-done: + return err + case <-ctxT.Done(): + return errors.New("terraformer import execution timeout") + } +} diff --git a/pkg/terraformer/terraformer.go b/pkg/terraformer/terraformer.go index 5a47b6789d1..a1d456fd7cf 100644 --- a/pkg/terraformer/terraformer.go +++ b/pkg/terraformer/terraformer.go @@ -16,6 +16,7 @@ import ( tfLogger "log" "github.com/Checkmarx/kics/pkg/terraformer/aws" + "github.com/Checkmarx/kics/pkg/terraformer/azure" importer "github.com/GoogleCloudPlatform/terraformer/cmd" "github.com/pkg/errors" "github.com/rs/zerolog/log" @@ -84,6 +85,8 @@ func Import(terraformerPath, destinationPath string) (string, error) { switch pathOptions.CloudProvider { case "aws": provider = aws.CloudProvider{} + case "azure": + provider = azure.CloudProvider{} default: return "", errors.New("unsupported Cloud Provider") } From b25ec64d522d752fa2c9469f37a9731db82e0440 Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Tue, 8 Feb 2022 10:19:26 +0000 Subject: [PATCH 2/4] add test --- .../azure/azure_cloud_provider_test.go | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 pkg/terraformer/azure/azure_cloud_provider_test.go diff --git a/pkg/terraformer/azure/azure_cloud_provider_test.go b/pkg/terraformer/azure/azure_cloud_provider_test.go new file mode 100644 index 00000000000..8f1392449b3 --- /dev/null +++ b/pkg/terraformer/azure/azure_cloud_provider_test.go @@ -0,0 +1,72 @@ +//go:build !dev +// +build !dev + +package azure + +import ( + "context" + "testing" + + importer "github.com/GoogleCloudPlatform/terraformer/cmd" + "github.com/GoogleCloudPlatform/terraformer/terraformutils" +) + +var mockOptions = &importer.ImportOptions{ + Resources: []string{"storage_account"}, + Excludes: []string{""}, + PathPattern: "destination", + PathOutput: "generated", + State: "local", + Bucket: "", + Profile: "", + Verbose: false, + Zone: "", + Regions: []string{"eastus"}, + Projects: []string{""}, + ResourceGroup: "", + Connect: true, + Compact: false, + Filter: []string{}, + Plan: false, + Output: "hcl", + RetryCount: 5, + RetrySleepMs: 300, +} + +func TestCloudProvider_Import(t *testing.T) { + type args struct { + ctx context.Context + options *importer.ImportOptions + destination string + } + tests := []struct { + name string + a CloudProvider + args args + wantErr bool + }{ + { + name: "test import", + a: CloudProvider{}, + args: args{ + ctx: context.Background(), + options: mockOptions, + destination: "destination", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := CloudProvider{} + ImporterFunc = mockImporter + if err := a.Import(tt.args.ctx, tt.args.options, tt.args.destination); (err != nil) != tt.wantErr { + t.Errorf("CloudProvider.Import() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +var mockImporter = func(provider terraformutils.ProviderGenerator, options importer.ImportOptions, args []string) error { + return nil +} From 403671dfde0d8b1f60ee2bf14e9e77f12dd2481f Mon Sep 17 00:00:00 2001 From: Rafaela Soares Date: Tue, 8 Feb 2022 10:20:21 +0000 Subject: [PATCH 3/4] added Terraformer integration with AZURE --- docs/integrations_terraformer.md | 153 ++++++++++++++++--------------- 1 file changed, 81 insertions(+), 72 deletions(-) diff --git a/docs/integrations_terraformer.md b/docs/integrations_terraformer.md index cfdf21f12b3..1dfe312d2a3 100644 --- a/docs/integrations_terraformer.md +++ b/docs/integrations_terraformer.md @@ -4,6 +4,7 @@ From version 1.5, KICS integrates with Terraformer to scan resources deployed in **Cloud providers supported:** - AWS +- AZURE ## Configure AWS Credentials @@ -35,6 +36,67 @@ $Env:AWS_SESSION_TOKEN="" ``` +## Configure AZURE Credentials +KICS provides two possibilities to use Terraformer with AZURE. Each one requires AZURE account credentials that you need to set as environment variables. + +#### Using Service Principal with Client Certificate + +MacOS and Linux: +```sh +export ARM_SUBSCRIPTION_ID="" +export ARM_CLIENT_ID="" +export ARM_CLIENT_CERTIFICATE_PATH="" +export ARM_CLIENT_CERTIFICATE_PASSWORD="" +export ARM_TENANT_ID="" +``` + +Windows: + +```sh +SET ARM_SUBSCRIPTION_ID= +SET ARM_CLIENT_ID="" +SET ARM_CLIENT_CERTIFICATE_PATH="" +SET ARM_CLIENT_CERTIFICATE_PASSWORD="" +SET ARM_TENANT_ID="" +``` + +Powershell: + +```sh +$Env:ARM_SUBSCRIPTION_ID="" +$Env:ARM_CLIENT_ID="" +$Env:ARM_CLIENT_CERTIFICATE_PATH="" +$Env:ARM_CLIENT_CERTIFICATE_PASSWORD="" +$Env:ARM_TENANT_ID="" +``` + +#### Service Principal with Client Secret + +MacOS and Linux: +```sh +export ARM_SUBSCRIPTION_ID="" +export ARM_CLIENT_ID="" +export ARM_CLIENT_SECRET="" +export ARM_TENANT_ID="" +``` + +Windows: + +```sh +SET ARM_SUBSCRIPTION_ID= +SET ARM_CLIENT_ID="" +SET ARM_CLIENT_SECRET="" +SET ARM_TENANT_ID="" +``` + +Powershell: + +```sh +$Env:ARM_SUBSCRIPTION_ID="" +$Env:ARM_CLIENT_ID="" +$Env:ARM_CLIENT_SECRET="" +$Env:ARM_TENANT_ID="" +``` ## KICS Terraformer Path Syntax @@ -46,10 +108,12 @@ terraformer::{CloudProvider}:{Resources}:{Regions} Possible values: - `aws` +- `azure` **Resources:** A slash-separated list of the resources intended to be imported and scanned. - -You can find a complete list of possible values [here](https://github.com/GoogleCloudPlatform/terraformer/blob/master/docs/aws.md#supported-services) +You can find a complete list of possible values in the links below: +- [aws](https://github.com/GoogleCloudPlatform/terraformer/blob/master/docs/aws.md#supported-services) +- [azure](https://github.com/GoogleCloudPlatform/terraformer/blob/master/docs/azure.md#list-of-supported-azure-resources) To import all resources please use: `*` @@ -74,7 +138,7 @@ If the flag `-o, --output-path` is passed the folder `kics-extract-terraformer` variables.tf ``` -### Docker +### [AWS] Run KICS Terraformer integration with Docker To run KICS Terraformer integration with Docker simply pass the AWS Credentials that were set as environment variables to the `docker run` command and use the terraformer path syntax @@ -87,92 +151,37 @@ docker run -e AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID -e AWS_SESSION_TOKEN ch docker run -e AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID -e AWS_SESSION_TOKEN -v ${PWD}:/path/ checkmarx/kics:latest scan -p "terraformer::aws:vpc:eu-west-2" -v --no-progress -o /path/results ``` - - -### Executable - - -### **Disclaimer:** In order to run terraformer with KICS executable please follow these prerequisites: - -### Install Terraform - -Follow the steps described in Hashicorp documentation https://learn.hashicorp.com/tutorials/terraform/install-cli#install-terraform to install terraform. -### Install AWS Provider Plugin +### [AZURE] Run KICS Terraformer integration with Docker +To run KICS Terraformer integration with Docker simply pass the AZURE Credentials that were set as environment variables to the docker run command and use the terraformer path syntax. Choose one of the following options: -It is required that the AWS Provider plugin for terraform to be present. - -To install AWS Provider plugin: -- Download the plugin from [Terraform Providers](https://releases.hashicorp.com/terraform-provider-aws/3.72.0/) according to your architecture. -- Unzip the file to: - -### Linux: -``` -$HOME/.terraform.d/plugins/linux_{arch}/ - -Example: -~/.terraform.d/plugins/linux_amd64/terraform-provider-aws_v3.71.0_x5 -``` - -### MacOS +#### Using Service Principal with Client Certificate +Note that you should fill the `` with the path that points to the directory where your certificate is located, and the `` should point to the certificate name located in ``. +```sh +docker run -v :/certificate -e ARM_CLIENT_CERTIFICATE_PATH=/certificate/.pfx -e ARM_CLIENT_CERTIFICATE_PASSWORD -e ARM_TENANT_ID -e ARM_CLIENT_ID -e ARM_SUBSCRIPTION_ID checkmarx/kics:latest scan -p "terraformer::azure:storage_account:eastus" -v --no-progress ``` -$HOME/.terraform.d/plugins/darwin_{arch} - -Example: -$HOME/.terraform.d/plugins/darwin_amd64/terraform-provider-aws_3.72.0_darwin_amd64 +```sh +docker run -v :/certificate -v ${PWD}:/path/ -e ARM_CLIENT_CERTIFICATE_PATH=/certificate/.pfx -e ARM_CLIENT_CERTIFICATE_PASSWORD -e ARM_TENANT_ID -e ARM_CLIENT_ID -e ARM_SUBSCRIPTION_ID checkmarx/kics:latest scan -p "terraformer::azure:storage_account:eastus" -v --no-progress -o /path/results ``` -### Windows: - -For Windows a little more work is required, since you can't globally install the AWS Provider plugin, you need to have it present in every directory you wish to import the resources to. - -Please follow these steps: - -- Create a versions.tf file in the folder you wish to run KICS and import the resources to. - -- Paste the code found under `USE PROVIDER` from terraform AWS Provider [Documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs) in the versions.tf file you just created. - -- run the command `terraform init` on the directory containing `versions.tf`. A new folder named `.terraform` should have been created containing the plugin. This folder must be present in every directory you wish to run KICS on using terraformer. +![client_certificate_terraformer_azure](https://user-images.githubusercontent.com/74001161/152843317-7e83b70c-2a44-4f22-8a5e-fa9434950269.gif) -**NOTE:** `.terraform.hcl.lock` can be deleted -Example tf file: - -```hcl -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "3.72.0" - } - } -} - -provider "aws" { - # Configuration options -} +#### Service Principal with Client Secret +```sh +docker run -e ARM_SUBSCRIPTION_ID -e ARM_CLIENT_ID -e ARM_CLIENT_SECRET -e ARM_TENANT_ID checkmarx/kics:latest scan -p "terraformer::azure:storage_account:eastus" -v --no-progress ``` - -## Examples: - -Example path: - ```sh -kics scan -p 'terraformer::aws:vpc/subnet:eu-west-2/eu-west-1' +docker run -e ARM_SUBSCRIPTION_ID -e ARM_CLIENT_ID -e ARM_CLIENT_SECRET -e ARM_TENANT_ID -v ${PWD}:/path/ checkmarx/kics:latest scan -p "terraformer::azure:storage_account:eastus" -v --no-progress -o /path/results ``` -These examples showcase KICS integration with terraformer for importing and scanning our VPCs in region `eu-west-2`. - -### Linux +![client_secret_terraformer_azure](https://user-images.githubusercontent.com/74001161/152833926-68b7cc56-23c0-4297-b308-56f4c6746e09.gif) - -### Windows - ## **NOTES** From 5213fe78b5c468581a9f5ef275bb7bb59c344ded Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Tue, 8 Feb 2022 11:07:05 +0000 Subject: [PATCH 4/4] correcting go lint issue --- pkg/terraformer/azure/azure_cloud_provider.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/terraformer/azure/azure_cloud_provider.go b/pkg/terraformer/azure/azure_cloud_provider.go index 2470f37a67b..56c974e926f 100644 --- a/pkg/terraformer/azure/azure_cloud_provider.go +++ b/pkg/terraformer/azure/azure_cloud_provider.go @@ -31,13 +31,11 @@ func (a CloudProvider) Import(ctx context.Context, options *importer.ImportOptio wg := sync.WaitGroup{} done := make(chan error, 1) - for _, region := range options.Regions { - wg.Add(1) - go func(region string) { - defer wg.Done() - done <- ImporterFunc(provider, *options, []string{options.ResourceGroup}) - }(region) - } + wg.Add(1) + go func() { + defer wg.Done() + done <- ImporterFunc(provider, *options, []string{options.ResourceGroup}) + }() go func() { defer close(done)