Improve examples, make them part of e2e tests

.github/actions/cleanup-aws/action.yaml

@@ -0,0 +1,73 @@
+name: "Cleanup AWS leftovers"
+description: "Cleanup any AWS leftovers of e2e test"
+inputs:
+  service_name:
+    required: true
+    description: "The full name of the service for filtering purposes"
+  aws_role_arn:
+    required: true
+    description: "The ARN of the AWS role to assume for AWS tests"
+  aws_region:
+    required: true
+    description: "The AWS region to use for AWS tests"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Configure AWS credentials
+      id: aws-auth
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: ${{ inputs.aws_role_arn }}
+        aws-region: ${{ inputs.aws_region }}
+
+    - name: Cleanup AWS
+      shell: bash
+      run: |
+        echo "::group::Deleting VPC Endpoints"
+        attempts=10
+        while [ $attempts -gt 0 ]; do
+          attempts=$((attempts - 1))
+          endpointids="$(aws ec2 --region eu-west-2 describe-vpc-endpoints|jq --arg name "${{ inputs.service_name }}" -r '.VpcEndpoints[] | select(.Tags[0].Value == $name) | .VpcEndpointId')"
+          if [ "$endpointids" == "" ]
+          then
+            break
+          fi
+
+          echo "Deleting endpoints $endpointids"
+          for endpointid in $endpointids
+          do
+            echo "Deleting vpc endpoint $endpointid"
+            aws ec2 --region "${{ inputs.aws_region }}" delete-vpc-endpoints --vpc-endpoint-ids "$endpointid"
+          done
+
+          sleep 60
+        done
+        echo "::endgroup::"
+
+        echo "::group::Deleting Security Groups"
+        sgids="$(aws ec2 --region "${{ inputs.aws_region }}" describe-security-groups|jq --arg name "${{ inputs.service_name }}" -r '.SecurityGroups[] | select(.Tags[0].Value == $name) | .GroupId')"
+        for sgid in $sgids
+        do
+          echo "Deleting SG $sgid"
+          aws ec2 --region "${{ inputs.aws_region }}" delete-security-group --group-id "$sgid"
+        done
+        echo "::endgroup::"
+
+        echo "::group::Deleting Subnets"
+        subnetids="$(aws ec2 --region "${{ inputs.aws_region }}" describe-subnets|jq --arg name "${{ inputs.service_name }}" -r '.Subnets[] | select(.Tags[0].Value == $name) | .SubnetId')"
+        for subnetid in $subnetids
+        do
+          echo "Deleting subnet $subnetid"
+          aws ec2 --region "${{ inputs.aws_region }}" delete-subnet --subnet-id "$subnetid"
+        done
+        echo "::endgroup::"
+
+        echo "::group::Deleting VPCs"
+        vpcids="$(aws ec2 --region "${{ inputs.aws_region }}" describe-vpcs|jq --arg name "${{ inputs.service_name }}" -r '.Vpcs[] | select(.Tags[0].Value == $name) | .VpcId')"
+        for vpcid in $vpcids
+        do
+          echo "Deleting vpc $vpcid"
+          aws ec2 --region "${{ inputs.aws_region }}" delete-vpc --vpc-id "$vpcid"
+        done
+        echo "::endgroup::"

.github/actions/cleanup/action.yaml

@@ -17,7 +17,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: cleanup
+    - name: cleanup clikchouse
       shell: bash
       run: |
         organization_id="${{ inputs.organization_id }}"

.github/actions/e2e/action.yaml

@@ -10,9 +10,9 @@ inputs:
   token_secret:
     required: true
     description: "The clickhouse token secret"
-  token:
+  service_name:
     required: true
-    description: "The unique token assigned to this e2e run"
+    description: "The unique name assigned to this test"
   test_name:
     required: true
     description: "The test name i.e. the name of the folder inside the examples dir"
@@ -30,12 +30,29 @@ inputs:
     required: false
     default: "false"
     description: "If true it runs tests out of last published terraform provider release"
-
+  aws_role_arn:
+    required: true
+    description: "The ARN of the AWS role to assume for AWS tests"
+  aws_region:
+    required: true
+    description: "The AWS region to use for AWS tests"
 runs:
   using: "composite"
   steps:
+    - shell: bash
+      id: defined
+      name: Check if test is defined for cloud provider
+      run: |
+        if [ -d "examples/full/${{ inputs.test_name }}/${{ inputs.cloud_provider }}" ]
+        then
+          echo "defined=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "Test ${{ inputs.test_name }} is not available for ${{ inputs.cloud_provider }}"
+        fi
+
     - name: Setup go
       uses: actions/setup-go@v5
+      if: ${{steps.defined.outputs.defined == 'true' }}
       with:
         go-version-file: 'go.mod'
         cache: true
@@ -47,40 +64,59 @@ runs:
         terraform_wrapper: false
 
     - shell: bash
+      if: ${{steps.defined.outputs.defined == 'true' }}
       name: Show terraform version
       run: terraform version
 
+    - name: Configure AWS credentials
+      id: aws-auth
+      uses: aws-actions/configure-aws-credentials@v4
+      if: ${{ inputs.cloud_provider == 'aws' }}
+      with:
+        role-to-assume: ${{ inputs.aws_role_arn }}
+        aws-region: ${{ inputs.aws_region }}
+        output-credentials: true
+
     - shell: bash
+      if: ${{steps.defined.outputs.defined == 'true' }}
       name: Set env variables
       run: |
         echo "::group::Setting env variables"
-        test_type=""
-        if [ "${{inputs.upgrade_test}}" == "true" ]
+
+        cat <<EOF >examples/full/${{ inputs.test_name }}/${{ inputs.cloud_provider }}/variables.tfvars
+        organization_id = "${{ inputs.organization_id }}"
+        token_key       = "${{ inputs.token_key }}"
+        token_secret    = "${{ inputs.token_secret }}"
+        service_name    = "${{ inputs.service_name }}"
+        EOF
+
+        if [ "${{ inputs.cloud_provider }}" == "aws" ]
         then
-          test_type=" upgrade"
-        fi
-        service_name="[e2e${test_type}]-${{ inputs.test_name }}-${{ inputs.tf_release }}-${{ inputs.cloud_provider }}-${{ inputs.token }}"
+          aws sts get-caller-identity
 
-        echo "TF_VAR_organization_id=${{ inputs.organization_id }}" >> $GITHUB_ENV
-        echo "TF_VAR_token_key=${{ inputs.token_key }}" >> $GITHUB_ENV
-        echo "TF_VAR_token_secret=${{ inputs.token_secret }}" >> $GITHUB_ENV
-        echo "TF_VAR_service_name=${service_name}" >> $GITHUB_ENV
-        echo "TF_VAR_cloud_provider=${{ inputs.cloud_provider }}" >> $GITHUB_ENV
+          cat <<EOF >>examples/full/${{ inputs.test_name }}/${{ inputs.cloud_provider }}/variables.tfvars
+        aws_region        = "${{ inputs.aws_region }}"
+        aws_key           = "${{ steps.aws-auth.outputs.aws-access-key-id }}"
+        aws_secret        = "${{ steps.aws-auth.outputs.aws-secret-access-key }}"
+        aws_session_token = "${{ steps.aws-auth.outputs.aws-session-token }}"
+        EOF
+        fi
+
         echo "::endgroup::"
 
     - shell: bash
-      if: ${{ inputs.upgrade_test == 'true' }}
+      if: ${{ inputs.upgrade_test == 'true' && steps.defined.outputs.defined == 'true' }}
       name: Upgrade test - Run terraform using latest stable version
       run: |
         echo "::group::Run terraform using latest stable version"
-        cd "examples/${{ inputs.test_name }}"
+        cd "examples/full/${{ inputs.test_name }}/${{ inputs.cloud_provider }}"
         terraform init -input=false
-        terraform plan -no-color
-        terraform apply -no-color -auto-approve
+        terraform plan -no-color -var-file=variables.tfvars
+        terraform apply -no-color -auto-approve -var-file=variables.tfvars
         echo "::endgroup::"
 
     - shell: bash
-      if: ${{ inputs.skip_build == 'false' }}
+      if: ${{ inputs.skip_build == 'false' && steps.defined.outputs.defined == 'true' }}
       name: Build provider from branch and create terraformrc to use it
       run: |
         echo "::group::Build provider from branch and create terraformrc to use it"
@@ -98,20 +134,23 @@ runs:
         echo "::endgroup::"
 
     - shell: bash
+      if: ${{steps.defined.outputs.defined == 'true' }}
       name: Run terraform
       run: |
         echo "::group::Run terraform"
-        cd "examples/${{ inputs.test_name }}"
+        cd "examples/full/${{ inputs.test_name }}/${{ inputs.cloud_provider }}"
+
         terraform init -input=false -upgrade
-        terraform plan -no-color
-        terraform apply -no-color -auto-approve
+        terraform plan -no-color -var-file=variables.tfvars
+        terraform apply -no-color -auto-approve -var-file=variables.tfvars
+        terraform refresh -no-color -var-file=variables.tfvars
         echo "::endgroup::"
 
     - shell: bash
       name: Terraform destroy
-      if: always()
+      if: ${{steps.defined.outputs.defined == 'true' }}
       run: |
         echo "::group::Run terraform destroy"
-        cd "examples/${{ inputs.test_name }}"
-        terraform destroy -no-color -auto-approve
+        cd "examples/full/${{ inputs.test_name }}/${{ inputs.cloud_provider }}"
+        terraform destroy -no-color -auto-approve -var-file=variables.tfvars
         echo "::endgroup::"

.github/workflows/e2e.yaml

@@ -2,14 +2,25 @@ name: E2E tests
 
 on:
   workflow_dispatch:
-    inputs: {}
+    inputs:
+      notify_errors:
+        type: boolean
+        default: false
+        description: "If checked, send any errors in e2e tests to the #proj-api-terraform slack channel"
+      skip_build_from_branch:
+        type: boolean
+        default: false
+        description: "If checked, the latest stable release of the clickhouse terraform plugin is used for the tests"
   schedule:
     - cron: "0 3 * * *"
 
 defaults:
   run:
     shell: bash
 
+env:
+  aws_region: "eu-west-2"
+
 jobs:
   # Generate a random token to tag the tests with
   token:
@@ -41,29 +52,50 @@ jobs:
     needs: [ "token", "find-tf-releases" ]
     runs-on: ubuntu-latest
     continue-on-error: true
+    permissions:
+      id-token: write
     strategy:
       fail-fast: false
       max-parallel: 5
       matrix:
-        test: [ "basic" ]
+        test: [ "basic", "private_endpoint" ]
         tf_release: ${{ fromJSON(needs.find-tf-releases.outputs.releases) }}
         cloud_provider: [ "aws", "gcp", "azure" ]
         upgrade_test: ["false"]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Generate test name
+        id: name
+        run: |
+          test_type=""
+          if [ "${{matrix.upgrade_test}}" == "true" ]
+          then
+            test_type=" upgrade"
+          fi
+
+          echo "test_name=[e2e${test_type}]-${{ matrix.test }}-${{ matrix.tf_release }}-${{ matrix.cloud_provider }}-${{ needs.token.outputs.token }}" >> $GITHUB_OUTPUT
       - name: e2e
         uses: ./.github/actions/e2e
         with:
           organization_id: ${{ secrets.TF_VAR_ORGANIZATION_ID }}
           token_key: ${{ secrets.TF_VAR_TOKEN_KEY }}
           token_secret: ${{ secrets.TF_VAR_TOKEN_SECRET }}
-          token: ${{needs.token.outputs.token}}
+          service_name: ${{steps.name.outputs.test_name}}
           test_name: ${{ matrix.test }}
           tf_release: ${{ matrix.tf_release }}
           cloud_provider: ${{ matrix.cloud_provider }}
           upgrade_test: ${{ matrix.upgrade_test }}
-          skip_build: "true"
+          skip_build: "${{ !contains(inputs.skip_build_from_branch, 'false') }}"
+          aws_region: ${{ env.aws_region }}
+          aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
+      - name: cleanup
+        if: ${{ always() && matrix.cloud_provider == 'aws' && matrix.test == 'private_endpoint' }}
+        uses: ./.github/actions/cleanup-aws
+        with:
+          service_name: ${{steps.name.outputs.test_name}}
+          aws_region: ${{ env.aws_region }}
+          aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
       - name: Mark error
         id: status
         if: failure()
@@ -76,19 +108,24 @@ jobs:
     if: ${{ needs.e2e.outputs.status == 'failure' }}
     steps:
       - name: Report Failure on slack
+        if: ${{ !contains(inputs.notify_errors, 'false') }}
         uses: ravsamhq/notify-slack-action@v2
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           status: ${{ job.status }}
-          notification_title: "E2E tests failed during release ${github.ref_name}"
-          footer: "{workflow_url}"
+          notification_title: "E2E tests failed for {branch}"
+          footer: "{run_url}"
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
+      - name: Report error on github UI
+        run: |
+          exit 1
 
   # Delete any leftover service that might have failed deleting
   cleanup:
     runs-on: ubuntu-latest
     needs: [ "e2e", "token" ]
+    if: always()
     continue-on-error: true
     env:
       organization_id: ${{ secrets.TF_VAR_ORGANIZATION_ID }}

.gitignore

@@ -1,7 +1,5 @@
-.terraform
-terraform.tfstate
-terraform.tfstate.backup
-.terraform.lock.hcl
+.terraform*
+terraform.tfstate*
 
 terraform-provider-clickhouse
 
@@ -10,3 +8,5 @@ secrets.tfvars
 dev/
 .idea/
 *.backup
+
+examples/full/**/variables.tfvars

README.md

@@ -4,9 +4,11 @@
 [![Dependabot Updates](https://github.com/ClickHouse/terraform-provider-clickhouse/actions/workflows/dependabot/dependabot-updates/badge.svg)](https://github.com/ClickHouse/terraform-provider-clickhouse/actions/workflows/dependabot/dependabot-updates)
 [![Unit tests](https://github.com/ClickHouse/terraform-provider-clickhouse/actions/workflows/test.yaml/badge.svg)](https://github.com/ClickHouse/terraform-provider-clickhouse/actions/workflows/test.yaml)
 
+This is the official terraform provider for [ClickHouse Cloud](https://clickhouse.com/docs/en/about-us/cloud).
+
 ## Usage
 
-This is the official terraform provider for [ClickHouse Cloud](https://clickhouse.com/cloud).
+You can find examples in the [examples/full](https://github.com/ClickHouse/terraform-provider-clickhouse/tree/main/examples/full) directory.
 
 Please refer to the [official docs](https://registry.terraform.io/providers/ClickHouse/clickhouse/latest/docs) for more details.
 
@@ -36,14 +38,14 @@ go install github.com/air-verse/air@latest
 
 Run `air` to automatically build the plugin binary every time you make changes to the code:
 
-```sh
+```bash
 $ air
 ```
 
 You can now run `terraform` and you'll be using the locally built binary. Please note that the `dev_overrides` make it so that you have to skip `terraform init`).
 For example, go to the `examples/basic` directory and :
 
-```
+```bash
 terraform apply -var-file="variables.tfvars"
 ╷
 │ Warning: Provider development overrides are in effect