Add postgres backend

.circleci/config.yml

@@ -100,13 +100,16 @@ jobs:
         command: |
           export KUBECONFIG=~/.k3d/k3s-default-config
           export KUBE_CONFIG_PATH=~/.k3d/k3s-default-config
-          k3d cluster create --image rancher/k3s:v1.21.8-k3s1 --port 8500:30058@server[0] # --k3s-server-arg "--kube-apiserver-arg=feature-gates=ServerSideApply=false"
+          k3d cluster create --image rancher/k3s:v1.21.8-k3s1 --port 8500:30058@server[0] --port 5432:32345@server[0] # --k3s-server-arg "--kube-apiserver-arg=feature-gates=ServerSideApply=false"
           k3d image import patoarvizu/amphibian:latest
           cd test/consul && helmfile sync
           cd ../consul-service && helmfile sync
           export CONSUL_HTTP_TOKEN=$(kubectl -n consul get secret consul-bootstrap-acl-token -o json | jq -r '.data.token' | base64 -d)
+          cd ../postgres && helmfile sync
+          sleep 30
           cd ../consul-state && terraform init && terraform apply -auto-approve
           cd ../kubernetes-state && terraform init && terraform apply -auto-approve
+          cd ../postgres-state && terraform init && terraform apply -auto-approve
           export AMP_CONSUL_TOKEN=${CONSUL_HTTP_TOKEN}
           cd ../secrets && helmfile sync
           cd ../amphibian && helmfile sync

.gitignore

@@ -16,4 +16,5 @@
 
 bin/
 
-.terraform/
+.terraform/
+**/.terraform.lock.hcl

README.md

@@ -13,6 +13,7 @@
     - [S3](#s3)
     - [Consul](#consul)
     - [Kubernetes](#kubernetes)
+    - [Postgres](#postgres)
   - [Target](#target)
     - [Values](#values)
 - [For security nerds](#for-security-nerds)
@@ -129,6 +130,21 @@ Additionally, the following options are not available, either because they're ir
 - `token`
 - `exec`
 
+#### Postgres
+
+- [Documentation](https://www.terraform.io/language/settings/backends/pg)
+- `type: pg`
+- Configuration block name: `postgresConfig`
+- **Note:** Postgres v10 and above support `scram-sha-256` as a password encryption mechanism, which is only supported on [Terraform 0.14](https://github.com/hashicorp/terraform/pull/26887)!
+
+The `pg` backend doesn't support configuration via environment variables in Terraform, but to avoid having to set credentials explicitly on `TerraformState` objects, Amphibian allows the injection of the connection string via the `AMP_PSQL_CONN_STR` environment variable. Note that this variable should be the full URL, including the `postgres://` prefix. The `schema_name` field is supported as documented in the link above.
+
+Additionally, the following options are not available since they're irrelevant for looking up remote states:
+
+- `skip_schema_creation`
+- `skip_table_creation`
+- `skip_index_creation`
+
 ### Target
 
 The `target` field represents the location and type of object where the outputs from the upstream state will be projected.

api/v1/terraformstate_types.go

@@ -97,13 +97,19 @@ type GCSConfig struct {
 	Prefix                    string `json:"prefix,omitempty"`
 }
 
+type PostgresConfig struct {
+	ConnStr    string `json:"connStr,omitempty"`
+	SchemaName string `json:"schemaName,omitempty"`
+}
+
 type TerraformStateSpec struct {
 	Type             string           `json:"type"`
 	RemoteConfig     RemoteConfig     `json:"remoteConfig,omitempty"`
 	S3Config         S3Config         `json:"s3Config,omitempty"`
 	ConsulConfig     ConsulConfig     `json:"consulConfig,omitempty"`
 	KubernetesConfig KubernetesConfig `json:"kubernetesConfig,omitempty"`
 	GCSConfig        GCSConfig        `json:"gcsConfig,omitempty"`
+	PostgresConfig   PostgresConfig   `json:"postgresConfig,omitempty"`
 	Target           Target           `json:"target"`
 }
 

config/crd/bases/terraform.patoarvizu.dev_terraformstates.yaml

@@ -91,6 +91,13 @@ spec:
               required:
               - secretSuffix
               type: object
+            postgresConfig:
+              properties:
+                connStr:
+                  type: string
+                schemaName:
+                  type: string
+              type: object
             remoteConfig:
               properties:
                 hostname:

controllers/terraformstate_controller.go

@@ -107,6 +107,8 @@ func (r *TerraformStateReconciler) Reconcile(req ctrl.Request) (ctrl.Result, err
 		dataBody.SetAttributeValue("config", createKubernetesBackendBody(state.Spec.KubernetesConfig))
 	case "gcs":
 		dataBody.SetAttributeValue("config", createGCSBackendBody(state.Spec.GCSConfig))
+	case "pg":
+		dataBody.SetAttributeValue("config", createPostgresBackendBody(state.Spec.PostgresConfig))
 	}
 	dataFile, err := os.Create(fmt.Sprintf("%s/data.tf", stateDir))
 	if err != nil {
@@ -421,6 +423,19 @@ func createGCSBackendBody(config terraformv1.GCSConfig) cty.Value {
 	return cty.ObjectVal(c)
 }
 
+func createPostgresBackendBody(config terraformv1.PostgresConfig) cty.Value {
+	c := make(map[string]cty.Value)
+	if len(config.ConnStr) > 0 {
+		c["conn_str"] = cty.StringVal(config.ConnStr)
+	} else if _, ok := os.LookupEnv("AMP_PSQL_CONN_STR"); ok {
+		c["conn_str"] = cty.StringVal(os.Getenv("AMP_PSQL_CONN_STR"))
+	}
+	if len(config.SchemaName) > 0 {
+		c["schema_name"] = cty.StringVal(config.SchemaName)
+	}
+	return cty.ObjectVal(c)
+}
+
 func createValueList(l []string) []cty.Value {
 	valueList := []cty.Value{}
 	for _, v := range l {

helm/amphibian/templates/crds/terraformstate.yaml

@@ -91,6 +91,13 @@ spec:
               required:
               - secretSuffix
               type: object
+            postgresConfig:
+              properties:
+                connStr:
+                  type: string
+                schemaName:
+                  type: string
+              type: object
             remoteConfig:
               properties:
                 hostname:

test/amphibian/values/values.yaml

@@ -23,6 +23,8 @@ authEnvVars:
   value: us-east-1
 - name: GOOGLE_BACKEND_CREDENTIALS
   value: /gcs-credentials/credentials.json
+- name: AMP_PSQL_CONN_STR
+  value: postgres://postgres:postgres123@localhost:5432/terraform_backend?sslmode=disable
 
 volumes:
 - name: gcs-credentials

test/consul-service/helmfile.yaml

@@ -3,6 +3,7 @@ releases:
   namespace: consul
   chart: ./chart
   version: 0.0.0
+  wait: true
 
 helmDefaults:
   kubeContext: k3d-k3s-default

test/e2e/terraformstate_test.go

@@ -191,6 +191,34 @@ func createGCSStateConfig(targetType string) (*terraformv1.TerraformState, error
 	return s, nil
 }
 
+func createPostgresStateConfig(targetType string) (*terraformv1.TerraformState, error) {
+	s := &terraformv1.TerraformState{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "TerraformState",
+			APIVersion: "terraform.patoarvizu.dev/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pg",
+			Namespace: "default",
+		},
+		Spec: terraformv1.TerraformStateSpec{
+			Type: "pg",
+			PostgresConfig: terraformv1.PostgresConfig{
+				ConnStr: "postgres://postgres:[email protected]:5432/terraform_backend?sslmode=disable",
+			},
+			Target: terraformv1.Target{
+				Type: targetType,
+				Name: "test-pg",
+			},
+		},
+	}
+	err := k8sClient.Create(context.TODO(), s)
+	if err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
 // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
 
@@ -347,6 +375,28 @@ var _ = Describe("With the controller running", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})
+	When("Deploying a TerraformState object with 'pg' config and target type 'configmap'", func() {
+		It("Should create the target ConfigMap", func() {
+			state, err = createPostgresStateConfig("configmap")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(state).ToNot(BeNil())
+			err = validateStateTargetConfigMap(state)
+			Expect(err).ToNot(HaveOccurred())
+			err = k8sClient.Delete(context.TODO(), state)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+	When("Deploying a TerraformState object with 'pg' config and target type 'secret'", func() {
+		It("Should create the target Secret", func() {
+			state, err = createPostgresStateConfig("secret")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(state).ToNot(BeNil())
+			err = validateStateTargetSecret(state)
+			Expect(err).ToNot(HaveOccurred())
+			err = k8sClient.Delete(context.TODO(), state)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
 })
 
 var _ = AfterSuite(func() {

test/postgres-state/outputs.tf

@@ -0,0 +1,14 @@
+output hello {
+  value = "world"
+}
+
+output map {
+  value = {
+    a = "b"
+    x = "y"
+  }
+}
+
+output list {
+  value = ["a", "b", "c"]
+}

test/postgres-state/remote_state.tf

@@ -0,0 +1,5 @@
+terraform {
+  backend "pg" {
+    conn_str = "postgres://postgres:postgres123@localhost:5432/terraform_backend?sslmode=disable"
+  }
+}

test/postgres/helmfile.yaml

@@ -0,0 +1,18 @@
+repositories:
+- name: bitnami
+  url: https://charts.bitnami.com/bitnami
+
+releases:
+- name: postgresql
+  namespace: pg
+  chart: bitnami/postgresql
+  version: 11.0.4
+  wait: true
+  values:
+  - ./values.yaml
+
+helmDefaults:
+  kubeContext: k3d-k3s-default
+  args:
+  - --kubeconfig
+  - {{ requiredEnv "HOME" }}/.k3d/k3s-default-config

test/postgres/values.yaml

@@ -0,0 +1,11 @@
+global:
+  postgresql:
+    auth:
+      postgresPassword: postgres123
+      database: terraform_backend
+
+primary:
+  service:
+    type: NodePort
+    nodePorts:
+      postgresql: 32345