vault: allow providing auth method path to the connect CA configuration

hashicorp · Feb 17, 2022 · 94094e2 · 94094e2
1 parent 6d32dfc
commit 94094e2
Show file tree

Hide file tree

Showing 10 changed files with 67 additions and 85 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -70,7 +70,7 @@ commands:
         type: string
       consul-k8s-image:
         type: string
-        default: "ishustava/consul-k8s-dev:02-08-2022-d21554d8"
+        default: "docker.mirror.hashicorp.services/hashicorpdev/consul-k8s-control-plane:latest"
       go-path:
         type: string
         default: "/home/circleci/.go_workspace"

diff --git a/acceptance/framework/consul/consul_cluster.go b/acceptance/framework/consul/consul_cluster.go
@@ -52,8 +52,6 @@ type HelmCluster struct {
 	noCleanupOnFailure bool
 	debugDirectory     string
 	logger             terratestLogger.TestLogger
-
-	ACLToken string
 }
 
 func NewHelmCluster(

diff --git a/acceptance/framework/helpers/helpers.go b/acceptance/framework/helpers/helpers.go
@@ -148,46 +148,3 @@ func MergeMaps(a, b map[string]string) {
 		a[k] = v
 	}
 }
-
-// VerifyFederation checks that the WAN federation between servers is successful
-// by first checking members are alive from the perspective of both servers.
-// If secure is true, it will also check that the ACL replication is running on the secondary server.
-func VerifyFederation(t *testing.T, primaryClient, secondaryClient *api.Client, releaseName string, secure bool) {
-	retrier := &retry.Timer{Timeout: 5 * time.Minute, Wait: 1 * time.Second}
-	start := time.Now()
-
-	// Check that server in dc1 is healthy from the perspective of the server in dc2, and vice versa.
-	// We're calling the Consul health API, as opposed to checking serf membership status,
-	// because we need to make sure that the federated servers can make API calls and forward requests
-	// from one server to another. From running tests in CI for a while and using serf membership status before,
-	// we've noticed that the status could be "alive" as soon as the server in the secondary cluster joins the primary
-	// and then switch to "failed". This would require us to check that the status is "alive" is showing consistently for
-	// some amount of time, which could be quite flakey. Calling the API in another datacenter allows us to check that
-	// each server can forward calls to another, which is what we need for connect.
-	retry.RunWith(retrier, t, func(r *retry.R) {
-		secondaryServerHealth, _, err := primaryClient.Health().Node(fmt.Sprintf("%s-consul-server-0", releaseName), &api.QueryOptions{Datacenter: "dc2"})
-		require.NoError(r, err)
-		require.Equal(r, secondaryServerHealth.AggregatedStatus(), api.HealthPassing)
-
-		primaryServerHealth, _, err := secondaryClient.Health().Node(fmt.Sprintf("%s-consul-server-0", releaseName), &api.QueryOptions{Datacenter: "dc1"})
-		require.NoError(r, err)
-		require.Equal(r, primaryServerHealth.AggregatedStatus(), api.HealthPassing)
-
-		if secure {
-			replicationStatus, _, err := secondaryClient.ACL().Replication(nil)
-			require.NoError(r, err)
-			require.True(r, replicationStatus.Enabled)
-			require.True(r, replicationStatus.Running)
-		}
-	})
-
-	logger.Logf(t, "Took %s to verify federation", time.Since(start))
-}
-
-// MergeMaps will merge the values in b with values in a and save in a.
-// If there are conflicts, the values in b will overwrite the values in a.
-func MergeMaps(a, b map[string]string) {
-	for k, v := range b {
-		a[k] = v
-	}
-}
diff --git a/acceptance/tests/vault/helpers.go b/acceptance/tests/vault/helpers.go
@@ -22,9 +22,9 @@ path "consul/data/secret/replication" {
   capabilities = ["read", "update"]
 }`
 
-	// connectCAPolicy allows Consul to bootstrap all certificates for the service mesh in Vault.
+	// connectCAPolicyTemplate allows Consul to bootstrap all certificates for the service mesh in Vault.
 	// Adapted from https://www.consul.io/docs/connect/ca/vault#consul-managed-pki-paths.
-	connectCAPolicy = `
+	connectCAPolicyTemplate = `
 path "/sys/mounts" {
   capabilities = [ "read" ]
 }
@@ -33,15 +33,15 @@ path "/sys/mounts/connect_root" {
   capabilities = [ "create", "read", "update", "delete", "list" ]
 }
 
-path "/sys/mounts/connect_inter" {
+path "/sys/mounts/%s/connect_inter" {
   capabilities = [ "create", "read", "update", "delete", "list" ]
 }
 
 path "/connect_root/*" {
   capabilities = [ "create", "read", "update", "delete", "list" ]
 }
 
-path "/connect_inter/*" {
+path "/%s/connect_inter/*" {
   capabilities = [ "create", "read", "update", "delete", "list" ]
 }
 `
@@ -118,7 +118,7 @@ func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consul
 	params = map[string]interface{}{
 		"bound_service_account_names":      consulServerServiceAccountName,
 		"bound_service_account_namespaces": ns,
-		"policies":                         fmt.Sprintf("consul-gossip,connect-ca,consul-server-%s,consul-replication-token", datacenter),
+		"policies":                         fmt.Sprintf("consul-gossip,connect-ca-%s,consul-server-%s,consul-replication-token", datacenter, datacenter),
 		"ttl":                              "24h",
 	}
 	_, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-server", authPath), params)
@@ -222,3 +222,11 @@ func configureReplicationTokenVaultSecret(t *testing.T, vaultClient *vapi.Client
 
 	return token
 }
+
+// createConnectCAPolicy creates the Vault Policy for the connect-ca in a given datacenter.
+func createConnectCAPolicy(t *testing.T, vaultClient *vapi.Client, datacenter string) {
+	err := vaultClient.Sys().PutPolicy(
+		fmt.Sprintf("connect-ca-%s", datacenter),
+		fmt.Sprintf(connectCAPolicyTemplate, datacenter, datacenter))
+	require.NoError(t, err)
+}
diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go
@@ -30,9 +30,7 @@ func TestVault(t *testing.T) {
 
 	gossipKey := configureGossipVaultSecret(t, vaultClient)
 
-	// Create the Vault Policy for the connect-ca.
-	err := vaultClient.Sys().PutPolicy("connect-ca", connectCAPolicy)
-	require.NoError(t, err)
+	createConnectCAPolicy(t, vaultClient, "dc1")
 
 	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1")
 
@@ -60,7 +58,7 @@ func TestVault(t *testing.T) {
 
 		"global.secretsBackend.vault.connectCA.address":             vaultCluster.Address(),
 		"global.secretsBackend.vault.connectCA.rootPKIPath":         "connect_root",
-		"global.secretsBackend.vault.connectCA.intermediatePKIPath": "connect_inter",
+		"global.secretsBackend.vault.connectCA.intermediatePKIPath": "dc1/connect_inter",
 
 		"global.acls.manageSystemACLs":       "true",
 		"global.tls.enabled":                 "true",

diff --git a/acceptance/tests/vault/vault_wan_fed_test.go b/acceptance/tests/vault/vault_wan_fed_test.go
@@ -118,6 +118,10 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 	replicationToken := configureReplicationTokenVaultSecret(t, vaultClient, consulReleaseName, ns, "kubernetes", "kubernetes-dc2")
 
+	// Create the Vault Policy for the Connect CA in both datacenters.
+	createConnectCAPolicy(t, vaultClient, "dc1")
+	createConnectCAPolicy(t, vaultClient, "dc2")
+
 	// Move Vault CA secret from primary to secondary so that we can mount it to pods in the
 	// secondary cluster.
 	vaultCASecretName := vault.CASecretName(vaultReleaseName)
@@ -164,13 +168,16 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"server.extraVolumes[0].load": "false",
 
 		// Vault config.
-		"global.secretsBackend.vault.enabled":              "true",
-		"global.secretsBackend.vault.consulServerRole":     "consul-server",
-		"global.secretsBackend.vault.consulClientRole":     "consul-client",
-		"global.secretsBackend.vault.consulCARole":         "consul-ca",
-		"global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init",
-		"global.secretsBackend.vault.ca.secretName":        vaultCASecretName,
-		"global.secretsBackend.vault.ca.secretKey":         "tls.crt",
+		"global.secretsBackend.vault.enabled":                       "true",
+		"global.secretsBackend.vault.consulServerRole":              "consul-server",
+		"global.secretsBackend.vault.consulClientRole":              "consul-client",
+		"global.secretsBackend.vault.consulCARole":                  "consul-ca",
+		"global.secretsBackend.vault.manageSystemACLsRole":          "server-acl-init",
+		"global.secretsBackend.vault.ca.secretName":                 vaultCASecretName,
+		"global.secretsBackend.vault.ca.secretKey":                  "tls.crt",
+		"global.secretsBackend.vault.connectCA.address":             primaryVaultCluster.Address(),
+		"global.secretsBackend.vault.connectCA.rootPKIPath":         "connect_root",
+		"global.secretsBackend.vault.connectCA.intermediatePKIPath": "dc1/connect_inter",
 	}
 
 	if cfg.UseKind {
@@ -216,14 +223,19 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		"server.extraConfig":          serverExtraConfig,
 
 		// Vault config.
-		"global.secretsBackend.vault.enabled":              "true",
-		"global.secretsBackend.vault.consulServerRole":     "consul-server",
-		"global.secretsBackend.vault.consulClientRole":     "consul-client",
-		"global.secretsBackend.vault.consulCARole":         "consul-ca",
-		"global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init",
-		"global.secretsBackend.vault.ca.secretName":        vaultCASecretName,
-		"global.secretsBackend.vault.ca.secretKey":         "tls.crt",
-		"global.secretsBackend.vault.agentAnnotations":     fmt.Sprintf("vault.hashicorp.com/tls-server-name: %s-vault", vaultReleaseName),
+		"global.secretsBackend.vault.enabled":                       "true",
+		"global.secretsBackend.vault.consulServerRole":              "consul-server",
+		"global.secretsBackend.vault.consulClientRole":              "consul-client",
+		"global.secretsBackend.vault.consulCARole":                  "consul-ca",
+		"global.secretsBackend.vault.manageSystemACLsRole":          "server-acl-init",
+		"global.secretsBackend.vault.ca.secretName":                 vaultCASecretName,
+		"global.secretsBackend.vault.ca.secretKey":                  "tls.crt",
+		"global.secretsBackend.vault.agentAnnotations":              fmt.Sprintf("vault.hashicorp.com/tls-server-name: %s-vault", vaultReleaseName),
+		"global.secretsBackend.vault.connectCA.address":             externalVaultAddress,
+		"global.secretsBackend.vault.connectCA.authMethodPath":      "kubernetes-dc2",
+		"global.secretsBackend.vault.connectCA.rootPKIPath":         "connect_root",
+		"global.secretsBackend.vault.connectCA.intermediatePKIPath": "dc2/connect_inter",
+		"global.secretsBackend.vault.connectCA.additionalConfig":    fmt.Sprintf(`"{"connect": [{"ca_config": [{"tls_server_name": "%s-vault"}]}]}"`, vaultReleaseName),
 	}
 
 	if cfg.UseKind {

diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl
@@ -73,20 +73,6 @@ as well as the global.name setting.
           {{ "{{" }}- end -{{ "}}" }}
 {{- end -}}
 
-{{- define "consul.vaultReplicationTokenTemplate" -}}
-|
-          {{ "{{" }}- with secret "{{ .Values.global.acls.replicationToken.secretName }}" -{{ "}}" }}
-          {{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}
-          {{ "{{" }}- end -{{ "}}" }}
-{{- end -}}
-
-{{- define "consul.vaultReplicationTokenConfigTemplate" -}}
-|
-          {{ "{{" }}- with secret "{{ .Values.global.acls.replicationToken.secretName }}" -{{ "}}" }}
-          acl { tokens { agent = "{{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}", replication = "{{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.replicationToken.secretKey }} -{{ "}}" }}" }}
-          {{ "{{" }}- end -{{ "}}" }}
-{{- end -}}
-
 {{/*
 Sets up the extra-from-values config file passed to consul and then uses sed to do any necessary
 substitution for HOST_IP/POD_IP/HOSTNAME. Useful for dogstats telemetry. The output file

diff --git a/charts/consul/templates/server-config-configmap.yaml b/charts/consul/templates/server-config-configmap.yaml
@@ -29,6 +29,7 @@ data:
               "root_pki_path": "{{ .connectCA.rootPKIPath }}",
               "auth_method": {
                 "type": "kubernetes",
+                "mount_path": "{{ .connectCA.authMethodPath }}",
                 "params": {
                   "role": "{{ .consulServerRole }}"
                 }

diff --git a/charts/consul/test/unit/server-config-configmap.bats b/charts/consul/test/unit/server-config-configmap.bats
@@ -375,7 +375,7 @@ load _helpers
       --set 'global.secretsBackend.vault.connectCA.intermediatePKIPath=int' \
       . | tee /dev/stderr |
       yq '.data["connect-ca-config.json"]' | tee /dev/stderr)
-  [ "${actual}" = '"{\n  \"connect\": [\n    {\n      \"ca_config\": [\n        {\n          \"address\": \"example.com\",\n          \"intermediate_pki_path\": \"int\",\n          \"root_pki_path\": \"root\",\n          \"auth_method\": {\n            \"type\": \"kubernetes\",\n            \"params\": {\n              \"role\": \"foo\"\n            }\n          }\n        }\n      ],\n      \"ca_provider\": \"vault\"\n    }\n  ]\n}\n"' ]
+  [ "${actual}" = '"{\n  \"connect\": [\n    {\n      \"ca_config\": [\n        {\n          \"address\": \"example.com\",\n          \"intermediate_pki_path\": \"int\",\n          \"root_pki_path\": \"root\",\n          \"auth_method\": {\n            \"type\": \"kubernetes\",\n            \"mount_path\": \"kubernetes\",\n            \"params\": {\n              \"role\": \"foo\"\n            }\n          }\n        }\n      ],\n      \"ca_provider\": \"vault\"\n    }\n  ]\n}\n"' ]
 
   local actual=$(helm template \
       -s templates/server-config-configmap.yaml  \
@@ -407,6 +407,25 @@ load _helpers
   [ "${actual}" = '"{\"hello\": \"world\"}\n"' ]
 }
 
+@test "server/ConfigMap: can set auth method mount path" {
+  cd `chart_dir`
+
+  local caConfig=$(helm template \
+      -s templates/server-config-configmap.yaml  \
+      --set 'global.secretsBackend.vault.enabled=true' \
+      --set 'global.secretsBackend.vault.consulServerRole=foo' \
+      --set 'global.secretsBackend.vault.consulClientRole=foo' \
+      --set 'global.secretsBackend.vault.connectCA.address=example.com' \
+      --set 'global.secretsBackend.vault.connectCA.rootPKIPath=root' \
+      --set 'global.secretsBackend.vault.connectCA.intermediatePKIPath=int' \
+      --set 'global.secretsBackend.vault.connectCA.authMethodPath=kubernetes2' \
+      . | tee /dev/stderr |
+      yq -r '.data["connect-ca-config.json"]' | tee /dev/stderr)
+
+  local actual=$(echo $caConfig |  jq -r .connect[0].ca_config[0].auth_method.mount_path)
+  [ "${actual}" = "kubernetes2" ]
+}
+
 @test "server/ConfigMap: doesn't set Vault CA cert in connect CA config by default" {
   cd `chart_dir`
 

diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
@@ -208,6 +208,9 @@ global:
         # The address of the Vault server.
         address: ""
 
+        # The mount path of the Kubernetes auth method in Vault.
+        authMethodPath: "kubernetes"
+
         # The path to a PKI secrets engine for the root certificate.
         # Please see https://www.consul.io/docs/connect/ca/vault#rootpkipath.
         rootPKIPath: ""