Allow NMA certs to be read from GSM

api/v1/helpers.go

@@ -553,31 +553,3 @@ func (v *VerticaDB) GetEncryptSpreadComm() string {
 func (v *VerticaDB) IsKSafetyCheckStrict() bool {
 	return vmeta.IsKSafetyCheckStrict(v.Annotations)
 }
-
-const gsmPrefix = "gsm://"
-
-// ReadSUPwdFromGSM returns true if superuser password has the prefix "gsm://". The prefix "gsm://" will
-// tell the operator to fetch superuser password from Google's secret manager instead of k8s meta-data.
-func (v *VerticaDB) ReadSUPwdFromGSM() bool {
-	return strings.HasPrefix(v.Spec.PasswordSecret, gsmPrefix)
-}
-
-// GetSUPwdSecretName returns secret name of the one that stores superuser password. If the secret name
-// has prefix "gsm://", we will remove it. This function will be used for processing GSM secrets.
-func (v *VerticaDB) GetSUPwdSecretName() string {
-	return strings.TrimPrefix(v.Spec.PasswordSecret, gsmPrefix)
-}
-
-// ReadCommunalCredsFromGSM returns true if communal access credentials has the prefix "gsm://".
-// The prefix "gsm://" will tell the operator to fetch communal access credentials from Google's
-// secret manager instead of k8s meta-data.
-func (v *VerticaDB) ReadCommunalCredsFromGSM() bool {
-	return strings.HasPrefix(v.Spec.Communal.CredentialSecret, gsmPrefix)
-}
-
-// GetCommunalCredsSecretName returns secret name of the one that stores communal access credentials.
-// If the secret name has prefix "gsm://", we will remove it. This function will be used for processing
-// GSM secrets.
-func (v *VerticaDB) GetCommunalCredsSecretName() string {
-	return strings.TrimPrefix(v.Spec.Communal.CredentialSecret, gsmPrefix)
-}

api/v1/verticadb_types.go

@@ -92,7 +92,10 @@ type VerticaDBSpec struct {
 	// An optional name for a secret that contains the password for the
 	// database's superuser. If this is not set, then we assume no such password
 	// is set for the database. If this is set, it is up the user to create this
-	// secret before deployment. The secret must have a key named password.
+	// secret before deployment. The secret must have a key named password. To
+	// store this secret outside of Kubernetes, you can use a secret path
+	// reference prefix, such as gsm://. Everything after the prefix is the name
+	// of the secret in the service you are storing.
 	PasswordSecret string `json:"passwordSecret,omitempty"`
 
 	// +kubebuilder:validation:Optional
@@ -258,12 +261,12 @@ type VerticaDBSpec struct {
 	// +kubebuilder:default:=""
 	// +kubebuilder:validation:Optional
 	// A secret that contains the TLS credentials to use for Vertica's node
-	// management agent (NMA).  If this is empty, the operator will create a
+	// management agent (NMA). If this is empty, the operator will create a
 	// secret to use and add the name of the generate secret in this field.
-	// When set, the secret must have the following keys defined:
-	// - tls.key: The private key to be used by the HTTP server
-	// - tls.crt: The signed certificate chain for the private key
-	// - ca.crt: The CA certificate
+	// When set, the secret must have the following keys defined: tls.key,
+	// tls.crt and ca.crt.  To store this secret outside of Kubernetes, you can
+	// use a secret path reference prefix, such as gsm://. Everything after the
+	// prefix is the name of the secret in the service you are storing.
 	NMATLSSecret string `json:"nmaTLSSecret,omitempty"`
 
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:hidden"
@@ -440,27 +443,17 @@ type CommunalStorage struct {
 
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:io.kubernetes:Secret"
-	// The name of a secret that contains the credentials to connect to the
-	// communal endpoint (only applies to s3://, gs:// or azb://). Certain keys
-	// need to be set, depending on the endpoint type:
-	// - s3:// or gs:// - If storing credentials in a secret, the secret must
-	//     have the following keys set: accesskey and secretkey.  When using
-	//     Google Cloud Storage, the IDs set in the secret are taken
-	//     from the hash-based message authentication code (HMAC) keys.
-	// - azb:// - It must have the following keys set:
-	//     accountName - Name of the Azure account
-	//     blobEndpoint - (Optional) Set this to the location of the endpoint.
-	//       If using an emulator like Azurite, it can be set to something like
-	//       'http://<IP-addr>:<port>'
-	//     accountKey - If accessing with an account key set it here
-	//     sharedAccessSignature - If accessing with a shared access signature,
-	//     	  set it here
-	//
-	// This field is optional. For AWS, authentication to communal storage can
-	// be provided through an attached IAM profile: attached to the EC2 instance
-	// or to a ServiceAccount with IRSA (see
-	// https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html).
-	// IRSA requires a Vertica server running at least with version >= 12.0.3.
+	// The name of an optional secret that contains the credentials to connect to the
+	// communal endpoint. This can be omitted if the communal storage uses some
+	// other form of authentication such as an attached IAM profile in AWS.
+	// Certain keys need to be set, depending on the endpoint type. If the
+	// communal storage starts with s3:// or gs://, the secret must have the
+	// following keys set: accesskey and secretkey. If the communal storage
+	// starts with azb://, the secret can have the following keys: accountName,
+	// blobEndpoint, accountKey, or sharedAccessSignature. To store this secret
+	// outside of Kubernetes, you can use a secret path reference prefix, such
+	// as gsm://. Everything after the prefix is the name of the secret in the
+	// service you are storing.
 	CredentialSecret string `json:"credentialSecret"`
 
 	// +kubebuilder:validation:Optional

changes/unreleased/Added-20231204-102917.yaml

@@ -0,0 +1,6 @@
+kind: Added
+body: Allow annotations to be added to the serviceAccount created through the helm
+  chart.
+time: 2023-12-04T10:29:17.162221413-04:00
+custom:
+  Issue: "618"

helm-charts/verticadb-operator/README.md

@@ -26,6 +26,8 @@ This helm chart will install the operator and an admission controller webhook.
 | rbac_proxy_image.name | Image name of Kubernetes RBAC proxy. | kubebuilder/kube-rbac-proxy:v0.13.1 |
 | rbac_proxy_image.repo | Repo server hosting rbac_proxy_image.name | gcr.io |
 | resources.\* | The resource requirements for the operator pod. | <pre>limits:<br>  cpu: 100m<br>  memory: 750Mi<br>requests:<br>  cpu: 100m<br>  memory: 20Mi</pre> |
+| serviceAccountAnnotations | A map of annotations that will be added to the serviceaccount created. | |
+| serviceAccountNameOverride | Controls the name given to the serviceaccount that is created. | |
 | tolerations | Any [tolerations and taints](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) used to influence where a pod is scheduled. This parameter is provided as a list. | Not set |
 | webhook.caBundle | A PEM encoded CA bundle that will be used to validate the webhook's server certificate.  This option is deprecated in favour of providing the CA bundle in the webhook.tlsSecret with the ca.crt key. This option will be removed in a future release.| |
 | webhook.certSource | The webhook requires a TLS certificate to work. This parm defines how the cert is supplied. Valid values are:<br><br>- **internal**: The certs are generated internally by the operator prior to starting the managing controller. The generated cert is self-signed. When it expires, the operator pod will need to be restarted in order to generate a new certificate. This is the default.<br><br>- **cert-manager**: The certs are generated using the cert-manager operator.  This operator needs to be deployed before deploying the operator. Deployment of this chart will create a self-signed cert through cert-manager. The advantage of this over 'internal' is that cert-manager will automatically handle private key rotation when the certificate is about to expire.<br><br>- **secret**: The certs are created prior to installation of this chart and are provided to the operator through a secret. This option gives you the most flexibility as it is entirely up to you how the cert is created.  This option requires the webhook.tlsSecret option to be set. For backwards compatibility, if webhook.tlsSecret is set, it is implicit that this mode is selected. | internal |

helm-charts/verticadb-operator/tests/serviceaccount_test.yaml

@@ -0,0 +1,24 @@
+suite: ServiceAccount tests
+templates:
+  - verticadb-operator-controller-manager-sa.yaml
+tests:
+  - it: should allow you to override the serviceaccount name
+    set:
+      serviceAccountNameOverride: opentext-sa
+    asserts:
+      - equal:
+          path: metadata.name
+          value: opentext-sa
+  - it: should allow you to add annotations to the serviceaccount
+    set:
+      serviceAccountAnnotations:
+        foo: "bar"
+        other: "value"
+        vertica.com/special: "yes"
+    asserts:
+      - equal:
+          path: metadata.annotations
+          value:
+            foo: "bar"
+            other: "value"
+            vertica.com/special: "yes"

helm-charts/verticadb-operator/values.yaml

@@ -138,6 +138,12 @@ priorityClassName: ""
 # See: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
 tolerations: []
 
+# Set this if you want to override the default name of the serviceaccount
+# serviceAccountNameOverride: ""
+
+# Annotations to be added to the serviceaccount that we create. This is a key/value map.
+serviceAccountAnnotations: {}
+
 prometheus:
   # Controls exposing of the prometheus metrics endpoint.  Valid options are:
   #

pkg/builder/builder.go

@@ -191,10 +191,11 @@ func buildVolumeMounts(vdb *vapi.VerticaDB) []corev1.VolumeMount {
 		volMnts = append(volMnts, buildSSHVolumeMounts()...)
 	}
 
-	if vmeta.UseVClusterOps(vdb.Annotations) && vmeta.UseNMACertsMount(vdb.Annotations) {
-		if vdb.Spec.NMATLSSecret != "" {
-			volMnts = append(volMnts, buildNMACertsVolumeMount()...)
-		}
+	if vmeta.UseVClusterOps(vdb.Annotations) &&
+		vmeta.UseNMACertsMount(vdb.Annotations) &&
+		vdb.Spec.NMATLSSecret != "" &&
+		cloud.IsK8sSecret(vdb.Spec.NMATLSSecret) {
+		volMnts = append(volMnts, buildNMACertsVolumeMount()...)
 	}
 
 	if vmeta.UseVClusterOps(vdb.Annotations) {
@@ -295,10 +296,11 @@ func buildVolumes(vdb *vapi.VerticaDB) []corev1.Volume {
 	if vdb.GetSSHSecretName() != "" {
 		vols = append(vols, buildSSHVolume(vdb))
 	}
-	if vmeta.UseVClusterOps(vdb.Annotations) && vmeta.UseNMACertsMount(vdb.Annotations) {
-		if vdb.Spec.NMATLSSecret != "" {
-			vols = append(vols, buildNMACertsSecretVolume(vdb))
-		}
+	if vmeta.UseVClusterOps(vdb.Annotations) &&
+		vmeta.UseNMACertsMount(vdb.Annotations) &&
+		vdb.Spec.NMATLSSecret != "" &&
+		cloud.IsK8sSecret(vdb.Spec.NMATLSSecret) {
+		vols = append(vols, buildNMACertsSecretVolume(vdb))
 	}
 	if vdb.IsDepotVolumeEmptyDir() {
 		vols = append(vols, buildDepotVolume())
@@ -586,7 +588,7 @@ func makeServerContainer(vdb *vapi.VerticaDB, sc *vapi.Subcluster) corev1.Contai
 	}...)
 
 	if vmeta.UseVClusterOps(vdb.Annotations) {
-		if vmeta.UseNMACertsMount(vdb.Annotations) {
+		if vmeta.UseNMACertsMount(vdb.Annotations) && cloud.IsK8sSecret(vdb.Spec.NMATLSSecret) {
 			envVars = append(envVars, []corev1.EnvVar{
 				// Provide the path to each of the certs that are mounted in the container.
 				{Name: NMARootCAEnv, Value: fmt.Sprintf("%s/%s", paths.NMACertsRoot, paths.HTTPServerCACrtName)},
@@ -595,8 +597,8 @@ func makeServerContainer(vdb *vapi.VerticaDB, sc *vapi.Subcluster) corev1.Contai
 			}...)
 		} else {
 			envVars = append(envVars, []corev1.EnvVar{
-				// The NMA will read the secrets directly from k8s. We provide the
-				// secret namespace and name for this reason.
+				// The NMA will read the secrets directly from the secret store.
+				// We provide the secret namespace and name for this reason.
 				{Name: NMASecretNamespaceEnv, Value: vdb.ObjectMeta.Namespace},
 				{Name: NMASecretNameEnv, Value: vdb.Spec.NMATLSSecret},
 			}...)
@@ -678,7 +680,7 @@ func makeDefaultReadinessOrStartupProbe(vdb *vapi.VerticaDB) *corev1.Probe {
 	// use the canary query then because that depends on having the password
 	// mounted in the file system. Default to just checking if the client port
 	// is being listened on.
-	if vdb.ReadSUPwdFromGSM() {
+	if cloud.IsGSMSecret(vdb.Spec.PasswordSecret) {
 		return makeVerticaClientPortProbe()
 	}
 	return makeCanaryQueryProbe(vdb)

pkg/cloud/secret.go

@@ -0,0 +1,115 @@
+/*
+ (c) Copyright [2021-2023] Open Text.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ You may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package cloud
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"hash/crc32"
+
+	gsm "cloud.google.com/go/secretmanager/apiv1"
+	"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
+	"github.com/go-logr/logr"
+	vapi "github.com/vertica/vertica-kubernetes/api/v1"
+	"github.com/vertica/vertica-kubernetes/pkg/events"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// MultiSourceSecretFetcher is secret reader that handles retrival from
+// different sources such as Kubernetes secret store and Google Secrets Manager
+// (GSM).
+type MultiSourceSecretFetcher struct {
+	client.Client
+	Log      logr.Logger
+	EVWriter events.EVWriter
+	VDB      *vapi.VerticaDB
+}
+
+// Fetch reads the secret from a secret store. The contents of the secret is successful.
+func (m *MultiSourceSecretFetcher) Fetch(ctx context.Context, secretName types.NamespacedName) (map[string][]byte, error) {
+	secretData, res, err := m.FetchAllowRequeue(ctx, secretName)
+	if res.Requeue && err == nil {
+		return secretData, fmt.Errorf("secret fetch ended with requeue but is not allowed in code path")
+	}
+	return secretData, err
+}
+
+// FetchAllowRequeue reads the secret from a secret store. This API has the
+// ability to requeue the reconcile iteration based on the error it finds.
+func (m *MultiSourceSecretFetcher) FetchAllowRequeue(ctx context.Context, secretName types.NamespacedName) (
+	map[string][]byte, ctrl.Result, error) {
+	switch {
+	case IsGSMSecret(secretName.Name):
+		return m.readFromGSM(ctx, secretName.Name)
+	case IsAWSSecretsManagerSecret(secretName.Name):
+		return nil, ctrl.Result{},
+			fmt.Errorf("fetching secret %s from Amazon Secrets Manager is not implemented", secretName.Name)
+	default:
+		return m.readFromK8s(ctx, secretName)
+	}
+}
+
+// readFromK8s reads the secret using the K8s Secret API.
+func (m *MultiSourceSecretFetcher) readFromK8s(ctx context.Context, secretName types.NamespacedName) (
+	map[string][]byte, ctrl.Result, error) {
+	tlsCerts := &corev1.Secret{}
+	err := m.Client.Get(ctx, secretName, tlsCerts)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			m.EVWriter.Eventf(m.VDB, corev1.EventTypeWarning, events.ObjectNotFound,
+				"Could not find the secret '%s'", secretName.Name)
+			return nil, ctrl.Result{Requeue: true}, nil
+		}
+		return nil, ctrl.Result{}, fmt.Errorf("could not fetch k8s secret named %s: %w", secretName, err)
+	}
+	return tlsCerts.Data, ctrl.Result{}, nil
+}
+
+// ReadFromGSM will fetch a secret from Google Secret Manager (GSM)
+func (m *MultiSourceSecretFetcher) readFromGSM(ctx context.Context, secName string) (map[string][]byte, ctrl.Result, error) {
+	clnt, err := gsm.NewClient(ctx)
+	if err != nil {
+		return nil, ctrl.Result{}, fmt.Errorf("failed to create secretmanager client")
+	}
+	defer clnt.Close()
+
+	req := &secretmanagerpb.AccessSecretVersionRequest{
+		Name: RemovePathReference(secName),
+	}
+	m.Log.Info("Reading secret from GSM", "name", req.Name)
+
+	result, err := clnt.AccessSecretVersion(ctx, req)
+	if err != nil {
+		return nil, ctrl.Result{}, fmt.Errorf("could not fetch secret: %w", err)
+	}
+
+	crc32c := crc32.MakeTable(crc32.Castagnoli)
+	checksum := int64(crc32.Checksum(result.Payload.Data, crc32c))
+	if checksum != *result.Payload.DataCrc32C {
+		return nil, ctrl.Result{}, fmt.Errorf("data corruption detected")
+	}
+	contents := make(map[string][]byte)
+	err = json.Unmarshal(result.Payload.Data, &contents)
+	if err != nil {
+		return nil, ctrl.Result{}, fmt.Errorf("failed to unmarshal the contents of the GSM secret '%s': %w", secName, err)
+	}
+	return contents, ctrl.Result{}, nil
+}