Support CMEK for ANF volumes.

* Added support for customer-managed encryption keys for ANF volumes.
NetApp · May 24, 2024 · 59bccb7 · 59bccb7
1 parent 286f8d6
commit 59bccb7
Show file tree

Hide file tree

Showing 13 changed files with 748 additions and 183 deletions.
diff --git a/cli/k8s_client/yaml_factory.go b/cli/k8s_client/yaml_factory.go
diff --git a/helm/trident-operator/templates/tridentconfigurator.yaml b/helm/trident-operator/templates/tridentconfigurator.yaml
@@ -27,5 +27,9 @@ spec:
     {{- range .Values.anfConfigurator.resourceGroups }}
     - {{ . }}
     {{- end }}
+  customerEncryptionKeys:
+    {{- range $key, $value := .Values.anfConfigurator.customerEncryptionKeys }}
+      {{ $key }}: {{ $value }}
+    {{- end }}
 {{- end }}
 
diff --git a/helm/trident-operator/values.yaml b/helm/trident-operator/values.yaml
@@ -160,3 +160,4 @@ anfConfigurator:
   capacityPools: []
   netappAccounts: []
   resourceGroups: []
+  customerEncryptionKeys: {}
diff --git a/operator/controllers/configurator/storage_drivers/anf.go b/operator/controllers/configurator/storage_drivers/anf.go
@@ -50,6 +50,9 @@ type ANFConfig struct {
 	ResourceGroups []string `json:"resourceGroups"`
 	VirtualNetwork string   `json:"virtualNetwork"`
 	Subnet         string   `json:"subnet"`
+
+	// Encryption: Map of NetApp accounts and customer keys.
+	CustomerEncryptionKeys map[string]string `json:"customerEncryptionKeys"`
 }
 
 func NewANFInstance(

diff --git a/operator/controllers/configurator/storage_drivers/yaml_factory.go b/operator/controllers/configurator/storage_drivers/yaml_factory.go
@@ -7,6 +7,8 @@ import (
 	"strings"
 
 	sa "github.com/netapp/trident/storage_attribute"
+
+	"github.com/netapp/trident/utils"
 )
 
 func getANFTBCYaml(anf *ANF, vPools, nasType string) string {
@@ -23,6 +25,7 @@ func getANFTBCYaml(anf *ANF, vPools, nasType string) string {
 	tbcYaml = strings.ReplaceAll(tbcYaml, "{SUBNET}", anf.Subnet)
 	tbcYaml = strings.ReplaceAll(tbcYaml, "{NAS_TYPE}", nasType)
 	tbcYaml = strings.ReplaceAll(tbcYaml, "{V_POOLS}", vPools)
+	tbcYaml = utils.ReplaceMultilineYAMLTag(tbcYaml, "CUSTOMER_ENCRYPTION_KEYS", constructEncryptionKeys(anf.CustomerEncryptionKeys))
 
 	if !anf.AMIEnabled && !anf.WorkloadIdentityEnabled {
 		tbcYaml = strings.ReplaceAll(tbcYaml, "{CLIENT_CREDENTIALS}", constructClientCredentials(anf.ClientCredentials))
@@ -55,10 +58,24 @@ spec:
     discovery: true
     method: true
     api: true
+  {CUSTOMER_ENCRYPTION_KEYS}
   storage:
   {V_POOLS}
 `
 
+func constructEncryptionKeys(m map[string]string) (encryptionKeys string) {
+	if len(m) == 0 {
+		return
+	}
+
+	encryptionKeys += "customerEncryptionKeys:\n"
+	for key, value := range m {
+		encryptionKeys += fmt.Sprintf("  %s: %s\n", key, value)
+	}
+
+	return encryptionKeys
+}
+
 func getANFVPoolYAML(serviceLevel, nasType, cPools string) string {
 	anfVPool := ANFVPoolYAML
 

diff --git a/operator/controllers/configurator/storage_drivers/yaml_factory_test.go b/operator/controllers/configurator/storage_drivers/yaml_factory_test.go
@@ -0,0 +1,42 @@
+package storage_drivers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConstructEncryptionKeys(t *testing.T) {
+	testCases := []struct {
+		name     string
+		input    map[string]string
+		expected string
+	}{
+		{
+			name:     "Empty Map",
+			input:    map[string]string{},
+			expected: "",
+		},
+		{
+			name:     "Single Element Map",
+			input:    map[string]string{"key1": "value1"},
+			expected: "customerEncryptionKeys:\n  key1: value1\n",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := constructEncryptionKeys(tc.input)
+			assert.Equal(t, tc.expected, result, "Incorrect string returned")
+		})
+	}
+}
+
+func TestConstructEncryptionKeys_MultiElementMap(t *testing.T) {
+	input := map[string]string{"key1": "value1", "key2": "value2"}
+	expected1 := "customerEncryptionKeys:\n  key1: value1\n  key2: value2\n"
+	expected2 := "customerEncryptionKeys:\n  key2: value2\n  key1: value1\n"
+
+	result := constructEncryptionKeys(input)
+	assert.True(t, result == expected1 || result == expected2, "Incorrect string returned")
+}
diff --git a/storage_drivers/azure/api/azure.go b/storage_drivers/azure/api/azure.go
@@ -631,6 +631,11 @@ func ParseSubvolumeID(
 	return
 }
 
+// CreateKeyVaultEndpoint to create KeyVault Endpoint ID
+func CreateKeyVaultEndpoint(subnet, resourceGroup, keyVaultEndpoint string) string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/privateEndpoints/%s", subnet, resourceGroup, keyVaultEndpoint)
+}
+
 // ///////////////////////////////////////////////////////////////////////////////
 // Functions to convert between ANF SDK & internal volume structs
 // ///////////////////////////////////////////////////////////////////////////////
@@ -737,29 +742,30 @@ func (c Client) newFileSystemFromVolume(ctx context.Context, vol *netapp.Volume)
 	}
 
 	return &FileSystem{
-		ID:                DerefString(vol.ID),
-		ResourceGroup:     resourceGroup,
-		NetAppAccount:     netappAccount,
-		CapacityPool:      cPoolName,
-		Name:              name,
-		FullName:          CreateVolumeFullName(resourceGroup, netappAccount, cPoolName, name),
-		Location:          DerefString(vol.Location),
-		Type:              DerefString(vol.Type),
-		ExportPolicy:      *exportPolicyImport(vol.Properties.ExportPolicy),
-		Labels:            c.getLabelsFromVolume(vol),
-		FileSystemID:      DerefString(vol.Properties.FileSystemID),
-		ProvisioningState: DerefString(vol.Properties.ProvisioningState),
-		CreationToken:     DerefString(vol.Properties.CreationToken),
-		ProtocolTypes:     DerefStringPtrArray(vol.Properties.ProtocolTypes),
-		QuotaInBytes:      DerefInt64(vol.Properties.UsageThreshold),
-		ServiceLevel:      cPool.ServiceLevel,
-		SnapshotDirectory: DerefBool(vol.Properties.SnapshotDirectoryVisible),
-		SubnetID:          DerefString(vol.Properties.SubnetID),
-		UnixPermissions:   DerefString(vol.Properties.UnixPermissions),
-		MountTargets:      c.getMountTargetsFromVolume(ctx, vol),
-		SubvolumesEnabled: c.getSubvolumesEnabledFromVolume(vol.Properties.EnableSubvolumes),
-		NetworkFeatures:   DerefNetworkFeatures(vol.Properties.NetworkFeatures),
-		KerberosEnabled:   DerefBool(vol.Properties.KerberosEnabled),
+		ID:                 DerefString(vol.ID),
+		ResourceGroup:      resourceGroup,
+		NetAppAccount:      netappAccount,
+		CapacityPool:       cPoolName,
+		Name:               name,
+		FullName:           CreateVolumeFullName(resourceGroup, netappAccount, cPoolName, name),
+		Location:           DerefString(vol.Location),
+		Type:               DerefString(vol.Type),
+		ExportPolicy:       *exportPolicyImport(vol.Properties.ExportPolicy),
+		Labels:             c.getLabelsFromVolume(vol),
+		FileSystemID:       DerefString(vol.Properties.FileSystemID),
+		ProvisioningState:  DerefString(vol.Properties.ProvisioningState),
+		CreationToken:      DerefString(vol.Properties.CreationToken),
+		ProtocolTypes:      DerefStringPtrArray(vol.Properties.ProtocolTypes),
+		QuotaInBytes:       DerefInt64(vol.Properties.UsageThreshold),
+		ServiceLevel:       cPool.ServiceLevel,
+		SnapshotDirectory:  DerefBool(vol.Properties.SnapshotDirectoryVisible),
+		SubnetID:           DerefString(vol.Properties.SubnetID),
+		UnixPermissions:    DerefString(vol.Properties.UnixPermissions),
+		MountTargets:       c.getMountTargetsFromVolume(ctx, vol),
+		SubvolumesEnabled:  c.getSubvolumesEnabledFromVolume(vol.Properties.EnableSubvolumes),
+		NetworkFeatures:    DerefNetworkFeatures(vol.Properties.NetworkFeatures),
+		KerberosEnabled:    DerefBool(vol.Properties.KerberosEnabled),
+		KeyVaultEndpointID: DerefString(vol.Properties.KeyVaultPrivateEndpointResourceID),
 	}, nil
 }
 
@@ -1138,6 +1144,14 @@ func (c Client) CreateVolume(ctx context.Context, request *FilesystemCreateReque
 		newVol.Properties.UnixPermissions = &request.UnixPermissions
 	}
 
+	encryptionKeySource := netapp.EncryptionKeySource(EncryptionKeyNetApp)
+	// Set Encryption Key Source and KeyVaultEndpointID if specified
+	if request.KeyVaultEndpointID != "" {
+		encryptionKeySource = netapp.EncryptionKeySource(EncryptionKeyVault)
+		newVol.Properties.EncryptionKeySource = &encryptionKeySource
+		newVol.Properties.KeyVaultPrivateEndpointResourceID = &request.KeyVaultEndpointID
+	}
+
 	Logc(ctx).WithFields(LogFields{
 		"name":          request.Name,
 		"creationToken": request.CreationToken,

diff --git a/storage_drivers/azure/api/azure_structs.go b/storage_drivers/azure/api/azure_structs.go
@@ -35,6 +35,9 @@ const (
 
 	NetworkFeaturesBasic    = "Basic"
 	NetworkFeaturesStandard = "Standard"
+
+	EncryptionKeyNetApp = "Microsoft.NetApp"
+	EncryptionKeyVault  = "Microsoft.KeyVault"
 )
 
 // AzureResources is the toplevel cache for the set of things we discover about our Azure environment.
@@ -107,49 +110,51 @@ type CapacityPool struct {
 
 // FileSystem records details of a discovered Azure Subnet.
 type FileSystem struct {
-	ID                string
-	ResourceGroup     string
-	NetAppAccount     string
-	CapacityPool      string
-	Name              string
-	FullName          string
-	Location          string
-	Type              string
-	ExportPolicy      ExportPolicy
-	Labels            map[string]string
-	FileSystemID      string
-	ProvisioningState string
-	CreationToken     string
-	ProtocolTypes     []string
-	QuotaInBytes      int64
-	ServiceLevel      string
-	SnapshotDirectory bool
-	UsedBytes         int
-	SubnetID          string
-	UnixPermissions   string
-	MountTargets      []MountTarget
-	SubvolumesEnabled bool
-	NetworkFeatures   string
-	KerberosEnabled   bool
+	ID                 string
+	ResourceGroup      string
+	NetAppAccount      string
+	CapacityPool       string
+	Name               string
+	FullName           string
+	Location           string
+	Type               string
+	ExportPolicy       ExportPolicy
+	Labels             map[string]string
+	FileSystemID       string
+	ProvisioningState  string
+	CreationToken      string
+	ProtocolTypes      []string
+	QuotaInBytes       int64
+	ServiceLevel       string
+	SnapshotDirectory  bool
+	UsedBytes          int
+	SubnetID           string
+	UnixPermissions    string
+	MountTargets       []MountTarget
+	SubvolumesEnabled  bool
+	NetworkFeatures    string
+	KerberosEnabled    bool
+	KeyVaultEndpointID string
 }
 
 // FilesystemCreateRequest embodies all the details of a volume to be created.
 type FilesystemCreateRequest struct {
-	ResourceGroup     string
-	NetAppAccount     string
-	CapacityPool      string
-	Name              string
-	SubnetID          string
-	CreationToken     string
-	ExportPolicy      ExportPolicy
-	Labels            map[string]string
-	ProtocolTypes     []string
-	QuotaInBytes      int64
-	SnapshotDirectory bool
-	SnapshotID        string
-	UnixPermissions   string
-	NetworkFeatures   string
-	KerberosEnabled   bool
+	ResourceGroup      string
+	NetAppAccount      string
+	CapacityPool       string
+	Name               string
+	SubnetID           string
+	CreationToken      string
+	ExportPolicy       ExportPolicy
+	Labels             map[string]string
+	ProtocolTypes      []string
+	QuotaInBytes       int64
+	SnapshotDirectory  bool
+	SnapshotID         string
+	UnixPermissions    string
+	NetworkFeatures    string
+	KerberosEnabled    bool
+	KeyVaultEndpointID string
 }
 
 // ExportPolicy records details of a discovered Azure volume export policy.

diff --git a/storage_drivers/azure/api/azure_test.go b/storage_drivers/azure/api/azure_test.go
@@ -1161,3 +1161,15 @@ func TestIsTerminalStateError(t *testing.T) {
 	assert.False(t, IsTerminalStateError(nil))
 	assert.False(t, IsTerminalStateError(errors.New("not terminal")))
 }
+
+func TestCreateKeyVaultEndpoint(t *testing.T) {
+	subnet := "sub123"
+	resourceGroup := "resourceGrp123"
+	keyVaultEndpoint := "keyVaultEP123"
+
+	expected := "/subscriptions/sub123/resourceGroups/resourceGrp123/providers/Microsoft.Network/privateEndpoints/keyVaultEP123"
+
+	result := CreateKeyVaultEndpoint(subnet, resourceGroup, keyVaultEndpoint)
+
+	assert.Equal(t, expected, result, "endpoint mismatch")
+}