Feature: ENIConfig set by custom annotation or label names (#280)

* Support for node labels to set eniConfig
* Annotations and labels can be configured with env variable
* Tests for annotations and labels config
* Custom annotation and label key environmental variables

README.md

@@ -96,6 +96,18 @@ Type: Boolean
 Default: `false`  
 Specifies that your pods may use subnets and security groups that are independent of your worker node's VPC configuration\. By default, pods share the same subnet and security groups as the worker node's primary interface\. Setting this variable to `true` causes `ipamD` to use the security groups and VPC subnet in a worker node's `ENIConfig` for elastic network interface allocation\. You must create an `ENIConfig` custom resource definition for each subnet that your pods will reside in, and then annotate each worker node to use a specific `ENIConfig` \(multiple worker nodes can be annotated with the same `ENIConfig`\)\. Worker nodes can only be annotated with a single `ENIConfig` at a time, and the subnet in the `ENIConfig` must belong to the same Availability Zone that the worker node resides in\. For more information, see [https://github.com/aws/amazon-vpc-cni-k8s/pull/165](https://github.com/aws/amazon-vpc-cni-k8s/pull/165)\.
 
+`ENI_CONFIG_ANNOTATION_DEF`
+Type: String
+Default: k8s.amazonaws.com/eniConfig
+Specifies node annotation key name. This should be used when AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true. Custom annotation value will be used to set eniConfig name. See ENI_CONFIG_LABEL_DEF for examples.
+
+`ENI_CONFIG_LABEL_DEF`
+Type: String
+Default: k8s.amazonaws.com/eniConfig
+Specifies node label key name. This should be used when AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true. Custom label value will be used to set eniConfig name. Note that annotations will take precedence over labels. To use labels, ensure default annotation k8s.amazonaws.com/eniConfig is not set on node and ENI_CONFIG_ANNOTATION_DEF is not used.
+For example, you can use custom node label key _example.com/eniConfig_ by setting ENI_CONFIG_LABEL_DEF=example.com/eniConfig. Then you can set that label on node with value of your custom eniConfig name like `eniConfig-us-east-1a`.
+In other example if your node has label _failure-domain.beta.kubernetes.io/zone_ and its value is set to availability zone `us-east-1a`, you can set ENI_CONFIG_LABEL_DEF=failure-domain.beta.kubernetes.io/zone. In such case eniConfig would be set to availability zone name `us-east-1a`, and you could use it to differentiate configs between zones.
+
 `AWS_VPC_K8S_CNI_EXTERNALSNAT`  
 Type: Boolean  
 Default: `false`  

pkg/eniconfig/eniconfig.go

@@ -35,8 +35,19 @@ import (
 )
 
 const (
-	eniConfigAnnotationDef = "k8s.amazonaws.com/eniConfig"
-	eniConfigDefault       = "default"
+	defaultEniConfigAnnotationDef = "k8s.amazonaws.com/eniConfig"
+	defaultEniConfigLabelDef      = "k8s.amazonaws.com/eniConfig"
+	eniConfigDefault              = "default"
+
+	// when "ENI_CONFIG_LABEL_DEF is defined, ENIConfigController will use that label key to
+	// search if is setting value for eniConfigLabelDef
+	// Example:
+	//   Node has set label k8s.amazonaws.com/eniConfigCustom=customConfig
+	//   We can get that value in controller by setting environmental variable ENI_CONFIG_LABEL_DEF
+	//   ENI_CONFIG_LABEL_DEF=k8s.amazonaws.com/eniConfigOverride
+	//   This will set eniConfigLabelDef to eniConfigOverride
+	envEniConfigAnnotationDef = "ENI_CONFIG_ANNOTATION_DEF"
+	envEniConfigLabelDef      = "ENI_CONFIG_LABEL_DEF"
 )
 
 type ENIConfig interface {
@@ -48,10 +59,12 @@ var ErrNoENIConfig = errors.New("eniconfig: eniconfig is not available")
 
 // ENIConfigController defines global context for ENIConfig controller
 type ENIConfigController struct {
-	eni        map[string]*v1alpha1.ENIConfigSpec
-	myENI      string
-	eniLock    sync.RWMutex
-	myNodeName string
+	eni                    map[string]*v1alpha1.ENIConfigSpec
+	myENI                  string
+	eniLock                sync.RWMutex
+	myNodeName             string
+	eniConfigAnnotationDef string
+	eniConfigLabelDef      string
 }
 
 // ENIConfigInfo returns locally cached ENIConfigs
@@ -66,6 +79,8 @@ func NewENIConfigController() *ENIConfigController {
 		myNodeName: os.Getenv("MY_NODE_NAME"),
 		eni:        make(map[string]*v1alpha1.ENIConfigSpec),
 		myENI:      eniConfigDefault,
+		eniConfigAnnotationDef: getEniConfigAnnotationDef(),
+		eniConfigLabelDef:      getEniConfigLabelDef(),
 	}
 }
 
@@ -104,22 +119,22 @@ func (h *Handler) Handle(ctx context.Context, event sdk.Event) error {
 
 	case *corev1.Node:
 
-		log.Infof("Handle corev1.Node: %s, %v", o.GetName(), o.GetAnnotations())
+		log.Infof("Handle corev1.Node: %s, %v, %v", o.GetName(), o.GetAnnotations(), o.GetLabels())
+		// Get annotations if not found get labels if not found fallback use default
 		if h.controller.myNodeName == o.GetName() {
-			annotation := o.GetAnnotations()
-
-			val, ok := annotation[eniConfigAnnotationDef]
-			if ok {
-				h.controller.eniLock.Lock()
-				defer h.controller.eniLock.Unlock()
-				h.controller.myENI = val
-				log.Infof(" Setting myENI to: %s", val)
-			} else {
-				h.controller.eniLock.Lock()
-				defer h.controller.eniLock.Unlock()
-				h.controller.myENI = eniConfigDefault
-				log.Infof(" Setting myENI to: %s", eniConfigDefault)
+
+			val, ok := o.GetAnnotations()[h.controller.eniConfigAnnotationDef]
+			if !ok {
+				val, ok = o.GetLabels()[h.controller.eniConfigLabelDef]
+				if !ok {
+					val = eniConfigDefault
+				}
 			}
+
+			h.controller.eniLock.Lock()
+			defer h.controller.eniLock.Unlock()
+			h.controller.myENI = val
+			log.Infof(" Setting myENI to: %s", val)
 		}
 	}
 	return nil
@@ -182,3 +197,33 @@ func (eniCfg *ENIConfigController) MyENIConfig() (*v1alpha1.ENIConfigSpec, error
 	}
 	return nil, ErrNoENIConfig
 }
+
+// getEniConfigAnnotationDef returns eniConfigAnnotation
+func getEniConfigAnnotationDef() string {
+	inputStr, found := os.LookupEnv(envEniConfigAnnotationDef)
+
+	if !found {
+		return defaultEniConfigAnnotationDef
+	}
+	if len(inputStr) > 0 {
+		log.Debugf("Using ENI_CONFIG_ANNOTATION_DEF %v", inputStr)
+		return inputStr
+	}
+
+	return defaultEniConfigAnnotationDef
+}
+
+// getEniConfigLabelDef returns eniConfigLabel name
+func getEniConfigLabelDef() string {
+	inputStr, found := os.LookupEnv(envEniConfigLabelDef)
+
+	if !found {
+		return defaultEniConfigLabelDef
+	}
+	if len(inputStr) > 0 {
+		log.Debugf("Using ENI_CONFIG_LABEL_DEF %v", inputStr)
+		return inputStr
+	}
+
+	return defaultEniConfigLabelDef
+}

pkg/eniconfig/eniconfig_test.go

@@ -60,6 +60,7 @@ func updateNodeAnnotation(hdlr sdk.Handler, nodeName string, configName string,
 		Deleted: toDelete,
 	}
 	eniAnnotations := make(map[string]string)
+	eniConfigAnnotationDef := getEniConfigAnnotationDef()
 
 	if !toDelete {
 		eniAnnotations[eniConfigAnnotationDef] = configName
@@ -68,6 +69,34 @@ func updateNodeAnnotation(hdlr sdk.Handler, nodeName string, configName string,
 	hdlr.Handle(nil, event)
 }
 
+func updateNodeLabel(hdlr sdk.Handler, nodeName string, configName string, toDelete bool) {
+
+	node := corev1.Node{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: nodeName,
+		},
+	}
+	accessor, err := meta.Accessor(&node)
+
+	if err != nil {
+		fmt.Printf("Failed to call meta.Access %v", err)
+	}
+
+	event := sdk.Event{
+		Object:  &node,
+		Deleted: toDelete,
+	}
+	eniLabels := make(map[string]string)
+	eniConfigLabelDef := getEniConfigLabelDef()
+
+	if !toDelete {
+		eniLabels[eniConfigLabelDef] = configName
+	}
+	accessor.SetLabels(eniLabels)
+	hdlr.Handle(nil, event)
+}
+
 func TestENIConfig(t *testing.T) {
 
 	testENIConfigController := NewENIConfigController()
@@ -105,7 +134,7 @@ func TestENIConfig(t *testing.T) {
 }
 
 func TestNodeENIConfig(t *testing.T) {
-	myNodeName := "testMyNode"
+	myNodeName := "testMyNodeWithAnnotation"
 	myENIConfig := "testMyENIConfig"
 	os.Setenv("MY_NODE_NAME", myNodeName)
 	testENIConfigController := NewENIConfigController()
@@ -144,3 +173,68 @@ func TestNodeENIConfig(t *testing.T) {
 	assert.Equal(t, defaultCfg, *outputCfg)
 
 }
+
+func TestNodeENIConfigLabel(t *testing.T) {
+	myNodeName := "testMyNodeWithLabel"
+	myENIConfig := "testMyENIConfig"
+	os.Setenv("MY_NODE_NAME", myNodeName)
+	testENIConfigController := NewENIConfigController()
+
+	testHandler := NewHandler(testENIConfigController)
+	updateNodeLabel(testHandler, myNodeName, myENIConfig, false)
+
+	// If there is no ENI config
+	_, err := testENIConfigController.MyENIConfig()
+	assert.Error(t, err)
+
+	// Add eniconfig for myENIConfig
+	group1Cfg := v1alpha1.ENIConfigSpec{
+		SecurityGroups: []string{"sg21-id", "sg22-id"},
+		Subnet:         "subnet21"}
+	updateENIConfig(testHandler, myENIConfig, group1Cfg, false)
+	outputCfg, err := testENIConfigController.MyENIConfig()
+	assert.NoError(t, err)
+	assert.Equal(t, group1Cfg, *outputCfg)
+
+	// Add default config
+	defaultSGs := []string{"sg1-id", "sg2-id"}
+	defaultSubnet := "subnet1"
+	defaultCfg := v1alpha1.ENIConfigSpec{
+		SecurityGroups: defaultSGs,
+		Subnet:         defaultSubnet}
+	updateENIConfig(testHandler, eniConfigDefault, defaultCfg, false)
+	outputCfg, err = testENIConfigController.MyENIConfig()
+	assert.NoError(t, err)
+	assert.Equal(t, group1Cfg, *outputCfg)
+
+	// Delete node's myENIConfig annotation, then the value should fallback to default
+	updateNodeLabel(testHandler, myNodeName, myENIConfig, true)
+	outputCfg, err = testENIConfigController.MyENIConfig()
+	assert.NoError(t, err)
+	assert.Equal(t, defaultCfg, *outputCfg)
+
+}
+
+func TestGetEniConfigAnnotationDefDefault(t *testing.T) {
+	os.Unsetenv(envEniConfigAnnotationDef)
+	eniConfigAnnotationDef := getEniConfigAnnotationDef()
+	assert.Equal(t, eniConfigAnnotationDef, defaultEniConfigAnnotationDef)
+}
+
+func TestGetEniConfigAnnotationlDefCustom(t *testing.T) {
+	os.Setenv(envEniConfigAnnotationDef, "k8s.amazonaws.com/eniConfigCustom")
+	eniConfigAnnotationDef := getEniConfigAnnotationDef()
+	assert.Equal(t, eniConfigAnnotationDef, "k8s.amazonaws.com/eniConfigCustom")
+}
+
+func TestGetEniConfigLabelDefDefault(t *testing.T) {
+	os.Unsetenv(envEniConfigLabelDef)
+	eniConfigLabelDef := getEniConfigLabelDef()
+	assert.Equal(t, eniConfigLabelDef, defaultEniConfigLabelDef)
+}
+
+func TestGetEniConfigLabelDefCustom(t *testing.T) {
+	os.Setenv(envEniConfigLabelDef, "k8s.amazonaws.com/eniConfigCustom")
+	eniConfigLabelDef := getEniConfigLabelDef()
+	assert.Equal(t, eniConfigLabelDef, "k8s.amazonaws.com/eniConfigCustom")
+}