feat: add pod controller to trigger EIP reconciles on pod replacement

api/v1alpha1/eip_types.go

@@ -17,6 +17,7 @@ package v1alpha1
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 type EIPAssignment struct {
@@ -68,6 +69,7 @@ type EIPStatus struct {
 
 	AssociationId string         `json:"associationId,omitempty"`
 	Assignment    *EIPAssignment `json:"assignment,omitempty"`
+	PodUID        types.UID      `json:"podUUID,omitempty"`
 }
 
 // +kubebuilder:object:root=true

config/rbac/role.yaml

@@ -11,38 +11,58 @@ rules:
   resources:
   - eips
   verbs:
+  - create
+  - delete
   - get
   - list
-  - watch
-  - create
-  - update
   - patch
-  - delete
+  - update
+  - watch
 - apiGroups:
   - aws.k8s.logmein.com
   resources:
   - eips/status
   verbs:
   - get
-  - update
   - patch
+  - update
 - apiGroups:
   - aws.k8s.logmein.com
   resources:
   - enis
   verbs:
+  - create
+  - delete
   - get
   - list
-  - watch
-  - create
-  - update
   - patch
-  - delete
+  - update
+  - watch
 - apiGroups:
   - aws.k8s.logmein.com
   resources:
   - enis/status
   verbs:
   - get
+  - patch
   - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
   - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/status
+  verbs:
+  - get
+  - patch
+  - update

controllers/eip_controller.go

@@ -95,14 +95,26 @@ func (r *EIPReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 
 		if status.State == "assigned" {
 			changed := false
-			if spec.Assignment == nil {
+			switch {
+			case spec.Assignment == nil:
 				// assignment was removed
 				status.State = "unassigning"
 				changed = true
-			} else if *(spec.Assignment) != *(status.Assignment) || addr.AssociationId == nil {
+			case *(spec.Assignment) != *(status.Assignment) || addr.AssociationId == nil:
 				// assignment was changed (in spec or in EC2)
 				status.State = "reassigning"
 				changed = true
+			case status.Assignment.PodName != "":
+				privateIP, podUID, err := r.getPodInfo(ctx, eip.Namespace, spec.Assignment.PodName)
+				if err != nil {
+					return ctrl.Result{}, err
+				}
+				if status.Assignment.PrivateIPAddress != privateIP || status.PodUID != podUID {
+					// pod was replaced
+					// (even if only the pod UID has changed, we can't be sure if the private IP is still attached to the same ENI)
+					status.State = "reassigning"
+					changed = true
+				}
 			}
 
 			if changed {
@@ -252,16 +264,16 @@ func (r *EIPReconciler) releaseEIP(ctx context.Context, eip *awsv1alpha1.EIP, lo
 	return nil
 }
 
-func (r *EIPReconciler) getPodPrivateIP(ctx context.Context, namespace, podName string) (string, error) {
+func (r *EIPReconciler) getPodInfo(ctx context.Context, namespace, podName string) (string, types.UID, error) {
 	pod := &corev1.Pod{}
 	if err := r.Client.Get(ctx, client.ObjectKey{
 		Namespace: namespace,
 		Name:      podName,
 	}, pod); err != nil {
-		return "", err
+		return "", types.UID(""), err
 	}
 
-	return pod.Status.PodIP, nil
+	return pod.Status.PodIP, pod.UID, nil
 }
 
 func (r *EIPReconciler) findENI(ctx context.Context, privateIP string) (string, error) {
@@ -285,7 +297,7 @@ func (r *EIPReconciler) findENI(ctx context.Context, privateIP string) (string,
 	}
 }
 
-func (r *EIPReconciler) getAssignmentTarget(ctx context.Context, eip *awsv1alpha1.EIP) (string, string, error) {
+func (r *EIPReconciler) getAssignmentTarget(ctx context.Context, eip *awsv1alpha1.EIP) (string, string, types.UID, error) {
 	modes := 0
 	if eip.Spec.Assignment.PodName != "" {
 		modes++
@@ -297,7 +309,7 @@ func (r *EIPReconciler) getAssignmentTarget(ctx context.Context, eip *awsv1alpha
 		modes++
 	}
 	if modes != 1 {
-		return "", "", fmt.Errorf("exactly one of podName, eni or privateIPAddress needs to be given in assignment")
+		return "", "", types.UID(""), fmt.Errorf("exactly one of podName, eni or privateIPAddress needs to be given in assignment")
 	}
 
 	if eip.Spec.Assignment.ENI != "" {
@@ -306,40 +318,42 @@ func (r *EIPReconciler) getAssignmentTarget(ctx context.Context, eip *awsv1alpha
 			Namespace: eip.Namespace,
 			Name:      eip.Spec.Assignment.ENI,
 		}, &eni); err != nil {
-			return "", "", err
+			return "", "", types.UID(""), err
 		}
 
 		index := eip.Spec.Assignment.ENIPrivateIPAddressIndex
 		if index >= len(eni.Status.PrivateIPAddresses) {
-			return "", "", fmt.Errorf("eniPrivateIPAddressIndex %d is out of range (ENI has %d addresses)", index, len(eni.Status.PrivateIPAddresses))
+			return "", "", types.UID(""), fmt.Errorf("eniPrivateIPAddressIndex %d is out of range (ENI has %d addresses)", index, len(eni.Status.PrivateIPAddresses))
 		}
-		return eni.Status.NetworkInterfaceID, eni.Status.PrivateIPAddresses[index], nil
+		return eni.Status.NetworkInterfaceID, eni.Status.PrivateIPAddresses[index], "", nil
 	}
 
 	privateIP := eip.Spec.Assignment.PrivateIPAddress
+	podUID := types.UID("")
 	if privateIP == "" {
-		ip, err := r.getPodPrivateIP(ctx, eip.Namespace, eip.Spec.Assignment.PodName)
+		ip, uid, err := r.getPodInfo(ctx, eip.Namespace, eip.Spec.Assignment.PodName)
 		if err != nil {
-			return "", "", err
+			return "", "", types.UID(""), err
 		}
 
 		if ip == "" {
-			return "", "", errors.New("Pod has no IP")
+			return "", "", types.UID(""), errors.New("Pod has no IP")
 		}
 
 		privateIP = ip
+		podUID = uid
 	}
 
 	eni, err := r.findENI(ctx, privateIP)
 	if err != nil {
-		return "", "", err
+		return "", "", types.UID(""), err
 	}
 
-	return eni, privateIP, nil
+	return eni, privateIP, podUID, nil
 }
 
 func (r *EIPReconciler) assignEIP(ctx context.Context, eip *awsv1alpha1.EIP, log logr.Logger) error {
-	eni, privateIP, err := r.getAssignmentTarget(ctx, eip)
+	eni, privateIP, podUID, err := r.getAssignmentTarget(ctx, eip)
 	if err != nil {
 		return err
 	}
@@ -362,6 +376,7 @@ func (r *EIPReconciler) assignEIP(ctx context.Context, eip *awsv1alpha1.EIP, log
 	eip.Status.AssociationId = aws.StringValue(resp.AssociationId)
 	eip.Status.Assignment = eip.Spec.Assignment
 	eip.Status.Assignment.PrivateIPAddress = privateIP
+	eip.Status.PodUID = podUID
 	if err := r.Update(ctx, eip); err != nil {
 		return err
 	}

controllers/pod_controller.go

@@ -0,0 +1,78 @@
+/*
+
+Copyright 2020 LogMeIn Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+*/
+
+package controllers
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	awsv1alpha1 "github.com/logmein/k8s-aws-operator/api/v1alpha1"
+)
+
+// PodReconciler reconciles a Pod object
+type PodReconciler struct {
+	client.Client
+	Log    logr.Logger
+	Scheme *runtime.Scheme
+}
+
+// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch
+
+func (r *PodReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
+	ctx := context.Background()
+	_ = r.Log.WithValues("pod", req.NamespacedName)
+
+	pod := corev1.Pod{}
+	if err := r.Get(ctx, req.NamespacedName, &pod); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	eips := &awsv1alpha1.EIPList{}
+	if err := r.List(ctx, eips); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	for _, eip := range eips.Items {
+		if eip.Status.Assignment != nil && eip.Status.Assignment.PodName == pod.Name && eip.Status.PodUID == pod.UID {
+			// pod UID doesn't match anymore -> pod was replaced; reset podUID and let the EIP controller do the rest
+			eip.Status.PodUID = types.UID("")
+			return ctrl.Result{}, r.Status().Update(ctx, &eip)
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Pod{}).
+		Complete(r)
+}

controllers/suite_test.go

@@ -22,14 +22,16 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	awsv1alpha1 "github.com/logmein/k8s-aws-operator/api/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	awsv1alpha1 "github.com/logmein/k8s-aws-operator/api/v1alpha1"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -63,6 +65,9 @@ var _ = BeforeSuite(func(done Done) {
 	err = awsv1alpha1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
 
+	err = corev1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
 	// +kubebuilder:scaffold:scheme
 
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})

main.go

@@ -23,13 +23,14 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	awsv1alpha1 "github.com/logmein/k8s-aws-operator/api/v1alpha1"
-	"github.com/logmein/k8s-aws-operator/controllers"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	awsv1alpha1 "github.com/logmein/k8s-aws-operator/api/v1alpha1"
+	"github.com/logmein/k8s-aws-operator/controllers"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -41,6 +42,7 @@ var (
 func init() {
 	corev1.AddToScheme(scheme)
 	awsv1alpha1.AddToScheme(scheme)
+	_ = corev1.AddToScheme(scheme)
 	// +kubebuilder:scaffold:scheme
 }
 
@@ -102,6 +104,14 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "ENI")
 		os.Exit(1)
 	}
+	if err = (&controllers.PodReconciler{
+		Client: mgr.GetClient(),
+		Log:    ctrl.Log.WithName("controllers").WithName("Pod"),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Pod")
+		os.Exit(1)
+	}
 	// +kubebuilder:scaffold:builder
 
 	setupLog.Info("starting manager")