Adding an option for --disable-nodegroup-eviction when deleting a cluster

pkg/actions/cluster/cluster.go

@@ -15,7 +15,7 @@ import (
 
 type Cluster interface {
 	Upgrade(dryRun bool) error
-	Delete(waitInterval time.Duration, wait, force bool) error
+	Delete(waitInterval time.Duration, wait, force, disableNodegroupEviction bool) error
 }
 
 func New(cfg *api.ClusterConfig, ctl *eks.ClusterProvider) (Cluster, error) {

pkg/actions/cluster/delete.go

@@ -10,8 +10,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/cloudformation"
 	"github.com/pkg/errors"
-
 	"github.com/weaveworks/eksctl/pkg/actions/nodegroup"
+
 	"github.com/weaveworks/eksctl/pkg/cfn/manager"
 	"github.com/weaveworks/eksctl/pkg/ctl/cmdutils"
 	"github.com/weaveworks/eksctl/pkg/fargate"
@@ -29,6 +29,11 @@ import (
 	"github.com/kris-nova/logger"
 )
 
+type NodeGroupDrainer interface {
+	Drain(input *nodegroup.DrainInput) error
+}
+type vpcCniDeleter func(clusterName string, ctl *eks.ClusterProvider, clientSet kubernetes.Interface)
+
 func deleteSharedResources(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, stackManager manager.StackManager, clusterOperable bool, clientSet kubernetes.Interface) error {
 	if clusterOperable {
 		if err := deleteFargateProfiles(cfg.Metadata, ctl, stackManager); err != nil {
@@ -159,7 +164,7 @@ func checkForUndeletedStacks(stackManager manager.StackManager) error {
 	return nil
 }
 
-func drainAllNodegroups(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, stackManager manager.StackManager, clientSet kubernetes.Interface, allStacks []manager.NodeGroupStack) error {
+func drainAllNodeGroups(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, clientSet kubernetes.Interface, allStacks []manager.NodeGroupStack, disableEviction bool, nodeGroupDrainer NodeGroupDrainer, vpcCniDeleter vpcCniDeleter) error {
 	if len(allStacks) == 0 {
 		return nil
 	}
@@ -172,11 +177,17 @@ func drainAllNodegroups(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, stackM
 	}
 
 	logger.Info("will drain %d unmanaged nodegroup(s) in cluster %q", len(cfg.NodeGroups), cfg.Metadata.Name)
-	nodeGroupManager := nodegroup.New(cfg, ctl, clientSet)
-	if err := nodeGroupManager.Drain(cmdutils.ToKubeNodeGroups(cfg), false, ctl.Provider.WaitTimeout(), 0, false, false); err != nil {
+
+	drainInput := &nodegroup.DrainInput{
+		NodeGroups:      cmdutils.ToKubeNodeGroups(cfg),
+		MaxGracePeriod:  ctl.Provider.WaitTimeout(),
+		DisableEviction: disableEviction,
+	}
+	if err := nodeGroupDrainer.Drain(drainInput); err != nil {
 		return err
 	}
-	attemptVpcCniDeletion(cfg.Metadata.Name, ctl, clientSet)
+
+	vpcCniDeleter(cfg.Metadata.Name, ctl, clientSet)
 	return nil
 }
 

pkg/actions/cluster/delete_test.go

@@ -0,0 +1,132 @@
+package cluster_test
+
+import (
+	"github.com/weaveworks/eksctl/pkg/actions/nodegroup"
+	"github.com/weaveworks/eksctl/pkg/ctl/cmdutils"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/stretchr/testify/mock"
+	"github.com/weaveworks/eksctl/pkg/actions/cluster"
+	api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
+	"github.com/weaveworks/eksctl/pkg/cfn/manager"
+	"github.com/weaveworks/eksctl/pkg/cfn/manager/fakes"
+	"github.com/weaveworks/eksctl/pkg/eks"
+	"github.com/weaveworks/eksctl/pkg/kubernetes"
+	"github.com/weaveworks/eksctl/pkg/testutils/mockprovider"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+type drainerMock struct {
+	mock.Mock
+}
+
+func (drainer *drainerMock) Drain(input *nodegroup.DrainInput) error {
+	args := drainer.Called(input)
+	return args.Error(0)
+}
+
+var _ = Describe("DrainAllNodeGroups", func() {
+	var (
+		clusterName      string
+		p                *mockprovider.MockProvider
+		cfg              *api.ClusterConfig
+		fakeStackManager *fakes.FakeStackManager
+		fakeClientSet    *fake.Clientset
+		ctl              *eks.ClusterProvider
+	)
+
+	BeforeEach(func() {
+		clusterName = "my-cluster"
+		p = mockprovider.NewMockProvider()
+		cfg = api.NewClusterConfig()
+		cfg.Metadata.Name = clusterName
+		fakeStackManager = new(fakes.FakeStackManager)
+		fakeClientSet = fake.NewSimpleClientset()
+		ctl = &eks.ClusterProvider{Provider: p, Status: &eks.ProviderStatus{}}
+	})
+
+	Context("draining node groups", func() {
+		When("disable eviction flag is set to false", func() {
+			It("drain the node groups", func() {
+				c := cluster.NewOwnedCluster(cfg, ctl, nil, fakeStackManager)
+				c.SetNewClientSet(func() (kubernetes.Interface, error) {
+					return fakeClientSet, nil
+				})
+
+				nodeGroupStacks := []manager.NodeGroupStack{{NodeGroupName: "ng-1"}}
+				mockedDrainInput := &nodegroup.DrainInput{
+					NodeGroups:     cmdutils.ToKubeNodeGroups(cfg),
+					MaxGracePeriod: ctl.Provider.WaitTimeout(),
+				}
+
+				mockedDrainer := &drainerMock{}
+				mockedDrainer.On("Drain", mockedDrainInput).Return(nil)
+				vpcCniDeleterCalled := 0
+				vpcCniDeleter := func(clusterName string, ctl *eks.ClusterProvider, clientSet kubernetes.Interface) {
+					vpcCniDeleterCalled++
+				}
+
+				err := cluster.DrainAllNodeGroups(cfg, ctl, fakeClientSet, nodeGroupStacks, false, mockedDrainer, vpcCniDeleter)
+				Expect(err).NotTo(HaveOccurred())
+				mockedDrainer.AssertNumberOfCalls(GinkgoT(), "Drain", 1)
+				Expect(vpcCniDeleterCalled).To(Equal(1))
+			})
+		})
+
+		When("disable eviction flag is set to true", func() {
+			It("drain the node groups", func() {
+				c := cluster.NewOwnedCluster(cfg, ctl, nil, fakeStackManager)
+				c.SetNewClientSet(func() (kubernetes.Interface, error) {
+					return fakeClientSet, nil
+				})
+
+				nodeGroupStacks := []manager.NodeGroupStack{{NodeGroupName: "ng-1"}}
+				mockedDrainInput := &nodegroup.DrainInput{
+					NodeGroups:      cmdutils.ToKubeNodeGroups(cfg),
+					MaxGracePeriod:  ctl.Provider.WaitTimeout(),
+					DisableEviction: true,
+				}
+
+				mockedDrainer := &drainerMock{}
+				mockedDrainer.On("Drain", mockedDrainInput).Return(nil)
+				vpcCniDeleterCalled := 0
+				vpcCniDeleter := func(clusterName string, ctl *eks.ClusterProvider, clientSet kubernetes.Interface) {
+					vpcCniDeleterCalled++
+				}
+
+				err := cluster.DrainAllNodeGroups(cfg, ctl, fakeClientSet, nodeGroupStacks, true, mockedDrainer, vpcCniDeleter)
+				Expect(err).NotTo(HaveOccurred())
+				mockedDrainer.AssertNumberOfCalls(GinkgoT(), "Drain", 1)
+				Expect(vpcCniDeleterCalled).To(Equal(1))
+			})
+		})
+
+		When("no node group stacks exist", func() {
+			It("does no draining at all", func() {
+				c := cluster.NewOwnedCluster(cfg, ctl, nil, fakeStackManager)
+				c.SetNewClientSet(func() (kubernetes.Interface, error) {
+					return fakeClientSet, nil
+				})
+
+				var nodeGroupStacks []manager.NodeGroupStack
+				mockedDrainInput := &nodegroup.DrainInput{
+					NodeGroups:     cmdutils.ToKubeNodeGroups(cfg),
+					MaxGracePeriod: ctl.Provider.WaitTimeout(),
+				}
+
+				mockedDrainer := &drainerMock{}
+				mockedDrainer.On("Drain", mockedDrainInput).Return(nil)
+				vpcCniDeleterCalled := 0
+				vpcCniDeleter := func(clusterName string, ctl *eks.ClusterProvider, clientSet kubernetes.Interface) {
+					vpcCniDeleterCalled++
+				}
+
+				err := cluster.DrainAllNodeGroups(cfg, ctl, fakeClientSet, nodeGroupStacks, false, mockedDrainer, vpcCniDeleter)
+				Expect(err).NotTo(HaveOccurred())
+				mockedDrainer.AssertNotCalled(GinkgoT(), "Drain")
+				Expect(vpcCniDeleterCalled).To(Equal(0))
+			})
+		})
+	})
+})

pkg/actions/cluster/export_test.go

@@ -1,6 +1,14 @@
 package cluster
 
-import "github.com/weaveworks/eksctl/pkg/kubernetes"
+import (
+	api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
+	"github.com/weaveworks/eksctl/pkg/eks"
+	"github.com/weaveworks/eksctl/pkg/kubernetes"
+)
+
+var (
+	DrainAllNodeGroups = drainAllNodeGroups
+)
 
 func (c *UnownedCluster) SetNewClientSet(newClientSet func() (kubernetes.Interface, error)) {
 	c.newClientSet = newClientSet
@@ -9,3 +17,11 @@ func (c *UnownedCluster) SetNewClientSet(newClientSet func() (kubernetes.Interfa
 func (c *OwnedCluster) SetNewClientSet(newClientSet func() (kubernetes.Interface, error)) {
 	c.newClientSet = newClientSet
 }
+
+func (c *UnownedCluster) SetNewNodeGroupManager(newNodeGroupManager func(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, clientSet kubernetes.Interface) NodeGroupDrainer) {
+	c.newNodeGroupManager = newNodeGroupManager
+}
+
+func (c *OwnedCluster) SetNewNodeGroupManager(newNodeGroupManager func(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, clientSet kubernetes.Interface) NodeGroupDrainer) {
+	c.newNodeGroupManager = newNodeGroupManager
+}

pkg/actions/cluster/owned.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/kris-nova/logger"
 	"github.com/pkg/errors"
+	"github.com/weaveworks/eksctl/pkg/actions/nodegroup"
 	api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
 	"github.com/weaveworks/eksctl/pkg/cfn/manager"
 	"github.com/weaveworks/eksctl/pkg/ctl/cmdutils"
@@ -15,11 +16,12 @@ import (
 )
 
 type OwnedCluster struct {
-	cfg          *api.ClusterConfig
-	ctl          *eks.ClusterProvider
-	clusterStack *manager.Stack
-	stackManager manager.StackManager
-	newClientSet func() (kubernetes.Interface, error)
+	cfg                 *api.ClusterConfig
+	ctl                 *eks.ClusterProvider
+	clusterStack        *manager.Stack
+	stackManager        manager.StackManager
+	newClientSet        func() (kubernetes.Interface, error)
+	newNodeGroupManager func(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, clientSet kubernetes.Interface) NodeGroupDrainer
 }
 
 func NewOwnedCluster(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, clusterStack *manager.Stack, stackManager manager.StackManager) *OwnedCluster {
@@ -31,6 +33,9 @@ func NewOwnedCluster(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, clusterSt
 		newClientSet: func() (kubernetes.Interface, error) {
 			return ctl.NewStdClientSet(cfg)
 		},
+		newNodeGroupManager: func(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, clientSet kubernetes.Interface) NodeGroupDrainer {
+			return nodegroup.New(cfg, ctl, clientSet)
+		},
 	}
 }
 
@@ -67,7 +72,7 @@ func (c *OwnedCluster) Upgrade(dryRun bool) error {
 	return nil
 }
 
-func (c *OwnedCluster) Delete(_ time.Duration, wait, force bool) error {
+func (c *OwnedCluster) Delete(_ time.Duration, wait, force, disableNodegroupEviction bool) error {
 	var (
 		clientSet kubernetes.Interface
 		oidc      *iamoidc.OpenIDConnectManager
@@ -105,8 +110,14 @@ func (c *OwnedCluster) Delete(_ time.Duration, wait, force bool) error {
 		if err != nil {
 			return err
 		}
-		if err := drainAllNodegroups(c.cfg, c.ctl, c.stackManager, clientSet, allStacks); err != nil {
-			return err
+
+		nodeGroupManager := c.newNodeGroupManager(c.cfg, c.ctl, clientSet)
+		if err := drainAllNodeGroups(c.cfg, c.ctl, clientSet, allStacks, disableNodegroupEviction, nodeGroupManager, attemptVpcCniDeletion); err != nil {
+			if !force {
+				return err
+			}
+
+			logger.Warning("an error occurred during nodegroups draining, force=true so proceeding with deletion: %q", err.Error())
 		}
 	}