From c731fb0f9a05a6a656e86b9412756e918a4a6624 Mon Sep 17 00:00:00 2001 From: Connor Catlett Date: Wed, 3 May 2023 10:17:16 -0400 Subject: [PATCH] Add Startup Taint Removal Feature Co-authored-by: Gengtao Xu Signed-off-by: Connor Catlett --- .../templates/clusterrole-csi-node.yaml | 2 +- charts/aws-ebs-csi-driver/templates/node.yaml | 2 + .../kubernetes/base/clusterrole-csi-node.yaml | 2 +- docs/install.md | 5 + hack/update-gomock | 4 + pkg/driver/constants.go | 6 + pkg/driver/mock_k8s_client.go | 799 ++++++++++++++++++ pkg/driver/mock_k8s_corev1.go | 486 +++++++++++ pkg/driver/node.go | 79 ++ pkg/driver/node_test.go | 126 +++ 10 files changed, 1509 insertions(+), 2 deletions(-) create mode 100644 pkg/driver/mock_k8s_client.go create mode 100644 pkg/driver/mock_k8s_corev1.go diff --git a/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml b/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml index 3ca368efb6..3e5382216c 100644 --- a/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml +++ b/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml @@ -8,4 +8,4 @@ metadata: rules: - apiGroups: [""] resources: ["nodes"] - verbs: ["get"] + verbs: ["get", "patch"] diff --git a/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/aws-ebs-csi-driver/templates/node.yaml index 520c09ea17..0045be73ee 100644 --- a/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/aws-ebs-csi-driver/templates/node.yaml @@ -43,6 +43,8 @@ spec: {{- with .Values.node.tolerations }} {{- toYaml . | nindent 8 }} {{- end }} + - key: "ebs.csi.aws.com/agent-not-ready" + operator: "Exists" {{- end }} {{- with .Values.node.securityContext }} securityContext: diff --git a/deploy/kubernetes/base/clusterrole-csi-node.yaml b/deploy/kubernetes/base/clusterrole-csi-node.yaml index f48b05092b..91c581e387 100644 --- a/deploy/kubernetes/base/clusterrole-csi-node.yaml +++ b/deploy/kubernetes/base/clusterrole-csi-node.yaml @@ -9,4 +9,4 @@ metadata: rules: - apiGroups: [""] resources: ["nodes"] - verbs: ["get"] + verbs: ["get", "patch"] diff --git a/docs/install.md b/docs/install.md index 5d241571d7..df7d63775c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -41,6 +41,11 @@ kubectl create secret generic aws-secret \ ### Configure driver toleration settings By default, the driver controller tolerates taint `CriticalAddonsOnly` and has `tolerationSeconds` configured as `300`; and the driver node tolerates all taints. If you don't want to deploy the driver node on all nodes, please set Helm `Value.node.tolerateAllTaints` to false before deployment. Add policies to `Value.node.tolerations` to configure customized toleration for nodes. +### Configure node startup taint +There are potential race conditions on node startup (especially when a node is first joining the cluster) where pods/processes that rely on the EBS CSI Driver can act on a node before the EBS CSI Driver is able to startup up and become fully ready. To combat this, the EBS CSI Driver contains a feature to automatically remove a taint from the node on startup. Users can taint their nodes when they join the cluster and/or on startup, to prevent other pods from running and/or being scheduled on the node prior to the EBS CSI Driver becoming ready. + +This feature is activated by default, and cluster administrators should use the taint `ebs.csi.aws.com/agent-not-ready:NoExecute` (any effect will work, but `NoExecute` is recommended). For example, EKS Managed Node Groups [support automatically tainting nodes](https://docs.aws.amazon.com/eks/latest/userguide/node-taints-managed-node-groups.html). + ### Deploy driver You may deploy the EBS CSI driver via Kustomize, Helm, or as an [Amazon EKS managed add-on](https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html). diff --git a/hack/update-gomock b/hack/update-gomock index 912b86fb66..1743c28f7e 100755 --- a/hack/update-gomock +++ b/hack/update-gomock @@ -22,6 +22,10 @@ mockgen -package cloud -destination=./pkg/cloud/mock_metadata.go -source pkg/clo mockgen -package driver -destination=./pkg/driver/mock_mount.go -source pkg/driver/mount.go mockgen -package mounter -destination=./pkg/mounter/mock_mount_windows.go -source pkg/mounter/safe_mounter_windows.go +# Reflection-based mocking for external dependencies +mockgen -package driver -destination=./pkg/driver/mock_k8s_client.go -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface +mockgen -package driver -destination=./pkg/driver/mock_k8s_corev1.go k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface + # Fixes "Mounter Type cannot implement 'Mounter' as it has a non-exported method and is defined in a different package" # See https://github.com/kubernetes/mount-utils/commit/a20fcfb15a701977d086330b47b7efad51eb608e for context. sed -i '/type MockMounter struct {/a \\tmount_utils.Interface' pkg/driver/mock_mount.go diff --git a/pkg/driver/constants.go b/pkg/driver/constants.go index d7e03d6bb7..3a29699827 100644 --- a/pkg/driver/constants.go +++ b/pkg/driver/constants.go @@ -167,3 +167,9 @@ var ( FSTypeNtfs: {}, } ) + +// constants for node k8s API use +const ( + // AgentNotReadyTaintKey contains the key of taints to be removed on driver startup + AgentNotReadyNodeTaintKey = "ebs.csi.aws.com/agent-not-ready" +) diff --git a/pkg/driver/mock_k8s_client.go b/pkg/driver/mock_k8s_client.go new file mode 100644 index 0000000000..3c0c12a0bc --- /dev/null +++ b/pkg/driver/mock_k8s_client.go @@ -0,0 +1,799 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: k8s.io/client-go/kubernetes (interfaces: Interface) + +// Package driver is a generated GoMock package. +package driver + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + discovery "k8s.io/client-go/discovery" + v1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1" + v1alpha1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1" + v1beta1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1" + v1alpha10 "k8s.io/client-go/kubernetes/typed/apiserverinternal/v1alpha1" + v10 "k8s.io/client-go/kubernetes/typed/apps/v1" + v1beta10 "k8s.io/client-go/kubernetes/typed/apps/v1beta1" + v1beta2 "k8s.io/client-go/kubernetes/typed/apps/v1beta2" + v11 "k8s.io/client-go/kubernetes/typed/authentication/v1" + v1alpha11 "k8s.io/client-go/kubernetes/typed/authentication/v1alpha1" + v1beta11 "k8s.io/client-go/kubernetes/typed/authentication/v1beta1" + v12 "k8s.io/client-go/kubernetes/typed/authorization/v1" + v1beta12 "k8s.io/client-go/kubernetes/typed/authorization/v1beta1" + v13 "k8s.io/client-go/kubernetes/typed/autoscaling/v1" + v2 "k8s.io/client-go/kubernetes/typed/autoscaling/v2" + v2beta1 "k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1" + v2beta2 "k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2" + v14 "k8s.io/client-go/kubernetes/typed/batch/v1" + v1beta13 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" + v15 "k8s.io/client-go/kubernetes/typed/certificates/v1" + v1beta14 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + v16 "k8s.io/client-go/kubernetes/typed/coordination/v1" + v1beta15 "k8s.io/client-go/kubernetes/typed/coordination/v1beta1" + v17 "k8s.io/client-go/kubernetes/typed/core/v1" + v18 "k8s.io/client-go/kubernetes/typed/discovery/v1" + v1beta16 "k8s.io/client-go/kubernetes/typed/discovery/v1beta1" + v19 "k8s.io/client-go/kubernetes/typed/events/v1" + v1beta17 "k8s.io/client-go/kubernetes/typed/events/v1beta1" + v1beta18 "k8s.io/client-go/kubernetes/typed/extensions/v1beta1" + v1alpha12 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1" + v1beta19 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta1" + v1beta20 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta2" + v1beta3 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3" + v110 "k8s.io/client-go/kubernetes/typed/networking/v1" + v1alpha13 "k8s.io/client-go/kubernetes/typed/networking/v1alpha1" + v1beta110 "k8s.io/client-go/kubernetes/typed/networking/v1beta1" + v111 "k8s.io/client-go/kubernetes/typed/node/v1" + v1alpha14 "k8s.io/client-go/kubernetes/typed/node/v1alpha1" + v1beta111 "k8s.io/client-go/kubernetes/typed/node/v1beta1" + v112 "k8s.io/client-go/kubernetes/typed/policy/v1" + v1beta112 "k8s.io/client-go/kubernetes/typed/policy/v1beta1" + v113 "k8s.io/client-go/kubernetes/typed/rbac/v1" + v1alpha15 "k8s.io/client-go/kubernetes/typed/rbac/v1alpha1" + v1beta113 "k8s.io/client-go/kubernetes/typed/rbac/v1beta1" + v1alpha16 "k8s.io/client-go/kubernetes/typed/resource/v1alpha1" + v114 "k8s.io/client-go/kubernetes/typed/scheduling/v1" + v1alpha17 "k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1" + v1beta114 "k8s.io/client-go/kubernetes/typed/scheduling/v1beta1" + v115 "k8s.io/client-go/kubernetes/typed/storage/v1" + v1alpha18 "k8s.io/client-go/kubernetes/typed/storage/v1alpha1" + v1beta115 "k8s.io/client-go/kubernetes/typed/storage/v1beta1" +) + +// MockKubernetesClient is a mock of Interface interface. +type MockKubernetesClient struct { + ctrl *gomock.Controller + recorder *MockKubernetesClientMockRecorder +} + +// MockKubernetesClientMockRecorder is the mock recorder for MockKubernetesClient. +type MockKubernetesClientMockRecorder struct { + mock *MockKubernetesClient +} + +// NewMockKubernetesClient creates a new mock instance. +func NewMockKubernetesClient(ctrl *gomock.Controller) *MockKubernetesClient { + mock := &MockKubernetesClient{ctrl: ctrl} + mock.recorder = &MockKubernetesClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockKubernetesClient) EXPECT() *MockKubernetesClientMockRecorder { + return m.recorder +} + +// AdmissionregistrationV1 mocks base method. +func (m *MockKubernetesClient) AdmissionregistrationV1() v1.AdmissionregistrationV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdmissionregistrationV1") + ret0, _ := ret[0].(v1.AdmissionregistrationV1Interface) + return ret0 +} + +// AdmissionregistrationV1 indicates an expected call of AdmissionregistrationV1. +func (mr *MockKubernetesClientMockRecorder) AdmissionregistrationV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdmissionregistrationV1", reflect.TypeOf((*MockKubernetesClient)(nil).AdmissionregistrationV1)) +} + +// AdmissionregistrationV1alpha1 mocks base method. +func (m *MockKubernetesClient) AdmissionregistrationV1alpha1() v1alpha1.AdmissionregistrationV1alpha1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdmissionregistrationV1alpha1") + ret0, _ := ret[0].(v1alpha1.AdmissionregistrationV1alpha1Interface) + return ret0 +} + +// AdmissionregistrationV1alpha1 indicates an expected call of AdmissionregistrationV1alpha1. +func (mr *MockKubernetesClientMockRecorder) AdmissionregistrationV1alpha1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdmissionregistrationV1alpha1", reflect.TypeOf((*MockKubernetesClient)(nil).AdmissionregistrationV1alpha1)) +} + +// AdmissionregistrationV1beta1 mocks base method. +func (m *MockKubernetesClient) AdmissionregistrationV1beta1() v1beta1.AdmissionregistrationV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdmissionregistrationV1beta1") + ret0, _ := ret[0].(v1beta1.AdmissionregistrationV1beta1Interface) + return ret0 +} + +// AdmissionregistrationV1beta1 indicates an expected call of AdmissionregistrationV1beta1. +func (mr *MockKubernetesClientMockRecorder) AdmissionregistrationV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdmissionregistrationV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).AdmissionregistrationV1beta1)) +} + +// AppsV1 mocks base method. +func (m *MockKubernetesClient) AppsV1() v10.AppsV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppsV1") + ret0, _ := ret[0].(v10.AppsV1Interface) + return ret0 +} + +// AppsV1 indicates an expected call of AppsV1. +func (mr *MockKubernetesClientMockRecorder) AppsV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppsV1", reflect.TypeOf((*MockKubernetesClient)(nil).AppsV1)) +} + +// AppsV1beta1 mocks base method. +func (m *MockKubernetesClient) AppsV1beta1() v1beta10.AppsV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppsV1beta1") + ret0, _ := ret[0].(v1beta10.AppsV1beta1Interface) + return ret0 +} + +// AppsV1beta1 indicates an expected call of AppsV1beta1. +func (mr *MockKubernetesClientMockRecorder) AppsV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppsV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).AppsV1beta1)) +} + +// AppsV1beta2 mocks base method. +func (m *MockKubernetesClient) AppsV1beta2() v1beta2.AppsV1beta2Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppsV1beta2") + ret0, _ := ret[0].(v1beta2.AppsV1beta2Interface) + return ret0 +} + +// AppsV1beta2 indicates an expected call of AppsV1beta2. +func (mr *MockKubernetesClientMockRecorder) AppsV1beta2() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppsV1beta2", reflect.TypeOf((*MockKubernetesClient)(nil).AppsV1beta2)) +} + +// AuthenticationV1 mocks base method. +func (m *MockKubernetesClient) AuthenticationV1() v11.AuthenticationV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AuthenticationV1") + ret0, _ := ret[0].(v11.AuthenticationV1Interface) + return ret0 +} + +// AuthenticationV1 indicates an expected call of AuthenticationV1. +func (mr *MockKubernetesClientMockRecorder) AuthenticationV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthenticationV1", reflect.TypeOf((*MockKubernetesClient)(nil).AuthenticationV1)) +} + +// AuthenticationV1alpha1 mocks base method. +func (m *MockKubernetesClient) AuthenticationV1alpha1() v1alpha11.AuthenticationV1alpha1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AuthenticationV1alpha1") + ret0, _ := ret[0].(v1alpha11.AuthenticationV1alpha1Interface) + return ret0 +} + +// AuthenticationV1alpha1 indicates an expected call of AuthenticationV1alpha1. +func (mr *MockKubernetesClientMockRecorder) AuthenticationV1alpha1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthenticationV1alpha1", reflect.TypeOf((*MockKubernetesClient)(nil).AuthenticationV1alpha1)) +} + +// AuthenticationV1beta1 mocks base method. +func (m *MockKubernetesClient) AuthenticationV1beta1() v1beta11.AuthenticationV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AuthenticationV1beta1") + ret0, _ := ret[0].(v1beta11.AuthenticationV1beta1Interface) + return ret0 +} + +// AuthenticationV1beta1 indicates an expected call of AuthenticationV1beta1. +func (mr *MockKubernetesClientMockRecorder) AuthenticationV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthenticationV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).AuthenticationV1beta1)) +} + +// AuthorizationV1 mocks base method. +func (m *MockKubernetesClient) AuthorizationV1() v12.AuthorizationV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AuthorizationV1") + ret0, _ := ret[0].(v12.AuthorizationV1Interface) + return ret0 +} + +// AuthorizationV1 indicates an expected call of AuthorizationV1. +func (mr *MockKubernetesClientMockRecorder) AuthorizationV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthorizationV1", reflect.TypeOf((*MockKubernetesClient)(nil).AuthorizationV1)) +} + +// AuthorizationV1beta1 mocks base method. +func (m *MockKubernetesClient) AuthorizationV1beta1() v1beta12.AuthorizationV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AuthorizationV1beta1") + ret0, _ := ret[0].(v1beta12.AuthorizationV1beta1Interface) + return ret0 +} + +// AuthorizationV1beta1 indicates an expected call of AuthorizationV1beta1. +func (mr *MockKubernetesClientMockRecorder) AuthorizationV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthorizationV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).AuthorizationV1beta1)) +} + +// AutoscalingV1 mocks base method. +func (m *MockKubernetesClient) AutoscalingV1() v13.AutoscalingV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AutoscalingV1") + ret0, _ := ret[0].(v13.AutoscalingV1Interface) + return ret0 +} + +// AutoscalingV1 indicates an expected call of AutoscalingV1. +func (mr *MockKubernetesClientMockRecorder) AutoscalingV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AutoscalingV1", reflect.TypeOf((*MockKubernetesClient)(nil).AutoscalingV1)) +} + +// AutoscalingV2 mocks base method. +func (m *MockKubernetesClient) AutoscalingV2() v2.AutoscalingV2Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AutoscalingV2") + ret0, _ := ret[0].(v2.AutoscalingV2Interface) + return ret0 +} + +// AutoscalingV2 indicates an expected call of AutoscalingV2. +func (mr *MockKubernetesClientMockRecorder) AutoscalingV2() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AutoscalingV2", reflect.TypeOf((*MockKubernetesClient)(nil).AutoscalingV2)) +} + +// AutoscalingV2beta1 mocks base method. +func (m *MockKubernetesClient) AutoscalingV2beta1() v2beta1.AutoscalingV2beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AutoscalingV2beta1") + ret0, _ := ret[0].(v2beta1.AutoscalingV2beta1Interface) + return ret0 +} + +// AutoscalingV2beta1 indicates an expected call of AutoscalingV2beta1. +func (mr *MockKubernetesClientMockRecorder) AutoscalingV2beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AutoscalingV2beta1", reflect.TypeOf((*MockKubernetesClient)(nil).AutoscalingV2beta1)) +} + +// AutoscalingV2beta2 mocks base method. +func (m *MockKubernetesClient) AutoscalingV2beta2() v2beta2.AutoscalingV2beta2Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AutoscalingV2beta2") + ret0, _ := ret[0].(v2beta2.AutoscalingV2beta2Interface) + return ret0 +} + +// AutoscalingV2beta2 indicates an expected call of AutoscalingV2beta2. +func (mr *MockKubernetesClientMockRecorder) AutoscalingV2beta2() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AutoscalingV2beta2", reflect.TypeOf((*MockKubernetesClient)(nil).AutoscalingV2beta2)) +} + +// BatchV1 mocks base method. +func (m *MockKubernetesClient) BatchV1() v14.BatchV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BatchV1") + ret0, _ := ret[0].(v14.BatchV1Interface) + return ret0 +} + +// BatchV1 indicates an expected call of BatchV1. +func (mr *MockKubernetesClientMockRecorder) BatchV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchV1", reflect.TypeOf((*MockKubernetesClient)(nil).BatchV1)) +} + +// BatchV1beta1 mocks base method. +func (m *MockKubernetesClient) BatchV1beta1() v1beta13.BatchV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BatchV1beta1") + ret0, _ := ret[0].(v1beta13.BatchV1beta1Interface) + return ret0 +} + +// BatchV1beta1 indicates an expected call of BatchV1beta1. +func (mr *MockKubernetesClientMockRecorder) BatchV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).BatchV1beta1)) +} + +// CertificatesV1 mocks base method. +func (m *MockKubernetesClient) CertificatesV1() v15.CertificatesV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CertificatesV1") + ret0, _ := ret[0].(v15.CertificatesV1Interface) + return ret0 +} + +// CertificatesV1 indicates an expected call of CertificatesV1. +func (mr *MockKubernetesClientMockRecorder) CertificatesV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CertificatesV1", reflect.TypeOf((*MockKubernetesClient)(nil).CertificatesV1)) +} + +// CertificatesV1beta1 mocks base method. +func (m *MockKubernetesClient) CertificatesV1beta1() v1beta14.CertificatesV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CertificatesV1beta1") + ret0, _ := ret[0].(v1beta14.CertificatesV1beta1Interface) + return ret0 +} + +// CertificatesV1beta1 indicates an expected call of CertificatesV1beta1. +func (mr *MockKubernetesClientMockRecorder) CertificatesV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CertificatesV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).CertificatesV1beta1)) +} + +// CoordinationV1 mocks base method. +func (m *MockKubernetesClient) CoordinationV1() v16.CoordinationV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CoordinationV1") + ret0, _ := ret[0].(v16.CoordinationV1Interface) + return ret0 +} + +// CoordinationV1 indicates an expected call of CoordinationV1. +func (mr *MockKubernetesClientMockRecorder) CoordinationV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CoordinationV1", reflect.TypeOf((*MockKubernetesClient)(nil).CoordinationV1)) +} + +// CoordinationV1beta1 mocks base method. +func (m *MockKubernetesClient) CoordinationV1beta1() v1beta15.CoordinationV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CoordinationV1beta1") + ret0, _ := ret[0].(v1beta15.CoordinationV1beta1Interface) + return ret0 +} + +// CoordinationV1beta1 indicates an expected call of CoordinationV1beta1. +func (mr *MockKubernetesClientMockRecorder) CoordinationV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CoordinationV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).CoordinationV1beta1)) +} + +// CoreV1 mocks base method. +func (m *MockKubernetesClient) CoreV1() v17.CoreV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CoreV1") + ret0, _ := ret[0].(v17.CoreV1Interface) + return ret0 +} + +// CoreV1 indicates an expected call of CoreV1. +func (mr *MockKubernetesClientMockRecorder) CoreV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CoreV1", reflect.TypeOf((*MockKubernetesClient)(nil).CoreV1)) +} + +// Discovery mocks base method. +func (m *MockKubernetesClient) Discovery() discovery.DiscoveryInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Discovery") + ret0, _ := ret[0].(discovery.DiscoveryInterface) + return ret0 +} + +// Discovery indicates an expected call of Discovery. +func (mr *MockKubernetesClientMockRecorder) Discovery() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Discovery", reflect.TypeOf((*MockKubernetesClient)(nil).Discovery)) +} + +// DiscoveryV1 mocks base method. +func (m *MockKubernetesClient) DiscoveryV1() v18.DiscoveryV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiscoveryV1") + ret0, _ := ret[0].(v18.DiscoveryV1Interface) + return ret0 +} + +// DiscoveryV1 indicates an expected call of DiscoveryV1. +func (mr *MockKubernetesClientMockRecorder) DiscoveryV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoveryV1", reflect.TypeOf((*MockKubernetesClient)(nil).DiscoveryV1)) +} + +// DiscoveryV1beta1 mocks base method. +func (m *MockKubernetesClient) DiscoveryV1beta1() v1beta16.DiscoveryV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiscoveryV1beta1") + ret0, _ := ret[0].(v1beta16.DiscoveryV1beta1Interface) + return ret0 +} + +// DiscoveryV1beta1 indicates an expected call of DiscoveryV1beta1. +func (mr *MockKubernetesClientMockRecorder) DiscoveryV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoveryV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).DiscoveryV1beta1)) +} + +// EventsV1 mocks base method. +func (m *MockKubernetesClient) EventsV1() v19.EventsV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EventsV1") + ret0, _ := ret[0].(v19.EventsV1Interface) + return ret0 +} + +// EventsV1 indicates an expected call of EventsV1. +func (mr *MockKubernetesClientMockRecorder) EventsV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EventsV1", reflect.TypeOf((*MockKubernetesClient)(nil).EventsV1)) +} + +// EventsV1beta1 mocks base method. +func (m *MockKubernetesClient) EventsV1beta1() v1beta17.EventsV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EventsV1beta1") + ret0, _ := ret[0].(v1beta17.EventsV1beta1Interface) + return ret0 +} + +// EventsV1beta1 indicates an expected call of EventsV1beta1. +func (mr *MockKubernetesClientMockRecorder) EventsV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EventsV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).EventsV1beta1)) +} + +// ExtensionsV1beta1 mocks base method. +func (m *MockKubernetesClient) ExtensionsV1beta1() v1beta18.ExtensionsV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExtensionsV1beta1") + ret0, _ := ret[0].(v1beta18.ExtensionsV1beta1Interface) + return ret0 +} + +// ExtensionsV1beta1 indicates an expected call of ExtensionsV1beta1. +func (mr *MockKubernetesClientMockRecorder) ExtensionsV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtensionsV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).ExtensionsV1beta1)) +} + +// FlowcontrolV1alpha1 mocks base method. +func (m *MockKubernetesClient) FlowcontrolV1alpha1() v1alpha12.FlowcontrolV1alpha1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FlowcontrolV1alpha1") + ret0, _ := ret[0].(v1alpha12.FlowcontrolV1alpha1Interface) + return ret0 +} + +// FlowcontrolV1alpha1 indicates an expected call of FlowcontrolV1alpha1. +func (mr *MockKubernetesClientMockRecorder) FlowcontrolV1alpha1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlowcontrolV1alpha1", reflect.TypeOf((*MockKubernetesClient)(nil).FlowcontrolV1alpha1)) +} + +// FlowcontrolV1beta1 mocks base method. +func (m *MockKubernetesClient) FlowcontrolV1beta1() v1beta19.FlowcontrolV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FlowcontrolV1beta1") + ret0, _ := ret[0].(v1beta19.FlowcontrolV1beta1Interface) + return ret0 +} + +// FlowcontrolV1beta1 indicates an expected call of FlowcontrolV1beta1. +func (mr *MockKubernetesClientMockRecorder) FlowcontrolV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlowcontrolV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).FlowcontrolV1beta1)) +} + +// FlowcontrolV1beta2 mocks base method. +func (m *MockKubernetesClient) FlowcontrolV1beta2() v1beta20.FlowcontrolV1beta2Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FlowcontrolV1beta2") + ret0, _ := ret[0].(v1beta20.FlowcontrolV1beta2Interface) + return ret0 +} + +// FlowcontrolV1beta2 indicates an expected call of FlowcontrolV1beta2. +func (mr *MockKubernetesClientMockRecorder) FlowcontrolV1beta2() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlowcontrolV1beta2", reflect.TypeOf((*MockKubernetesClient)(nil).FlowcontrolV1beta2)) +} + +// FlowcontrolV1beta3 mocks base method. +func (m *MockKubernetesClient) FlowcontrolV1beta3() v1beta3.FlowcontrolV1beta3Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FlowcontrolV1beta3") + ret0, _ := ret[0].(v1beta3.FlowcontrolV1beta3Interface) + return ret0 +} + +// FlowcontrolV1beta3 indicates an expected call of FlowcontrolV1beta3. +func (mr *MockKubernetesClientMockRecorder) FlowcontrolV1beta3() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlowcontrolV1beta3", reflect.TypeOf((*MockKubernetesClient)(nil).FlowcontrolV1beta3)) +} + +// InternalV1alpha1 mocks base method. +func (m *MockKubernetesClient) InternalV1alpha1() v1alpha10.InternalV1alpha1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InternalV1alpha1") + ret0, _ := ret[0].(v1alpha10.InternalV1alpha1Interface) + return ret0 +} + +// InternalV1alpha1 indicates an expected call of InternalV1alpha1. +func (mr *MockKubernetesClientMockRecorder) InternalV1alpha1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InternalV1alpha1", reflect.TypeOf((*MockKubernetesClient)(nil).InternalV1alpha1)) +} + +// NetworkingV1 mocks base method. +func (m *MockKubernetesClient) NetworkingV1() v110.NetworkingV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NetworkingV1") + ret0, _ := ret[0].(v110.NetworkingV1Interface) + return ret0 +} + +// NetworkingV1 indicates an expected call of NetworkingV1. +func (mr *MockKubernetesClientMockRecorder) NetworkingV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkingV1", reflect.TypeOf((*MockKubernetesClient)(nil).NetworkingV1)) +} + +// NetworkingV1alpha1 mocks base method. +func (m *MockKubernetesClient) NetworkingV1alpha1() v1alpha13.NetworkingV1alpha1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NetworkingV1alpha1") + ret0, _ := ret[0].(v1alpha13.NetworkingV1alpha1Interface) + return ret0 +} + +// NetworkingV1alpha1 indicates an expected call of NetworkingV1alpha1. +func (mr *MockKubernetesClientMockRecorder) NetworkingV1alpha1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkingV1alpha1", reflect.TypeOf((*MockKubernetesClient)(nil).NetworkingV1alpha1)) +} + +// NetworkingV1beta1 mocks base method. +func (m *MockKubernetesClient) NetworkingV1beta1() v1beta110.NetworkingV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NetworkingV1beta1") + ret0, _ := ret[0].(v1beta110.NetworkingV1beta1Interface) + return ret0 +} + +// NetworkingV1beta1 indicates an expected call of NetworkingV1beta1. +func (mr *MockKubernetesClientMockRecorder) NetworkingV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkingV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).NetworkingV1beta1)) +} + +// NodeV1 mocks base method. +func (m *MockKubernetesClient) NodeV1() v111.NodeV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeV1") + ret0, _ := ret[0].(v111.NodeV1Interface) + return ret0 +} + +// NodeV1 indicates an expected call of NodeV1. +func (mr *MockKubernetesClientMockRecorder) NodeV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeV1", reflect.TypeOf((*MockKubernetesClient)(nil).NodeV1)) +} + +// NodeV1alpha1 mocks base method. +func (m *MockKubernetesClient) NodeV1alpha1() v1alpha14.NodeV1alpha1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeV1alpha1") + ret0, _ := ret[0].(v1alpha14.NodeV1alpha1Interface) + return ret0 +} + +// NodeV1alpha1 indicates an expected call of NodeV1alpha1. +func (mr *MockKubernetesClientMockRecorder) NodeV1alpha1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeV1alpha1", reflect.TypeOf((*MockKubernetesClient)(nil).NodeV1alpha1)) +} + +// NodeV1beta1 mocks base method. +func (m *MockKubernetesClient) NodeV1beta1() v1beta111.NodeV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeV1beta1") + ret0, _ := ret[0].(v1beta111.NodeV1beta1Interface) + return ret0 +} + +// NodeV1beta1 indicates an expected call of NodeV1beta1. +func (mr *MockKubernetesClientMockRecorder) NodeV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).NodeV1beta1)) +} + +// PolicyV1 mocks base method. +func (m *MockKubernetesClient) PolicyV1() v112.PolicyV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PolicyV1") + ret0, _ := ret[0].(v112.PolicyV1Interface) + return ret0 +} + +// PolicyV1 indicates an expected call of PolicyV1. +func (mr *MockKubernetesClientMockRecorder) PolicyV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PolicyV1", reflect.TypeOf((*MockKubernetesClient)(nil).PolicyV1)) +} + +// PolicyV1beta1 mocks base method. +func (m *MockKubernetesClient) PolicyV1beta1() v1beta112.PolicyV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PolicyV1beta1") + ret0, _ := ret[0].(v1beta112.PolicyV1beta1Interface) + return ret0 +} + +// PolicyV1beta1 indicates an expected call of PolicyV1beta1. +func (mr *MockKubernetesClientMockRecorder) PolicyV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PolicyV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).PolicyV1beta1)) +} + +// RbacV1 mocks base method. +func (m *MockKubernetesClient) RbacV1() v113.RbacV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RbacV1") + ret0, _ := ret[0].(v113.RbacV1Interface) + return ret0 +} + +// RbacV1 indicates an expected call of RbacV1. +func (mr *MockKubernetesClientMockRecorder) RbacV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RbacV1", reflect.TypeOf((*MockKubernetesClient)(nil).RbacV1)) +} + +// RbacV1alpha1 mocks base method. +func (m *MockKubernetesClient) RbacV1alpha1() v1alpha15.RbacV1alpha1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RbacV1alpha1") + ret0, _ := ret[0].(v1alpha15.RbacV1alpha1Interface) + return ret0 +} + +// RbacV1alpha1 indicates an expected call of RbacV1alpha1. +func (mr *MockKubernetesClientMockRecorder) RbacV1alpha1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RbacV1alpha1", reflect.TypeOf((*MockKubernetesClient)(nil).RbacV1alpha1)) +} + +// RbacV1beta1 mocks base method. +func (m *MockKubernetesClient) RbacV1beta1() v1beta113.RbacV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RbacV1beta1") + ret0, _ := ret[0].(v1beta113.RbacV1beta1Interface) + return ret0 +} + +// RbacV1beta1 indicates an expected call of RbacV1beta1. +func (mr *MockKubernetesClientMockRecorder) RbacV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RbacV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).RbacV1beta1)) +} + +// ResourceV1alpha1 mocks base method. +func (m *MockKubernetesClient) ResourceV1alpha1() v1alpha16.ResourceV1alpha1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResourceV1alpha1") + ret0, _ := ret[0].(v1alpha16.ResourceV1alpha1Interface) + return ret0 +} + +// ResourceV1alpha1 indicates an expected call of ResourceV1alpha1. +func (mr *MockKubernetesClientMockRecorder) ResourceV1alpha1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourceV1alpha1", reflect.TypeOf((*MockKubernetesClient)(nil).ResourceV1alpha1)) +} + +// SchedulingV1 mocks base method. +func (m *MockKubernetesClient) SchedulingV1() v114.SchedulingV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SchedulingV1") + ret0, _ := ret[0].(v114.SchedulingV1Interface) + return ret0 +} + +// SchedulingV1 indicates an expected call of SchedulingV1. +func (mr *MockKubernetesClientMockRecorder) SchedulingV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SchedulingV1", reflect.TypeOf((*MockKubernetesClient)(nil).SchedulingV1)) +} + +// SchedulingV1alpha1 mocks base method. +func (m *MockKubernetesClient) SchedulingV1alpha1() v1alpha17.SchedulingV1alpha1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SchedulingV1alpha1") + ret0, _ := ret[0].(v1alpha17.SchedulingV1alpha1Interface) + return ret0 +} + +// SchedulingV1alpha1 indicates an expected call of SchedulingV1alpha1. +func (mr *MockKubernetesClientMockRecorder) SchedulingV1alpha1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SchedulingV1alpha1", reflect.TypeOf((*MockKubernetesClient)(nil).SchedulingV1alpha1)) +} + +// SchedulingV1beta1 mocks base method. +func (m *MockKubernetesClient) SchedulingV1beta1() v1beta114.SchedulingV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SchedulingV1beta1") + ret0, _ := ret[0].(v1beta114.SchedulingV1beta1Interface) + return ret0 +} + +// SchedulingV1beta1 indicates an expected call of SchedulingV1beta1. +func (mr *MockKubernetesClientMockRecorder) SchedulingV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SchedulingV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).SchedulingV1beta1)) +} + +// StorageV1 mocks base method. +func (m *MockKubernetesClient) StorageV1() v115.StorageV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StorageV1") + ret0, _ := ret[0].(v115.StorageV1Interface) + return ret0 +} + +// StorageV1 indicates an expected call of StorageV1. +func (mr *MockKubernetesClientMockRecorder) StorageV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageV1", reflect.TypeOf((*MockKubernetesClient)(nil).StorageV1)) +} + +// StorageV1alpha1 mocks base method. +func (m *MockKubernetesClient) StorageV1alpha1() v1alpha18.StorageV1alpha1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StorageV1alpha1") + ret0, _ := ret[0].(v1alpha18.StorageV1alpha1Interface) + return ret0 +} + +// StorageV1alpha1 indicates an expected call of StorageV1alpha1. +func (mr *MockKubernetesClientMockRecorder) StorageV1alpha1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageV1alpha1", reflect.TypeOf((*MockKubernetesClient)(nil).StorageV1alpha1)) +} + +// StorageV1beta1 mocks base method. +func (m *MockKubernetesClient) StorageV1beta1() v1beta115.StorageV1beta1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StorageV1beta1") + ret0, _ := ret[0].(v1beta115.StorageV1beta1Interface) + return ret0 +} + +// StorageV1beta1 indicates an expected call of StorageV1beta1. +func (mr *MockKubernetesClientMockRecorder) StorageV1beta1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StorageV1beta1", reflect.TypeOf((*MockKubernetesClient)(nil).StorageV1beta1)) +} diff --git a/pkg/driver/mock_k8s_corev1.go b/pkg/driver/mock_k8s_corev1.go new file mode 100644 index 0000000000..58069c8c6c --- /dev/null +++ b/pkg/driver/mock_k8s_corev1.go @@ -0,0 +1,486 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: k8s.io/client-go/kubernetes/typed/core/v1 (interfaces: CoreV1Interface,NodeInterface) + +// Package driver is a generated GoMock package. +package driver + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "k8s.io/api/core/v1" + v10 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + v11 "k8s.io/client-go/applyconfigurations/core/v1" + v12 "k8s.io/client-go/kubernetes/typed/core/v1" + rest "k8s.io/client-go/rest" +) + +// MockCoreV1Interface is a mock of CoreV1Interface interface. +type MockCoreV1Interface struct { + ctrl *gomock.Controller + recorder *MockCoreV1InterfaceMockRecorder +} + +// MockCoreV1InterfaceMockRecorder is the mock recorder for MockCoreV1Interface. +type MockCoreV1InterfaceMockRecorder struct { + mock *MockCoreV1Interface +} + +// NewMockCoreV1Interface creates a new mock instance. +func NewMockCoreV1Interface(ctrl *gomock.Controller) *MockCoreV1Interface { + mock := &MockCoreV1Interface{ctrl: ctrl} + mock.recorder = &MockCoreV1InterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCoreV1Interface) EXPECT() *MockCoreV1InterfaceMockRecorder { + return m.recorder +} + +// ComponentStatuses mocks base method. +func (m *MockCoreV1Interface) ComponentStatuses() v12.ComponentStatusInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ComponentStatuses") + ret0, _ := ret[0].(v12.ComponentStatusInterface) + return ret0 +} + +// ComponentStatuses indicates an expected call of ComponentStatuses. +func (mr *MockCoreV1InterfaceMockRecorder) ComponentStatuses() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ComponentStatuses", reflect.TypeOf((*MockCoreV1Interface)(nil).ComponentStatuses)) +} + +// ConfigMaps mocks base method. +func (m *MockCoreV1Interface) ConfigMaps(arg0 string) v12.ConfigMapInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigMaps", arg0) + ret0, _ := ret[0].(v12.ConfigMapInterface) + return ret0 +} + +// ConfigMaps indicates an expected call of ConfigMaps. +func (mr *MockCoreV1InterfaceMockRecorder) ConfigMaps(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigMaps", reflect.TypeOf((*MockCoreV1Interface)(nil).ConfigMaps), arg0) +} + +// Endpoints mocks base method. +func (m *MockCoreV1Interface) Endpoints(arg0 string) v12.EndpointsInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Endpoints", arg0) + ret0, _ := ret[0].(v12.EndpointsInterface) + return ret0 +} + +// Endpoints indicates an expected call of Endpoints. +func (mr *MockCoreV1InterfaceMockRecorder) Endpoints(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Endpoints", reflect.TypeOf((*MockCoreV1Interface)(nil).Endpoints), arg0) +} + +// Events mocks base method. +func (m *MockCoreV1Interface) Events(arg0 string) v12.EventInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Events", arg0) + ret0, _ := ret[0].(v12.EventInterface) + return ret0 +} + +// Events indicates an expected call of Events. +func (mr *MockCoreV1InterfaceMockRecorder) Events(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Events", reflect.TypeOf((*MockCoreV1Interface)(nil).Events), arg0) +} + +// LimitRanges mocks base method. +func (m *MockCoreV1Interface) LimitRanges(arg0 string) v12.LimitRangeInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LimitRanges", arg0) + ret0, _ := ret[0].(v12.LimitRangeInterface) + return ret0 +} + +// LimitRanges indicates an expected call of LimitRanges. +func (mr *MockCoreV1InterfaceMockRecorder) LimitRanges(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LimitRanges", reflect.TypeOf((*MockCoreV1Interface)(nil).LimitRanges), arg0) +} + +// Namespaces mocks base method. +func (m *MockCoreV1Interface) Namespaces() v12.NamespaceInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Namespaces") + ret0, _ := ret[0].(v12.NamespaceInterface) + return ret0 +} + +// Namespaces indicates an expected call of Namespaces. +func (mr *MockCoreV1InterfaceMockRecorder) Namespaces() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Namespaces", reflect.TypeOf((*MockCoreV1Interface)(nil).Namespaces)) +} + +// Nodes mocks base method. +func (m *MockCoreV1Interface) Nodes() v12.NodeInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Nodes") + ret0, _ := ret[0].(v12.NodeInterface) + return ret0 +} + +// Nodes indicates an expected call of Nodes. +func (mr *MockCoreV1InterfaceMockRecorder) Nodes() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Nodes", reflect.TypeOf((*MockCoreV1Interface)(nil).Nodes)) +} + +// PersistentVolumeClaims mocks base method. +func (m *MockCoreV1Interface) PersistentVolumeClaims(arg0 string) v12.PersistentVolumeClaimInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PersistentVolumeClaims", arg0) + ret0, _ := ret[0].(v12.PersistentVolumeClaimInterface) + return ret0 +} + +// PersistentVolumeClaims indicates an expected call of PersistentVolumeClaims. +func (mr *MockCoreV1InterfaceMockRecorder) PersistentVolumeClaims(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PersistentVolumeClaims", reflect.TypeOf((*MockCoreV1Interface)(nil).PersistentVolumeClaims), arg0) +} + +// PersistentVolumes mocks base method. +func (m *MockCoreV1Interface) PersistentVolumes() v12.PersistentVolumeInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PersistentVolumes") + ret0, _ := ret[0].(v12.PersistentVolumeInterface) + return ret0 +} + +// PersistentVolumes indicates an expected call of PersistentVolumes. +func (mr *MockCoreV1InterfaceMockRecorder) PersistentVolumes() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PersistentVolumes", reflect.TypeOf((*MockCoreV1Interface)(nil).PersistentVolumes)) +} + +// PodTemplates mocks base method. +func (m *MockCoreV1Interface) PodTemplates(arg0 string) v12.PodTemplateInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PodTemplates", arg0) + ret0, _ := ret[0].(v12.PodTemplateInterface) + return ret0 +} + +// PodTemplates indicates an expected call of PodTemplates. +func (mr *MockCoreV1InterfaceMockRecorder) PodTemplates(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PodTemplates", reflect.TypeOf((*MockCoreV1Interface)(nil).PodTemplates), arg0) +} + +// Pods mocks base method. +func (m *MockCoreV1Interface) Pods(arg0 string) v12.PodInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Pods", arg0) + ret0, _ := ret[0].(v12.PodInterface) + return ret0 +} + +// Pods indicates an expected call of Pods. +func (mr *MockCoreV1InterfaceMockRecorder) Pods(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pods", reflect.TypeOf((*MockCoreV1Interface)(nil).Pods), arg0) +} + +// RESTClient mocks base method. +func (m *MockCoreV1Interface) RESTClient() rest.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RESTClient") + ret0, _ := ret[0].(rest.Interface) + return ret0 +} + +// RESTClient indicates an expected call of RESTClient. +func (mr *MockCoreV1InterfaceMockRecorder) RESTClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RESTClient", reflect.TypeOf((*MockCoreV1Interface)(nil).RESTClient)) +} + +// ReplicationControllers mocks base method. +func (m *MockCoreV1Interface) ReplicationControllers(arg0 string) v12.ReplicationControllerInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReplicationControllers", arg0) + ret0, _ := ret[0].(v12.ReplicationControllerInterface) + return ret0 +} + +// ReplicationControllers indicates an expected call of ReplicationControllers. +func (mr *MockCoreV1InterfaceMockRecorder) ReplicationControllers(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReplicationControllers", reflect.TypeOf((*MockCoreV1Interface)(nil).ReplicationControllers), arg0) +} + +// ResourceQuotas mocks base method. +func (m *MockCoreV1Interface) ResourceQuotas(arg0 string) v12.ResourceQuotaInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResourceQuotas", arg0) + ret0, _ := ret[0].(v12.ResourceQuotaInterface) + return ret0 +} + +// ResourceQuotas indicates an expected call of ResourceQuotas. +func (mr *MockCoreV1InterfaceMockRecorder) ResourceQuotas(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourceQuotas", reflect.TypeOf((*MockCoreV1Interface)(nil).ResourceQuotas), arg0) +} + +// Secrets mocks base method. +func (m *MockCoreV1Interface) Secrets(arg0 string) v12.SecretInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Secrets", arg0) + ret0, _ := ret[0].(v12.SecretInterface) + return ret0 +} + +// Secrets indicates an expected call of Secrets. +func (mr *MockCoreV1InterfaceMockRecorder) Secrets(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Secrets", reflect.TypeOf((*MockCoreV1Interface)(nil).Secrets), arg0) +} + +// ServiceAccounts mocks base method. +func (m *MockCoreV1Interface) ServiceAccounts(arg0 string) v12.ServiceAccountInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ServiceAccounts", arg0) + ret0, _ := ret[0].(v12.ServiceAccountInterface) + return ret0 +} + +// ServiceAccounts indicates an expected call of ServiceAccounts. +func (mr *MockCoreV1InterfaceMockRecorder) ServiceAccounts(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ServiceAccounts", reflect.TypeOf((*MockCoreV1Interface)(nil).ServiceAccounts), arg0) +} + +// Services mocks base method. +func (m *MockCoreV1Interface) Services(arg0 string) v12.ServiceInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Services", arg0) + ret0, _ := ret[0].(v12.ServiceInterface) + return ret0 +} + +// Services indicates an expected call of Services. +func (mr *MockCoreV1InterfaceMockRecorder) Services(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Services", reflect.TypeOf((*MockCoreV1Interface)(nil).Services), arg0) +} + +// MockNodeInterface is a mock of NodeInterface interface. +type MockNodeInterface struct { + ctrl *gomock.Controller + recorder *MockNodeInterfaceMockRecorder +} + +// MockNodeInterfaceMockRecorder is the mock recorder for MockNodeInterface. +type MockNodeInterfaceMockRecorder struct { + mock *MockNodeInterface +} + +// NewMockNodeInterface creates a new mock instance. +func NewMockNodeInterface(ctrl *gomock.Controller) *MockNodeInterface { + mock := &MockNodeInterface{ctrl: ctrl} + mock.recorder = &MockNodeInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockNodeInterface) EXPECT() *MockNodeInterfaceMockRecorder { + return m.recorder +} + +// Apply mocks base method. +func (m *MockNodeInterface) Apply(arg0 context.Context, arg1 *v11.NodeApplyConfiguration, arg2 v10.ApplyOptions) (*v1.Node, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Apply", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Node) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Apply indicates an expected call of Apply. +func (mr *MockNodeInterfaceMockRecorder) Apply(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockNodeInterface)(nil).Apply), arg0, arg1, arg2) +} + +// ApplyStatus mocks base method. +func (m *MockNodeInterface) ApplyStatus(arg0 context.Context, arg1 *v11.NodeApplyConfiguration, arg2 v10.ApplyOptions) (*v1.Node, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ApplyStatus", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Node) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ApplyStatus indicates an expected call of ApplyStatus. +func (mr *MockNodeInterfaceMockRecorder) ApplyStatus(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyStatus", reflect.TypeOf((*MockNodeInterface)(nil).ApplyStatus), arg0, arg1, arg2) +} + +// Create mocks base method. +func (m *MockNodeInterface) Create(arg0 context.Context, arg1 *v1.Node, arg2 v10.CreateOptions) (*v1.Node, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Node) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create. +func (mr *MockNodeInterfaceMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockNodeInterface)(nil).Create), arg0, arg1, arg2) +} + +// Delete mocks base method. +func (m *MockNodeInterface) Delete(arg0 context.Context, arg1 string, arg2 v10.DeleteOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockNodeInterfaceMockRecorder) Delete(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockNodeInterface)(nil).Delete), arg0, arg1, arg2) +} + +// DeleteCollection mocks base method. +func (m *MockNodeInterface) DeleteCollection(arg0 context.Context, arg1 v10.DeleteOptions, arg2 v10.ListOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCollection", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteCollection indicates an expected call of DeleteCollection. +func (mr *MockNodeInterfaceMockRecorder) DeleteCollection(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCollection", reflect.TypeOf((*MockNodeInterface)(nil).DeleteCollection), arg0, arg1, arg2) +} + +// Get mocks base method. +func (m *MockNodeInterface) Get(arg0 context.Context, arg1 string, arg2 v10.GetOptions) (*v1.Node, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Node) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockNodeInterfaceMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockNodeInterface)(nil).Get), arg0, arg1, arg2) +} + +// List mocks base method. +func (m *MockNodeInterface) List(arg0 context.Context, arg1 v10.ListOptions) (*v1.NodeList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", arg0, arg1) + ret0, _ := ret[0].(*v1.NodeList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockNodeInterfaceMockRecorder) List(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockNodeInterface)(nil).List), arg0, arg1) +} + +// Patch mocks base method. +func (m *MockNodeInterface) Patch(arg0 context.Context, arg1 string, arg2 types.PatchType, arg3 []byte, arg4 v10.PatchOptions, arg5 ...string) (*v1.Node, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2, arg3, arg4} + for _, a := range arg5 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Patch", varargs...) + ret0, _ := ret[0].(*v1.Node) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Patch indicates an expected call of Patch. +func (mr *MockNodeInterfaceMockRecorder) Patch(arg0, arg1, arg2, arg3, arg4 interface{}, arg5 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2, arg3, arg4}, arg5...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockNodeInterface)(nil).Patch), varargs...) +} + +// PatchStatus mocks base method. +func (m *MockNodeInterface) PatchStatus(arg0 context.Context, arg1 string, arg2 []byte) (*v1.Node, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PatchStatus", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Node) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PatchStatus indicates an expected call of PatchStatus. +func (mr *MockNodeInterfaceMockRecorder) PatchStatus(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PatchStatus", reflect.TypeOf((*MockNodeInterface)(nil).PatchStatus), arg0, arg1, arg2) +} + +// Update mocks base method. +func (m *MockNodeInterface) Update(arg0 context.Context, arg1 *v1.Node, arg2 v10.UpdateOptions) (*v1.Node, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Node) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockNodeInterfaceMockRecorder) Update(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockNodeInterface)(nil).Update), arg0, arg1, arg2) +} + +// UpdateStatus mocks base method. +func (m *MockNodeInterface) UpdateStatus(arg0 context.Context, arg1 *v1.Node, arg2 v10.UpdateOptions) (*v1.Node, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateStatus", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Node) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateStatus indicates an expected call of UpdateStatus. +func (mr *MockNodeInterfaceMockRecorder) UpdateStatus(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateStatus", reflect.TypeOf((*MockNodeInterface)(nil).UpdateStatus), arg0, arg1, arg2) +} + +// Watch mocks base method. +func (m *MockNodeInterface) Watch(arg0 context.Context, arg1 v10.ListOptions) (watch.Interface, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Watch", arg0, arg1) + ret0, _ := ret[0].(watch.Interface) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Watch indicates an expected call of Watch. +func (mr *MockNodeInterfaceMockRecorder) Watch(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockNodeInterface)(nil).Watch), arg0, arg1) +} diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 68308fc803..c4e2cba475 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -18,6 +18,7 @@ package driver import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -30,6 +31,9 @@ import ( "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/volume" ) @@ -89,6 +93,13 @@ func newNodeService(driverOptions *DriverOptions) nodeService { panic(err) } + // Remove taint from node to indicate driver startup success + // This is done at the last possible moment to prevent race conditions or false positive removals + err = removeNotReadyTaint(cloud.DefaultKubernetesAPIClient) + if err != nil { + klog.ErrorS(err, "Unexpected failure when attempting to remove node taint(s)") + } + return nodeService{ metadata: metadata, mounter: nodeMounter, @@ -799,3 +810,71 @@ func collectMountOptions(fsType string, mntFlags []string) []string { } return options } + +// Struct for JSON patch operations +type JSONPatch struct { + OP string `json:"op,omitempty"` + Path string `json:"path,omitempty"` + Value interface{} `json:"value"` +} + +// removeNotReadyTaint removes the taint ebs.csi.aws.com/agent-not-ready from the local node +// This taint can be optionally applied by users to prevent startup race conditions such as +// https://github.com/kubernetes/kubernetes/issues/95911 +func removeNotReadyTaint(k8sClient cloud.KubernetesAPIClient) error { + nodeName := os.Getenv("CSI_NODE_NAME") + if nodeName == "" { + klog.V(4).InfoS("CSI_NODE_NAME missing, skipping taint removal") + return nil + } + + clientset, err := k8sClient() + if err != nil { + klog.V(4).InfoS("Failed to communicate with k8s API, skipping taint removal") + return nil //lint:ignore nilerr Failing to communicate with k8s API is a soft failure + } + + node, err := clientset.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + var taintsToKeep []corev1.Taint + for _, taint := range node.Spec.Taints { + if taint.Key != AgentNotReadyNodeTaintKey { + taintsToKeep = append(taintsToKeep, taint) + } else { + klog.V(4).InfoS("Queued taint for removal", "key", taint.Key, "effect", taint.Effect) + } + } + + if len(taintsToKeep) == len(node.Spec.Taints) { + klog.V(4).InfoS("No taints to remove on node, skipping taint removal") + return nil + } + + patchRemoveTaints := []JSONPatch{ + { + OP: "test", + Path: "/spec/taints", + Value: node.Spec.Taints, + }, + { + OP: "replace", + Path: "/spec/taints", + Value: taintsToKeep, + }, + } + + patch, err := json.Marshal(patchRemoveTaints) + if err != nil { + return err + } + + _, err = clientset.CoreV1().Nodes().Patch(context.Background(), nodeName, k8stypes.JSONPatchType, patch, metav1.PatchOptions{}) + if err != nil { + return err + } + klog.InfoS("Removed taint(s) from local node", "node", nodeName) + return nil +} diff --git a/pkg/driver/node_test.go b/pkg/driver/node_test.go index 5803661bbf..6114c54ffc 100644 --- a/pkg/driver/node_test.go +++ b/pkg/driver/node_test.go @@ -22,6 +22,7 @@ package driver import ( "context" "errors" + "fmt" "io/fs" "os" "reflect" @@ -35,6 +36,8 @@ import ( "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver/internal" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" ) var ( @@ -2165,6 +2168,129 @@ func TestNodeGetInfo(t *testing.T) { } } +func TestRemoveNotReadyTaint(t *testing.T) { + nodeName := "test-node-123" + testCases := []struct { + name string + setup func(t *testing.T, mockCtl *gomock.Controller) func() (kubernetes.Interface, error) + expResult error + }{ + { + name: "missing CSI_NODE_NAME", + setup: func(t *testing.T, mockCtl *gomock.Controller) func() (kubernetes.Interface, error) { + return func() (kubernetes.Interface, error) { + t.Fatalf("Unexpected call to k8s client getter") + return nil, nil + } + }, + expResult: nil, + }, + { + name: "failed to setup k8s client", + setup: func(t *testing.T, mockCtl *gomock.Controller) func() (kubernetes.Interface, error) { + t.Setenv("CSI_NODE_NAME", nodeName) + return func() (kubernetes.Interface, error) { + return nil, fmt.Errorf("Failed setup!") + } + }, + expResult: nil, + }, + { + name: "failed to get node", + setup: func(t *testing.T, mockCtl *gomock.Controller) func() (kubernetes.Interface, error) { + t.Setenv("CSI_NODE_NAME", nodeName) + getNodeMock, _ := getNodeMock(mockCtl, nodeName, nil, fmt.Errorf("Failed to get node!")) + + return func() (kubernetes.Interface, error) { + return getNodeMock, nil + } + }, + expResult: fmt.Errorf("Failed to get node!"), + }, + { + name: "no taints to remove", + setup: func(t *testing.T, mockCtl *gomock.Controller) func() (kubernetes.Interface, error) { + t.Setenv("CSI_NODE_NAME", nodeName) + getNodeMock, _ := getNodeMock(mockCtl, nodeName, &corev1.Node{}, nil) + + return func() (kubernetes.Interface, error) { + return getNodeMock, nil + } + }, + expResult: nil, + }, + { + name: "failed to patch node", + setup: func(t *testing.T, mockCtl *gomock.Controller) func() (kubernetes.Interface, error) { + t.Setenv("CSI_NODE_NAME", nodeName) + getNodeMock, mockNode := getNodeMock(mockCtl, nodeName, &corev1.Node{ + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: AgentNotReadyNodeTaintKey, + Effect: "NoExecute", + }, + }, + }, + }, nil) + mockNode.EXPECT().Patch(gomock.Any(), gomock.Eq(nodeName), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("Failed to patch node!")) + + return func() (kubernetes.Interface, error) { + return getNodeMock, nil + } + }, + expResult: fmt.Errorf("Failed to patch node!"), + }, + { + name: "success", + setup: func(t *testing.T, mockCtl *gomock.Controller) func() (kubernetes.Interface, error) { + t.Setenv("CSI_NODE_NAME", nodeName) + getNodeMock, mockNode := getNodeMock(mockCtl, nodeName, &corev1.Node{ + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: AgentNotReadyNodeTaintKey, + Effect: "NoSchedule", + }, + }, + }, + }, nil) + mockNode.EXPECT().Patch(gomock.Any(), gomock.Eq(nodeName), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + + return func() (kubernetes.Interface, error) { + return getNodeMock, nil + } + }, + expResult: nil, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + + k8sClientGetter := tc.setup(t, mockCtl) + result := removeNotReadyTaint(k8sClientGetter) + + if !reflect.DeepEqual(result, tc.expResult) { + t.Fatalf("Expected result `%v`, got result `%v`", tc.expResult, result) + } + }) + } +} + +func getNodeMock(mockCtl *gomock.Controller, nodeName string, returnNode *corev1.Node, returnError error) (kubernetes.Interface, *MockNodeInterface) { + mockClient := NewMockKubernetesClient(mockCtl) + mockCoreV1 := NewMockCoreV1Interface(mockCtl) + mockNode := NewMockNodeInterface(mockCtl) + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).MinTimes(1) + mockCoreV1.EXPECT().Nodes().Return(mockNode).MinTimes(1) + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(returnNode, returnError).MinTimes(1) + + return mockClient, mockNode +} + func expectErr(t *testing.T, actualErr error, expectedCode codes.Code) { if actualErr == nil { t.Fatalf("Expect error but got no error")