Add kubeAPIServerOverride option to antrea-controller

To override the address of Kubernetes apiserver for antrea-controller, the option, kubeAPIServerOverride, is added to antrea-controller.conf. It's typically used when kube-proxy is not deployed (substituted by AntreaProxy) and kube-controller-manager does not run NodeIPAMController (substituted by Antrea NodeIPAM). Signed-off-by: Quan Tian <[email protected]>
antrea-io · Jun 1, 2023 · 2535bb4 · 2535bb4
1 parent 7688240
commit 2535bb4
Show file tree

Hide file tree

Showing 15 changed files with 164 additions and 16 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -214,8 +214,13 @@ jobs:
         run: |
           mkdir log
           mkdir test-e2e-encap-all-features-enabled-coverage
-          # FlowExporter requires the FlowAggregator, so we keep it disabled.
-          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=true,AllBeta=true,FlowExporter=false --proxy-all --multicast
+          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh \
+            --encap-mode encap \
+            --coverage \
+            --feature-gates AllAlpha=true,AllBeta=true \
+            --proxy-all \
+            --node-ipam \
+            --multicast
       - name: Tar coverage files
         run: tar -czf test-e2e-encap-all-features-enabled-coverage.tar.gz test-e2e-encap-all-features-enabled-coverage
       - name: Upload coverage for test-e2e-encap-all-features-enabled-coverage

diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -248,6 +248,7 @@ nodePortLocal:
 {{- end }}
 
 # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+# It is typically used when kube-proxy is not deployed (replaced by AntreaProxy).
 # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
 kubeAPIServerOverride: {{ .Values.kubeAPIServerOverride | quote }}
 

diff --git a/build/charts/antrea/conf/antrea-controller.conf b/build/charts/antrea/conf/antrea-controller.conf
@@ -78,6 +78,12 @@ tlsMinVersion: {{ .Values.tlsMinVersion | quote }}
 # certificates.
 clientCAFile: {{ .Values.clientCAFile | quote }}
 
+# Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+# It is typically used when kube-proxy is not deployed (replaced by AntreaProxy) and kube-controller-manager
+# does not run NodeIPAMController (replaced by Antrea NodeIPAM).
+# Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+kubeAPIServerOverride: {{ .Values.kubeAPIServerOverride | quote }}
+
 nodeIPAM:
 {{- with .Values.nodeIPAM }}
   # Enable the integrated Node IPAM controller within the Antrea controller.

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3210,6 +3210,7 @@ data:
       portRange: "61000-62000"
 
     # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (replaced by AntreaProxy).
     # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
     kubeAPIServerOverride: ""
 
@@ -3423,6 +3424,12 @@ data:
     # certificates.
     clientCAFile: ""
 
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (replaced by AntreaProxy) and kube-controller-manager
+    # does not run NodeIPAMController (replaced by Antrea NodeIPAM).
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    kubeAPIServerOverride: ""
+
     nodeIPAM:
       # Enable the integrated Node IPAM controller within the Antrea controller.
       enableNodeIPAM: false
@@ -4371,7 +4378,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: abf7cb1c21b730664510e8a762d967df5c620467f12bf3e0bae41df73489de65
+        checksum/config: 994e75167f0126a535cea63fc65a5ab86361648a20bcacb06d3c588f06f6e5f6
       labels:
         app: antrea
         component: antrea-agent
@@ -4612,7 +4619,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: abf7cb1c21b730664510e8a762d967df5c620467f12bf3e0bae41df73489de65
+        checksum/config: 994e75167f0126a535cea63fc65a5ab86361648a20bcacb06d3c588f06f6e5f6
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3210,6 +3210,7 @@ data:
       portRange: "61000-62000"
 
     # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (replaced by AntreaProxy).
     # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
     kubeAPIServerOverride: ""
 
@@ -3423,6 +3424,12 @@ data:
     # certificates.
     clientCAFile: ""
 
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (replaced by AntreaProxy) and kube-controller-manager
+    # does not run NodeIPAMController (replaced by Antrea NodeIPAM).
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    kubeAPIServerOverride: ""
+
     nodeIPAM:
       # Enable the integrated Node IPAM controller within the Antrea controller.
       enableNodeIPAM: false
@@ -4371,7 +4378,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: abf7cb1c21b730664510e8a762d967df5c620467f12bf3e0bae41df73489de65
+        checksum/config: 994e75167f0126a535cea63fc65a5ab86361648a20bcacb06d3c588f06f6e5f6
       labels:
         app: antrea
         component: antrea-agent
@@ -4613,7 +4620,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: abf7cb1c21b730664510e8a762d967df5c620467f12bf3e0bae41df73489de65
+        checksum/config: 994e75167f0126a535cea63fc65a5ab86361648a20bcacb06d3c588f06f6e5f6
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3210,6 +3210,7 @@ data:
       portRange: "61000-62000"
 
     # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (replaced by AntreaProxy).
     # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
     kubeAPIServerOverride: ""
 
@@ -3423,6 +3424,12 @@ data:
     # certificates.
     clientCAFile: ""
 
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (replaced by AntreaProxy) and kube-controller-manager
+    # does not run NodeIPAMController (replaced by Antrea NodeIPAM).
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    kubeAPIServerOverride: ""
+
     nodeIPAM:
       # Enable the integrated Node IPAM controller within the Antrea controller.
       enableNodeIPAM: false
@@ -4371,7 +4378,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: e119a0592b2cef130a915258e9f23a8e4ed0aa28685841dc6f0d4b72a6983beb
+        checksum/config: 04761c3e699fa0f59516b557f366049b0f1acf2f390d94e6753ee017cdffcfd9
       labels:
         app: antrea
         component: antrea-agent
@@ -4610,7 +4617,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: e119a0592b2cef130a915258e9f23a8e4ed0aa28685841dc6f0d4b72a6983beb
+        checksum/config: 04761c3e699fa0f59516b557f366049b0f1acf2f390d94e6753ee017cdffcfd9
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3223,6 +3223,7 @@ data:
       portRange: "61000-62000"
 
     # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (replaced by AntreaProxy).
     # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
     kubeAPIServerOverride: ""
 
@@ -3436,6 +3437,12 @@ data:
     # certificates.
     clientCAFile: ""
 
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (replaced by AntreaProxy) and kube-controller-manager
+    # does not run NodeIPAMController (replaced by Antrea NodeIPAM).
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    kubeAPIServerOverride: ""
+
     nodeIPAM:
       # Enable the integrated Node IPAM controller within the Antrea controller.
       enableNodeIPAM: false
@@ -4384,7 +4391,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 373bbb8d6a42a8f3e546ea446fd078c04e947093e68e0c1c973f4c696bf8d607
+        checksum/config: 6b902d5d6e9a2c0e2fde41aedf349eeee38d4530330381b01849815211ad1dd8
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -4669,7 +4676,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 373bbb8d6a42a8f3e546ea446fd078c04e947093e68e0c1c973f4c696bf8d607
+        checksum/config: 6b902d5d6e9a2c0e2fde41aedf349eeee38d4530330381b01849815211ad1dd8
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3210,6 +3210,7 @@ data:
       portRange: "61000-62000"
 
     # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (replaced by AntreaProxy).
     # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
     kubeAPIServerOverride: ""
 
@@ -3423,6 +3424,12 @@ data:
     # certificates.
     clientCAFile: ""
 
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (replaced by AntreaProxy) and kube-controller-manager
+    # does not run NodeIPAMController (replaced by Antrea NodeIPAM).
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    kubeAPIServerOverride: ""
+
     nodeIPAM:
       # Enable the integrated Node IPAM controller within the Antrea controller.
       enableNodeIPAM: false
@@ -4371,7 +4378,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7e1d1a60a99fdbe25ff10b92e85ff234a5769c02bd9ae0ead56a3fe0a8ad118d
+        checksum/config: 1bc87d7d5b568beb91ad2b29510cb5cff3613c68e51fb82abdf545046767f679
       labels:
         app: antrea
         component: antrea-agent
@@ -4610,7 +4617,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7e1d1a60a99fdbe25ff10b92e85ff234a5769c02bd9ae0ead56a3fe0a8ad118d
+        checksum/config: 1bc87d7d5b568beb91ad2b29510cb5cff3613c68e51fb82abdf545046767f679
       labels:
         app: antrea
         component: antrea-controller

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -102,6 +102,7 @@ func run(o *Options) error {
 	if err != nil {
 		return fmt.Errorf("error creating K8s clients: %v", err)
 	}
+	k8s.OverrideKubeAPIServer(o.config.KubeAPIServerOverride)
 
 	informerFactory := informers.NewSharedInformerFactory(k8sClient, informerDefaultResync)
 	crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, informerDefaultResync)

diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go
@@ -120,10 +120,11 @@ func run(o *Options) error {
 	// Create K8s Clientset, Aggregator Clientset, CRD Clientset and SharedInformerFactory for the given config.
 	// Aggregator Clientset is used to update the CABundle of the APIServices backed by antrea-controller so that
 	// the aggregator can verify its serving certificate.
-	client, aggregatorClient, crdClient, apiExtensionClient, mcClient, err := k8s.CreateClients(o.config.ClientConnection, "")
+	client, aggregatorClient, crdClient, apiExtensionClient, mcClient, err := k8s.CreateClients(o.config.ClientConnection, o.config.KubeAPIServerOverride)
 	if err != nil {
 		return fmt.Errorf("error creating K8s clients: %v", err)
 	}
+	k8s.OverrideKubeAPIServer(o.config.KubeAPIServerOverride)
 	informerFactory := informers.NewSharedInformerFactory(client, informerDefaultResync)
 	crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, informerDefaultResync)
 	podInformer := informerFactory.Core().V1().Pods()

diff --git a/docs/antrea-ipam.md b/docs/antrea-ipam.md
@@ -58,8 +58,15 @@ Below is a sample of needed changes in the Antrea deployment YAML:
     ...
 ```
 
-Note that, prior to v1.12, a feature gate, `NodeIPAM` must also be enabled for
-`antrea-controller`.
+When running Antrea NodeIPAM in a particular version or scenario, you may need to
+be aware of the following:
+
+* Prior to v1.12, a feature gate, `NodeIPAM` must also be enabled for
+  `antrea-controller`.
+* Prior to v1.13, running Antrea NodeIPAM without kube-proxy is not supported.
+  Starting with v1.13, the `kubeAPIServerOverride` option in the `antrea-controller`
+  configuration must be set to the address of Kubernetes apiserver when kube-proxy
+  is not deployed.
 
 ## Antrea Flexible IPAM
 

diff --git a/pkg/config/agent/config.go b/pkg/config/agent/config.go
@@ -148,6 +148,7 @@ type AgentConfig struct {
 	// FlowExporter configuration options.
 	FlowExporter FlowExporterConfig `yaml:"flowExporter,omitempty"`
 	// Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+	// It is typically used when kube-proxy is not deployed (replaced by AntreaProxy).
 	// Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
 	KubeAPIServerOverride string `yaml:"kubeAPIServerOverride,omitempty"`
 	// Provide the address of DNS server, to override the kube-dns service. It's used to resolve hostname in FQDN policy.

diff --git a/pkg/config/controller/config.go b/pkg/config/controller/config.go
@@ -66,6 +66,11 @@ type ControllerConfig struct {
 	ClientCAFile string `yaml:"clientCAFile,omitempty"`
 	// Legacy CRD mirroring (deprecated).
 	LegacyCRDMirroring *bool `yaml:"legacyCRDMirroring,omitempty"`
+	// Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+	// It is typically used when kube-proxy is not deployed (replaced by AntreaProxy) and kube-controller-manager
+	// does not run NodeIPAMController (replaced by Antrea NodeIPAM).
+	// Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+	KubeAPIServerOverride string `yaml:"kubeAPIServerOverride,omitempty"`
 	// NodeIPAM Configuration
 	NodeIPAM NodeIPAMConfig `yaml:"nodeIPAM"`
 	// IPsec CSR signer configuration

diff --git a/pkg/util/k8s/client.go b/pkg/util/k8s/client.go
@@ -16,6 +16,9 @@ package k8s
 
 import (
 	"fmt"
+	"net"
+	"os"
+	"strings"
 
 	netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
 	discovery "k8s.io/api/discovery/v1"
@@ -32,6 +35,11 @@ import (
 	crdclientset "antrea.io/antrea/pkg/client/clientset/versioned"
 )
 
+const (
+	kubeServiceHostEnvKey = "KUBERNETES_SERVICE_HOST"
+	kubeServicePortEnvKey = "KUBERNETES_SERVICE_PORT"
+)
+
 // CreateClients creates kube clients from the given config.
 func CreateClients(config componentbaseconfig.ClientConnectionConfiguration, kubeAPIServerOverride string) (
 	clientset.Interface, aggregatorclientset.Interface, crdclientset.Interface, apiextensionclientset.Interface, mcclientset.Interface, error) {
@@ -80,7 +88,6 @@ func CreateNetworkAttachDefClient(config componentbaseconfig.ClientConnectionCon
 		return nil, err
 	}
 	return netAttachDefClient, nil
-
 }
 
 func createRestConfig(config componentbaseconfig.ClientConnectionConfiguration, kubeAPIServerOverride string) (*rest.Config, error) {
@@ -110,7 +117,25 @@ func createRestConfig(config componentbaseconfig.ClientConnectionConfiguration,
 	kubeConfig.Burst = int(config.Burst)
 
 	return kubeConfig, nil
+}
 
+// OverrideKubeAPIServer overrides the env vars related to the kubernetes service used by InClusterConfig.
+// It's required because some K8s libraries like DelegatingAuthenticationOptions and DelegatingAuthorizationOptions
+// read the information from env vars and don't support overriding via parameters.
+func OverrideKubeAPIServer(kubeAPIServerOverride string) {
+	if len(kubeAPIServerOverride) == 0 {
+		return
+	}
+	hostPort := strings.Replace(kubeAPIServerOverride, "https://", "", -1)
+	var host, port string
+	var err error
+	if host, port, err = net.SplitHostPort(hostPort); err != nil {
+		// if SplitHostPort returns an error, the entire hostport is considered as host
+		host = hostPort
+		port = "443"
+	}
+	os.Setenv(kubeServiceHostEnvKey, host)
+	os.Setenv(kubeServicePortEnvKey, port)
 }
 
 func EndpointSliceAPIAvailable(k8sClient clientset.Interface) (bool, error) {