Add kubeAPIServerOverride option to antrea-controller

To override the address of Kubernetes apiserver for antrea-controller,
the option, kubeAPIServerOverride, is added to antrea-controller.conf.
It's typically used when kube-proxy is not deployed (substituted by
AntreaProxy) and kube-controller-manager does not run NodeIPAMController
(substituted by Antrea NodeIPAM).

Signed-off-by: Quan Tian <[email protected]>

.github/workflows/kind.yml

@@ -215,7 +215,13 @@ jobs:
           mkdir log
           mkdir test-e2e-encap-all-features-enabled-coverage
           # FlowExporter requires the FlowAggregator, so we keep it disabled.
-          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --coverage --feature-gates AllAlpha=true,AllBeta=true,FlowExporter=false --proxy-all --multicast
+          ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-all-features-enabled-coverage ./ci/kind/test-e2e-kind.sh \
+            --encap-mode encap \
+            --coverage \
+            --feature-gates AllAlpha=true,AllBeta=true,FlowExporter=false \
+            --proxy-all \
+            --node-ipam \
+            --multicast
       - name: Tar coverage files
         run: tar -czf test-e2e-encap-all-features-enabled-coverage.tar.gz test-e2e-encap-all-features-enabled-coverage
       - name: Upload coverage for test-e2e-encap-all-features-enabled-coverage

build/charts/antrea/conf/antrea-agent.conf

@@ -239,6 +239,7 @@ nodePortLocal:
 {{- end }}
 
 # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+# It is typically used when kube-proxy is not deployed (substituted by AntreaProxy).
 # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
 kubeAPIServerOverride: {{ .Values.kubeAPIServerOverride | quote }}
 

build/charts/antrea/conf/antrea-controller.conf

@@ -78,6 +78,12 @@ tlsMinVersion: {{ .Values.tlsMinVersion | quote }}
 # certificates.
 clientCAFile: {{ .Values.clientCAFile | quote }}
 
+# Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+# It is typically used when kube-proxy is not deployed (substituted by AntreaProxy) and kube-controller-manager
+# does not run NodeIPAMController (substituted by Antrea NodeIPAM).
+# Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+kubeAPIServerOverride: {{ .Values.kubeAPIServerOverride | quote }}
+
 nodeIPAM:
 {{- with .Values.nodeIPAM }}
   # Enable the integrated Node IPAM controller within the Antrea controller.

build/yamls/antrea-aks.yml

@@ -3203,6 +3203,7 @@ data:
       portRange: "61000-62000"
 
     # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (substituted by AntreaProxy).
     # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
     kubeAPIServerOverride: ""
 
@@ -3416,6 +3417,12 @@ data:
     # certificates.
     clientCAFile: ""
 
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (substituted by AntreaProxy) and kube-controller-manager
+    # does not run NodeIPAMController (substituted by Antrea NodeIPAM).
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    kubeAPIServerOverride: ""
+
     nodeIPAM:
       # Enable the integrated Node IPAM controller within the Antrea controller.
       enableNodeIPAM: false
@@ -4364,7 +4371,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7b3f20e4be884c2def359ef222cf07498761ff76b66e893d9afa325761354c9f
+        checksum/config: d63eed74f086329f0ac9a44b73db0a396f22ab6f1ae142c7de3710400f8d2dd5
       labels:
         app: antrea
         component: antrea-agent
@@ -4605,7 +4612,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7b3f20e4be884c2def359ef222cf07498761ff76b66e893d9afa325761354c9f
+        checksum/config: d63eed74f086329f0ac9a44b73db0a396f22ab6f1ae142c7de3710400f8d2dd5
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-eks.yml

@@ -3203,6 +3203,7 @@ data:
       portRange: "61000-62000"
 
     # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (substituted by AntreaProxy).
     # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
     kubeAPIServerOverride: ""
 
@@ -3416,6 +3417,12 @@ data:
     # certificates.
     clientCAFile: ""
 
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (substituted by AntreaProxy) and kube-controller-manager
+    # does not run NodeIPAMController (substituted by Antrea NodeIPAM).
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    kubeAPIServerOverride: ""
+
     nodeIPAM:
       # Enable the integrated Node IPAM controller within the Antrea controller.
       enableNodeIPAM: false
@@ -4364,7 +4371,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7b3f20e4be884c2def359ef222cf07498761ff76b66e893d9afa325761354c9f
+        checksum/config: d63eed74f086329f0ac9a44b73db0a396f22ab6f1ae142c7de3710400f8d2dd5
       labels:
         app: antrea
         component: antrea-agent
@@ -4606,7 +4613,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7b3f20e4be884c2def359ef222cf07498761ff76b66e893d9afa325761354c9f
+        checksum/config: d63eed74f086329f0ac9a44b73db0a396f22ab6f1ae142c7de3710400f8d2dd5
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-gke.yml

@@ -3203,6 +3203,7 @@ data:
       portRange: "61000-62000"
 
     # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (substituted by AntreaProxy).
     # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
     kubeAPIServerOverride: ""
 
@@ -3416,6 +3417,12 @@ data:
     # certificates.
     clientCAFile: ""
 
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (substituted by AntreaProxy) and kube-controller-manager
+    # does not run NodeIPAMController (substituted by Antrea NodeIPAM).
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    kubeAPIServerOverride: ""
+
     nodeIPAM:
       # Enable the integrated Node IPAM controller within the Antrea controller.
       enableNodeIPAM: false
@@ -4364,7 +4371,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: b7f4a912f5e4d42314ea1667b8e2f3d97a7666e7379ed4f65f9299a3f37399c2
+        checksum/config: f8a3177fc8722b9cdaade9fcdfb96cc5c907514320d435eae5f7fa4049b1fa32
       labels:
         app: antrea
         component: antrea-agent
@@ -4603,7 +4610,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: b7f4a912f5e4d42314ea1667b8e2f3d97a7666e7379ed4f65f9299a3f37399c2
+        checksum/config: f8a3177fc8722b9cdaade9fcdfb96cc5c907514320d435eae5f7fa4049b1fa32
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-ipsec.yml

@@ -3216,6 +3216,7 @@ data:
       portRange: "61000-62000"
 
     # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (substituted by AntreaProxy).
     # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
     kubeAPIServerOverride: ""
 
@@ -3429,6 +3430,12 @@ data:
     # certificates.
     clientCAFile: ""
 
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (substituted by AntreaProxy) and kube-controller-manager
+    # does not run NodeIPAMController (substituted by Antrea NodeIPAM).
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    kubeAPIServerOverride: ""
+
     nodeIPAM:
       # Enable the integrated Node IPAM controller within the Antrea controller.
       enableNodeIPAM: false
@@ -4377,7 +4384,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: d5789c48750f03a8652da56fc0e7f6cd4b12911fff41a84c8426245270fd5ec2
+        checksum/config: 1fe02636168501e0a689729a3ebfa5a8b6c68eadb09de2ad7312b3886049ced3
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -4662,7 +4669,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: d5789c48750f03a8652da56fc0e7f6cd4b12911fff41a84c8426245270fd5ec2
+        checksum/config: 1fe02636168501e0a689729a3ebfa5a8b6c68eadb09de2ad7312b3886049ced3
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea.yml

@@ -3203,6 +3203,7 @@ data:
       portRange: "61000-62000"
 
     # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (substituted by AntreaProxy).
     # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
     kubeAPIServerOverride: ""
 
@@ -3416,6 +3417,12 @@ data:
     # certificates.
     clientCAFile: ""
 
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # It is typically used when kube-proxy is not deployed (substituted by AntreaProxy) and kube-controller-manager
+    # does not run NodeIPAMController (substituted by Antrea NodeIPAM).
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    kubeAPIServerOverride: ""
+
     nodeIPAM:
       # Enable the integrated Node IPAM controller within the Antrea controller.
       enableNodeIPAM: false
@@ -4364,7 +4371,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 1f7ec3f7c131b06c35ae624655ebbf81ca332c08abcfcddd434dd3c0a5387dab
+        checksum/config: ffa13a9f8a1a624c7e622a79418366c40acdda15502a3d3d8d57af858e855060
       labels:
         app: antrea
         component: antrea-agent
@@ -4603,7 +4610,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 1f7ec3f7c131b06c35ae624655ebbf81ca332c08abcfcddd434dd3c0a5387dab
+        checksum/config: ffa13a9f8a1a624c7e622a79418366c40acdda15502a3d3d8d57af858e855060
       labels:
         app: antrea
         component: antrea-controller

cmd/antrea-agent/agent.go

@@ -102,6 +102,7 @@ func run(o *Options) error {
 	if err != nil {
 		return fmt.Errorf("error creating K8s clients: %v", err)
 	}
+	k8s.OverrideKubeAPIServer(o.config.KubeAPIServerOverride)
 
 	informerFactory := informers.NewSharedInformerFactory(k8sClient, informerDefaultResync)
 	crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, informerDefaultResync)

cmd/antrea-controller/controller.go

@@ -120,10 +120,11 @@ func run(o *Options) error {
 	// Create K8s Clientset, Aggregator Clientset, CRD Clientset and SharedInformerFactory for the given config.
 	// Aggregator Clientset is used to update the CABundle of the APIServices backed by antrea-controller so that
 	// the aggregator can verify its serving certificate.
-	client, aggregatorClient, crdClient, apiExtensionClient, mcClient, err := k8s.CreateClients(o.config.ClientConnection, "")
+	client, aggregatorClient, crdClient, apiExtensionClient, mcClient, err := k8s.CreateClients(o.config.ClientConnection, o.config.KubeAPIServerOverride)
 	if err != nil {
 		return fmt.Errorf("error creating K8s clients: %v", err)
 	}
+	k8s.OverrideKubeAPIServer(o.config.KubeAPIServerOverride)
 	informerFactory := informers.NewSharedInformerFactory(client, informerDefaultResync)
 	crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, informerDefaultResync)
 	podInformer := informerFactory.Core().V1().Pods()

docs/antrea-ipam.md

@@ -58,8 +58,15 @@ Below is a sample of needed changes in the Antrea deployment YAML:
     ...
 ```
 
-Note that, prior to v1.12, a feature gate, `NodeIPAM` must also be enabled for
-`antrea-controller`.
+When running Antrea NodeIPAM in a particular version or scenario, you may need to
+be aware of the following:
+
+* Prior to v1.12, a feature gate, `NodeIPAM` must also be enabled for
+  `antrea-controller`.
+* Prior to v1.13, running Antrea NodeIPAM without kube-proxy is not supported.
+  Starting with v1.13, the `kubeAPIServerOverride` option in the `antrea-controller`
+  configuration must be set to the address of Kubernetes apiserver when kube-proxy
+  is not deployed.
 
 ## Antrea Flexible IPAM
 

pkg/config/agent/config.go

@@ -171,6 +171,7 @@ type AgentConfig struct {
 	// NodePortLocal (NPL) configuration options.
 	NodePortLocal NodePortLocalConfig `yaml:"nodePortLocal,omitempty"`
 	// Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+	// It is typically used when kube-proxy is not deployed (substituted by AntreaProxy).
 	// Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
 	KubeAPIServerOverride string `yaml:"kubeAPIServerOverride,omitempty"`
 	// Provide the address of DNS server, to override the kube-dns service. It's used to resolve hostname in FQDN policy.

pkg/config/controller/config.go

@@ -66,6 +66,11 @@ type ControllerConfig struct {
 	ClientCAFile string `yaml:"clientCAFile,omitempty"`
 	// Legacy CRD mirroring (deprecated).
 	LegacyCRDMirroring *bool `yaml:"legacyCRDMirroring,omitempty"`
+	// Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+	// It is typically used when kube-proxy is not deployed (substituted by AntreaProxy) and kube-controller-manager
+	// does not run NodeIPAMController (substituted by Antrea NodeIPAM).
+	// Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+	KubeAPIServerOverride string `yaml:"kubeAPIServerOverride,omitempty"`
 	// NodeIPAM Configuration
 	NodeIPAM NodeIPAMConfig `yaml:"nodeIPAM"`
 	// IPsec CSR signer configuration

pkg/util/k8s/client.go

@@ -16,6 +16,9 @@ package k8s
 
 import (
 	"fmt"
+	"net"
+	"os"
+	"strings"
 
 	netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
 	discovery "k8s.io/api/discovery/v1"
@@ -32,6 +35,11 @@ import (
 	crdclientset "antrea.io/antrea/pkg/client/clientset/versioned"
 )
 
+const (
+	kubeServiceHostEnvKey = "KUBERNETES_SERVICE_HOST"
+	kubeServicePortEnvKey = "KUBERNETES_SERVICE_PORT"
+)
+
 // CreateClients creates kube clients from the given config.
 func CreateClients(config componentbaseconfig.ClientConnectionConfiguration, kubeAPIServerOverride string) (
 	clientset.Interface, aggregatorclientset.Interface, crdclientset.Interface, apiextensionclientset.Interface, mcclientset.Interface, error) {
@@ -80,7 +88,6 @@ func CreateNetworkAttachDefClient(config componentbaseconfig.ClientConnectionCon
 		return nil, err
 	}
 	return netAttachDefClient, nil
-
 }
 
 func createRestConfig(config componentbaseconfig.ClientConnectionConfiguration, kubeAPIServerOverride string) (*rest.Config, error) {
@@ -110,7 +117,25 @@ func createRestConfig(config componentbaseconfig.ClientConnectionConfiguration,
 	kubeConfig.Burst = int(config.Burst)
 
 	return kubeConfig, nil
+}
 
+// OverrideKubeAPIServer overrides the env vars related to the kubernetes service used by InClusterConfig.
+// It's required because some K8s libraries like DelegatingAuthenticationOptions and DelegatingAuthorizationOptions
+// read the information from env vars and don't support overriding via parameters.
+func OverrideKubeAPIServer(kubeAPIServerOverride string) {
+	if len(kubeAPIServerOverride) == 0 {
+		return
+	}
+	hostPort := strings.Replace(kubeAPIServerOverride, "https://", "", -1)
+	var host, port string
+	var err error
+	if host, port, err = net.SplitHostPort(hostPort); err != nil {
+		// if SplitHostPort returns an error, the entire hostport is considered as host
+		host = hostPort
+		port = "443"
+	}
+	os.Setenv(kubeServiceHostEnvKey, host)
+	os.Setenv(kubeServicePortEnvKey, port)
 }
 
 func EndpointSliceAPIAvailable(k8sClient clientset.Interface) (bool, error) {