Expose ‘—pod-network-cidr’ argument in minikube

cmd/minikube/cmd/start.go

@@ -43,6 +43,7 @@ import (
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
 	cmdutil "k8s.io/minikube/cmd/util"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	cfg "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/console"
@@ -146,7 +147,8 @@ func init() {
 	startCmd.Flags().Var(&extraOptions, "extra-config",
 		`A set of key=value pairs that describe configuration that may be passed to different components.
 		The key should be '.' separated, and the first part before the dot is the component to apply the configuration to.
-		Valid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler.`)
+		Valid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler
+		Valid kubeadm parameters: `+fmt.Sprintf("%s, %s", strings.Join(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], ", "), strings.Join(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], ",")))
 	startCmd.Flags().String(uuid, "", "Provide VM UUID to restore MAC address (only supported with Hyperkit driver).")
 	startCmd.Flags().String(vpnkitSock, "", "Location of the VPNKit socket used for networking. If empty, disables Hyperkit VPNKitSock, if 'auto' uses Docker for Mac VPNKit connection, otherwise uses the specified VSock.")
 	startCmd.Flags().StringSlice(vsockPorts, []string{}, "List of guest VSock ports that should be exposed as sockets on the host (Only supported on with hyperkit now).")
@@ -337,6 +339,14 @@ func validateConfig() {
 	if viper.GetBool(hidden) && viper.GetString(vmDriver) != "kvm2" {
 		exit.Usage("Sorry, the --hidden feature is currently only supported with --vm-driver=kvm2")
 	}
+
+	// check that kubeadm extra args contain only whitelisted parameters
+	for param := range extraOptions.AsMap().Get(kubeadm.Kubeadm) {
+		if !pkgutil.ContainsString(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) &&
+			!pkgutil.ContainsString(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) {
+			exit.Usage("Sorry, the kubeadm.%s parameter is currently not supported by --extra-config", param)
+		}
+	}
 }
 
 // doCacheBinaries caches Kubernetes binaries in the foreground

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -43,6 +43,35 @@ import (
 	"k8s.io/minikube/pkg/util"
 )
 
+// enum to differentiate kubeadm command line parameters from kubeadm config file parameters (see the
+// KubeadmExtraArgsWhitelist variable below for more info)
+const (
+	KubeadmCmdParam    = iota
+	KubeadmConfigParam = iota
+)
+
+// KubeadmExtraArgsWhitelist is a whitelist of supported kubeadm params that can be supplied to kubeadm through
+// minikube's ExtraArgs parameter. The list is split into two parts - params that can be supplied as flags on the
+// command line and params that have to be inserted into the kubeadm config file. This is because of a kubeadm
+// constraint which allows only certain params to be provided from the command line when the --config parameter
+// is specified
+var KubeadmExtraArgsWhitelist = map[int][]string{
+	KubeadmCmdParam: {
+		"ignore-preflight-errors",
+		"dry-run",
+		"kubeconfig",
+		"kubeconfig-dir",
+		"node-name",
+		"cri-socket",
+		"experimental-upload-certs",
+		"certificate-key",
+		"rootfs",
+	},
+	KubeadmConfigParam: {
+		"pod-network-cidr",
+	},
+}
+
 // SkipPreflights are preflight checks we always skip.
 var SkipPreflights = []string{
 	// We use --ignore-preflight-errors=DirAvailable since we have our own custom addons
@@ -163,18 +192,29 @@ func (k *Bootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string
 	}
 }
 
+// createFlagsFromExtraArgs converts kubeadm extra args into flags to be supplied from the commad linne
+func createFlagsFromExtraArgs(extraOptions util.ExtraOptionSlice) string {
+	kubeadmExtraOpts := extraOptions.AsMap().Get(Kubeadm)
+
+	// kubeadm allows only a small set of parameters to be supplied from the command line when the --config param
+	// is specified, here we remove those that are not allowed
+	for opt := range kubeadmExtraOpts {
+		if !util.ContainsString(KubeadmExtraArgsWhitelist[KubeadmCmdParam], opt) {
+			// kubeadmExtraOpts is a copy so safe to delete
+			delete(kubeadmExtraOpts, opt)
+		}
+	}
+	return convertToFlags(kubeadmExtraOpts)
+}
+
 // StartCluster starts the cluster
 func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
 
-	extraOpts, err := ExtraConfigForComponent(Kubeadm, k8s.ExtraOptions, version)
-	if err != nil {
-		return errors.Wrap(err, "generating extra configuration for kubelet")
-	}
-	extraFlags := convertToFlags(extraOpts)
+	extraFlags := createFlagsFromExtraArgs(k8s.ExtraOptions)
 
 	r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime})
 	if err != nil {
@@ -474,6 +514,25 @@ sudo systemctl start kubelet
 	return nil
 }
 
+// createExtraComponentConfig generates a map of component to extra args for all of the components except kubeadm
+func createExtraComponentConfig(extraOptions util.ExtraOptionSlice, version semver.Version, componentFeatureArgs string) ([]ComponentExtraArgs, error) {
+	extraArgsSlice, err := NewComponentExtraArgs(extraOptions, version, componentFeatureArgs)
+	if err != nil {
+		return nil, err
+	}
+
+	// kubeadm extra args should not be included in the kubeadm config in the extra args section (instead, they must
+	// be inserted explicitly in the appropriate places or supplied from the command line); here we remove all of the
+	// kubeadm extra args from the slice
+	for i, extraArgs := range extraArgsSlice {
+		if extraArgs.Component == Kubeadm {
+			extraArgsSlice = append(extraArgsSlice[:i], extraArgsSlice[i+1:]...)
+			break
+		}
+	}
+	return extraArgsSlice, nil
+}
+
 func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, error) {
 	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
@@ -486,8 +545,7 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, er
 		return "", errors.Wrap(err, "parses feature gate config for kubeadm and component")
 	}
 
-	// generates a map of component to extra args for apiserver, controller-manager, and scheduler
-	extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version, componentFeatureArgs)
+	extraComponentConfig, err := createExtraComponentConfig(k8s.ExtraOptions, version, componentFeatureArgs)
 	if err != nil {
 		return "", errors.Wrap(err, "generating extra component config for kubeadm")
 	}
@@ -501,6 +559,7 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, er
 	opts := struct {
 		CertDir           string
 		ServiceCIDR       string
+		PodSubnet         string
 		AdvertiseAddress  string
 		APIServerPort     int
 		KubernetesVersion string
@@ -514,6 +573,7 @@ func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) (string, er
 	}{
 		CertDir:           util.DefaultCertPath,
 		ServiceCIDR:       util.DefaultServiceCIDR,
+		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
 		AdvertiseAddress:  k8s.NodeIP,
 		APIServerPort:     nodePort,
 		KubernetesVersion: k8s.KubernetesVersion,

pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go

@@ -153,6 +153,24 @@ func TestGenerateConfig(t *testing.T) {
 			Key:       "scheduler-name",
 			Value:     "mini-scheduler",
 		},
+		util.ExtraOption{
+			Component: Kubeadm,
+			Key:       "ignore-preflight-errors",
+			Value:     "true",
+		},
+		util.ExtraOption{
+			Component: Kubeadm,
+			Key:       "dry-run",
+			Value:     "true",
+		},
+	}
+
+	extraOptsPodCidr := util.ExtraOptionSlice{
+		util.ExtraOption{
+			Component: Kubeadm,
+			Key:       "pod-network-cidr",
+			Value:     "192.168.32.0/20",
+		},
 	}
 
 	// Test version policy: Last 4 major releases (slightly looser than our general policy)
@@ -177,6 +195,7 @@ func TestGenerateConfig(t *testing.T) {
 		{"crio-options-gates", "crio", false, config.KubernetesConfig{ExtraOptions: extraOpts, FeatureGates: "a=b"}},
 		{"unknown-component", "docker", true, config.KubernetesConfig{ExtraOptions: util.ExtraOptionSlice{util.ExtraOption{Component: "not-a-real-component", Key: "killswitch", Value: "true"}}}},
 		{"containerd-api-port", "containerd", false, config.KubernetesConfig{NodePort: 12345}},
+		{"containerd-pod-network-cidr", "containerd", false, config.KubernetesConfig{ExtraOptions: extraOptsPodCidr}},
 		{"image-repository", "docker", false, config.KubernetesConfig{ImageRepository: "test/repo"}},
 	}
 	for vname, version := range versions {

pkg/minikube/bootstrapper/kubeadm/templates.go

@@ -85,7 +85,7 @@ etcd:
 kubernetesVersion: {{.KubernetesVersion}}
 networking:
   dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: {{if .PodSubnet}}{{.PodSubnet}}{{else}}""{{end}}
   serviceSubnet: {{.ServiceCIDR}}
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/kubeadm/testdata/containerd-pod-network-cidr__default.yaml

@@ -0,0 +1,43 @@
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: InitConfiguration
+localAPIEndpoint:
+  advertiseAddress: 1.1.1.1
+  bindPort: 8443
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+nodeRegistration:
+  criSocket: /run/containerd/containerd.sock
+  name: mk
+  taints: []
+---
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: ClusterConfiguration
+apiServer:
+  extraArgs:
+    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
+certificatesDir: /var/lib/minikube/certs/
+clusterName: kubernetes
+controlPlaneEndpoint: localhost:8443
+dns:
+  type: CoreDNS
+etcd:
+  local:
+    dataDir: /data/minikube
+kubernetesVersion: v1.14.1
+networking:
+  dnsDomain: cluster.local
+  podSubnet: ""
+  serviceSubnet: 10.96.0.0/12
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+imageGCHighThresholdPercent: 100
+evictionHard:
+  nodefs.available: "0%"
+  nodefs.inodesFree: "0%"
+  imagefs.available: "0%"

pkg/minikube/bootstrapper/kubeadm/testdata/containerd-pod-network-cidr__new.yaml

@@ -0,0 +1,43 @@
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: InitConfiguration
+localAPIEndpoint:
+  advertiseAddress: 1.1.1.1
+  bindPort: 8443
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+nodeRegistration:
+  criSocket: /run/containerd/containerd.sock
+  name: mk
+  taints: []
+---
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: ClusterConfiguration
+apiServer:
+  extraArgs:
+    enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
+certificatesDir: /var/lib/minikube/certs/
+clusterName: kubernetes
+controlPlaneEndpoint: localhost:8443
+dns:
+  type: CoreDNS
+etcd:
+  local:
+    dataDir: /data/minikube
+kubernetesVersion: v1.14.0
+networking:
+  dnsDomain: cluster.local
+  podSubnet: ""
+  serviceSubnet: 10.96.0.0/12
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+imageGCHighThresholdPercent: 100
+evictionHard:
+  nodefs.available: "0%"
+  nodefs.inodesFree: "0%"
+  imagefs.available: "0%"

pkg/minikube/bootstrapper/kubeadm/testdata/containerd-pod-network-cidr__obsolete.yaml

@@ -0,0 +1,17 @@
+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+noTaintMaster: true
+api:
+  advertiseAddress: 1.1.1.1
+  bindPort: 8443
+  controlPlaneEndpoint: localhost
+kubernetesVersion: v1.10.0
+certificatesDir: /var/lib/minikube/certs/
+networking:
+  serviceSubnet: 10.96.0.0/12
+etcd:
+  dataDir: /data/minikube
+nodeName: mk
+criSocket: /run/containerd/containerd.sock
+apiServerExtraArgs:
+  admission-control: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/kubeadm/testdata/containerd-pod-network-cidr__old.yaml

@@ -0,0 +1,39 @@
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: InitConfiguration
+apiEndpoint:
+  advertiseAddress: 1.1.1.1
+  bindPort: 8443
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+nodeRegistration:
+  criSocket: /run/containerd/containerd.sock
+  name: mk
+  taints: []
+---
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: ClusterConfiguration
+apiServerExtraArgs:
+  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
+certificatesDir: /var/lib/minikube/certs/
+clusterName: kubernetes
+controlPlaneEndpoint: localhost:8443
+etcd:
+  local:
+    dataDir: /data/minikube
+kubernetesVersion: v1.12.0
+networking:
+  dnsDomain: cluster.local
+  podSubnet: 192.168.32.0/20
+  serviceSubnet: 10.96.0.0/12
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+evictionHard:
+  nodefs.available: "0%"
+  nodefs.inodesFree: "0%"
+  imagefs.available: "0%"

pkg/minikube/bootstrapper/kubeadm/testdata/containerd-pod-network-cidr__recent.yaml

@@ -0,0 +1,39 @@
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: InitConfiguration
+apiEndpoint:
+  advertiseAddress: 1.1.1.1
+  bindPort: 8443
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+nodeRegistration:
+  criSocket: /run/containerd/containerd.sock
+  name: mk
+  taints: []
+---
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: ClusterConfiguration
+apiServerExtraArgs:
+  enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
+certificatesDir: /var/lib/minikube/certs/
+clusterName: kubernetes
+controlPlaneEndpoint: localhost:8443
+etcd:
+  local:
+    dataDir: /data/minikube
+kubernetesVersion: v1.13.0
+networking:
+  dnsDomain: cluster.local
+  podSubnet: 192.168.32.0/20
+  serviceSubnet: 10.96.0.0/12
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+evictionHard:
+  nodefs.available: "0%"
+  nodefs.inodesFree: "0%"
+  imagefs.available: "0%"

pkg/minikube/bootstrapper/kubeadm/testdata/crio-options-gates__default.yaml

@@ -26,9 +26,6 @@ controllerManager:
   extraArgs:
     feature-gates: "a=b"
     kube-api-burst: "32"
-kubeadm:
-  extraArgs:
-    feature-gates: "a=b"
 scheduler:
   extraArgs:
     feature-gates: "a=b"