From 6d624c7ad58de9112bddcd62c3ae2e83c460c8dd Mon Sep 17 00:00:00 2001 From: Kashif Saadat Date: Fri, 6 Oct 2017 17:52:19 +0100 Subject: [PATCH] Add Calico v2.5 support for Kubernetes v1.8+ --- .../k8s-1.8.yaml.template | 439 ++++++++++++++++++ .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 32 +- 2 files changed, 464 insertions(+), 7 deletions(-) create mode 100644 upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.8.yaml.template diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.8.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.8.yaml.template new file mode 100644 index 0000000000000..eea3a281d3e0e --- /dev/null +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.8.yaml.template @@ -0,0 +1,439 @@ +# This ConfigMap can be used to configure a self-hosted Canal installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: canal-config + namespace: kube-system +data: + # The interface used by canal for host <-> host communication. + # If left blank, then the interface is chosen using the node's + # default route. + canal_iface: "" + + # Whether or not to masquerade traffic to destinations not within + # the pod network. + masquerade: "true" + + # The CNI network configuration to install on each node. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.1.0", + "type": "calico", + "log_level": "info", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "ipam": { + "type": "host-local", + "subnet": "usePodCidr" + }, + "policy": { + "type": "k8s", + "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" + }, + "kubernetes": { + "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + } + + # Flannel network configuration. Mounted into the flannel container. + net-conf.json: | + { + "Network": "{{ .NonMasqueradeCIDR }}", + "Backend": { + "Type": "vxlan" + } + } + +--- + +# This manifest installs the calico/node container, as well +# as the Calico CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: canal + namespace: kube-system + labels: + k8s-app: canal +spec: + selector: + matchLabels: + k8s-app: canal + template: + metadata: + labels: + k8s-app: canal + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + serviceAccountName: canal + tolerations: + # Allow the pod to run on the master. This is required for + # the master to communicate with pods. + - key: node-role.kubernetes.io/master + effect: NoSchedule + # Mark the pod as a critical add-on for rescheduling. + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + # Runs calico/node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: quay.io/calico/node:v2.5.1 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Enable felix logging. + - name: FELIX_LOGSEVERITYSYS + value: "info" + # Don't enable BGP. + - name: CALICO_NETWORKING_BACKEND + value: "none" + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "kops,canal" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Period, in seconds, at which felix re-applies all iptables state + - name: FELIX_IPTABLESREFRESHINTERVAL + value: "60" + # Disable IPV6 support in Felix. + - name: FELIX_IPV6SUPPORT + value: "false" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # No IP address needed. + - name: IP + value: "" + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "{{- or .Networking.Canal.DefaultEndpointToHostAction "ACCEPT" }}" + # Controls whether Felix inserts rules to the top of iptables chains, or appends to the bottom + - name: FELIX_CHAININSERTMODE + value: "{{- or .Networking.Canal.ChainInsertMode "insert" }}" + # Set to enable the experimental Prometheus metrics server + - name: FELIX_PROMETHEUSMETRICSENABLED + value: "{{- or .Networking.Canal.PrometheusMetricsEnabled "false" }}" + # TCP port that the Prometheus metrics server should bind to + - name: FELIX_PROMETHEUSMETRICSPORT + value: "{{- or .Networking.Canal.PrometheusMetricsPort "9091" }}" + # Enable Prometheus Go runtime metrics collection + - name: FELIX_PROMETHEUSGOMETRICSENABLED + value: "{{- or .Networking.Canal.PrometheusGoMetricsEnabled "true" }}" + # Enable Prometheus process metrics collection + - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED + value: "{{- or .Networking.Canal.PrometheusProcessMetricsEnabled "true" }}" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + limits: + cpu: 250m + requests: + cpu: 250m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /readiness + port: 9099 + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: quay.io/calico/cni:v1.10.0 + command: ["/install-cni.sh"] + env: + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: canal-config + key: cni_network_config + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + # This container runs flannel using the kube-subnet-mgr backend + # for allocating subnets. + - name: kube-flannel + image: quay.io/coreos/flannel:v0.8.0 + command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] + securityContext: + privileged: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: FLANNELD_IFACE + valueFrom: + configMapKeyRef: + name: canal-config + key: canal_iface + - name: FLANNELD_IP_MASQ + valueFrom: + configMapKeyRef: + name: canal-config + key: masquerade + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: run + mountPath: /run + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + # Used by calico/node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Used by flannel. + - name: run + hostPath: + path: /run + - name: flannel-cfg + configMap: + name: canal-config + + +# Create all the CustomResourceDefinitions needed for +# Calico policy-only mode. +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global Felix Configuration +kind: CustomResourceDefinition +metadata: + name: globalfelixconfigs.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalFelixConfig + plural: globalfelixconfigs + singular: globalfelixconfig + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global BGP Configuration +kind: CustomResourceDefinition +metadata: + name: globalbgpconfigs.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalBGPConfig + plural: globalbgpconfigs + singular: globalbgpconfig + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico IP Pools +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPPool + plural: ippools + singular: ippool + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global Network Policies +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkPolicy + plural: globalnetworkpolicies + singular: globalnetworkpolicy + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: canal + namespace: kube-system + +--- + +# Calico Roles +# Pulled from https://docs.projectcalico.org/v2.5/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico + namespace: kube-system +rules: + - apiGroups: [""] + resources: + - namespaces + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - pods/status + verbs: + - update + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - update + - watch + - apiGroups: ["extensions"] + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - bgppeers + - globalbgpconfigs + - ippools + - globalnetworkpolicies + verbs: + - create + - get + - list + - update + - watch + +--- + +# Flannel roles +# Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- + +# Bind the flannel ClusterRole to the canal ServiceAccount. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: canal-flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: +- kind: ServiceAccount + name: canal + namespace: kube-system + +--- + +# Bind the calico ClusterRole to the canal ServiceAccount. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: canal-calico +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico +subjects: +- kind: ServiceAccount + name: canal + namespace: kube-system diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index a49a6030521f4..8ae29007ea121 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -443,16 +443,19 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri if b.cluster.Spec.Networking.Canal != nil { key := "networking.projectcalico.org.canal" - // Locking canal addon version to 2.4.1 (same as Calico node). Best to maintain lockstep for sanity - version := "2.4.1" + versions := map[string]string{ + "pre-k8s-1.6": "2.4.1", + "k8s-1.6": "2.4.1", + "k8s-1.8": "2.5.1", + } { - location := key + "/pre-k8s-1.6.yaml" id := "pre-k8s-1.6" + location := key + "/" + id + ".yaml" addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ Name: fi.String(key), - Version: fi.String(version), + Version: fi.String(versions[id]), Selector: networkingSelector, Manifest: fi.String(location), KubernetesVersion: "<1.6.0", @@ -462,15 +465,30 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri } { - location := key + "/k8s-1.6.yaml" id := "k8s-1.6" + location := key + "/" + id + ".yaml" addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ Name: fi.String(key), - Version: fi.String(version), + Version: fi.String(versions[id]), Selector: networkingSelector, Manifest: fi.String(location), - KubernetesVersion: ">=1.6.0", + KubernetesVersion: ">=1.6.0 <1.8.0", + Id: id, + }) + manifests[key+"-"+id] = "addons/" + location + } + + { + id := "k8s-1.8" + location := key + "/" + id + ".yaml" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(versions[id]), + Selector: networkingSelector, + Manifest: fi.String(location), + KubernetesVersion: ">=1.8.0", Id: id, }) manifests[key+"-"+id] = "addons/" + location