diff --git a/charts/kubefed/charts/controllermanager/templates/kubefedconfig.yaml b/charts/kubefed/charts/controllermanager/templates/kubefedconfig.yaml index 293175cf42..4357feaed1 100644 --- a/charts/kubefed/charts/controllermanager/templates/kubefedconfig.yaml +++ b/charts/kubefed/charts/controllermanager/templates/kubefedconfig.yaml @@ -24,12 +24,13 @@ spec: {{- if .Values.featureGates }} - name: PushReconciler configuration: {{ .Values.featureGates.PushReconciler | default "Enabled" | quote }} - - name: RawResourceStatusCollection - configuration: {{ .Values.featureGates.RawResourceStatusCollection | default "Disabled" | quote }} - name: SchedulerPreferences configuration: {{ .Values.featureGates.SchedulerPreferences | default "Enabled" | quote }} - name: CrossClusterServiceDiscovery configuration: {{ .Values.featureGates.CrossClusterServiceDiscovery | default "Disabled" | quote }} - name: FederatedIngress configuration: {{ .Values.featureGates.FederatedIngress | default "Disabled" | quote }} + # NOTE: Commented feature gate to fix https://github.com/kubernetes-sigs/kubefed/issues/1333 + #- name: RawResourceStatusCollection + # configuration: {{ .Values.featureGates.RawResourceStatusCollection | default "Disabled" | quote }} {{- end }} diff --git a/charts/kubefed/charts/controllermanager/templates/post-install-job.yaml b/charts/kubefed/charts/controllermanager/templates/post-install-job.yaml new file mode 100644 index 0000000000..730c2e1b29 --- /dev/null +++ b/charts/kubefed/charts/controllermanager/templates/post-install-job.yaml @@ -0,0 +1,103 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: "{{ .Release.Name }}-kubefed-config-hook" + namespace: "{{ .Release.Namespace }}" + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +data: + setup.sh: |- + #!/bin/bash + set -euo pipefail + + kubectl patch kubefedconfig -n {{ .Release.Namespace }} kubefed --type='json' -p='[{"op": "add", "path": "/spec/featureGates", "value":[{"configuration": {{ .Values.featureGates.PushReconciler | default "Enabled" | quote }},"name":"PushReconciler"},{"configuration": {{ .Values.featureGates.CrossClusterServiceDiscovery | default "Disabled" | quote }},"name":"CrossClusterServiceDiscovery"},{"configuration": {{ .Values.featureGates.RawResourceStatusCollection | default "Disabled" | quote }},"name":"RawResourceStatusCollection"},{"configuration": {{ .Values.featureGates.FederatedIngress | default "Disabled" | quote }},"name":"FederatedIngress"},{"configuration": {{ .Values.featureGates.SchedulerPreferences | default "Enabled" | quote }},"name":"SchedulerPreferences"}]}]' + + echo "Kubefedconfig patched successfully!" + + kubectl rollout restart deployment/kubefed-controller-manager -n {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ .Release.Name }}-{{ randAlphaNum 10 | lower }}" + namespace: "{{ .Release.Namespace }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "-4" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +spec: + template: + metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + spec: + restartPolicy: Never + serviceAccountName: kubefed-config-hook + automountServiceAccountToken: true + containers: + - name: post-install-job + image: "bitnami/kubectl:1.17.16" + command: ["/bin/bash"] + args: ["/opt/scripts/setup.sh"] + volumeMounts: + - name: "scripts" + mountPath: "/opt/scripts" + volumes: + - name: "scripts" + configMap: + name: "{{ .Release.Name }}-kubefed-config-hook" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubefed-config-hook + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +rules: + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["patch","get"] + - apiGroups: ["core.kubefed.io"] + resources: ["kubefedconfigs"] + verbs: ["patch","get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubefed-config-hook + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubefed-config-hook +subjects: + - kind: ServiceAccount + name: kubefed-config-hook +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubefed-config-hook + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation diff --git a/scripts/deploy-kubefed.sh b/scripts/deploy-kubefed.sh index 18ba03244e..64e50a5aff 100755 --- a/scripts/deploy-kubefed.sh +++ b/scripts/deploy-kubefed.sh @@ -92,27 +92,6 @@ function kubefed-admission-webhook-ready() { [[ "${readyReplicas}" -ge "1" ]] } -function deployment-image-as-expected() { - local namespace="${1}" - local deployment="${2}" - local container="${3}" - local expected_image="${4}" - - local deployed_image - deployed_image="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.template.spec.containers[?(@.name=="'"${container}"'")].image}')" - [[ "${deployed_image}" == "${expected_image}" ]] -} - -function check-command-installed() { - local cmdName="${1}" - - command -v "${cmdName}" >/dev/null 2>&1 || - { - echo "${cmdName} command not found. Please download dependencies using ${BASH_SOURCE%/*}/download-binaries.sh and install it in your PATH." >&2 - exit 1 - } -} - NS="${KUBEFED_NAMESPACE:-kube-federation-system}" IMAGE_NAME="${1:-}" NAMESPACED="${NAMESPACED:-}" diff --git a/scripts/pre-commit.sh b/scripts/pre-commit.sh index 0632c56701..72ed7121b5 100755 --- a/scripts/pre-commit.sh +++ b/scripts/pre-commit.sh @@ -38,6 +38,8 @@ E2E_TEST_CMD="${TEMP_DIR}/e2e-${PLATFORM} ${COMMON_TEST_ARGS}" # given control plane scope. IN_MEMORY_E2E_TEST_CMD="go test -v -timeout 900s -race ./test/e2e -args ${COMMON_TEST_ARGS} -in-memory-controllers=true -limited-scope-in-memory-controllers=false" +KUBEFED_UPGRADE_TEST_NS="upgrade-test" + function build-binaries() { ${MAKE_CMD} hyperfed ${MAKE_CMD} controller @@ -62,6 +64,43 @@ function run-e2e-tests() { ${E2E_TEST_CMD} } +function run-e2e-upgrade-test() { + HOST_CLUSTER="$(kubectl config current-context)" + + echo "Adding a repo to install an older kubefed version" + helm repo add kubefed-charts https://raw.githubusercontent.com/kubernetes-sigs/kubefed/master/charts + helm repo update + + # Get the previous version prior to our latest stable version + KUBEFED_UPGRADE_TEST_VERSION=$(helm search repo kubefed-charts/kubefed --versions | awk '{print $2}' | head -3 | tail -1) + + echo "Installing an older kubefed version v${KUBEFED_UPGRADE_TEST_VERSION}" + helm install kubefed kubefed-charts/kubefed --namespace ${KUBEFED_UPGRADE_TEST_NS} --version=v${KUBEFED_UPGRADE_TEST_VERSION} --create-namespace --wait + + deployment-image-as-expected "${KUBEFED_UPGRADE_TEST_NS}" kubefed-admission-webhook admission-webhook "quay.io/kubernetes-multicluster/kubefed:v${KUBEFED_UPGRADE_TEST_VERSION}" + deployment-image-as-expected "${KUBEFED_UPGRADE_TEST_NS}" kubefed-controller-manager controller-manager "quay.io/kubernetes-multicluster/kubefed:v${KUBEFED_UPGRADE_TEST_VERSION}" + + echo "Upgrading kubefed to current version" + IMAGE_NAME="local/kubefed:e2e" + local repository=${IMAGE_NAME%/*} + local image_tag=${IMAGE_NAME##*/} + local image=${image_tag%:*} + local tag=${image_tag#*:} + + helm upgrade -i kubefed charts/kubefed --namespace ${KUBEFED_UPGRADE_TEST_NS} \ + --set controllermanager.controller.repository=${repository} \ + --set controllermanager.controller.image=${image} \ + --set controllermanager.controller.tag=${tag} \ + --set controllermanager.webhook.repository=${repository} \ + --set controllermanager.webhook.image=${image} \ + --set controllermanager.webhook.tag=${tag} \ + --set controllermanager.featureGates.CrossClusterServiceDiscovery=Enabled,controllermanager.featureGates.FederatedIngress=Enabled,controllermanager.featureGates.RawResourceStatusCollection=Enabled \ + --wait + + deployment-image-as-expected "${KUBEFED_UPGRADE_TEST_NS}" kubefed-admission-webhook admission-webhook "local/kubefed:e2e" + deployment-image-as-expected "${KUBEFED_UPGRADE_TEST_NS}" kubefed-controller-manager controller-manager "local/kubefed:e2e" +} + function run-e2e-tests-with-in-memory-controllers() { ${IN_MEMORY_E2E_TEST_CMD} } @@ -138,7 +177,7 @@ echo "Downloading e2e test dependencies" KIND_TAG="v1.19.4@sha256:796d09e217d93bed01ecf8502633e48fd806fe42f9d02fdd468b81cd4e3bd40b" ./scripts/create-clusters.sh -declare -a join_cluster_list=() +declare -a join_cluster_list=() if [[ -z "${JOIN_CLUSTERS}" ]]; then for i in $(seq 2 "${NUM_CLUSTERS}"); do join_cluster_list+=("cluster${i}") @@ -178,3 +217,9 @@ run-namespaced-e2e-tests echo "Deleting namespace-scoped kubefed" KUBEFED_NAMESPACE=foo NAMESPACED=y DELETE_CLUSTER_RESOURCE=y ./scripts/delete-kubefed.sh + +echo "Running e2e upgrade test" +run-e2e-upgrade-test + +echo "Deleting kubefed" +KUBEFED_NAMESPACE=${KUBEFED_UPGRADE_TEST_NS} DELETE_CLUSTER_RESOURCE=y ./scripts/delete-kubefed.sh diff --git a/scripts/util.sh b/scripts/util.sh index 5cab71f9eb..3787fb3bfb 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -98,3 +98,24 @@ function util::wait-for-condition() { fi } readonly -f util::wait-for-condition + +function deployment-image-as-expected() { + local namespace="${1}" + local deployment="${2}" + local container="${3}" + local expected_image="${4}" + + local deployed_image + deployed_image="$(kubectl -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.template.spec.containers[?(@.name=="'"${container}"'")].image}')" + [[ "${deployed_image}" == "${expected_image}" ]] +} + +function check-command-installed() { + local cmdName="${1}" + + command -v "${cmdName}" >/dev/null 2>&1 || + { + echo "${cmdName} command not found. Please download dependencies using ${BASH_SOURCE%/*}/download-binaries.sh and install it in your PATH." >&2 + exit 1 + } +}