(knative, kfserving, cert-manager) Build working solution for kfservi…

…ng integration (#212) * (knative, kfserving, cert-manager) Build working solution for kfserving integration * (knative) kustomize knative using expanded manifest and namespace patch * (cert-manager) Adjust cert-manager and kubeflow-issuer for v1.3
GoogleCloudPlatform · Apr 21, 2021 · 0505dd6 · 0505dd6
1 parent bf3c450
commit 0505dd6
Show file tree

Hide file tree

Showing 16 changed files with 4,243 additions and 37 deletions.
diff --git a/kubeflow/Makefile b/kubeflow/Makefile
@@ -73,18 +73,9 @@ apply-kubeflow: validate-values check-name
 	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/namespaces  ./common/kubeflow-namespace
 	kubectl --context=$(KFCTXT) apply -f ./$(BUILD_DIR)/namespaces
 
-	# Kubeflow kubeflow-istio
-	rm -rf $(BUILD_DIR)/kubeflow-istio && mkdir -p $(BUILD_DIR)/kubeflow-istio
-	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/kubeflow-istio ./common/istio
-	kubectl --context=$(KFCTXT) apply -f ./$(BUILD_DIR)/kubeflow-istio
+	$(MAKE) apply-istio
 
-	# Common cert-manager
-	rm -rf $(BUILD_DIR)/cert-manager && mkdir -p $(BUILD_DIR)/cert-manager
-	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/cert-manager ./common/cert-manager
-	kubectl --context=$(KFCTXT) apply -f ./$(BUILD_DIR)/cert-manager
-	kubectl --context=$(KFCTXT) -n cert-manager wait --for=condition=Available --timeout=600s deploy cert-manager-webhook
-	kubectl --context=$(KFCTXT) -n cert-manager wait --for=condition=Available --timeout=600s deploy cert-manager
-	kubectl --context=$(KFCTXT) -n cert-manager wait --for=condition=Available --timeout=600s deploy cert-manager-cainjector
+	$(MAKE) apply-cert-manager
 
 	# Contrib metacontroller
 	rm -rf $(BUILD_DIR)/metacontroller && mkdir -p $(BUILD_DIR)/metacontroller
@@ -161,10 +152,9 @@ apply-kubeflow: validate-values check-name
 	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/pipeline ./apps/pipeline
 	kubectl --context=$(KFCTXT) apply -f ./$(BUILD_DIR)/pipeline
 
-	# App kfserving
-	rm -rf $(BUILD_DIR)/kfserving && mkdir -p $(BUILD_DIR)/kfserving
-	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/kfserving ./apps/kfserving
-	kubectl --context=$(KFCTXT) apply -f ./$(BUILD_DIR)/kfserving
+	$(MAKE) apply-knative 
+
+	$(MAKE) apply-kfserving
 
 	# Common user-namespace
 	rm -rf $(BUILD_DIR)/user-namespace && mkdir -p $(BUILD_DIR)/user-namespace
@@ -176,10 +166,7 @@ apply-kubeflow: validate-values check-name
 	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/katib ./apps/katib
 	kubectl --context=$(KFCTXT) apply -f ./$(BUILD_DIR)/katib
 
-	# Common cert-manager kubeflow-issuer
-	rm -rf $(BUILD_DIR)/kubeflow-issuer && mkdir -p $(BUILD_DIR)/kubeflow-issuer
-	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/kubeflow-issuer ./common/cert-manager/upstream/cert-manager/kubeflow-issuer
-	kubectl --context=$(KFCTXT) apply -f ./$(BUILD_DIR)/kubeflow-issuer
+	$(MAKE) apply-kubeflow-issuer
 
 	# Kick the IAP pod because we will reset the policy and need to patch it.
 	# TODO(https://github.com/kubeflow/gcp-blueprints/issues/14)
@@ -188,6 +175,52 @@ apply-kubeflow: validate-values check-name
 	# https://github.com/kubeflow/gcp-blueprints/issues/160
 	kubectl --context=$(KFCTXT) -n istio-system delete pods -l service=backend-updater
 
+.PHONY: apply-knative
+apply-knative:
+	# Common Knative
+	# It has error when trying to patch the resource downloaded from knative because it doesn't match kustomize requirement.
+	# Original release of knative manifest can not be patched by kustomize because of `already registered id`. For now we run kubectl directly.
+	rm -rf $(BUILD_DIR)/knative && mkdir -p $(BUILD_DIR)/knative
+	kustomize build -o $(BUILD_DIR)/knative ./common/knative
+	kubectl --context=${KFCTXT} apply -f ./$(BUILD_DIR)/knative/*v1_namespace_knative-serving.yaml
+	kubectl --context=$(KFCTXT) apply --recursive=true -f ./$(BUILD_DIR)/knative
+
+.PHONY: apply-kfserving
+apply-kfserving:
+	# App kfserving
+	rm -rf $(BUILD_DIR)/kfserving && mkdir -p $(BUILD_DIR)/kfserving
+	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/kfserving ./apps/kfserving
+	kubectl --context=$(KFCTXT) apply -f ./$(BUILD_DIR)/kfserving
+	kubectl --context=$(KFCTXT) patch cm config-domain --namespace knative-serving --type merge -p '{"data":{"$(NAME).endpoints.$(PROJECT).cloud.goog": ""}}'
+
+.PHONY: apply-cert-manager
+apply-cert-manager:
+	# Common cert-manager
+	rm -rf $(BUILD_DIR)/cert-manager && mkdir -p $(BUILD_DIR)/cert-manager
+	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/cert-manager ./common/cert-manager
+# Try kpt live apply to simplify this steps.
+	kubectl --context=$(KFCTXT) apply -f ./$(BUILD_DIR)/cert-manager/*v1_namespace_cert-manager.yaml
+	kubectl --context=$(KFCTXT) apply --recursive=true -f ./$(BUILD_DIR)/cert-manager
+	kubectl --context=$(KFCTXT) -n cert-manager wait --for=condition=Available --timeout=600s deploy cert-manager-webhook
+	kubectl --context=$(KFCTXT) -n cert-manager wait --for=condition=Available --timeout=600s deploy cert-manager
+	kubectl --context=$(KFCTXT) -n cert-manager wait --for=condition=Available --timeout=600s deploy cert-manager-cainjector
+
+.PHONY: apply-kubeflow-issuer
+apply-kubeflow-issuer:
+	# Common kubeflow-issuer for cert-manager
+	rm -rf $(BUILD_DIR)/kubeflow-issuer && mkdir -p $(BUILD_DIR)/kubeflow-issuer
+	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/kubeflow-issuer ./common/cert-manager/cert-manager-1-3/cert-manager/kubeflow-issuer
+# kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/kubeflow-issuer ./common/cert-manager/upstream/cert-manager/kubeflow-issuer
+	kubectl --context=$(KFCTXT) apply --recursive=true -f ./$(BUILD_DIR)/kubeflow-issuer
+
+
+.PHONY: apply-istio
+apply-istio: 
+	# Kubeflow kubeflow-istio
+	rm -rf $(BUILD_DIR)/kubeflow-istio && mkdir -p $(BUILD_DIR)/kubeflow-istio
+	kustomize build --load-restrictor LoadRestrictionsNone -o $(BUILD_DIR)/kubeflow-istio ./common/istio
+	kubectl --context=$(KFCTXT) apply -f ./$(BUILD_DIR)/kubeflow-istio
+
 .PHONY: apply-v2
 apply-v2: clean-build check-name check-iap apply-gcp wait-gcp create-ctxt apply-asm iap-secret apply-kubeflow
 ifeq ($(PRIVATE_GKE),true)

diff --git a/kubeflow/common/cert-manager/cert-manager-1-3/cert-manager/base/kustomization.yaml b/kubeflow/common/cert-manager/cert-manager-1-3/cert-manager/base/kustomization.yaml
@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- https://github.com/jetstack/cert-manager/releases/download/v1.3.0/cert-manager.yaml
+
+images:
+- name: quay.io/jetstack/cert-manager-controller
+  newName: quay.io/jetstack/cert-manager-controller
+  newTag: v1.3.0
+- name: quay.io/jetstack/cert-manager-cainjector
+  newName: quay.io/jetstack/cert-manager-cainjector
+  newTag: v1.3.0
+- name: quay.io/jetstack/cert-manager-webhook
+  newName: quay.io/jetstack/cert-manager-webhook
+  newTag: v1.3.0
diff --git a/...low/common/cert-manager/cert-manager-1-3/cert-manager/kubeflow-issuer/cluster-issuer.yaml b/...low/common/cert-manager/cert-manager-1-3/cert-manager/kubeflow-issuer/cluster-issuer.yaml
@@ -0,0 +1,6 @@
+apiVersion: cert-manager.io/v1alpha2
+kind: ClusterIssuer
+metadata:
+  name: kubeflow-self-signing-issuer
+spec:
+  selfSigned: {}
diff --git a/...flow/common/cert-manager/cert-manager-1-3/cert-manager/kubeflow-issuer/kustomization.yaml b/...flow/common/cert-manager/cert-manager-1-3/cert-manager/kubeflow-issuer/kustomization.yaml
@@ -0,0 +1,10 @@
+# Define the self-signed issuer for Kubeflow
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: cert-manager
+commonLabels:
+  kustomize.component: cert-manager
+  app.kubernetes.io/component: cert-manager
+  app.kubernetes.io/name: cert-manager
+resources:
+- cluster-issuer.yaml
diff --git a/...ommon/cert-manager/cert-manager-1-3/cert-manager/overlays/letsencrypt/cluster-issuer.yaml b/...ommon/cert-manager/cert-manager-1-3/cert-manager/overlays/letsencrypt/cluster-issuer.yaml
@@ -0,0 +1,11 @@
+apiVersion: cert-manager.io/v1alpha2
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    email: $(acmeEmail)
+    http01: {}
+    privateKeySecretRef:
+      name: letsencrypt-prod-secret
+    server: $(acmeUrl)
diff --git a/...common/cert-manager/cert-manager-1-3/cert-manager/overlays/letsencrypt/kustomization.yaml b/...common/cert-manager/cert-manager-1-3/cert-manager/overlays/letsencrypt/kustomization.yaml
@@ -0,0 +1,35 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../../base
+namespace: cert-manager
+resources:
+- cluster-issuer.yaml
+commonLabels:
+  kustomize.component: cert-manager
+  app.kubernetes.io/component: cert-manager
+  app.kubernetes.io/name: cert-manager
+configMapGenerator:
+- name: cert-manager-parameters
+  behavior: merge
+  envs:
+  - params.env
+generatorOptions:
+  disableNameSuffixHash: true
+vars:
+- name: acmeEmail
+  objref:
+    kind: ConfigMap
+    name: cert-manager-parameters
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.acmeEmail
+- name: acmeUrl
+  objref:
+    kind: ConfigMap
+    name: cert-manager-parameters
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.acmeUrl
+configurations:
+- params.yaml
diff --git a/kubeflow/common/cert-manager/cert-manager-1-3/cert-manager/overlays/letsencrypt/params.env b/kubeflow/common/cert-manager/cert-manager-1-3/cert-manager/overlays/letsencrypt/params.env
@@ -0,0 +1,2 @@
+acmeEmail=
+acmeUrl=https://acme-v02.api.letsencrypt.org/directory
diff --git a/kubeflow/common/cert-manager/cert-manager-1-3/cert-manager/overlays/letsencrypt/params.yaml b/kubeflow/common/cert-manager/cert-manager-1-3/cert-manager/overlays/letsencrypt/params.yaml
@@ -0,0 +1,5 @@
+varReference:
+- path: spec/acme/email
+  kind: ClusterIssuer
+- path: spec/acme/server
+  kind: ClusterIssuer
diff --git a/...common/cert-manager/cert-manager-1-3/cert-manager/overlays/self-signed/kustomization.yaml b/...common/cert-manager/cert-manager-1-3/cert-manager/overlays/self-signed/kustomization.yaml
@@ -0,0 +1,13 @@
+# TODO(https://github.com/kubeflow/manifests/issues/1052) clean up
+# the manifests after the refactor is done. We should move
+# cluster-issuer into the kubeflow-issuer package.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../../base
+resources:
+- ../../kubeflow-issuer
+commonLabels:
+  kustomize.component: cert-manager
+  app.kubernetes.io/component: cert-manager
+  app.kubernetes.io/name: cert-manager
diff --git a/kubeflow/common/cert-manager/kustomization.yaml b/kubeflow/common/cert-manager/kustomization.yaml
@@ -1,6 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ./application # {"$kpt-set":"cert-manager-cert-manager-v3"}
-- ./upstream/cert-manager-crds/base
-- ./upstream/cert-manager-kube-system-resources/base # {"$kpt-set":"cert-manager-cert-manager-kube-system-resources-base"}
+# - ./application # {"$kpt-set":"cert-manager-cert-manager-v3"}
+# - ./upstream/cert-manager-crds/base
+# - ./upstream/cert-manager-kube-system-resources/base # {"$kpt-set":"cert-manager-cert-manager-kube-system-resources-base"}
+- ./cert-manager-1-3/cert-manager/base