Sync kubeflow pipelines manifests 2.1.0

README.md

@@ -57,7 +57,7 @@ This repo periodically syncs all official Kubeflow components from their respect
 | Katib | apps/katib/upstream | [v0.16.0](https://github.com/kubeflow/katib/tree/v0.16.0/manifests/v1beta1) |
 | KServe | contrib/kserve/kserve | [v0.11.2](https://github.com/kserve/kserve/tree/v0.11.2/install/v0.11.2) |
 | KServe Models Web App | contrib/kserve/models-web-app | [v0.10.0](https://github.com/kserve/models-web-app/tree/v0.10.0/config) |
-| Kubeflow Pipelines | apps/pipeline/upstream | [2.0.5](https://github.com/kubeflow/pipelines/tree/2.0.5/manifests/kustomize) |
+| Kubeflow Pipelines | apps/pipeline/upstream | [2.1.0](https://github.com/kubeflow/pipelines/tree/2.1.0/manifests/kustomize) |
 | Kubeflow Tekton Pipelines | apps/kfp-tekton/upstream | [2.0.5](https://github.com/kubeflow/kfp-tekton/tree/2.0.5/manifests/kustomize) |
 | Kubeflow Model Registry | apps/model-registry/upstream | [main](https://github.com/kubeflow/model-registry/tree/main/manifests/kustomize) |
 

apps/pipeline/upstream/base/cache-deployer/kustomization.yaml

@@ -8,4 +8,4 @@ commonLabels:
   app: cache-deployer
 images:
   - name: gcr.io/ml-pipeline/cache-deployer
-    newTag: 2.0.5
+    newTag: 2.1.0

apps/pipeline/upstream/base/cache/kustomization.yaml

@@ -10,4 +10,4 @@ commonLabels:
   app: cache-server
 images:
   - name: gcr.io/ml-pipeline/cache-server
-    newTag: 2.0.5
+    newTag: 2.1.0

apps/pipeline/upstream/base/installs/generic/kustomization.yaml

@@ -45,4 +45,4 @@ vars:
   fieldref:
     fieldpath: data.defaultPipelineRoot
 configurations:
-- params.yaml
+ - params.yaml

apps/pipeline/upstream/base/installs/generic/pipeline-install-config.yaml

@@ -11,7 +11,7 @@ data:
     until the changes take effect. A quick way to restart all deployments in a
     namespace: `kubectl rollout restart deployment -n <your-namespace>`.
   appName: pipeline
-  appVersion: 2.0.5
+  appVersion: 2.1.0
   dbHost: mysql # relic to be removed after release
   dbPort: "3306" # relic to be removed after release
   dbType: mysql
@@ -93,3 +93,4 @@ data:
   ## If this value doesn't include a unit abbreviation, the units will be assumed
   ## to be nanoseconds.
   ConMaxLifeTime: "120s"
+  LOG_LEVEL: "info"

apps/pipeline/upstream/base/installs/generic/postgres/kustomization.yaml

@@ -0,0 +1,48 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+bases:
+- ../../../postgresql/pipeline
+- ../../../postgresql/cache
+- ../../../cache-deployer
+resources:
+- pipeline-install-config.yaml
+- postgres-secret-extended.yaml
+vars:
+- name: kfp-namespace
+  objref:
+    kind: Deployment
+    apiVersion: apps/v1
+    name: ml-pipeline
+  fieldref:
+    fieldpath: metadata.namespace
+- name: kfp-app-name
+  objref:
+    kind: ConfigMap
+    name: pipeline-install-config
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.appName
+- name: kfp-app-version
+  objref:
+    kind: ConfigMap
+    name: pipeline-install-config
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.appVersion
+- name: kfp-artifact-bucket-name
+  objref:
+    kind: ConfigMap
+    name: pipeline-install-config
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.bucketName
+- name: kfp-default-pipeline-root
+  objref:
+    kind: ConfigMap
+    name: pipeline-install-config
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.defaultPipelineRoot
+configurations:
+ - params.yaml

apps/pipeline/upstream/base/installs/generic/postgres/params.yaml

@@ -0,0 +1,10 @@
+# Allow Kustomize var to replace following fields.
+varReference:
+- path: data/config
+  kind: ConfigMap
+- path: data/defaultPipelineRoot
+  kind: ConfigMap
+- path: metadata/name
+  kind: Application
+- path: spec/descriptor/version
+  kind: Application

apps/pipeline/upstream/base/installs/generic/postgres/pipeline-install-config.yaml

@@ -0,0 +1,95 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pipeline-install-config
+data:
+  warning: |
+    1. Do not use kubectl to edit this configmap, because some values are used
+    during kustomize build. Instead, change the configmap and apply the entire
+    kustomize manifests again.
+    2. After updating the configmap, some deployments may need to be restarted
+    until the changes take effect. A quick way to restart all deployments in a
+    namespace: `kubectl rollout restart deployment -n <your-namespace>`.
+  appName: pipeline
+  appVersion: 2.0.0
+  dbHost: postgres  # relic to be removed after release
+  dbPort: "5432"    # relic to be removed after release
+  dbType: postgres  
+  postgresHost: postgres
+  postgresPort: "5432"
+  mlmdDb: metadb
+  cacheDb: cachedb
+  pipelineDb: mlpipeline
+  bucketName: mlpipeline
+  ## defaultPipelineRoot: Optional. Default pipeline root in v2 compatible mode.
+  ## https://www.kubeflow.org/docs/components/pipelines/sdk/v2/v2-compatibility/
+  ##
+  ## If the field is not set, kfp-launcher configmaps won't be created and 
+  ## v2 compatible mode defaults to minio://mlpipeline/v2/artifacts as pipeline
+  ## root.
+  ##
+  ## When not in Kubeflow Pipelines multi-user mode, the config works as you
+  ## would normally expect.
+  ##
+  ## In Kubeflow Pipelines multi-user mode, the config creates default
+  ## kfp-launcher configmaps in each user's namespace. Users can edit the
+  ## kfp-launcher configmap's defaultPipelineRoot field afterwards to configure
+  ## namespace-specific default pipeline root. The namespace specific changes in
+  ## kfp-launcher configmap won't be overridden by pipeline-install-config.
+  ##
+  ## Caveat: when you update the config from a non-empty value, only new
+  ## namespaces get the updated config by default. Owners of existing namespaces
+  ## must delete the kfp-launcher configmap to get the new default config value.
+  ##
+  ## Examples:
+  ## defaultPipelineRoot: minio://mlpipeline/v2/artifacts
+  ## defaultPipelineRoot: gs://your-bucket/path/to/artifacts
+  ## defaultPipelineRoot: s3://your-bucket/path/to/artifacts
+  ##
+  ## V2 Compatible Mode Feature stage:
+  ## [Beta](https://github.com/kubeflow/pipelines/blob/master/docs/release/feature-stages.md#beta)
+  defaultPipelineRoot: ""
+  ## autoUpdatePipelineDefaultVersion: States if the pipeline version
+  ## should be updated by defult for a versioned pipeline or not when a new
+  ## version is uploaded. This sets the deployment wide definition.
+  autoUpdatePipelineDefaultVersion: "true"
+  ## cronScheduleTimezone: States the timezone which should be used for
+  ## the cron scheduler. If not specified the local timezone of the
+  ## cluster will be used. Valid values are UTC, Local or values according to
+  ## the IANA Time Zone database, such as "America/New_York" and "Asia/Shanghai".
+  ## Feature stage:
+  ## [Alpha](https://github.com/kubeflow/pipelines/blob/master/docs/release/feature-stages.md#alpha)
+  cronScheduleTimezone: "UTC"
+  ## cacheImage is the image that the mutating webhook will use to patch
+  ## cached steps with. Will be used to echo a message announcing that 
+  ## the cached step result will be used. If not set it will default to 
+  ## 'gcr.io/google-containers/busybox'
+  cacheImage: "gcr.io/google-containers/busybox"
+  ## cacheNodeRestrictions the dummy container runing if output is cached
+  ## will run with the same affinity and node selector as the default pipeline
+  ## step. This is defaulted to 'false' to allow the pod to be scheduled on 
+  ## any node and avoid defaulting to specific nodes. Allowed values are:
+  ## 'false' and 'true'.
+  cacheNodeRestrictions: "false"
+  ## MAXIMUM_CACHE_STALENESS configures caching according to
+  ## https://www.kubeflow.org/docs/components/pipelines/overview/caching/ and
+  ## https://www.kubeflow.org/docs/components/pipelines/overview/caching-v2/.
+  ## Larger than MAXIMUM_CACHE_STALENESS per pipeline user set values are 
+  ## reduced to MAXIMUM_CACHE_STALENESS.
+  ## The administrator of the storage backend can rely on it to delete old cache
+  ## artifacts.
+  MAXIMUM_CACHE_STALENESS: ""
+  ## MAXIMUM_CACHE_STALENESS: "P30D"
+  ## DEFAULT_CACHE_STALENESS configures caching according to
+  ## https://www.kubeflow.org/docs/components/pipelines/overview/caching/ and
+  ## https://www.kubeflow.org/docs/components/pipelines/overview/caching-v2/.
+  ## This value is used if the user did not set a value in the pipeline.
+  DEFAULT_CACHE_STALENESS: ""
+  ## DEFAULT_CACHE_STALENESS: "P7D"
+  ## ConMaxLifeTime will set the connection max lifetime for MySQL
+  ## this is very important to setup when using external databases.
+  ## See this issue for more details: https://github.com/kubeflow/pipelines/issues/5329
+  ## Note: this value should be a string that can be parsed by `time.ParseDuration`.
+  ## If this value doesn't include a unit abbreviation, the units will be assumed
+  ## to be nanoseconds.
+  ConMaxLifeTime: "120s"

apps/pipeline/upstream/base/installs/generic/postgres/postgres-secret-extended.yaml

@@ -0,0 +1,7 @@
+kind: Secret
+apiVersion: v1
+metadata:
+  name: postgres-secret-extended
+stringData:
+  username: user
+  password: "password"

apps/pipeline/upstream/base/installs/multi-user/persistence-agent/cluster-role.yaml

@@ -27,8 +27,9 @@ rules:
   verbs:
   - report
 - apiGroups:
-  - ''
+    - pipelines.kubeflow.org
   resources:
-  - namespaces
+    - runs
   verbs:
-  - get
+    - reportMetrics
+    - readArtifact

apps/pipeline/upstream/base/installs/multi-user/persistence-agent/deployment-patch.yaml

@@ -7,14 +7,7 @@ spec:
     spec:
       containers:
       - name: ml-pipeline-persistenceagent
-        envFrom:
-        - configMapRef:
-            name: persistenceagent-config
         env:
         - name: NAMESPACE
           value: ''
           valueFrom: null
-        - name: KUBEFLOW_USERID_HEADER
-          value: kubeflow-userid
-        - name: KUBEFLOW_USERID_PREFIX
-          value: ""

apps/pipeline/upstream/base/installs/multi-user/persistence-agent/kustomization.yaml

@@ -3,7 +3,3 @@ kind: Kustomization
 resources:
 - cluster-role.yaml
 - cluster-role-binding.yaml
-configMapGenerator:
-- name: persistenceagent-config
-  envs:
-  - params.env

apps/pipeline/upstream/base/metadata/base/kustomization.yaml

@@ -9,4 +9,4 @@ resources:
   - metadata-grpc-sa.yaml
 images:
   - name: gcr.io/ml-pipeline/metadata-envoy
-    newTag: 2.0.5
+    newTag: 2.1.0

apps/pipeline/upstream/base/metadata/base/metadata-grpc-deployment.yaml

@@ -22,7 +22,7 @@ spec:
         # * .cloudbuild.yaml and .release.cloudbuild.yaml
         # * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml
         # * test/tag_for_hosted.sh
-        image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0
+        image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0
         env:
         - name: DBCONFIG_USER
           valueFrom:

apps/pipeline/upstream/base/metadata/overlays/postgres/kustomization.yaml

@@ -0,0 +1,38 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+
+bases:
+- ../../base
+resources:
+- metadata-db-pvc.yaml
+- metadata-db-deployment.yaml
+- metadata-db-service.yaml
+
+patchesStrategicMerge:
+- patches/metadata-grpc-deployment.yaml
+
+configMapGenerator:
+- name: metadata-postgres-db-parameters
+  envs:
+  - params.env
+secretGenerator:
+- name: metadata-postgres-db-secrets
+  envs:
+  - secrets.env
+generatorOptions:
+  disableNameSuffixHash: true
+
+images:
+- name: postgres
+  newName: postgres
+  newTag: 14.7-alpine3.17
+
+vars:
+- name: MLMD_DB_HOST
+  objref:
+    kind: Service
+    name: metadata-postgres-db
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.name

apps/pipeline/upstream/base/metadata/overlays/postgres/metadata-db-deployment.yaml

@@ -0,0 +1,43 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metadata-postgres-db
+  labels:
+    component: db
+spec:
+  selector:
+    matchLabels:
+      component: db
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      name: db
+      labels:
+        component: db
+      annotations:
+        sidecar.istio.io/inject: "false"
+    spec:
+      containers:
+      - name: db-container
+        image: postgres
+        env:
+          - name: PGDATA
+            value: /var/lib/postgresql/data/pgdata
+        envFrom:
+        - configMapRef:
+            name: metadata-postgres-db-parameters
+        - secretRef:
+            name: metadata-postgres-db-secrets
+        ports:
+        - name: postgres
+          containerPort: 5432
+        volumeMounts:
+        - name: metadata-postgres
+          mountPath: /var/lib/postgresql/data
+      volumes:
+      - name: metadata-postgres
+        persistentVolumeClaim:
+          claimName: metadata-postgres
+