Establish release process (kubeflow#5746)

* prow_config: Run all tests when version changes Signed-off-by: Yannis Zarkadas <[email protected]> * prow_config: Build and push all images with the release tag when the version changes Signed-off-by: Yannis Zarkadas <[email protected]> * releasing: Add releasing helpers Signed-off-by: Yannis Zarkadas <[email protected]> * testing: Add parameter to workflow for pushing images with release tags Signed-off-by: Yannis Zarkadas <[email protected]>
StatCan · Mar 24, 2021 · 50902c6 · 50902c6
1 parent 3fccec8
commit 50902c6
Show file tree

Hide file tree

Showing 5 changed files with 445 additions and 4 deletions.
diff --git a/prow_config.yaml b/prow_config.yaml
@@ -12,6 +12,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/crud-web-apps/common/frontend/kubeflow-common-lib/*
     kwargs: {}
   # Run unittests for access management
@@ -20,6 +21,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/access-management/*
     kwargs: {}
   # Run unittests for admission webhook
@@ -28,6 +30,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/admission-webhook/*
     kwargs: {}
   # Run unittests for the central dashboard
@@ -36,6 +39,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/centraldashboard/*
     kwargs: {}
   # Run unittests for the Jupyter Web App
@@ -44,6 +48,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/crud-web-apps/common/frontend/kubeflow-common-lib/*
       - components/crud-web-apps/common/backend/*
       - components/crud-web-apps/jupyter/*
@@ -54,6 +59,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/common/*
       - components/notebook-controller/*
     kwargs: {}
@@ -63,6 +69,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/profile-controller/*
     kwargs: {}
   # Run unittests for the Tensorboard Controller
@@ -71,6 +78,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/common/*
       - components/tensorboard-controller/*
     kwargs: {}
@@ -80,6 +88,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/crud-web-apps/common/frontend/kubeflow-common-lib/*
       - components/crud-web-apps/common/backend/*
       - components/crud-web-apps/tensorboards/*
@@ -90,6 +99,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/crud-web-apps/common/frontend/kubeflow-common-lib/*
       - components/crud-web-apps/common/backend/*
       - components/crud-web-apps/volumes/*
@@ -100,6 +110,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/base/*
     kwargs: {}
   # Run build test for notebook-server-jupyter OCI image
@@ -108,6 +119,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/jupyter/*
     kwargs: {}
   # Create and push notebook-server-rstudio OCI image to ECR
@@ -116,6 +128,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/rstudio/*
     kwargs: {}
   # Run build test for notebook-server-codeserver OCI image
@@ -124,6 +137,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/codeserver/*
     kwargs: {}
   # Run build test for notebook-server-jupyter-pytorch OCI image
@@ -132,6 +146,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/jupyter-pytorch/*
     kwargs: {}
   # Run build test for notebook-server-jupyter-tensorflow OCI images
@@ -140,6 +155,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/jupyter-tensorflow/*
     kwargs: {}
   # Run build test for notebook-server-jupyter-pytorch-full OCI images
@@ -148,6 +164,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/jupyter-pytorch-full/*
     kwargs: {}
   # Run build test for notebook-server-jupyter-tensorflow-full OCI images
@@ -156,6 +173,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/jupyter-tensorflow-full/*
     kwargs: {}
   # Run build test for notebook-server-jupyter-scipy OCI image
@@ -164,6 +182,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/jupyter-scipy/*
     kwargs: {}
   # Run build test for notebook-server-codeserver-python OCI image
@@ -172,6 +191,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/codeserver-python/*
     kwargs: {}
   # Run build test for notebook-server-rstudio-tidyverse OCI image
@@ -180,6 +200,7 @@ workflows:
     job_types:
       - presubmit
     include_dirs:
+      - releasing/version/*
       - components/example-notebook-servers/rstudio-tidyverse/*
     kwargs: {}
 #######################################
@@ -353,3 +374,187 @@ workflows:
     include_dirs:
       - components/example-notebook-servers/rstudio-tidyverse/*
     kwargs: {}
+
+#######################################
+# Postsubmit jobs for releasing
+#######################################
+  # Create and push Jupyter Web App OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.jwa.create_workflow
+    name: jwa-build
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push Access Management OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.access_management.create_workflow
+    name: ac-mgr-build
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push Admission Webhook OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.admission_webhook.create_workflow
+    name: adm-wh-build
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push Central Dashboard OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.central_dashboard.create_workflow
+    name: cdash-build
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push Tensorboard Web App OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.twa.create_workflow
+    name: twa-build
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push Notebook Controller OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_controller.create_workflow
+    name: nb-c-build
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push Profile Controller OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.profile_controller.create_workflow
+    name: pro-c-build
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push Tensorboard Controller OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.tensorboard_controller.create_workflow
+    name: tb-c-build
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push Volumes Web App OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.vwa.create_workflow
+    name: vwa-build
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-base OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_base.create_workflow
+    name: nb-base
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-jupyter OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_jupyter.create_workflow
+    name: nb-jupyter
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-rstudio OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_rstudio.create_workflow
+    name: nb-rstudio
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-codeserver OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_codeserver.create_workflow
+    name: nb-code
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-jupyter-pytorch OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_jupyter_pytorch.create_workflow
+    name: nb-j-pt
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-jupyter-tensorflow OCI images to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_jupyter_tensorflow.create_workflow
+    name: nb-j-tf
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-jupyter-pytorch-full OCI images to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_jupyter_pytorch_full.create_workflow
+    name: nb-j-pt-f
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-jupyter-tensorflow OCI images to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_jupyter_tensorflow_full.create_workflow
+    name: nb-j-tf-f
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-jupyter-scipy OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_jupyter_scipy.create_workflow
+    name: nb-j-sp
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-codeserver-python OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_codeserver_python.create_workflow
+    name: nb-code-p
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
+  # Create and push notebook-server-rstudio-tidyverse OCI image to ECR
+  - py_func: kubeflow.kubeflow.cd.notebook_servers.notebook_server_rstudio_tidyverse.create_workflow
+    name: nb-rs-tidy
+    job_types:
+      - postsubmit
+    include_dirs:
+      - releasing/version/*
+    kwargs:
+      release: true
diff --git a/py/kubeflow/kubeflow/ci/workflow_utils.py b/py/kubeflow/kubeflow/ci/workflow_utils.py
@@ -29,14 +29,15 @@
 
 class ArgoTestBuilder:
     def __init__(self, name=None, namespace=None, bucket=None,
-                 test_target_name=None,
+                 test_target_name=None, release=False,
                  **kwargs):
         self.name = name
         self.namespace = namespace
         self.bucket = bucket
         self.template_label = "argo_test"
         self.test_target_name = test_target_name
         self.mkdir_task_name = "make-artifacts-dir"
+        self.release = release
 
         # *********************************************************************
         #
@@ -213,9 +214,14 @@ def create_kaniko_task(self, task_template, dockerfile, context,
 
         # append the tag base-commit[0:7]
         if ":" not in destination:
-            sha = os.getenv("PULL_BASE_SHA", "12341234kanikotest")
-            base = os.getenv("PULL_BASE_REF", "master")
-            destination += ":%s-%s" % (base, sha[0:8])
+            if self.release:
+                with open(os.path.join("/src/kubeflow/kubeflow", "releasing/version/VERSION")) as f:
+                    version = f.read().strip()
+                destination += ":%s" % version
+            else:
+                sha = os.getenv("PULL_BASE_SHA", "12341234kanikotest")
+                base = os.getenv("PULL_BASE_REF", "master")
+                destination += ":%s-%s" % (base, sha[0:8])
 
         # add short UUID to step name to ensure it is unique
         kaniko["name"] = "kaniko-build-push-" + ''.join(random.choices(alphabet, k=8))