From f84844c541bd9d38f4ddcc5f3deb6f61d8d697ef Mon Sep 17 00:00:00 2001 From: Christie Wilson Date: Fri, 15 Mar 2019 15:36:05 -0700 Subject: [PATCH] Add Task for publishing tekton pipeline images + yaml Add a `Task` which invokes `ko` to build and publish all images and yaml config required for installing Tekton Pipelines. This Task will: * Build and publish the "base image" using Kaniko * Generate a .ko.yaml * Invoke ko to build/publish images and generate a release.yaml * Parse the release.yaml for built images; ensuring that the expected images were built (and no more) * Tag the built images with the correct version and also tag in all regions (us, asia, eu) This should be the same functionality that could previously be seen in https://github.com/tektoncd/pipeline/blob/master/hack/release.sh (which used https://github.com/knative/test-infra/blob/master/scripts/release.sh). We can remove release.sh once we have completed #530 as well. Some functionality has been implemented in a python script, which has its own tests. Since it is currently difficult to update the pull request test logic to do additional things (such as run python unit tests), I'm hoping we are okay with waiting until #532 to add automatic running of these tests). Fixes #528 Fixes #529 --- .gitignore | 3 + .ko.yaml | 2 +- docs/resources.md | 4 +- hack/release.md | 2 + tekton/README.md | 71 ++++++ tekton/koparse/koparse.py | 116 +++++++++ tekton/koparse/test_koparse.py | 71 ++++++ tekton/koparse/test_release.yaml | 392 +++++++++++++++++++++++++++++++ tekton/publish-run.yaml | 186 +++++++++++++++ tekton/publish.yaml | 183 +++++++++++++++ 10 files changed, 1028 insertions(+), 2 deletions(-) create mode 100644 tekton/README.md create mode 100755 tekton/koparse/koparse.py create mode 100755 tekton/koparse/test_koparse.py create mode 100644 tekton/koparse/test_release.yaml create mode 100644 tekton/publish-run.yaml create mode 100644 tekton/publish.yaml diff --git a/.gitignore b/.gitignore index a0c1e5dd767..8f511d1e2f8 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ # JetBrains IDE config .idea + +# Python +*.pyc diff --git a/.ko.yaml b/.ko.yaml index 4174b994834..9b34cc27bf9 100644 --- a/.ko.yaml +++ b/.ko.yaml @@ -1,5 +1,5 @@ baseImageOverrides: - # TODO(jasonhall): Use build-base in the build-pipeline path, when it's build/released. + # TODO(christiewilson): Use our built base image github.com/tektoncd/pipeline/cmd/creds-init: gcr.io/knative-nightly/github.com/knative/build/build-base:latest github.com/tektoncd/pipeline/cmd/git-init: gcr.io/knative-nightly/github.com/knative/build/build-base:latest github.com/tektoncd/pipeline/cmd/bash: busybox # image should have shell in $PATH diff --git a/docs/resources.md b/docs/resources.md index ffd57fa31be..9d6251d5178 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -309,6 +309,8 @@ spec: value: gcs - name: location value: gs://some-bucket + - name: dir + value: "y" # This can have any value to be considered "true" ``` Params that can be added are the following: @@ -361,7 +363,7 @@ service account. - name: location value: gs://some-private-bucket - name: dir - value: "directory" + value: "y" secrets: - fieldName: GOOGLE_APPLICATION_CREDENTIALS secretName: bucket-sa diff --git a/hack/release.md b/hack/release.md index cdbc8adb922..9aed6ea7213 100644 --- a/hack/release.md +++ b/hack/release.md @@ -1,5 +1,7 @@ # Creating a new Tekton Pipeline release +**Note: we are transitioning to a Pipelines based test and release process, see [tekton/README.md](../tekton.README.md).** + The `release.sh` script automates the creation of Tekton Pipeline releases, either nightly or versioned ones. diff --git a/tekton/README.md b/tekton/README.md new file mode 100644 index 00000000000..e2469468740 --- /dev/null +++ b/tekton/README.md @@ -0,0 +1,71 @@ +# Tekton Repo CI/CD + +We dogfood our project by using Tekton Pipelines to build, test and release Tekton Pipelines! + +This directory contains the [`Tasks`](https://github.com/knative/build-pipeline/blob/master/docs/tasks.md) +and [`Pipelines`](https://github.com/knative/build-pipeline/blob/master/docs/pipelines.md) that we (will) +use. + +TODO(#538): In #538 or #537 we will update [Prow](https://github.com/knative/build-pipeline/blob/master/CONTRIBUTING.md#pull-request-process) +to invoke these `Pipelines` automatically, but for now we will have to invoke them manually. + +## Release Pipeline + +The `Tasks` which make up our release `Pipeline` are: + +* [`publish.yaml`](publish.yaml) - This `Task` uses [`kaniko`](https://github.com/GoogleContainerTools/kaniko) + to build and publish base images, and uses [`ko`](https://github.com/google/go-containerregistry/tree/master/cmd/ko) + to build all of the container images we release and generate the `release.yaml` + +### Running + +To run these `Pipelines` and `Tasks`, you must have Tekton Pipelines installed, either via +[an official release](https://github.com/knative/build-pipeline/blob/master/docs/install.md) +or [from `HEAD`](https://github.com/knative/build-pipeline/blob/master/DEVELOPMENT.md#install-pipeline). + +TODO(#531): Add the Pipeline, for now all we have are `Tasks` which we can invoke individually +by creating [`TaskRuns`](https://github.com/knative/build-pipeline/blob/master/docs/taskruns.md) +and [`PipelineResources`](https://github.com/knative/build-pipeline/blob/master/docs/resources.md). + +TODO(#569): Normally we'd use the image `PipelineResources` to control which image registry the images are pushed to. +However since we have so many images, all going to the same registry, we are cheating and using a parameter +for the image registry instead. + +* [`publish-run.yaml`](publish-run.yaml) - This example `TaskRun` and `PipelineResources` demonstrate + how to invoke `publish.yaml`: + + ```bash + kubectl apply -f tekton/publish.yaml + kubectl apply -f tekton/publish-run.yaml + ``` + +### Authentication + +Users executing the publish task must be able to: + +* Push to the image registry (production registry is `gcr.io/tekton-releases`) +* Write to the GCS bucket (production bucket is `gs://tekton-releases`) + +To be able to publish images via `kaniko` or `ko`, you must be able to push to your image registry. +At the moment, the publish `Task` will try to use your default service account in the namespace where +you create the `TaskRun`. If that default service account is able to push to your image registry, +you are good to go. Otherwise, you need to use [a secret annotated with your docker registry +credentials](https://github.com/tektoncd/pipeline/blob/master/docs/auth.md#basic-authentication-docker). + +TODO(#631) Ensure that we are supporting folks using credentials other than the cluster defaults; not +sure how this will play out with publishing to our prod registry! + +#### Production credentials + +TODO(dlorenc, bobcatfish): We need to setup a group which users can be added to, as well as guidelines +around who should be added to this group. + +For now, users who need access to our production registry (`gcr.io/tekton-releases`) and production +GCS bucket (`gs://tekton-releases`) should ping @bobcatfish or @dlorenc to get added to the authorized +users. + +## Supporting scripts + +Some supporting scripts have been written using Python 2.7: + +* [koparse](./koparse) - Contains logic for parsing `release.yaml` files created by `ko` \ No newline at end of file diff --git a/tekton/koparse/koparse.py b/tekton/koparse/koparse.py new file mode 100755 index 00000000000..4c44d373fdb --- /dev/null +++ b/tekton/koparse/koparse.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python2.7 + +""" +koparse.py parses release.yaml files from `ko` + +The `ko` tool (https://github.com/google/go-containerregistry/tree/master/cmd/ko) +builds images and embeds the full names of the built images in the resulting +yaml files. + +This script does two things: + +* Parses those image names out of the release.yaml, including their digests, and + outputs those to stdout +* Verifies the list of built images against an expected list, to be sure that all + expected images were built (and no extra images were built) +""" + +import argparse +import os +import re +import string +import sys + + +DIGEST_MARKER = "@sha256" + + +class ImagesMismatchError(Exception): + def __init__(self, missing, extra): + self.missing = missing + self.extra = extra + + def __str__(self): + errs = [] + if self.missing: + errs.append("Images %s were expected but missing." % self.missing) + if self.extra: + errs.append("Images %s were present but not expected." % + self.extra) + return string.join(errs, " ") + + +class BadActualImageFormatError(Exception): + def __init__(self, image): + self.image = image + + def __str__(self): + return "Format of image %s was unexpected, did not contain %s" % (self.image, DIGEST_MARKER) + + +def parse_release(base, path): + """Extracts built images from the release.yaml at path + + Args: + base: The built images will be expected to start with this string, + other images will be ignored + path: The path to the file (release.yaml) that will contain the built images + Returns: + list of the images parsed from the file + """ + images = [] + with open(path) as f: + for line in f: + match = re.search(base + ".*@sha256:[0-9a-f]*", line) + if match: + images.append(match.group(0)) + return images + + +def compare_expected_images(expected, actual): + """Ensures that the list of actual images includes only the expected images + + Args: + expected: A list of all of the names of images that are expected to have + been built, including the path to the image without the digest + actual: A list of the names of the built images, including the path to the + image and the digest + """ + for image in actual: + if DIGEST_MARKER not in image: + raise BadActualImageFormatError(image) + + actual_no_digest = [string.split(image, DIGEST_MARKER)[0] + for image in actual] + + missing = set(expected) - set(actual_no_digest) + extra = set(actual_no_digest) - set(expected) + + if missing or extra: + raise ImagesMismatchError(list(missing), list(extra)) + + +if __name__ == "__main__": + arg_parser = argparse.ArgumentParser( + description="Parse expected built images from a release.yaml created by `ko`") + arg_parser.add_argument("--path", type=str, required=True, + help="Path to the release.yaml") + arg_parser.add_argument("--base", type=str, required=True, + help="String prefix which is used to find images within the release.yaml") + arg_parser.add_argument("--images", type=str, required=True, nargs="+", + help="List of all images expected to be built, without digests") + args = arg_parser.parse_args() + + try: + images = parse_release(args.base, args.path) + compare_expected_images(args.images, images) + except (IOError, BadActualImageFormatError) as e: + sys.stderr.write("Error determining built images: %s\n" % e) + sys.exit(1) + except (ImagesMismatchError) as e: + sys.stderr.write("Expected images did not match: %s\n" % e) + with open(args.path) as f: + sys.stderr.write(f.read()) + sys.exit(1) + + print("\n".join(images)) diff --git a/tekton/koparse/test_koparse.py b/tekton/koparse/test_koparse.py new file mode 100755 index 00000000000..a44a096b8f3 --- /dev/null +++ b/tekton/koparse/test_koparse.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python2.7 + +import os +import unittest + +import koparse + + +IMAGE_BASE = "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/" +PATH_TO_TEST_RELEASE_YAML = os.path.join(os.path.dirname( + os.path.abspath(__file__)), "test_release.yaml") +PATH_TO_WRONG_FILE = os.path.join(os.path.dirname( + os.path.abspath(__file__)), "koparse.py") +BUILT_IMAGES = [ + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/kubeconfigwriter@sha256:68453f5bb4b76c0eab98964754114d4f79d3a50413872520d8919a6786ea2b35", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/creds-init@sha256:67448da79e4731ab534b91df08da547bc434ab08e41d905858f2244e70290f48", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/git-init@sha256:7d5520efa2d55e1346c424797988c541327ee52ef810a840b5c6f278a9de934a", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/nop@sha256:3784d6b8f73043a29d2c1d6196801bee46fe808fbb94ba4fd21ca52dce503183", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/bash@sha256:d55917ef5c92627027e3755bfc577fbfa2fb783cccfb13a98632cb6ba6088cd6", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/gsutil@sha256:421a261436e16af4057b4a069fdae8a5aca6e37269952209ad9932a774aa0003", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/controller@sha256:bdc6f22a44944c829983c30213091b60f490b41f89577e8492f6a2936be0df41", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/webhook@sha256:cca7069a11aaf0d9d214306d456bc40b2e33e5839429bf07c123ad964d495d8a", +] +EXPECTED_IMAGES = [ + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/kubeconfigwriter", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/creds-init", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/git-init", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/nop", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/bash", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/gsutil", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/controller", + "gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/webhook", +] + + +class TestKoparse(unittest.TestCase): + + def test_parse_release(self): + images = koparse.parse_release(IMAGE_BASE, PATH_TO_TEST_RELEASE_YAML) + self.assertListEqual(images, BUILT_IMAGES) + + def test_parse_release_no_file(self): + with self.assertRaises(IOError): + koparse.parse_release(IMAGE_BASE, "whoops") + + def test_parse_release_wrong_contents(self): + images = koparse.parse_release(IMAGE_BASE, PATH_TO_WRONG_FILE) + self.assertEqual(images, []) + + def test_compare_expected_images(self): + koparse.compare_expected_images(EXPECTED_IMAGES, BUILT_IMAGES) + + def test_compare_expected_images_bad_format(self): + with self.assertRaises(koparse.BadActualImageFormatError): + koparse.compare_expected_images(EXPECTED_IMAGES, EXPECTED_IMAGES) + + def test_compare_expected_images_missing(self): + extra_expected = (EXPECTED_IMAGES[:] + + ["gcr.io/knative-releases/something-else"]) + with self.assertRaises(koparse.ImagesMismatchError): + koparse.compare_expected_images(extra_expected, BUILT_IMAGES) + + def test_compare_expected_images_too_many(self): + extra_actual = (BUILT_IMAGES[:] + + ["gcr.io/knative-releases/something-else@sha256:somedigest"]) + with self.assertRaises(koparse.ImagesMismatchError): + koparse.compare_expected_images(EXPECTED_IMAGES, extra_actual) + + +if __name__ == "__main__": + unittest.main() diff --git a/tekton/koparse/test_release.yaml b/tekton/koparse/test_release.yaml new file mode 100644 index 00000000000..eb80e19148b --- /dev/null +++ b/tekton/koparse/test_release.yaml @@ -0,0 +1,392 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: tekton-pipelines +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tekton-pipelines-admin +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + - secrets + - events + - serviceaccounts + - configmaps + - persistentvolumeclaims + verbs: + - get + - list + - create + - update + - delete + - patch + - watch +- apiGroups: + - extensions + resources: + - deployments + verbs: + - get + - list + - create + - update + - delete + - patch + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - get + - list + - create + - update + - delete + - patch + - watch +- apiGroups: + - tekton.dev + resources: + - tasks + - clustertasks + - taskruns + - pipelines + - pipelineruns + - pipelineresources + verbs: + - get + - list + - create + - update + - delete + - patch + - watch +- apiGroups: + - tekton.dev + resources: + - tasks/status + - clustertasks/status + - taskruns/status + - pipelines/status + - pipelineruns/status + - pipelineresources/status + verbs: + - get + - list + - create + - update + - delete + - patch + - watch +- apiGroups: + - build.knative.dev + resources: + - builds + - buildtemplates + - clusterbuildtemplates + verbs: + - get + - list + - create + - update + - delete + - patch + - watch +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tekton-pipelines-controller + namespace: tekton-pipelines +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: tekton-pipelines-controller-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tekton-pipelines-admin +subjects: +- kind: ServiceAccount + name: tekton-pipelines-controller + namespace: tekton-pipelines +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clustertasks.tekton.dev +spec: + group: tekton.dev + names: + categories: + - all + - knative + - tekton-pipelines + kind: ClusterTask + plural: clustertasks + scope: Cluster + subresources: + status: {} + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: images.caching.internal.knative.dev +spec: + group: caching.internal.knative.dev + names: + categories: + - all + - knative-internal + - caching + kind: Image + plural: images + shortNames: + - img + singular: image + scope: Namespaced + subresources: + status: {} + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pipelines.tekton.dev +spec: + group: tekton.dev + names: + categories: + - all + - knative + - tekton-pipelines + kind: Pipeline + plural: pipelines + scope: Namespaced + subresources: + status: {} + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pipelineruns.tekton.dev +spec: + group: tekton.dev + names: + categories: + - all + - knative + - tekton-pipelines + kind: PipelineRun + plural: pipelineruns + scope: Namespaced + subresources: + status: {} + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pipelineresources.tekton.dev +spec: + group: tekton.dev + names: + categories: + - all + - knative + - tekton-pipelines + kind: PipelineResource + plural: pipelineresources + scope: Namespaced + subresources: + status: {} + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tasks.tekton.dev +spec: + group: tekton.dev + names: + categories: + - all + - knative + - tekton-pipelines + kind: Task + plural: tasks + scope: Namespaced + subresources: + status: {} + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: taskruns.tekton.dev +spec: + group: tekton.dev + names: + categories: + - all + - knative + - tekton-pipelines + kind: TaskRun + plural: taskruns + scope: Namespaced + subresources: + status: {} + version: v1alpha1 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: tekton-pipelines-controller + name: tekton-pipelines-controller + namespace: tekton-pipelines +spec: + ports: + - name: metrics + port: 9090 + protocol: TCP + targetPort: 9090 + selector: + app: tekton-pipelines-controller +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: tekton-pipelines-webhook + name: tekton-pipelines-webhook + namespace: tekton-pipelines +spec: + ports: + - port: 443 + targetPort: 443 + selector: + app: tekton-pipelines-webhook +--- +apiVersion: v1 +data: null +kind: ConfigMap +metadata: + name: config-artifact-bucket + namespace: tekton-pipelines +--- +apiVersion: v1 +data: + image: gcr.io/k8s-prow/entrypoint@sha256:7c7cd8906ce4982ffee326218e9fc75da2d4896d53cabc9833b9cc8d2d6b2b8f +kind: ConfigMap +metadata: + name: config-entrypoint + namespace: tekton-pipelines +--- +apiVersion: v1 +data: + loglevel.controller: info + loglevel.webhook: info + zap-logger-config: | + { + "level": "info", + "development": false, + "sampling": { + "initial": 100, + "thereafter": 100 + }, + "outputPaths": ["stdout"], + "errorOutputPaths": ["stderr"], + "encoding": "json", + "encoderConfig": { + "timeKey": "", + "levelKey": "level", + "nameKey": "logger", + "callerKey": "caller", + "messageKey": "msg", + "stacktraceKey": "stacktrace", + "lineEnding": "", + "levelEncoder": "", + "timeEncoder": "", + "durationEncoder": "", + "callerEncoder": "" + } + } +kind: ConfigMap +metadata: + name: config-logging + namespace: tekton-pipelines +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: tekton-pipelines-controller + namespace: tekton-pipelines +spec: + replicas: 1 + template: + metadata: + labels: + app: tekton-pipelines-controller + spec: + containers: + - args: + - -logtostderr + - -stderrthreshold + - INFO + - -kubeconfig-writer-image + - gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/kubeconfigwriter@sha256:68453f5bb4b76c0eab98964754114d4f79d3a50413872520d8919a6786ea2b35 + - -creds-image + - gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/creds-init@sha256:67448da79e4731ab534b91df08da547bc434ab08e41d905858f2244e70290f48 + - -git-image + - gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/git-init@sha256:7d5520efa2d55e1346c424797988c541327ee52ef810a840b5c6f278a9de934a + - -nop-image + - gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/nop@sha256:3784d6b8f73043a29d2c1d6196801bee46fe808fbb94ba4fd21ca52dce503183 + - -bash-noop-image + - gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/bash@sha256:d55917ef5c92627027e3755bfc577fbfa2fb783cccfb13a98632cb6ba6088cd6 + - -gsutil-image + - gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/gsutil@sha256:421a261436e16af4057b4a069fdae8a5aca6e37269952209ad9932a774aa0003 + image: gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/controller@sha256:bdc6f22a44944c829983c30213091b60f490b41f89577e8492f6a2936be0df41 + name: tekton-pipelines-controller + volumeMounts: + - mountPath: /etc/config-logging + name: config-logging + serviceAccountName: tekton-pipelines-controller + volumes: + - configMap: + name: config-logging + name: config-logging +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: tekton-pipelines-webhook + namespace: tekton-pipelines +spec: + replicas: 1 + template: + metadata: + labels: + app: tekton-pipelines-webhook + spec: + containers: + - image: gcr.io/knative-releases/github.com/knative/build-pipeline/cmd/webhook@sha256:cca7069a11aaf0d9d214306d456bc40b2e33e5839429bf07c123ad964d495d8a + name: webhook + volumeMounts: + - mountPath: /etc/config-logging + name: config-logging + serviceAccountName: tekton-pipelines-controller + volumes: + - configMap: + name: config-logging + name: config-logging diff --git a/tekton/publish-run.yaml b/tekton/publish-run.yaml new file mode 100644 index 00000000000..849f9a0800a --- /dev/null +++ b/tekton/publish-run.yaml @@ -0,0 +1,186 @@ +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: tekton-pipelines +spec: + type: git + params: + - name: url + value: https://github.com/bobcatfish/pipeline + - name: revision + value: dogfood_image_release # Replace this with the commit you want to release from +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: tekton-bucket +spec: + type: storage + params: + - name: type + value: gcs + - name: location + value: gs://tekton-test-release # TODO(#527) Default here should be actual bucket we're publishing to + - name: dir + value: "y" +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: base-image +spec: + type: image + params: + - name: url + value: build-base # Registry is provided via parameter, this is a hack see #569 +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: entrypoint-image +spec: + type: image + params: + - name: url + value: cmd/entrypoint # Registry is provided via parameter, this is a hack see #569 +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: kubeconfigwriter-image +spec: + type: image + params: + - name: url + value: cmd/kubeconfigwriter # Registry is provided via parameter, this is a hack see #569 +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: creds-init-image +spec: + type: image + params: + - name: url + value: cmd/creds-init # Registry is provided via parameter, this is a hack see #569 +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: git-init-image +spec: + type: image + params: + - name: url + value: cmd/git-init # Registry is provided via parameter, this is a hack see #569 +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: nop-image +spec: + type: image + params: + - name: url + value: cmd/nop # Registry is provided via parameter, this is a hack see #569 +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: bash-image +spec: + type: image + params: + - name: url + value: cmd/bash # Registry is provided via parameter, this is a hack see #569 +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: gsutil-image +spec: + type: image + params: + - name: url + value: cmd/gsutil # Registry is provided via parameter, this is a hack see #569 +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: controller-image +spec: + type: image + params: + - name: url + value: cmd/controller # Registry is provided via parameter, this is a hack see #569 +--- +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + name: webhook-image +spec: + type: image + params: + - name: url + value: cmd/webhook # Registry is provided via parameter, this is a hack see #569 +--- +apiVersion: tekton.dev/v1alpha1 +kind: TaskRun +metadata: + name: publish-run +spec: + #serviceAccount: powerful-account + taskRef: + name: publish-tekton-pipelines + trigger: + type: manual + inputs: + resources: + - name: source + resourceRef: + name: tekton-pipelines + - name: bucket + resourceRef: + name: tekton-bucket + params: + - name: versionTag + value: 0.2.0 # Update with the version you want to release, for nightly releases, we might want to use `vYYYYMMDD-commit` + - name: imageRegistry + value: gcr.io/christiewilson-catfactory # TODO(#527) Default here should be actual registry we're publishing to + - name: pathToProject + value: github.com/tektoncd/pipeline + outputs: + resources: + - name: bucket + resourceRef: + name: tekton-bucket + - name: builtBaseImage + resourceRef: + name: base-image + - name: builtEntrypointImage + resourceRef: + name: entrypoint-image + - name: builtKubeconfigWriterImage + resourceRef: + name: kubeconfigwriter-image + - name: builtCredsInitImage + resourceRef: + name: creds-init-image + - name: builtGitInitImage + resourceRef: + name: git-init-image + - name: builtNopImage + resourceRef: + name: nop-image + - name: builtBashImage + resourceRef: + name: bash-image + - name: builtGsutilImage + resourceRef: + name: gsutil-image + - name: builtControllerImage + resourceRef: + name: controller-image + - name: builtWebhookImage + resourceRef: + name: webhook-image \ No newline at end of file diff --git a/tekton/publish.yaml b/tekton/publish.yaml new file mode 100644 index 00000000000..ab04ea0b11e --- /dev/null +++ b/tekton/publish.yaml @@ -0,0 +1,183 @@ +apiVersion: tekton.dev/v1alpha1 +kind: Task +metadata: + name: publish-tekton-pipelines +spec: + inputs: + resources: + - name: source + type: git + targetPath: go/src/github.com/tektoncd/pipeline + - name: bucket + type: storage + params: + - name: versionTag + description: The X.Y.Z version that the artifacts should be tagged with + - name: imageRegistry + description: TODO(#569) This is a hack to make it easy for folks to switch the registry being used by the many many image outputs + - name: pathToProject + description: The path to the folder in the go/src dir that contains the project, which is used by `ko` to name the resulting images + outputs: + resources: + - name: bucket + type: storage + - name: builtBaseImage + type: image + - name: builtEntrypointImage + type: image + - name: builtKubeconfigWriterImage + type: image + - name: builtCredsInitImage + type: image + - name: builtGitInitImage + type: image + - name: builtNopImage + type: image + - name: builtBashImage + type: image + - name: builtGsutilImage + type: image + - name: builtControllerImage + type: image + - name: builtWebhookImage + type: image + steps: + + - name: build-push-base-images + image: gcr.io/kaniko-project/executor + command: + - /kaniko/executor + args: + - --dockerfile=/workspace/go/src/github.com/tektoncd/pipeline/images/Dockerfile + - --destination=${inputs.params.imageRegistry}/${inputs.params.pathToProject}/${outputs.resources.builtBaseImage.url} + - --context=/workspace/go/src/github.com/tektoncd/pipeline + + - name: create-ko-yaml + image: busybox + command: + - /bin/sh + args: + - -ce + - | + set -e + set -x + + cat < /workspace/go/src/github.com/tektoncd/pipeline/.ko.yaml + baseImageOverrides: + ${inputs.params.pathToProject}/${outputs.resources.builtCredsInitImage.url}: ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/build-base:latest + ${inputs.params.pathToProject}/${outputs.resources.builtGitInitImage.url}: ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/build-base:latest + ${inputs.params.pathToProject}/${outputs.resources.builtBashImage.url}: busybox # image should have shell in $PATH + ${inputs.params.pathToProject}/${outputs.resources.builtGsutilImage.url}: google/cloud-sdk:alpine # image should have gsutil in $PATH + EOF + + cat /workspace/go/src/github.com/tektoncd/pipeline/.ko.yaml + + # TODO(#631): publish a `ko` image + - name: install-ko + image: golang + env: + - name: GOBIN + value: /workspace + command: ["go"] + args: + - "get" + - "github.com/google/go-containerregistry/cmd/ko" + + # TODO(#631): publish a `ko` image (which has golang) + - name: install-go + image: golang + env: + command: + - /bin/sh + args: + - -ce + - | + set -e + set -x + # TODO(#631): this is a hack to make the go binary available in a container that has gcloud + mkdir -p /workspace/golang + cp /usr/local/go/bin/go /workspace/golang/go + cp -R /usr/local/go /workspace/golang/localgo + + - name: ensure-release-dirs-exist + image: busybox + command: ["mkdir"] + args: + - "-p" + - "/workspace/bucket/latest/" + - "/workspace/bucket/previous/" + + - name: run-ko + image: google/cloud-sdk + env: + - name: KO_DOCKER_REPO + value: ${inputs.params.imageRegistry} + - name: GOBIN + value: /workspace/golang + - name: GOPATH + value: /workspace/go + command: + - /bin/sh + args: + - -ce + - | + set -e + set -x + + # TODO(#631) Hacks to have the go binary available in this container + ls -lA /workspace/golang + cp -R /workspace/golang/localgo/ /usr/local/go + cp /workspace/golang/go /usr/bin/go + + # TODO(#631) This is a hack to auth with the default creds, need a solution that supports service accounts + gcloud auth configure-docker + + # Publish images and create release.yaml + /workspace/ko resolve --preserve-import-paths -f /workspace/go/src/github.com/tektoncd/pipeline/config/ > /workspace/bucket/latest/release.yaml + + - name: copy-to-tagged-bucket + image: busybox + workingDir: "/workspace/bucket" + command: + - /bin/sh + args: + - -ce + - | + mkdir -p /workspace/bucket/previous/${inputs.params.versionTag}/ + cp /workspace/bucket/latest/release.yaml /workspace/bucket/previous/${inputs.params.versionTag}/release.yaml + + # TODO(#216) Hopefully once we can avoid doing this parsing; on the other hand, `ko` doesn't + # really fit super well into our declarative model + - name: tag-images + #image: gcr.io/cloud-builders/gcloud + image: google/cloud-sdk + command: + - /bin/bash + args: + - -ce + - | + set -e + set -x + + REGIONS=(us eu asia) + IMAGES=( + ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/${outputs.resources.builtEntrypointImage.url} + ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/${outputs.resources.builtKubeconfigWriterImage.url} + ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/${outputs.resources.builtCredsInitImage.url} + ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/${outputs.resources.builtGitInitImage.url} + ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/${outputs.resources.builtNopImage.url} + ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/${outputs.resources.builtBashImage.url} + ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/${outputs.resources.builtGsutilImage.url} + ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/${outputs.resources.builtControllerImage.url} + ${inputs.params.imageRegistry}/${inputs.params.pathToProject}/${outputs.resources.builtWebhookImage.url} + ) + BUILT_IMAGES=( $(/workspace/go/src/github.com/tektoncd/pipeline/tekton/koparse/koparse.py --path /workspace/bucket/latest/release.yaml --base ${inputs.params.imageRegistry}/${inputs.params.pathToProject} --images ${IMAGES[@]}) ) + + for IMAGE in "${BUILT_IMAGES[@]}" + do + for REGION in "${REGIONS[@]}" + do + IMAGE_WITHOUT_SHA=${IMAGE%%@*} + gcloud -q container images add-tag ${IMAGE} ${REGION}.${IMAGE_WITHOUT_SHA}:${inputs.params.versionTag} + done + done \ No newline at end of file