diff --git a/.prow.sh b/.prow.sh index b92678291..b18c53581 100755 --- a/.prow.sh +++ b/.prow.sh @@ -1,23 +1,7 @@ #! /bin/bash -e - -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # This is for testing csi-release-tools itself in Prow. All other # repos use prow.sh for that, but as csi-release-tools isn't a normal # repo with some Go code in it, it has a custom Prow test script. ./verify-shellcheck.sh "$(pwd)" -./verify-spelling.sh "$(pwd)" -./verify-boilerplate.sh "$(pwd)" diff --git a/README.md b/README.md index 6315a736a..696be47a9 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,11 @@ The expected repository layout is: Dockerfile in the root when only building a single command - `Makefile` - includes `release-tools/build.make` and sets configuration variables - - `.travis.yml` - a symlink to `release-tools/.travis.yml` + - `.prow.sh` script which imports `release-tools/prow.sh` + and may contain further customization + - `.cloudbuild.sh` and `cloudbuild.yaml` as symlinks to + the corresponding files in `release-tools` or (if necessary) + as custom files To create a release, tag a certain revision with a name that starts with `v`, for example `v1.0.0`, then `make push` @@ -44,17 +48,10 @@ is the recommended way of maintaining a copy of the rules inside the changes also locally, test them and then push them back to the shared repository at a later time. -We no longer care about importing the full commit history, so `--squash` should be used -when submitting a `release-tools` update. Also make sure that the PR for that -contains the automatically generated commit message in the PR description. -It contains the list of individual commits that were squashed. The script from -https://github.com/kubernetes-csi/csi-release-tools/issues/7 can create such -PRs automatically. - Cheat sheet: -- `git subtree add --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once) -- `git subtree pull --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes) +- `git subtree add --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once) +- `git subtree pull --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes) - edit, `git commit`, `git subtree push --prefix=release-tools git@github.com:/csi-release-tools.git ` - push to a new branch before submitting a PR verify-shellcheck.sh diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 3aee7a81b..4575eb819 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -11,7 +11,7 @@ The release manager must: kubernetes/org to request membership * Be a top level approver for the repository. To become a top level approver, the candidate must demonstrate ownership and deep knowledge of the repository - through active maintenance, responding to and fixing issues, reviewing PRs, + through active maintainence, responding to and fixing issues, reviewing PRs, test triage. * Be part of the maintainers or admin group for the repository. admin is a superset of maintainers, only maintainers level is required for cutting a diff --git a/boilerplate/boilerplate.Dockerfile.txt b/boilerplate/boilerplate.Dockerfile.txt deleted file mode 100644 index 34cb349c4..000000000 --- a/boilerplate/boilerplate.Dockerfile.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright YEAR The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/boilerplate/boilerplate.Makefile.txt b/boilerplate/boilerplate.Makefile.txt deleted file mode 100644 index d0d526523..000000000 --- a/boilerplate/boilerplate.Makefile.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright YEAR The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/boilerplate/boilerplate.bzl.txt b/boilerplate/boilerplate.bzl.txt deleted file mode 100644 index d0d526523..000000000 --- a/boilerplate/boilerplate.bzl.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright YEAR The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/boilerplate/boilerplate.go.txt b/boilerplate/boilerplate.go.txt deleted file mode 100644 index 3249913bd..000000000 --- a/boilerplate/boilerplate.go.txt +++ /dev/null @@ -1,15 +0,0 @@ -/* -Copyright YEAR The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ \ No newline at end of file diff --git a/boilerplate/boilerplate.py b/boilerplate/boilerplate.py deleted file mode 100755 index 5618b9ab8..000000000 --- a/boilerplate/boilerplate.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import argparse -import difflib -import glob -import os -import re -import sys -from datetime import date - -parser = argparse.ArgumentParser() -parser.add_argument( - "filenames", - help="list of files to check, all files if unspecified", - nargs='*') - -# Rootdir defaults to the directory **above** the repo-infra dir. -rootdir = os.path.dirname(__file__) + "./../../" -rootdir = os.path.abspath(rootdir) -parser.add_argument( - "--rootdir", default=rootdir, help="root directory to examine") - -default_boilerplate_dir = os.path.abspath(os.path.dirname(__file__)) - -parser.add_argument( - "--boilerplate-dir", default=default_boilerplate_dir) - -parser.add_argument( - "-v", "--verbose", - help="give verbose output regarding why a file does not pass", - action="store_true") - -args = parser.parse_args() - -verbose_out = sys.stderr if args.verbose else open("/dev/null", "w") - -def get_refs(): - refs = {} - - for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")): - extension = os.path.basename(path).split(".")[1] - - ref_file = open(path, 'r') - ref = ref_file.read().splitlines() - ref_file.close() - refs[extension] = ref - - return refs - -def file_passes(filename, refs, regexs): - try: - f = open(filename, 'r') - except Exception as exc: - print("Unable to open %s: %s" % (filename, exc), file=verbose_out) - return False - - data = f.read() - f.close() - - basename = os.path.basename(filename) - extension = file_extension(filename) - if extension != "": - ref = refs[extension] - else: - ref = refs[basename] - - # remove build tags from the top of Go files - if extension == "go": - p = regexs["go_build_constraints"] - (data, found) = p.subn("", data, 1) - - # remove shebang from the top of shell files - if extension == "sh" or extension == "py": - p = regexs["shebang"] - (data, found) = p.subn("", data, 1) - - data = data.splitlines() - - # if our test file is smaller than the reference it surely fails! - if len(ref) > len(data): - print('File %s smaller than reference (%d < %d)' % - (filename, len(data), len(ref)), - file=verbose_out) - return False - - # trim our file to the same number of lines as the reference file - data = data[:len(ref)] - - p = regexs["year"] - for d in data: - if p.search(d): - print('File %s is missing the year' % filename, file=verbose_out) - return False - - # Replace all occurrences of the regex "CURRENT_YEAR|...|2016|2015|2014" with "YEAR" - p = regexs["date"] - for i, d in enumerate(data): - (data[i], found) = p.subn('YEAR', d) - if found != 0: - break - - # if we don't match the reference at this point, fail - if ref != data: - print("Header in %s does not match reference, diff:" % filename, file=verbose_out) - if args.verbose: - print(file=verbose_out) - for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''): - print(line, file=verbose_out) - print(file=verbose_out) - return False - - return True - -def file_extension(filename): - return os.path.splitext(filename)[1].split(".")[-1].lower() - -skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', - 'cluster/env.sh', 'vendor', 'test/e2e/generated/bindata.go', - 'repo-infra/verify/boilerplate/test', '.glide'] - -def normalize_files(files): - newfiles = [] - for pathname in files: - if any(x in pathname for x in skipped_dirs): - continue - newfiles.append(pathname) - return newfiles - -def get_files(extensions): - files = [] - if len(args.filenames) > 0: - files = args.filenames - else: - for root, dirs, walkfiles in os.walk(args.rootdir): - # don't visit certain dirs. This is just a performance improvement - # as we would prune these later in normalize_files(). But doing it - # cuts down the amount of filesystem walking we do and cuts down - # the size of the file list - for d in skipped_dirs: - if d in dirs: - dirs.remove(d) - - for name in walkfiles: - pathname = os.path.join(root, name) - files.append(pathname) - - files = normalize_files(files) - - outfiles = [] - for pathname in files: - basename = os.path.basename(pathname) - extension = file_extension(pathname) - if extension in extensions or basename in extensions: - outfiles.append(pathname) - return outfiles - -def get_regexs(): - regexs = {} - # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing - regexs["year"] = re.compile( 'YEAR' ) - # dates can be 2014, 2015, 2016, ..., CURRENT_YEAR, company holder names can be anything - years = range(2014, date.today().year + 1) - regexs["date"] = re.compile( '(%s)' % "|".join(map(lambda l: str(l), years)) ) - # strip // +build \n\n build constraints - regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE) - # strip #!.* from shell scripts - regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) - return regexs - - - -def main(): - regexs = get_regexs() - refs = get_refs() - filenames = get_files(refs.keys()) - - for filename in filenames: - if not file_passes(filename, refs, regexs): - print(filename, file=sys.stdout) - - return 0 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/boilerplate/boilerplate.py.txt b/boilerplate/boilerplate.py.txt deleted file mode 100644 index 34cb349c4..000000000 --- a/boilerplate/boilerplate.py.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright YEAR The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/boilerplate/boilerplate.sh.txt b/boilerplate/boilerplate.sh.txt deleted file mode 100644 index d0d526523..000000000 --- a/boilerplate/boilerplate.sh.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright YEAR The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/build.make b/build.make index 9b8306c06..c17a1c194 100644 --- a/build.make +++ b/build.make @@ -20,7 +20,7 @@ # This is the default. It can be overridden in the main Makefile after # including build.make. -REGISTRY_NAME?=quay.io/k8scsi +REGISTRY_NAME=quay.io/k8scsi # Can be set to -mod=vendor to ensure that the "vendor" directory is used. GOFLAGS_VENDOR= @@ -69,17 +69,12 @@ endif # toolchain. BUILD_PLATFORMS = -# Add go ldflags using LDFLAGS at the time of compilation. -IMPORTPATH_LDFLAGS = -X main.version=$(REV) -EXT_LDFLAGS = -extldflags "-static" -LDFLAGS = -FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS) # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. $(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ - if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \ + if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \ echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ @@ -275,16 +270,3 @@ test-shellcheck: .PHONY: check-go-version-% check-go-version-%: ./release-tools/verify-go-version.sh "$*" - -# Test for spelling errors. -.PHONY: test-spelling -test-spelling: - @ echo; echo "### $@:" - @ ./release-tools/verify-spelling.sh "$(pwd)" - -# Test the boilerplates of the files. -.PHONY: test-boilerplate -test-boilerplate: - @ echo; echo "### $@:" - @ ./release-tools/verify-boilerplate.sh "$(pwd)" - diff --git a/cloudbuild.sh b/cloudbuild.sh index 1edda4d3f..3ba11ecad 100755 --- a/cloudbuild.sh +++ b/cloudbuild.sh @@ -1,19 +1,5 @@ #! /bin/bash -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # shellcheck disable=SC1091 . release-tools/prow.sh diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 7f0a4341c..0ca456131 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -16,16 +16,14 @@ # To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). -# Building three images in external-snapshotter takes roughly half an hour, -# sometimes more. -timeout: 3600s +timeout: 1800s # This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, # or any new substitutions added in the future. options: substitution_option: ALLOW_LOOSE steps: # The image must contain bash and curl. Ideally it should also contain - # the desired version of Go (currently defined in release-tools/travis.yml), + # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' entrypoint: ./.cloudbuild.sh diff --git a/prow.sh b/prow.sh index ec695ec6d..0b17f8246 100755 --- a/prow.sh +++ b/prow.sh @@ -1,5 +1,5 @@ #! /bin/bash - +# # Copyright 2019 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -65,18 +65,6 @@ get_versioned_variable () { echo "$value" } -# This takes a version string like CSI_PROW_KUBERNETES_VERSION and -# maps it to the corresponding git tag, branch or commit. -version_to_git () { - version="$1" - shift - case "$version" in - latest|master) echo "master";; - release-*) echo "$version";; - *) echo "v$version";; - esac -} - configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only @@ -85,14 +73,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64 # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -# Go versions can be specified separately for different tasks -# If the pre-installed Go is missing or a different -# version, the required version here will get installed -# from https://golang.org/dl/. -go_from_travis_yml () { - grep "^ *- go:" "${RELEASE_TOOLS_ROOT}/travis.yml" | sed -e 's/.*go: *//' -} -configvar CSI_PROW_GO_VERSION_BUILD "$(go_from_travis_yml)" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.16" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below @@ -220,7 +201,16 @@ configvar CSI_PROW_DRIVER_CANARY_REGISTRY "gcr.io/k8s-staging-sig-storage" "regi # all generated files are present. # # CSI_PROW_E2E_REPO=none disables E2E testing. -configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION}")" "E2E version" +tag_from_version () { + version="$1" + shift + case "$version" in + latest) echo "master";; + release-*) echo "$version";; + *) echo "v$version";; + esac +} +configvar CSI_PROW_E2E_VERSION "$(tag_from_version "${CSI_PROW_KUBERNETES_VERSION}")" "E2E version" configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo" configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" @@ -230,8 +220,8 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2 -configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" +configvar CSI_PROW_SANITY_VERSION 5421d9f3c37be3b95b241b44a094a3db11bee789 "csi-test version" # latest master +configvar CSI_PROW_SANITY_IMPORT_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI driver" @@ -251,16 +241,11 @@ configvar CSI_PROW_DEP_VERSION v0.5.1 "golang dep version to be used for vendor # # Unknown or unsupported entries are ignored. # -# Testing of alpha features is only supported for CSI_PROW_KUBERNETES_VERSION=latest -# because CSI_PROW_E2E_ALPHA and CSI_PROW_E2E_ALPHA_GATES are not set for -# older Kubernetes releases. The script supports that, it just isn't done because -# it is not needed and would cause additional maintenance effort. -# # Sanity testing with csi-sanity only covers the CSI driver itself and # thus only makes sense in repos which provide their own CSI # driver. Repos can enable sanity testing by setting # CSI_PROW_TESTS_SANITY=sanity. -configvar CSI_PROW_TESTS "unit parallel serial $(if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ]; then echo parallel-alpha serial-alpha; fi) sanity" "tests to run" +configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha sanity" "tests to run" tests_enabled () { local t1 t2 # We want word-splitting here, so ignore: Quote to prevent word splitting, or split robustly with mapfile or read -a. @@ -296,7 +281,7 @@ configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non- configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests" # Serial vs. parallel is always determined by these regular expressions. -# Individual regular expressions are separated by spaces for readability +# Individual regular expressions are seperated by spaces for readability # and expected to not contain spaces. Use dots instead. The complete # regex for Ginkgo will be created by joining the individual terms. configvar CSI_PROW_E2E_SERIAL '\[Serial\] \[Disruptive\]' "tags for serial E2E tests" @@ -333,14 +318,7 @@ configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'GenericEphemeralVolume=true,CSIStorag configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment -default_csi_snapshotter_version () { - if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ] || [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then - echo "master" - else - echo "v3.0.2" - fi -} -configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external-snapshotter version tag" +configvar CSI_SNAPSHOTTER_VERSION 'v3.0.0' "external-snapshotter version tag" # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply @@ -468,22 +446,20 @@ git_checkout () { # This clones a repo ("https://github.com/kubernetes/kubernetes") # in a certain location ("$GOPATH/src/k8s.io/kubernetes") at -# a the head of a specific branch (i.e., release-1.13, master), -# tag (v1.20.0) or commit. -# -# The directory must not exist. -git_clone () { - local repo path name parent +# a the head of a specific branch (i.e., release-1.13, master). +# The directory cannot exist. +git_clone_branch () { + local repo path branch parent repo="$1" shift path="$1" shift - name="$1" + branch="$1" shift parent="$(dirname "$path")" mkdir -p "$parent" - (cd "$parent" && run git clone --single-branch --branch "$name" "$repo" "$path") || die "cloning $repo" failed + (cd "$parent" && run git clone --single-branch --branch "$branch" "$repo" "$path") || die "cloning $repo" failed # This is useful for local testing or when switching between different revisions in the same # repo. (cd "$path" && run git clean -fdx) || die "failed to clean $path" @@ -572,11 +548,10 @@ start_cluster () { else type="docker" fi - git_clone https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$(version_to_git "$version")" || die "checking out Kubernetes $version failed" + git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed" go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" - # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910 - (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" + run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes" || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" @@ -705,10 +680,6 @@ install_csi_driver () { install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd" - if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then - CRD_BASE_DIR="${REPO_DIR}/client/config/crd" - fi - echo "Installing snapshot CRDs from ${CRD_BASE_DIR}" kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false @@ -728,16 +699,7 @@ install_snapshot_crds() { # Install snapshot controller and associated RBAC, retrying until the pod is running. install_snapshot_controller() { - CONTROLLER_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}" - if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then - CONTROLLER_DIR="${REPO_DIR}" - fi - SNAPSHOT_RBAC_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" - echo "kubectl apply -f ${SNAPSHOT_RBAC_YAML}" - # Ignore: Double quote to prevent globbing and word splitting. - # shellcheck disable=SC2086 - kubectl apply -f ${SNAPSHOT_RBAC_YAML} - + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" cnt=0 until kubectl get clusterrolebinding snapshot-controller-role; do if [ $cnt -gt 30 ]; then @@ -751,67 +713,14 @@ install_snapshot_controller() { sleep 10 done - SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" - if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then - # snapshot-controller image built from the PR will get a "csiprow" tag. - # Load it into the "kind" cluster so that we can deploy it. - NEW_TAG="csiprow" - NEW_IMG="snapshot-controller:${NEW_TAG}" - echo "kind load docker-image --name csi-prow ${NEW_IMG}" - kind load docker-image --name csi-prow ${NEW_IMG} || die "could not load the snapshot-controller:csiprow image into the kind cluster" - - # deploy snapshot-controller - echo "Deploying snapshot-controller" - # Replace image in SNAPSHOT_CONTROLLER_YAML with snapshot-controller:csiprow and deploy - # NOTE: This logic is similar to the logic here: - # https://github.com/kubernetes-csi/csi-driver-host-path/blob/v1.4.0/deploy/util/deploy-hostpath.sh#L155 - # Ignore: Double quote to prevent globbing and word splitting. - # shellcheck disable=SC2086 - # Ignore: Use find instead of ls to better handle non-alphanumeric filenames. - # shellcheck disable=SC2012 - for i in $(ls ${SNAPSHOT_CONTROLLER_YAML} | sort); do - echo " $i" - # Ignore: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead. - # shellcheck disable=SC2002 - # Ignore: See if you can use ${variable//search/replace} instead. - # shellcheck disable=SC2001 - modified="$(cat "$i" | while IFS= read -r line; do - nocomments="$(echo "$line" | sed -e 's/ *#.*$//')" - if echo "$nocomments" | grep -q '^[[:space:]]*image:[[:space:]]*'; then - # Split 'image: k8s.gcr.io/sig-storage/snapshot-controller:v3.0.0' - # into image (snapshot-controller:v3.0.0), - # name (snapshot-controller), - # tag (v3.0.0). - image=$(echo "$nocomments" | sed -e 's;.*image:[[:space:]]*;;') - name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;') - tag=$(echo "$image" | sed -e 's;.*:;;') - - # Now replace registry and/or tag - NEW_TAG="csiprow" - line="$(echo "$nocomments" | sed -e "s;$image;${name}:${NEW_TAG};")" - echo " using $line" >&2 - fi - echo "$line" - done)" - if ! echo "$modified" | kubectl apply -f -; then - echo "modified version of $i:" - echo "$modified" - exit 1 - fi - echo "kubectl apply -f ${SNAPSHOT_CONTROLLER_YAML}(modified)" - done - else - echo "kubectl apply -f $SNAPSHOT_CONTROLLER_YAML" - kubectl apply -f "$SNAPSHOT_CONTROLLER_YAML" - fi + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" cnt=0 - expected_running_pods=$(kubectl apply --dry-run=client -o "jsonpath={.spec.replicas}" -f "$SNAPSHOT_CONTROLLER_YAML") - expected_namespace=$(kubectl apply --dry-run=client -o "jsonpath={.metadata.namespace}" -f "$SNAPSHOT_CONTROLLER_YAML") - while [ "$(kubectl get pods -n "$expected_namespace" -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do + expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-) + while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do if [ $cnt -gt 30 ]; then echo "snapshot-controller pod status:" - kubectl describe pods -n "$expected_namespace" -l app=snapshot-controller + kubectl describe pods -l app=snapshot-controller echo >&2 "ERROR: snapshot controller not ready after over 5 min" exit 1 fi @@ -885,8 +794,8 @@ install_sanity () ( return fi - git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_PACKAGE_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed" - ( cd "${GOPATH}/src/${CSI_PROW_SANITY_PACKAGE_PATH}/cmd/csi-sanity" && run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go build -o "${CSI_PROW_WORK}/csi-sanity" ) || die "building csi-sanity failed" + git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_IMPORT_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed" + run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed" ) # Captures pod output while running some other command. @@ -1004,7 +913,7 @@ make_test_to_junit () { echo "$line" # pass through if echo "$line" | grep -q "^### [^ ]*:$"; then if [ "$testname" ]; then - # previous test successful + # previous test succesful echo " " >>"$out" echo " " >>"$out" fi diff --git a/travis.yml b/travis.yml deleted file mode 100644 index 1ab13aef2..000000000 --- a/travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: go -sudo: required -services: - - docker -git: - depth: false -matrix: - include: - - go: 1.15 -before_script: -- mkdir -p bin -- wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep -- chmod u+x bin/dep -- export PATH=$PWD/bin:$PATH -script: -- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ) -after_success: - - if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then - docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io; - make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' ); - fi diff --git a/verify-boilerplate.sh b/verify-boilerplate.sh deleted file mode 100755 index 815939574..000000000 --- a/verify-boilerplate.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -echo "Verifying boilerplate" - -if [[ -z "$(command -v python)" ]]; then - echo "Cannot find python. Make link to python3..." - update-alternatives --install /usr/bin/python python /usr/bin/python3 1 -fi - -# The csi-release-tools directory (absolute path). -TOOLS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -# Directory to check. Default is the parent of the tools themselves. -ROOT="${1:-${TOOLS}/..}" - -boiler="${TOOLS}/boilerplate/boilerplate.py" - -mapfile -t files_need_boilerplate < <("${boiler}" --rootdir="${ROOT}" --verbose) - -# Run boilerplate.py unit tests -unitTestOut="$(mktemp)" -trap cleanup EXIT -cleanup() { - rm "${unitTestOut}" -} - -# Run boilerplate check -if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then - for file in "${files_need_boilerplate[@]}"; do - echo "Boilerplate header is wrong for: ${file}" - done - - exit 1 -fi - -echo "Done" diff --git a/verify-go-version.sh b/verify-go-version.sh index f242e769d..266946904 100755 --- a/verify-go-version.sh +++ b/verify-go-version.sh @@ -30,7 +30,7 @@ version=$("$GO" version) || die "determining version of $GO failed" # shellcheck disable=SC2001 majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/') # shellcheck disable=SC2001 -expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/') +expected=$(. release-tools/prow.sh && echo "$$CSI_PROW_GO_VERSION_BUILD") if [ "$majorminor" != "$expected" ]; then cat >&2 < "${ERROR_LOG}" -if [[ -s "${ERROR_LOG}" ]]; then - sed 's/^/error: /' "${ERROR_LOG}" # add 'error' to each line to highlight in e2e status - echo "Found spelling errors!" - RES=1 -fi -exit "${RES}" diff --git a/verify-subtree.sh b/verify-subtree.sh index aa72194a4..f04a9fa26 100755 --- a/verify-subtree.sh +++ b/verify-subtree.sh @@ -1,5 +1,5 @@ #! /bin/sh -e - +# # Copyright 2019 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License");