From fa15d3983b4616516c350bb633335d2f27b9a674 Mon Sep 17 00:00:00 2001 From: Damien Martin-Guillerez Date: Tue, 28 Jul 2015 15:54:40 +0000 Subject: [PATCH] Bazel release process scripts Those scripts are made to handle the soon to be Bazel release process. It is still missing the CI part that deploys built artifacts and sends announcement mails. It uses the RELNOTES: tag (RELNOTES for a simple change, RELNOTES[NEW] for a new feature, RELNOTES[INC] for an incompatible change) to create the CHANGELOG.md file. The main entry point is scripts/release/release.sh and should be the only script called directly. -- Change-Id: I6e066fda4d862ce793cc03dc8bdf0b8cb34f7c2c Reviewed-on: https://bazel-review.googlesource.com/#/c/1580/ MOS_MIGRATED_REVID=99282250 --- scripts/release/BUILD | 21 ++ scripts/release/common.sh | 61 ++++++ scripts/release/release.sh | 338 ++++++++++++++++++++++++++++++++ scripts/release/release_test.sh | 249 +++++++++++++++++++++++ 4 files changed, 669 insertions(+) create mode 100755 scripts/release/common.sh create mode 100755 scripts/release/release.sh create mode 100755 scripts/release/release_test.sh diff --git a/scripts/release/BUILD b/scripts/release/BUILD index b2fb039097911c..7ccd9d5596c303 100644 --- a/scripts/release/BUILD +++ b/scripts/release/BUILD @@ -20,3 +20,24 @@ sh_test( ":relnotes", ], ) + +sh_library( + name = "release", + srcs = [ + "common.sh", + "release.sh", + ], + deps = [":relnotes"], +) + +sh_test( + name = "release_test", + srcs = ["release_test.sh"], + data = [ + "testenv.sh", + "//:git", + "//src/test/shell:bashunit", + ], + tags = ["need_git"], + deps = [":release"], +) diff --git a/scripts/release/common.sh b/scripts/release/common.sh new file mode 100755 index 00000000000000..f1afd809d28ef8 --- /dev/null +++ b/scripts/release/common.sh @@ -0,0 +1,61 @@ +#!/bin/bash -eu + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Some common method for release scripts + +# A release candidate is created from a branch named "release-%name%" +# where %name% is the name of the release. Once promoted to a release, +# A tag %name% will be created from this branch and the corresponding +# branch removed. +# The last commit of the release branch is always a commit containing +# the release notes in the commit message and updating the CHANGELOG.md. +# This last commit will be cherry-picked back in the master branch +# when the release candidate is promoted to a release. +# To follow tracks and to support how CI systems fetch the refs, we +# store two commit notes: the release name and the candidate number. + +# Returns the branch name of the current git repository +function git_get_branch() { + git symbolic-ref --short HEAD +} + +# Show the commit message of the ref specified in argument +function git_commit_msg() { + git show -s --pretty=format:%B "$@" +} + +# Extract the release candidate number from the git notes +function get_release_candidate() { + git notes --ref=release-candidate show "$@" 2>/dev/null | xargs echo || true +} + +# Extract the release name from the git notes +function get_release_name() { + git notes --ref=release show "$@" 2>/dev/null | xargs echo || true +} + +# Returns the info from the branch of the release. It is the current branch +# but it errors out if the current branch is not a release branch. This +# method returns the tag of the release and the number of the current +# candidate in this release. +function get_release_branch() { + local branch_name=$(git_get_branch) + if [ -z "$(get_release_name)" -o -z "$(get_release_candidate)" ]; then + echo "Not a release branch: ${branch_name}." >&2 + exit 1 + fi + echo "${branch_name}" +} diff --git a/scripts/release/release.sh b/scripts/release/release.sh new file mode 100755 index 00000000000000..bebc2114d56086 --- /dev/null +++ b/scripts/release/release.sh @@ -0,0 +1,338 @@ +#!/bin/bash -eu + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate the release branches and handle the release tags. + +set -eu + +# Repositories to push the release branch and the release tag. +: ${RELEASE_REPOSITORIES:="https://github.com/google/bazel"} + +# Repositories to push the master branch +: ${MASTER_REPOSITORIES:="https://github.com/google/bazel https://bazel.googlesource.com/bazel"} + +# Name of the default editor +: ${EDITOR=vi} + +# Author of the release commits +: ${RELEASE_AUTHOR="Bazel Release System "} + +# Load relnotes.sh +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source ${SCRIPT_DIR}/relnotes.sh + +# Load common.sh +source ${SCRIPT_DIR}/common.sh + +# Editing release notes info for the user +RELEASE_NOTE_MESSAGE='# Editing release notes +# Modify the release notes to make them suitable for the release. +# Every line starting with a # will be removed as well as every +# empty line at the start and at the end. +' + +# Set the release name $1 (and eventually the candidate number $2). +function set_release_name() { + git notes --ref=release remove 2>/dev/null || true + git notes --ref=release-candidate remove 2>/dev/null || true + git notes --ref=release append -m "$1" + [ -z "${2-}" ] || git notes --ref=release-candidate append -m "$2" +} + +# Trim empty lines at the beginning and the end of the buffer. +function trim_empty_lines() { + local f="$(echo $'\f')" # linefeed because OSX sed does not support \f + # Replace all new line by a linefeed, then using sed, remove the leading + # and trailing linefeeds and convert them back to newline + tr '\n' '\f' | sed -e "s/^$f*//" -e "s/$f*$//" | tr '\f' '\n' +} + +# Launch the editor and return the edited release notes +function release_note_editor() { + local tmpfile="$1" + local origin_branch="$2" + local branch_name="${3-}" + $EDITOR ${tmpfile} || { + echo "Editor failed, cancelling release creation..." >&2 + git checkout -q ${origin_branch} >/dev/null + [ -z "${branch_name}" ] || git branch -D ${branch_name} + exit 1 + } + # Stripping the release notes + local relnotes="$(cat ${tmpfile} | grep -v '^#' | trim_empty_lines)" + if [ -z "${relnotes}" ]; then + echo "Release notes are empty, cancelling release creation..." >&2 + git checkout -q ${origin_branch} >/dev/null + [ -z "${branch_name}" ] || git branch -D ${branch_name} + exit 1 + fi + echo "${relnotes}" >${tmpfile} +} + +# Create the release commit by changing the CHANGELOG file +function create_release_commit() { + local release_title="$1" + local release_name="$2" + local relnotes="$3" + local tmpfile="$4" + local baseline="$5" + shift 5 + local cherrypicks=$@ + local changelog_path="$PWD/CHANGELOG.md" + + version_info=$(create_revision_information $baseline $cherrypicks) + # CHANGELOG.md + cat >${tmpfile} <>${tmpfile} <>${tmpfile} <>${tmpfile} + fi + cat ${tmpfile} > ${changelog_path} + git add ${changelog_path} + # Commit message + cat >${tmpfile} </dev/null || { + echo "Failed to cherry-pick $i. please resolve the conflict and exit." >&2 + echo " Use 'git cherry-pick --abort; exit' to abort the cherry-picks." >&2 + echo " Use 'git cherry-pick --continue; exit' to resolve the conflict." >&2 + bash + if [ "$(git rev-parse HEAD)" == "${previous_head}" ]; then + echo "Cherry-pick aborted, aborting the whole command..." >&2 + return 1 + fi + } + done + return 0 +} + +# Execute the create command: +# Create a new release named "$1" with "$2" as the baseline commit. +function create_release() { + local release_name="$1" + local baseline="$2" + shift 2 + local origin_branch=$(git_get_branch) + local branch_name="release-${release_name}" + local release_title="Release ${release_name} ($(date +%Y-%m-%d))" + local tmpfile=$(mktemp ${TMPDIR:-/tmp}/relnotes-XXXXXXXX) + local tmpfile2=$(mktemp ${TMPDIR:-/tmp}/relnotes-XXXXXXXX) + trap 'rm -f ${tmpfile} ${tmpfile2}' EXIT + + # Get the rc number (1 by default) + local rc=1 + if [ -n "$(git branch --list --column ${branch_name})" ]; then + rc=$(($(get_release_candidate "${branch_name}")+1)) + fi + + # Save the changelog so we compute the relnotes against HEAD. + git show master:CHANGELOG.md >${tmpfile2} 2>/dev/null || echo >${tmpfile2} + + echo "Creating new release branch ${branch_name} for release ${release_name}" + git checkout -B ${branch_name} ${baseline} + + apply_cherry_picks $@ || { + git checkout ${origin_branch} + git branch -D ${branch_name} + exit 1 + } + + echo "Creating release notes" + echo "${RELEASE_NOTE_MESSAGE}" > ${tmpfile} + echo "# ${release_title}" >> ${tmpfile} + echo >> ${tmpfile} + create_release_notes "${tmpfile2}" >> ${tmpfile} + release_note_editor ${tmpfile} "${origin_branch}" "${branch_name}" + local relnotes="$(cat ${tmpfile})" + + echo "Creating the release commit" + create_release_commit "${release_title}" "${release_name}" \ + "${relnotes}" "${tmpfile}" "${baseline}" $@ + set_release_name "${release_name}" "${rc}" + + rm -f ${tmpfile} ${tmpfile2} + trap - EXIT +} + +# Push the release branch to the release repositories so a release +# candidate can be created. +function push_release_candidate() { + local branch="$(get_release_branch)" + for repo in ${RELEASE_REPOSITORIES}; do + git push -f ${repo} +${branch} + git push -f ${repo} +refs/notes/release + git push -f ${repo} +refs/notes/release-candidate + done +} + +# Deletes the release branch after a release or abandoning the release +function cleanup_branches() { + local tag_name=$1 + local i + echo "Destroying the release branches for release ${tag_name}" + # Destroy branch, ignoring if it doesn't exists. + git branch -D release-${tag_name} &>/dev/null || true + for i in $RELEASE_REPOSITORIES; do + git push -f $i :release-${tag_name} &>/dev/null || true + done +} + +# Releases the current release branch, creating the necessary tag, +# destroying the release branch, updating the master's CHANGELOG.md +# and pushing everything to GitHub. +function do_release() { + local branch=$(get_release_branch) + local tag_name=$(get_release_name) + + echo -n "You are about to release branch ${branch} in tag ${tag_name}, confirm? [y/N] " + read answer + if [ "$answer" = "y" -o "$answer" = "Y" ]; then + # Remove release "candidate" + set_release_name "${tag_name}" + echo "Creating the tag" + git tag ${tag_name} + + echo "Cherry-picking CHANGELOG.md modification into master" + git checkout master + # Ensuring we are up to date for master + git pull --rebase $(echo "$MASTER_REPOSITORIES" | cut -d " " -f 1) master + # We do not cherry-pick because we might have conflict if the baseline + # does not contains the latest CHANGELOG.md file, so trick it. + local changelog_path="$PWD/CHANGELOG.md" + git show ${branch}:CHANGELOG.md >${changelog_path} + local tmpfile=$(mktemp ${TMPDIR:-/tmp}/relnotes-XXXXXXXX) + trap 'rm -f ${tmpfile}' EXIT + git_commit_msg ${branch} >${tmpfile} + git add ${changelog_path} + git commit -F ${tmpfile} --no-edit --author "${RELEASE_AUTHOR}" + rm -f ${tmpfile} + trap - EXIT + + echo "Pushing the change to remote repositories" + for i in $MASTER_REPOSITORIES; do + git push $i +master + done + for i in $RELEASE_REPOSITORIES; do + git push $i +refs/tags/${tag_name} + git push $i +refs/notes/release-candidate + git push $i +refs/notes/release + done + cleanup_branches ${tag_name} + fi +} + +# Abandon the current release, deleting the branch on the local +# repository and on GitHub, discarding all changes +function abandon_release() { + local branch_info=$(get_release_branch) + local tag_name=$(get_release_name) + echo -n "You are about to abandon release ${tag_name}, confirm? [y/N] " + read answer + if [ "$answer" = "y" -o "$answer" = "Y" ]; then + git checkout -q master >/dev/null + cleanup_branches ${tag_name} + fi +} + +function usage() { + cat >&2 <&2 + echo "Please commit or stash them before using that script." >&2 + exit 1 +} + +[ "$(git rev-parse --show-toplevel)" == "$PWD" ] || { + echo "You should run this script from the root of the git repository." >&2 + exit 1 +} + +progname=$0 +cmd=${1-} +shift || usage $progname + +case $cmd in + create) + (( $# >= 2 )) || usage $progname + create_release "$@" + ;; + push) + push_release_candidate + ;; + release) + do_release + ;; + abandon) + abandon_release + ;; + *) + usage $progname + ;; +esac diff --git a/scripts/release/release_test.sh b/scripts/release/release_test.sh new file mode 100755 index 00000000000000..dc3b2c5956fa4a --- /dev/null +++ b/scripts/release/release_test.sh @@ -0,0 +1,249 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Tests release notes generation (relnotes.sh) +set -eu + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source ${SCRIPT_DIR}/testenv.sh || { echo "testenv.sh not found!" >&2; exit 1; } + +source ${SCRIPT_DIR}/common.sh || { echo "common.sh not found!" >&2; exit 1; } + +RELEASE_SCRIPT=${SCRIPT_DIR}/release.sh + +GERRIT_ROOT=${TEST_TMPDIR}/git/gerrit +GITHUB_ROOT=${TEST_TMPDIR}/git/github +WORKSPACE=${TEST_TMPDIR}/git/workspace +export RELEASE_REPOSITORIES="${GITHUB_ROOT}" +export MASTER_REPOSITORIES="${GITHUB_ROOT} ${GERRIT_ROOT}" + +setup_git_repository + +function set_up() { + # Clean previous clones + rm -fr ${GERRIT_ROOT} ${GITHUB_ROOT} ${WORKSPACE} + # Now creates the clones + git clone -l --bare -q ${MASTER_ROOT} ${GERRIT_ROOT} + git clone -l --bare -q ${MASTER_ROOT} ${GITHUB_ROOT} + # And the working copy + git clone -l -q ${GERRIT_ROOT} ${WORKSPACE} + cd ${WORKSPACE} + # Avoid committer message + cat >>.git/config <$TEST_log +} + +function push() { + local branch=$(git_get_branch) + ${RELEASE_SCRIPT} push || fail "Failed to push release branch $branch" + git --git-dir=${GITHUB_ROOT} branch >$TEST_log + expect_log "$branch" + git --git-dir=${GERRIT_ROOT} branch >$TEST_log + expect_not_log "$branch" + assert_equals "$(git show -s --pretty=format:%B $branch)" \ + "$(git --git-dir=${GITHUB_ROOT} show -s --pretty=format:%B $branch)" +} + +function release() { + local tag=$1 + local branch=$(git_get_branch) + local changelog=$(cat CHANGELOG.md) + local commit=$(git show -s --pretty=format:%B $branch) + echo y | ${RELEASE_SCRIPT} release || fail "Failed to release ${branch}" + assert_equals master "$(git_get_branch)" + git tag >$TEST_log + expect_log $tag + git --git-dir=${GITHUB_ROOT} tag >$TEST_log + expect_log $tag + git --git-dir=${GERRIT_ROOT} tag >$TEST_log + expect_not_log $tag + # Test commit is everywhere + assert_equals "$commit" "$(git show -s --pretty=format:%B $tag)" + assert_equals "$commit" "$(git show -s --pretty=format:%B master)" + assert_equals "$commit" \ + "$(git --git-dir=${GITHUB_ROOT} show -s --pretty=format:%B $tag)" + assert_equals "$commit" \ + "$(git --git-dir=${GITHUB_ROOT} show -s --pretty=format:%B master)" + assert_equals "$commit" \ + "$(git --git-dir=${GERRIT_ROOT} show -s --pretty=format:%B master)" + + # Now test for CHANGELOG.md file in master branch + assert_equals "$changelog" "$(git show $tag:CHANGELOG.md)" + assert_equals "$changelog" "$(git show master:CHANGELOG.md)" + assert_equals "$changelog" \ + "$(git --git-dir=${GITHUB_ROOT} show $tag:CHANGELOG.md)" + assert_equals "$changelog" \ + "$(git --git-dir=${GITHUB_ROOT} show master:CHANGELOG.md)" + assert_equals "$changelog" \ + "$(git --git-dir=${GERRIT_ROOT} show master:CHANGELOG.md)" + +} + +function abandon() { + local tag="$1" + local branch=$(git_get_branch) + local changelog="$(git show master:CHANGELOG.md)" + local master_sha1=$(git rev-parse master) + echo y | ${RELEASE_SCRIPT} abandon || fail "Failed to abandon release ${branch}" + assert_equals master "$(git_get_branch)" + + # test release was not tagged + git tag >$TEST_log + expect_not_log $tag + git --git-dir=${GITHUB_ROOT} tag >$TEST_log + expect_not_log $tag + git --git-dir=${GERRIT_ROOT} tag >$TEST_log + expect_not_log $tag + + # Test branch was deleted + git branch >$TEST_log + expect_not_log $branch + git --git-dir=${GITHUB_ROOT} branch >$TEST_log + expect_not_log $branch + + # Test the master branch commit hasn't changed + assert_equals "$(git rev-parse master)" "${master_sha1}" + + # Now test for CHANGELOG.md file in master branch hasn't changed + assert_equals "$changelog" "$(git show master:CHANGELOG.md)" + assert_equals "$changelog" \ + "$(git --git-dir=${GITHUB_ROOT} show master:CHANGELOG.md)" + assert_equals "$changelog" \ + "$(git --git-dir=${GERRIT_ROOT} show master:CHANGELOG.md)" + +} + +function test_release_workflow() { + export EDITOR=true + # Initial release + create v0 965c392 + expect_log "Release v0" + expect_log "Initial release" + # Push the release branch + push + # Do the initial release + release v0 + + # Second release. + + # First we need to edit the logs + export EDITOR=${TEST_TMPDIR}/editor.sh + local RELNOTES='Incompatible changes: + + - Remove deprecated "make var" INCDIR + +Important changes: + + - Use a default implementation of a progress message, rather than + defaulting to null for all SpawnActions.' + + cat >${TEST_TMPDIR}/expected.log <${TEST_TMPDIR}/replacement.log + + cat >${EDITOR} <&2 + cat ${TEST_TMPDIR}/expected.log >&2 + echo "Got:" >&2 + cat \$1 >&2 + exit 1 +fi + +# 2. write the replacement in the input file +cat ${TEST_TMPDIR}/replacement.log >\$1 +EOF + chmod +x ${EDITOR} + create v1 1170dc6 0540fde + local header='Release v1 ('$(date +%Y-%m-%d)') + +Baseline: 1170dc6 + + 0540fde: Extract version numbers that look like "..._1.2.3_..." + from BUILD_EMBED_LABEL into Info.plist. + +' + assert_equals "${header}Test replacement" "$(cat ${TEST_log})" + push + + # Test creating a second candidate + echo "#!/usr/bin/true" >${EDITOR} + create v1 1170dc6 0540fde cef25c4 + header='Release v1 ('$(date +%Y-%m-%d)') + +Baseline: 1170dc6 + + 0540fde: Extract version numbers that look like "..._1.2.3_..." + from BUILD_EMBED_LABEL into Info.plist. + + cef25c4: RELNOTES: Attribute error messages related to Android + resources are easier to understand now. + +' + RELNOTES="${RELNOTES}"' + - Attribute error messages related to Android resources are easier + to understand now.' + assert_equals "${header}${RELNOTES}" "$(cat ${TEST_log})" + assert_equals 2 "$(get_release_candidate)" + + # Push the release + push + release v1 + + # Third release to test abandon + cat >${EDITOR} <\$1 +EOF + # Create release + create v2 2464526 + expect_log "Release v2" + expect_log "Baseline: 2464526" + # Abandon it + abandon v2 + # Re-create release + create v2 2464526 + expect_log "Release v2" + expect_log "Baseline: 2464526" + # Push + push + # Abandon it + abandon v2 +} + +run_suite "Release tests"