From 185efe40d0639d76c28e1e5cdaec80fbc8674207 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Sun, 28 May 2023 10:17:41 +0100 Subject: [PATCH] ci: Add sync workflow Signed-off-by: Ryan Northey --- .github/workflows/envoy-sync.yaml | 43 ++++++++++++++ README.md | 9 ++- ci/sync_envoy.sh | 95 +++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/envoy-sync.yaml create mode 100755 ci/sync_envoy.sh diff --git a/.github/workflows/envoy-sync.yaml b/.github/workflows/envoy-sync.yaml new file mode 100644 index 0000000000..8b04690ae9 --- /dev/null +++ b/.github/workflows/envoy-sync.yaml @@ -0,0 +1,43 @@ +name: Sync Envoy + +on: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + sync: + runs-on: ubuntu-20.04 + permissions: + contents: write + if: | + ${{ + !contains(github.actor, '[bot]') + || github.actor == 'sync-envoy[bot]' + }} + steps: + # Checkout the repo + - name: 'Checkout Repository' + uses: actions/checkout@v3 + with: + ref: main + fetch-depth: 0 + + # Checkout the Envoy repo + - name: 'Checkout Repository' + uses: actions/checkout@v3 + with: + repository: envoyproxy/envoy + ref: main + fetch-depth: 0 + path: upstream + + - run: mv upstream ../envoy + - run: ci/sync_envoy.sh + env: + ENVOY_SRC_DIR: ../envoy diff --git a/README.md b/README.md index 6d261f5223..6efa1804fe 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,11 @@ This repository contains a Go-based implementation of an API server that implements the discovery service APIs defined in [data-plane-api](https://github.com/envoyproxy/data-plane-api). +## Proto files + +The go proto files are synced from the upstream Envoy repository (https://github.com/envoyproxy/envoy) on every upstream commit. + +Synchronization is triggered using the `envoy-sync.yaml` workflow. ## Scope @@ -61,8 +66,8 @@ The Envoy xDS APIs follow a well defined [versioning scheme](https://www.envoypr ### Deprecated -`V2` control-plane code has been removed and will no longer be supported. For previous conversations on support for various xDS versions, see here: -- [here](https://docs.google.com/document/d/1ZkHpz6DwEUmAlG0kb2Mgu4iaeQC2Bbb0egMbECoNNKY/edit?ts=5e602993#heading=h.15nsmgmjaaml) +`V2` control-plane code has been removed and will no longer be supported. For previous conversations on support for various xDS versions, see here: +- [here](https://docs.google.com/document/d/1ZkHpz6DwEUmAlG0kb2Mgu4iaeQC2Bbb0egMbECoNNKY/edit?ts=5e602993#heading=h.15nsmgmjaaml) - [here](https://envoyproxy.slack.com/archives/C7LDJTM6Z/p1582925082005300) *Note*: It is recommended to use a previous SHA if there is still a need for `V2`. diff --git a/ci/sync_envoy.sh b/ci/sync_envoy.sh new file mode 100755 index 0000000000..d68f1bba2d --- /dev/null +++ b/ci/sync_envoy.sh @@ -0,0 +1,95 @@ +#!/bin/bash -e + +set -o pipefail + +MIRROR_MSG="Mirrored from envoyproxy/envoy" +SRCS=(envoy contrib) +GO_TARGETS=(@envoy_api//...) +IMPORT_BASE="github.com/envoyproxy/go-control-plane" +COMMITTER_NAME="envoy-sync[bot]" +COMMITTER_EMAIL="envoy-sync[bot]@users.noreply.github.com" + + +if [[ -z "$ENVOY_SRC_DIR" ]]; then + echo "ENVOY_SRC_DIR not set, it should point to a cloned Envoy repo" >&2 + exit 1 +elif [[ ! -e "$ENVOY_SRC_DIR" ]]; then + echo "ENVOY_SRC_DIR ($ENVOY_SRC_DIR) not found, did you clone it?" >&2 + exit 1 +fi + + +build_protos () { + # TODO(phlax): use envoy do_ci target once https://github.com/envoyproxy/envoy/pull/27675 lands + local go_protos go_proto go_file rule_dir proto input_dir output_dir + echo "Building go protos ..." + cd "${ENVOY_SRC_DIR}" || exit 1 + + # shellcheck disable=SC1091 + . ci/setup_cache.sh + + read -a go_protos <<< "$(bazel query "kind('go_proto_library', ${GO_TARGETS[*]})" | tr '\n' ' ')" + bazel build \ + --experimental_proto_descriptor_sets_include_source_info \ + "${go_protos[@]}" + rm -rf build_go + mkdir -p build_go + for go_proto in "${go_protos[@]}"; do + # strip @envoy_api// + rule_dir="$(echo "${go_proto:12}" | cut -d: -f1)" + proto="$(echo "${go_proto:12}" | cut -d: -f2)" + input_dir="bazel-bin/external/envoy_api/${rule_dir}/${proto}_/${IMPORT_BASE}/${rule_dir}" + output_dir="build_go/${rule_dir}" + mkdir -p "$output_dir" + while read -r go_file; do + cp -a "$go_file" "$output_dir" + done <<< "$(find "$input_dir" -name "*.go")" + done + cd - || exit 1 +} + +get_last_envoy_sha () { + git log \ + --grep="$MIRROR_MSG" -n 1 \ + | grep "$MIRROR_MSG" \ + | tail -n 1 \ + | sed -e "s#.*$MIRROR_MSG @ ##" +} + +sync_protos () { + local src envoy_src + echo "Syncing go protos ..." + for src in "${SRCS[@]}"; do + envoy_src="${ENVOY_SRC_DIR}/build_go/${src}" + rm -rf "$src" + echo "Copying ${envoy_src} -> ${src}" + cp -a "$envoy_src" "$src" + git add "$src" + done +} + +commit_changes () { + local last_envoy_sha changes changed + echo "Committing changes ..." + changed="$(git diff HEAD --name-only | grep -v envoy/COMMIT || :)" + if [[ -z "$changed" ]]; then + echo "Nothing changed, not committing" + return + fi + last_envoy_sha="$(get_last_envoy_sha)" + changes="$(git -C "${ENVOY_SRC_DIR}" rev-list "${last_envoy_sha}"..HEAD)" + echo "Changes detected: " + echo "$changes" + latest_commit="$(git -C "${ENVOY_SRC_DIR}" rev-list "${last_envoy_sha}"..HEAD | head -n1)" + echo "$latest_commit" > envoy/COMMIT + git config --global user.email "$COMMITTER_EMAIL" + git config --global user.name "$COMMITTER_NAME" + git add envoy contrib + git commit --allow-empty -s -m "${MIRROR_MSG} @ {latest_commit}" + git push origin main +} + + +build_protos +sync_protos +commit_changes