From cfeb08bf6f770414e516f9faee90d481f6f10c86 Mon Sep 17 00:00:00 2001 From: Junya Sasaki Date: Tue, 10 Sep 2024 20:32:49 +0900 Subject: [PATCH] feat(create-prs-to-update-vcs-repositories): new github action (#317) * feat(create_prs_for_vcs_repositories_update): add initial files Signed-off-by: Junya Sasaki * renamed from create_prs_for_vcs_repositories_update to create-prs-for-vcs-repositories-update Signed-off-by: Junya Sasaki * fix(create-prs-for-vcs-repositories-update): fix a wrong branch name in README.md Signed-off-by: Junya Sasaki * fix(create_prs_for_vcs_repositories_update): add missing shell property Signed-off-by: Junya Sasaki * fix(create_prs_for_vcs_repositories_update): bug fix in missing using property Signed-off-by: Junya Sasaki * fix(create_prs_for_vcs_repositories_update): bug fix for missing back-slash Signed-off-by: Junya Sasaki * doc(create-prs-for-vcs-repositories-update): fix missing description in README.md Signed-off-by: Junya Sasaki * doc(create-prs-for-vcs-repositories-update): fix temporary names Signed-off-by: Junya Sasaki * style(pre-commit): autofix * fix(create-prs-for-vcs-repositories-update): fix yamllint errors Signed-off-by: Junya Sasaki * fix(create-prs-for-vcs-repositories-update): Avoid cspell check fail due to a word "Github" Signed-off-by: Junya Sasaki * fix(create-prs-for-vcs-repositories-update): fix wrong comments * noqa does not work for cspell check Signed-off-by: Junya Sasaki * fix(create-prs-for-vcs-repositories-update): use "-" not "_" Signed-off-by: Junya Sasaki * fix(create-prs-for-vcs-repositories-update): renamed as follows except folder * create-prs-for-vcs-repositories-update -> create-prs-to-update-vcs-repositories Signed-off-by: Junya Sasaki * fix(create-prs-to-update-vcs-repositories): renamed folder as follows * create-prs-for-vcs-repositories-update -> create-prs-to-update-vcs-repositories Signed-off-by: Junya Sasaki * fix(create-prs-to-update-vcs-repositories): rename a function to be context aware Signed-off-by: Junya Sasaki * fix(create-prs-to-update-vcs-repositories): do not declare logger outside of used scope Signed-off-by: Junya Sasaki * fix(create-prs-to-update-vcs-repositories): parse args inside of used scope Signed-off-by: Junya Sasaki * fix(create-prs-to-update-vcs-repositories): typos Signed-off-by: Junya Sasaki --------- Signed-off-by: Junya Sasaki Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../README.md | 60 ++++ .../action.yaml | 69 ++++ .../create_prs_to_update_vcs_repositories.py | 340 ++++++++++++++++++ 3 files changed, 469 insertions(+) create mode 100644 create-prs-to-update-vcs-repositories/README.md create mode 100644 create-prs-to-update-vcs-repositories/action.yaml create mode 100644 create-prs-to-update-vcs-repositories/create_prs_to_update_vcs_repositories.py diff --git a/create-prs-to-update-vcs-repositories/README.md b/create-prs-to-update-vcs-repositories/README.md new file mode 100644 index 00000000..3a4d4e3b --- /dev/null +++ b/create-prs-to-update-vcs-repositories/README.md @@ -0,0 +1,60 @@ +# create-prs-to-update-vcs-repositories + +## Description + +This action creates pull requests to update the vcs repositories in the autoware repository. + +## Initial setup (within `autowarefoundation` org) + +This action uses the app to create pull requests. + +### Secrets + +For this action to use this bot, it requires the following secrets: + +- `APP_ID`: The app ID of the bot. +- `PRIVATE_KEY`: The private key of the bot. + +These secrets are already set if inside of the autoware repository. + +## Usage + +```yaml +jobs: + sync-files: + runs-on: ubuntu-latest + steps: + - name: Generate token + id: generate-token + uses: tibdex/github-app-token@v1 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.PRIVATE_KEY }} + + - name: Run + uses: autowarefoundation/autoware-github-actions/create-prs-to-update-vcs-repositories@v1 + with: + token: ${{ steps.generate-token.outputs.token }} + repo_name: autowarefoundation/autoware + parent_dir: . + base_branch: main + new_branch_prefix: feat/update- + autoware_repos_file_name: autoware.repos + verbosity: 0 +``` + +## Inputs + +| Name | Required | Default | Description | +| ------------------------ | -------- | -------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| token | true | | The token for pull requests. | +| repo_name | true | | The name of the repository to create pull requests. | +| parent_dir | false | . | The parent directory of the repository. | +| base_branch | false | main | The base branch to create pull requests. | +| new_branch_prefix | false | feat/update- | The prefix of the new branch name. The branch name will be `{new_branch_prefix}-{user_name}/{repository_name}/{new_version}`. | +| autoware_repos_file_name | false | autoware.repos | The name of the vcs imported repository's file (e.g. autoware.repos). | +| verbosity | false | 0 | The verbosity level (0 - 2). | + +## Outputs + +None. diff --git a/create-prs-to-update-vcs-repositories/action.yaml b/create-prs-to-update-vcs-repositories/action.yaml new file mode 100644 index 00000000..d972b5f7 --- /dev/null +++ b/create-prs-to-update-vcs-repositories/action.yaml @@ -0,0 +1,69 @@ +name: create-prs-to-update-vcs-repositories +description: Create PRs to update VCS repositories + +inputs: + token: + description: GitHub token + required: true + repo_name: + description: Repository name + required: true + parent_dir: + description: Parent directory + required: false + default: ./ + base_branch: + description: Base branch + required: false + default: main + new_branch_prefix: + description: New branch prefix. The branch name will be "-//". + required: false + default: feat/update- + autoware_repos_file_name: + description: Autoware repositories file name + required: false + default: autoware.repos + verbosity: + description: Verbosity level + required: false + default: 0 + +runs: + using: composite + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + architecture: x64 + + - name: Install dependencies + shell: bash + run: python -m pip install --upgrade ruamel.yaml PyGithub GitPython packaging + + - name: Set git config + uses: autowarefoundation/autoware-github-actions/set-git-config@v1 + with: + token: ${{ inputs.token }} + + - name: Fetch all branches + shell: bash + run: git fetch --all + + - name: Run Python + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.token }} + run: | + VERBOSITY="" + for i in $(seq 1 ${{ inputs.verbosity }}); do + VERBOSITY="$VERBOSITY -v" + done + python ${GITHUB_ACTION_PATH}/create_prs_to_update_vcs_repositories.py \ + --repo_name ${{ inputs.repo_name }} \ + --parent_dir ${{ inputs.parent_dir }} \ + --base_branch ${{ inputs.base_branch }} \ + --new_branch_prefix ${{ inputs.new_branch_prefix }} \ + --autoware_repos_file_name ${{ inputs.autoware_repos_file_name }} \ + $VERBOSITY diff --git a/create-prs-to-update-vcs-repositories/create_prs_to_update_vcs_repositories.py b/create-prs-to-update-vcs-repositories/create_prs_to_update_vcs_repositories.py new file mode 100644 index 00000000..bf519e20 --- /dev/null +++ b/create-prs-to-update-vcs-repositories/create_prs_to_update_vcs_repositories.py @@ -0,0 +1,340 @@ +import os +import re +import argparse +import logging +from typing import Optional + +from ruamel.yaml import YAML +from packaging import version +import git +import github +from github import Github # cspell: ignore Github + + +class AutowareRepos: + """ + This class gets information from autoware.repos and updates it + + Attributes: + autoware_repos_file_name (str): the path to autoware.repos. e.g. "./autoware.repos" + autoware_repos (dict): the content of autoware.repos + """ + def __init__(self, autoware_repos_file_name: str): + self.autoware_repos_file_name: str = autoware_repos_file_name + + self.yaml = YAML() + + # Keep comments in the file + self.yaml.preserve_quotes = True + + with open(self.autoware_repos_file_name, "r") as file: + self.autoware_repos = self.yaml.load(file) + + def _parse_repos(self) -> dict[str, str]: + """ + parse autoware.repos and return a dictionary of GitHub repository URLs and versions + + Returns: + repository_url_version_dict (dict[str, str]): a dictionary of GitHub repository URLs and versions. e.g. {'https://github.com/tier4/glog.git': 'v0.6.0'} + """ + repository_url_version_dict: dict[str, str] = { + repository_info["url"]: repository_info["version"] + for repository_info in self.autoware_repos["repositories"].values() + } + return repository_url_version_dict + + def pickup_semver_repositories(self, semantic_version_pattern: str) -> dict[str, str]: + """ + pick up repositories with semantic version tags + + Args: + semantic_version_pattern (str): a regular expression pattern for semantic version. e.g. r'(v\d+\.\d+\.\d+)' + + Returns: + repositories_url_semantic_version_dict (dict[str, str]): a dictionary of GitHub repository URLs and semantic versions. e.g. {'https://github.com/tier4/glog.git': 'v0.6.0'} + + """ + repository_url_version_dict = self._parse_repos() + + repositories_url_semantic_version_dict: dict[str, Optional[str]] = { + url: (match.group(1) if (match := re.search(semantic_version_pattern, version)) else None) + for url, version in repository_url_version_dict.items() + } + return repositories_url_semantic_version_dict + + def update_repository_version(self, url: str, new_version: str) -> None: + """ + update the version of the repository specified by the URL + + Args: + url (str): the URL of the repository to be updated + new_version (str): the new version to be set + """ + for repository_relative_path, repository_info in self.autoware_repos["repositories"].items(): + if repository_info["url"] == url: + target_repository_relative_path: str = repository_relative_path + + self.autoware_repos["repositories"][target_repository_relative_path]["version"] = new_version + + with open(self.autoware_repos_file_name, "w") as file: + self.yaml.dump(self.autoware_repos, file) + + +class GitHubInterface: + + # Pattern for GitHub repository URL + URL_PATTERN = r'https://github.com/([^/]+)/([^/]+?)(?:\.git)?$' + + def __init__(self, token: str): + self.g = Github(token) # cspell: ignore Github + + def url_to_repository_name(self, url:str) -> str: + # Get repository name from url + match = re.search(GitHubInterface.URL_PATTERN, url) # cspell: ignore Github + assert match is not None, f"URL {url} is invalid" + user_name = match.group(1) + repo_name = match.group(2) + + return str(f'{user_name}/{repo_name}') + + def get_tags_by_url(self, url: str) -> list[str]: + # Extract repository's name from URL + repo_name = self.url_to_repository_name(url) + + # Get tags + tags: github.PaginatedList.PaginatedList = self.g.get_repo(repo_name).get_tags() + + return [tag.name for tag in tags] + + def create_pull_request(self, repo_name: str, title: str, body: str, head: str, base: str) -> None: + # Create a PR from head to base + self.g.get_repo(repo_name).create_pull( + title=title, + body=body, + head=head, + base=base, + ) + + +def parse_args() -> argparse.Namespace: + + # Parse arguments + parser = argparse.ArgumentParser(description="Create a PR to update version in autoware.repos") + + # Verbosity count + parser.add_argument("-v", "--verbose", action="count", default=0, help="Verbosity level") + + # Repository information + args_repo = parser.add_argument_group("Repository information") + args_repo.add_argument("--parent_dir", type=str, default="./", help="The parent directory of the repository") + args_repo.add_argument("--repo_name", type=str, default="autowarefoundation/autoware_dummy_repository", help="The repository name to create a PR") + args_repo.add_argument("--base_branch", type=str, default="main", help="The base branch of autoware.repos") + args_repo.add_argument("--new_branch_prefix", type=str, default="feat/update-", help="The prefix of the new branch name") + + ''' + Following default pattern = r'\b(v?\d+\.\d+(?:\.\d+)?(?:-\w+)?(?:\+\w+(\.\d+)?)?)\b' + can parse the following example formats: + "0.0.1", + "v0.0.1", + "ros2-v0.0.4", + "xxx-1.0.0-yyy", + "2.3.4", + "v1.2.3-beta", + "v1.0", + "v2", + "1.0.0-alpha+001", + "v1.0.0-rc1+build.1", + "2.0.0+build.1848", + "2.0.1-alpha.1227", + "1.0.0-alpha.beta", + "ros_humble-v0.10.2" + ''' + args_repo.add_argument( + "--semantic_version_pattern", + type=str, + default=r'\b(v?\d+\.\d+(?:\.\d+)?(?:-\w+)?(?:\+\w+(\.\d+)?)?)\b', + help="The pattern of semantic version" + ) + + # For the Autoware + args_aw = parser.add_argument_group("Autoware") + args_aw.add_argument("--autoware_repos_file_name", type=str, default="autoware.repos", help="The path to autoware.repos") + + return parser.parse_args() + + +def get_logger(verbose: int) -> logging.Logger: + + # Initialize logger depending on the verbosity level + if verbose == 0: + logging.basicConfig(level=logging.WARNING) + elif verbose == 1: + logging.basicConfig(level=logging.INFO) + elif verbose >= 2: + logging.basicConfig(level=logging.DEBUG) + + return logging.getLogger(__name__) + + +def get_latest_tag(tags: list[str], current_version: str) -> Optional[str]: + ''' + Description: + Get the latest tag from the list of tags + + Args: + tags (list[str]): a list of tags + current_version (str): the current version of the repository + ''' + latest_tag = None + for tag in tags: + # Exclude parse failed ones such as 'tier4/universe', 'main', ... etc + try: + version.parse(tag) + except (version.InvalidVersion, TypeError): + continue + + # OK, it's a valid version + if latest_tag is None: + latest_tag = tag + else: + if version.parse(tag) > version.parse(latest_tag): + latest_tag = tag + + return latest_tag + + +def create_one_branch(repo: git.Repo, branch_name: str, logger: logging.Logger) -> bool: + + # Check if the branch already exists + if branch_name in repo.heads: + logger.info(f"Branch '{branch_name}' already exists.") + return False + else: + # Create a new branch and checkout + repo.create_head(branch_name) + logger.info(f"Created a new branch '{branch_name}'") + return True + + +def main(args: argparse.Namespace) -> None: + + # Get logger + logger = get_logger(args.verbose) + + # Get GitHub token + github_token: str = os.getenv("GITHUB_TOKEN", default=None) + if github_token == "None": + raise ValueError("Please set GITHUB_TOKEN as an environment variable") + github_interface = GitHubInterface(token = github_token) # cspell: ignore Github + + autoware_repos: AutowareRepos = AutowareRepos(autoware_repos_file_name = args.autoware_repos_file_name) + + # Get the repositories with semantic version tags + repositories_url_semantic_version_dict: dict[str, str] = autoware_repos.pickup_semver_repositories(semantic_version_pattern = args.semantic_version_pattern) + + # Get reference to the repository + repo = git.Repo(args.parent_dir) + + # Get all the branches + branches = [r.remote_head for r in repo.remote().refs] + + for url, current_version in repositories_url_semantic_version_dict.items(): + ''' + Description: + In this loop, the script will create a PR to update the version of the repository specified by the URL. + The step is as follows: + 1. Get tags of the repository + 2. Check if the current version is the latest + 3. Get the latest tag + 4. Create a new branch + 5. Update autoware.repos + 6. Commit and push + 7. Create a PR + ''' + + # get tags of the repository + tags: list[str] = github_interface.get_tags_by_url(url) + + latest_tag: Optional[str] = get_latest_tag(tags, current_version) + + # Skip if the expected format is not found + if latest_tag is None: + logger.debug(f"The latest tag with expected format is not found in the repository {url}. Skip for this repository.") + continue + + # Exclude parse failed ones such as 'tier4/universe', 'main', ... etc + try: + # If current version is a valid version, compare with the current version + logger.debug(f"url: {url}, latest_tag: {latest_tag}, current_version: {current_version}") + if version.parse(latest_tag) > version.parse(current_version): + # OK, the latest tag is newer than the current version + pass + else: + # The current version is the latest + logger.debug(f"Repository {url} has the latest version {current_version}. Skip for this repository.") + continue + except (version.InvalidVersion, TypeError): + # If the current version is not a valid version and the latest tag is a valid version, let's update + pass + + # Get repository name + repo_name: str = github_interface.url_to_repository_name(url) + + # Set branch name + branch_name: str = f"{args.new_branch_prefix}{repo_name}/{latest_tag}" + + # Check if the remote branch already exists + if branch_name in branches: + logger.info(f"Branch '{branch_name}' already exists on the remote.") + continue + + # First, create a branch + create_one_branch(repo, branch_name, logger) + + # Switch to the branch + repo.heads[branch_name].checkout() + + # Change version in autoware.repos + autoware_repos.update_repository_version(url, latest_tag) + + # Add + repo.index.add([args.autoware_repos_file_name]) + + # Commit + commit_message = f"feat(autoware.repos): update {repo_name} to {latest_tag}" + repo.git.commit(m=commit_message, s=True) + + # Push + origin = repo.remote(name='origin') + origin.push(branch_name) + + # Switch back to base branch + repo.heads[args.base_branch].checkout() + + # Create a PR + github_interface.create_pull_request( + repo_name = args.repo_name, + title = f"feat(autoware.repos): update {repo_name} to {latest_tag}", + body = f"This PR updates the version of the repository {repo_name} in autoware.repos", + head = branch_name, + base = args.base_branch + ) + + # Switch back to base branch + repo.heads[args.base_branch].checkout() + + # Reset any changes + repo.git.reset('--hard', f'origin/{args.base_branch}') + + # Clean untracked files + repo.git.clean('-fd') + + # Restore base's autoware.repos + autoware_repos: AutowareRepos = AutowareRepos(autoware_repos_file_name = args.autoware_repos_file_name) + + # Loop end + + +if __name__ == "__main__": + + main(parse_args())