diff --git a/BREEZE.rst b/BREEZE.rst index 341f8b8dbeede4..d1528485fe4d46 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1589,6 +1589,15 @@ All the command parameters are here: :width: 100% :alt: Breeze verify-provider-packages +Generating Provider Issue +......................... + +You can use Breeze to generate a provider issue when you release new providers. + +.. image:: ./images/breeze/output_release-management_generate-issue-content.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/images/breeze/output_release-management_generate-issue-content.svg + :width: 100% + :alt: Breeze generate-issue-content Preparing airflow packages .......................... diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 920527acf0b4fc..af09164873f944 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -392,7 +392,7 @@ set as your environment variable. You can also pass the token as `--github-token` option in the script. ```shell script -./dev/provider_packages/prepare_provider_packages.py generate-issue-content --only-available-in-dist +breeze release-management generate-issue-content --only-available-in-dist ``` You can also generate the token by following @@ -401,7 +401,7 @@ You can also generate the token by following If you are preparing release for RC2/RC3 candidates, you should add `--suffix` parameter: ```shell script -./dev/provider_packages/prepare_provider_packages.py generate-issue-content --only-available-in-dist --suffix rc2 +breeze release-management generate-issue-content --only-available-in-dist --suffix rc2 ``` diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 6872f4378abaaf..0c86259305a19a 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: 99e484ad56c10cbba1755bb3a994f55d4acacc477ea73e23bbddd1e2f95f477f7fe873b0f8f20873a1b3d78e173d713660e59ab3c6f292ef836489043740c061 +Package config hash: f28f0d555b81a0f48d6b29b3cf8bba132b8c6a8f3d290a25ad4fd62019a9adbf86c0dc913c474e23ae110f3f433db0214bf46b21000f0d2bdd0884134923ae91 --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg index eaddc76e172613..c6c6d8dee9a4b5 100644 --- a/dev/breeze/setup.cfg +++ b/dev/breeze/setup.cfg @@ -57,12 +57,14 @@ install_requires = click inputimeout importlib-metadata>=4.4; python_version < "3.8" + jinja2 pendulum pre-commit psutil pytest pytest-xdist pyyaml + PyGithub requests rich>=12.6.0 rich-click>=1.5 diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index b319ebaa782178..0534ea8cf1d1c0 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -16,14 +16,22 @@ # under the License. from __future__ import annotations +import json +import os +import re import shlex import sys +import textwrap import time from copy import deepcopy +from datetime import datetime +from pathlib import Path from re import match -from typing import IO +from typing import IO, NamedTuple import click +from rich.progress import Progress +from rich.syntax import Syntax from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed from airflow_breeze.global_constants import ( @@ -75,7 +83,7 @@ check_async_run_results, run_with_pool, ) -from airflow_breeze.utils.path_utils import cleanup_python_generated_files +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, cleanup_python_generated_files from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.run_utils import ( RunCommandResult, @@ -692,3 +700,190 @@ def release_prod_images( f"{dockerhub_repo}:{airflow_version}", f"{dockerhub_repo}:latest", ) + + +def is_package_in_dist(dist_files: list[str], package: str) -> bool: + """Check if package has been prepared in dist folder.""" + for file in dist_files: + if file.startswith(f'apache_airflow_providers_{package.replace(".","_")}') or file.startswith( + f'apache-airflow-providers-{package.replace(".","-")}' + ): + return True + return False + + +def get_prs_for_package(package_id: str) -> list[int]: + import yaml + + pr_matcher = re.compile(r".*\(#([0-9]*)\)``$") + changelog_path = ( + AIRFLOW_SOURCES_ROOT / "airflow" / "providers" / package_id.replace(".", os.sep) / "CHANGELOG.rst" + ) + # load yaml from file + provider_yaml_dict = yaml.safe_load( + ( + AIRFLOW_SOURCES_ROOT + / "airflow" + / "providers" + / package_id.replace(r".", os.sep) + / "provider.yaml" + ).read_text() + ) + current_release_version = provider_yaml_dict["versions"][0] + prs = [] + with open(changelog_path) as changelog_file: + changelog_lines = changelog_file.readlines() + extract_prs = False + skip_line = False + for line in changelog_lines: + if skip_line: + # Skip first "....." header + skip_line = False + continue + if line.strip() == current_release_version: + extract_prs = True + skip_line = True + continue + if extract_prs: + if len(line) > 1 and all(c == "." for c in line.strip()): + # Header for next version reached + break + if line.startswith(".. Below changes are excluded from the changelog"): + # The reminder of PRs is not important skipping it + break + match_result = pr_matcher.match(line.strip()) + if match_result: + prs.append(int(match_result.group(1))) + return prs + + +@release_management.command( + name="generate-issue-content", help="Generates content for issue to test the release." +) +@click.option( + "--github-token", + envvar="GITHUB_TOKEN", + help=textwrap.dedent( + """ + GitHub token used to authenticate. + You can set omit it if you have GITHUB_TOKEN env variable set. + Can be generated with: + https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status""" + ), +) +@click.option("--suffix", default="rc1", help="Suffix to add to the version prepared") +@click.option( + "--only-available-in-dist", + is_flag=True, + help="Only consider package ids with packages prepared in the dist folder", +) +@click.option("--excluded-pr-list", type=str, help="Coma-separated list of PRs to exclude from the issue.") +@argument_packages +def generate_issue_content( + packages: list[str], + github_token: str, + suffix: str, + only_available_in_dist: bool, + excluded_pr_list: str, +): + import jinja2 + import yaml + from github import Github, Issue, PullRequest, UnknownObjectException + + class ProviderPRInfo(NamedTuple): + provider_package_id: str + pypi_package_name: str + version: str + pr_list: list[PullRequest.PullRequest | Issue.Issue] + + provider_dependencies: dict[str, dict[str, list[str]]] = json.loads( + (AIRFLOW_SOURCES_ROOT / "generated" / "provider_dependencies.json").read_text() + ) + if not packages: + packages = list(provider_dependencies.keys()) + with ci_group("Generates GitHub issue content with people who can test it"): + if excluded_pr_list: + excluded_prs = [int(pr) for pr in excluded_pr_list.split(",")] + else: + excluded_prs = [] + all_prs: set[int] = set() + provider_prs: dict[str, list[int]] = {} + if only_available_in_dist: + files_in_dist = os.listdir(str(APACHE_AIRFLOW_GITHUB_REPOSITORY / "dist")) + prepared_package_ids = [] + for package_id in packages: + if not only_available_in_dist or is_package_in_dist(files_in_dist, package_id): + get_console().print(f"Extracting PRs for provider {package_id}") + prepared_package_ids.append(package_id) + else: + get_console.print( + f"Skipping extracting PRs for provider {package_id} as it is missing in dist" + ) + continue + prs = get_prs_for_package(package_id) + provider_prs[package_id] = list(filter(lambda pr: pr not in excluded_prs, prs)) + all_prs.update(provider_prs[package_id]) + g = Github(github_token) + repo = g.get_repo("apache/airflow") + pull_requests: dict[int, PullRequest.PullRequest | Issue.Issue] = {} + with Progress(console=get_console()) as progress: + task = progress.add_task(f"Retrieving {len(all_prs)} PRs ", total=len(all_prs)) + pr_list = list(all_prs) + for i in range(len(pr_list)): + pr_number = pr_list[i] + progress.console.print( + f"Retrieving PR#{pr_number}: https://github.com/apache/airflow/pull/{pr_number}" + ) + try: + pull_requests[pr_number] = repo.get_pull(pr_number) + except UnknownObjectException: + # Fallback to issue if PR not found + try: + pull_requests[pr_number] = repo.get_issue(pr_number) # (same fields as PR) + except UnknownObjectException: + get_console().print(f"[red]The PR #{pr_number} could not be found[/]") + progress.advance(task) + providers: dict[str, ProviderPRInfo] = {} + for package_id in prepared_package_ids: + pull_request_list = [pull_requests[pr] for pr in provider_prs[package_id] if pr in pull_requests] + provider_yaml_dict = yaml.safe_load( + ( + AIRFLOW_SOURCES_ROOT + / "airflow" + / "providers" + / package_id.replace(".", os.sep) + / "provider.yaml" + ).read_text() + ) + if pull_request_list: + providers[package_id] = ProviderPRInfo( + version=provider_yaml_dict["versions"][0], + provider_package_id=package_id, + pypi_package_name=provider_yaml_dict["package-name"], + pr_list=pull_request_list, + ) + template = jinja2.Template( + (Path(__file__).parents[1] / "provider_issue_TEMPLATE.md.jinja2").read_text() + ) + issue_content = template.render(providers=providers, date=datetime.now(), suffix=suffix) + get_console().print() + get_console().print( + "[green]Below you can find the issue content that you can use " + "to ask contributor to test providers![/]" + ) + get_console().print() + get_console().print() + get_console().print( + "Issue title: [yellow]Status of testing Providers that were " + f"prepared on { datetime.now().strftime('%B %d, %Y') }[/]" + ) + get_console().print() + syntax = Syntax(issue_content, "markdown", theme="ansi_dark") + get_console().print(syntax) + get_console().print() + users: set[str] = set() + for provider_info in providers.values(): + for pr in provider_info.pr_list: + users.add("@" + pr.user.login) + get_console().print("All users involved in the PRs:") + get_console().print(" ".join(users)) diff --git a/dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2 b/dev/breeze/src/airflow_breeze/provider_issue_TEMPLATE.md.jinja2 similarity index 57% rename from dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2 rename to dev/breeze/src/airflow_breeze/provider_issue_TEMPLATE.md.jinja2 index caab34d824edb4..416d0e43a85ccf 100644 --- a/dev/provider_packages/PROVIDER_ISSUE_TEMPLATE.md.jinja2 +++ b/dev/breeze/src/airflow_breeze/provider_issue_TEMPLATE.md.jinja2 @@ -5,9 +5,9 @@ Let us know in the comment, whether the issue is addressed. Those are providers that require testing as there were some substantial changes introduced: -{% for provider_id, provider_pr_info in interesting_providers.items() %} -## Provider [{{ provider_id }}: {{ provider_pr_info.provider_details.versions[0] }}{{ suffix }}](https://pypi.org/project/{{ provider_pr_info.provider_details.pypi_package_name }}/{{ provider_pr_info.provider_details.versions[0] }}{{ suffix }}) -{%- for pr in provider_pr_info.pr_list %} +{% for provider_id, provider_info in providers.items() %} +## Provider [{{ provider_id }}: {{ provider_info.version }}{{ suffix }}](https://pypi.org/project/{{ provider_info.pypi_package_name }}/{{ provider_info.version }}{{ suffix }}) +{%- for pr in provider_info.pr_list %} - [ ] [{{ pr.title }} (#{{ pr.number }})]({{ pr.html_url }}): @{{ pr.user.login }} {%- endfor %} {%- endfor %} @@ -20,11 +20,11 @@ The guidelines on how to test providers can be found in NOTE TO RELEASE MANAGER: -Please move here the providers that have doc-only changes or for which changes are trivial and -you could asses that they are OK. In case +Please move here the providers that have doc-only changes or for which changes are trivial, and +you could assess that they are OK. In case -The providers are automatically installed on Airflow 2.1 and latest `main` during the CI, so we know they -are installable. Also all classes within the providers are imported during the CI run so we know all +The providers are automatically installed on Airflow 2.3 and latest `main` during the CI, so we know they +are installable. Also, all classes within the providers are imported during the CI run so we know all providers can be imported. --> diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index 2ef0859c8980e9..e642c413495b41 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -41,15 +41,13 @@ from pathlib import Path from random import choice from shutil import copyfile -from typing import Any, Generator, Iterable, NamedTuple, Union +from typing import Any, Generator, Iterable, NamedTuple import jsonschema import rich_click as click import semver as semver -from github import Github, Issue, PullRequest, UnknownObjectException from packaging.version import Version from rich.console import Console -from rich.progress import Progress from rich.syntax import Syntax from yaml import safe_load @@ -1870,166 +1868,6 @@ def update_changelogs(changelog_files: list[str], git_update: bool, base_branch: _update_changelog(package_id=package_id, base_branch=base_branch, verbose=verbose) -def get_prs_for_package(package_id: str) -> list[int]: - pr_matcher = re.compile(r".*\(#([0-9]*)\)``$") - verify_provider_package(package_id) - changelog_path = verify_changelog_exists(package_id) - provider_details = get_provider_details(package_id) - current_release_version = provider_details.versions[0] - prs = [] - with open(changelog_path) as changelog_file: - changelog_lines = changelog_file.readlines() - extract_prs = False - skip_line = False - for line in changelog_lines: - if skip_line: - # Skip first "....." header - skip_line = False - continue - if line.strip() == current_release_version: - extract_prs = True - skip_line = True - continue - if extract_prs: - if len(line) > 1 and all(c == "." for c in line.strip()): - # Header for next version reached - break - if line.startswith(".. Below changes are excluded from the changelog"): - # The reminder of PRs is not important skipping it - break - match_result = pr_matcher.match(line.strip()) - if match_result: - prs.append(int(match_result.group(1))) - return prs - - -PullRequestOrIssue = Union[PullRequest.PullRequest, Issue.Issue] - - -class ProviderPRInfo(NamedTuple): - provider_details: ProviderPackageDetails - pr_list: list[PullRequestOrIssue] - - -def is_package_in_dist(dist_files: list[str], package: str) -> bool: - """Check if package has been prepared in dist folder.""" - for file in dist_files: - if file.startswith(f'apache_airflow_providers_{package.replace(".","_")}') or file.startswith( - f'apache-airflow-providers-{package.replace(".","-")}' - ): - return True - return False - - -@cli.command() -@click.option( - "--github-token", - envvar="GITHUB_TOKEN", - help=textwrap.dedent( - """ - GitHub token used to authenticate. - You can set omit it if you have GITHUB_TOKEN env variable set. - Can be generated with: - https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status""" - ), -) -@click.option("--suffix", default="rc1") -@click.option( - "--only-available-in-dist", - is_flag=True, - help="Only consider package ids with packages prepared in the dist folder", -) -@click.option("--excluded-pr-list", type=str, help="Coma-separated list of PRs to exclude from the issue.") -@argument_package_ids -def generate_issue_content( - package_ids: list[str], - github_token: str, - suffix: str, - only_available_in_dist: bool, - excluded_pr_list: str, -): - if not package_ids: - package_ids = get_all_providers() - """Generates content for issue to test the release.""" - with with_group("Generates GitHub issue content with people who can test it"): - if excluded_pr_list: - excluded_prs = [int(pr) for pr in excluded_pr_list.split(",")] - else: - excluded_prs = [] - all_prs: set[int] = set() - provider_prs: dict[str, list[int]] = {} - if only_available_in_dist: - files_in_dist = os.listdir(str(DIST_PATH)) - prepared_package_ids = [] - for package_id in package_ids: - if not only_available_in_dist or is_package_in_dist(files_in_dist, package_id): - console.print(f"Extracting PRs for provider {package_id}") - prepared_package_ids.append(package_id) - else: - console.print(f"Skipping extracting PRs for provider {package_id} as it is missing in dist") - continue - prs = get_prs_for_package(package_id) - provider_prs[package_id] = list(filter(lambda pr: pr not in excluded_prs, prs)) - all_prs.update(provider_prs[package_id]) - g = Github(github_token) - repo = g.get_repo("apache/airflow") - pull_requests: dict[int, PullRequestOrIssue] = {} - with Progress(console=console) as progress: - task = progress.add_task(f"Retrieving {len(all_prs)} PRs ", total=len(all_prs)) - pr_list = list(all_prs) - for i in range(len(pr_list)): - pr_number = pr_list[i] - progress.console.print( - f"Retrieving PR#{pr_number}: https://github.com/apache/airflow/pull/{pr_number}" - ) - try: - pull_requests[pr_number] = repo.get_pull(pr_number) - except UnknownObjectException: - # Fallback to issue if PR not found - try: - pull_requests[pr_number] = repo.get_issue(pr_number) # (same fields as PR) - except UnknownObjectException: - console.print(f"[red]The PR #{pr_number} could not be found[/]") - progress.advance(task) - interesting_providers: dict[str, ProviderPRInfo] = {} - non_interesting_providers: dict[str, ProviderPRInfo] = {} - for package_id in prepared_package_ids: - pull_request_list = [pull_requests[pr] for pr in provider_prs[package_id] if pr in pull_requests] - provider_details = get_provider_details(package_id) - if pull_request_list: - interesting_providers[package_id] = ProviderPRInfo(provider_details, pull_request_list) - else: - non_interesting_providers[package_id] = ProviderPRInfo(provider_details, pull_request_list) - context = { - "interesting_providers": interesting_providers, - "date": datetime.now(), - "suffix": suffix, - "non_interesting_providers": non_interesting_providers, - } - issue_content = render_template(template_name="PROVIDER_ISSUE", context=context, extension=".md") - console.print() - console.print( - "[green]Below you can find the issue content that you can use " - "to ask contributor to test providers![/]" - ) - console.print() - console.print() - console.print( - "Issue title: [yellow]Status of testing Providers that were " - f"prepared on { datetime.now().strftime('%B %d, %Y') }[/]" - ) - console.print() - syntax = Syntax(issue_content, "markdown", theme="ansi_dark") - console.print(syntax) - console.print() - users: set[str] = set() - for provider_info in interesting_providers.values(): - for pr in provider_info.pr_list: - users.add("@" + pr.user.login) - console.print("All users involved in the PRs:") - console.print(" ".join(users)) - - if __name__ == "__main__": # The cli exit code is: # * 0 in case of success diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index e5dd45394ef945..eaa5f17713a7b3 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -36,18 +36,19 @@ prod-image:pull:e3c89dd908fc44adf6e159c2950ebdd0 prod-image:verify:31bc5efada1d70a0a31990025db1a093 prod-image:4f98deab35e53ebddbdc9950a50555a4 release-management:generate-constraints:ae30d6ad49a1b2c15b61cb29080fd957 +release-management:generate-issue-content:24218438f9e85e7c92258aadebbb19de release-management:prepare-airflow-package:3ac14ea6d2b09614959c0ec4fd564789 release-management:prepare-provider-documentation:3fe5ead9887c518d1b397d1103dc0025 release-management:prepare-provider-packages:40144cb01afc56f6a4f92d9e117e546e release-management:release-prod-images:c9bc40938e0efad49e51ef66e83f9527 release-management:verify-provider-packages:88bd609aff6d09d52ab8d80d6e055e7b -release-management:82eaa53824d5e9231ccadbc73042bed2 +release-management:eafd13da512a5a7c0fb1499cf7ff1d63 setup:autocomplete:03343478bf1d0cf9c101d454cdb63b68 setup:config:3ffcd35dd24b486ddf1d08b797e3d017 -setup:regenerate-command-images:255746830d7b5d1337d13b8e101f7f83 +setup:regenerate-command-images:ab2d83c339fa3a42b0c819b6b6cc88ae setup:self-upgrade:d02f70c7a230eae3463ceec2056b63fa setup:version:123b462a421884dc2320ffc5e54b2478 -setup:2e9e4ab1729c5420b7a2b78cbee7539a +setup:fbabee281b69f818091d780b24bd815a shell:76e0f530b7af514a2aad3032b6516c46 start-airflow:06d4aeb5f1b65f6b975f3f915558d0b3 static-checks:f45ad432bdc47a2256fdb0277b19d816 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 22001729e6f7fe..c8763961adb6bb 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -35,8 +35,8 @@ .breeze-help-r1 { fill: #c5c8c6;font-weight: bold } .breeze-help-r2 { fill: #c5c8c6 } .breeze-help-r3 { fill: #d0b344;font-weight: bold } -.breeze-help-r4 { fill: #68a0b3;font-weight: bold } -.breeze-help-r5 { fill: #868887 } +.breeze-help-r4 { fill: #868887 } +.breeze-help-r5 { fill: #68a0b3;font-weight: bold } .breeze-help-r6 { fill: #98a84b;font-weight: bold } .breeze-help-r7 { fill: #8d7b39 } @@ -187,49 +187,49 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).│ -│(cassandra | kerberos | mongo | pinot | celery | trino | all)│ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ -│stop              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing                Tools that developers can use to run tests                                                 │ -│ci-image               Tools that developers can use to manually manage CI images                                 │ -│k8s                    Tools that developers use to run Kubernetes tests                                          │ -│prod-image             Tools that developers can use to manually manage PROD images                               │ -│setup                  Tools that developers can use to configure Breeze                                          │ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).│ +│(cassandra | kerberos | mongo | pinot | celery | trino | all)│ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documentation in the container.                                                           │ +│stop              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing                Tools that developers can use to run tests                                                 │ +│ci-image               Tools that developers can use to manually manage CI images                                 │ +│k8s                    Tools that developers use to run Kubernetes tests                                          │ +│prod-image             Tools that developers can use to manually manage PROD images                               │ +│setup                  Tools that developers can use to configure Breeze                                          │ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management.svg b/images/breeze/output_release-management.svg index 39ecbd6956753a..5be8c4f4fa8a33 100644 --- a/images/breeze/output_release-management.svg +++ b/images/breeze/output_release-management.svg @@ -1,4 +1,4 @@ - + - + @@ -89,9 +89,12 @@ + + + - Command: release-management + Command: release-management @@ -102,21 +105,22 @@ -Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... +Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... Tools that release managers can use to prepare and manage Airflow releases -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ -│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ -│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ -│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ -│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ -│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ +│generate-issue-content           Generates content for issue to test the release.                                 │ +│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ +│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ +│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ +│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ +│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_generate-issue-content.svg b/images/breeze/output_release-management_generate-issue-content.svg new file mode 100644 index 00000000000000..4dfa2f36492e2e --- /dev/null +++ b/images/breeze/output_release-management_generate-issue-content.svg @@ -0,0 +1,176 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management generate-issue-content + + + + + + + + + + +Usage: breeze release-management generate-issue-content [OPTIONS] [airbyte | alibaba | amazon | apache.beam | +                                                        apache.cassandra | apache.drill | apache.druid | apache.hdfs | +                                                        apache.hive | apache.kylin | apache.livy | apache.pig | +                                                        apache.pinot | apache.spark | apache.sqoop | arangodb | asana +                                                        | atlassian.jira | celery | cloudant | cncf.kubernetes | +                                                        common.sql | databricks | datadog | dbt.cloud | dingding | +                                                        discord | docker | elasticsearch | exasol | facebook | ftp | +                                                        github | google | grpc | hashicorp | http | imap | influxdb | +                                                        jdbc | jenkins | microsoft.azure | microsoft.mssql | +                                                        microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | +                                                        odbc | openfaas | opsgenie | oracle | pagerduty | papermill | +                                                        plexus | postgres | presto | qubole | redis | salesforce | +                                                        samba | segment | sendgrid | sftp | singularity | slack | +                                                        snowflake | sqlite | ssh | tableau | tabular | telegram | +                                                        trino | vertica | yandex | zendesk]... + +Generates content for issue to test the release. + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-tokenGitHub token used to authenticate. You can set omit it if you have GITHUB_TOKEN env   │ +│variable set. Can be generated with:                                                  │ +│https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status   │ +│(TEXT)                                                                                │ +│--suffixSuffix to add to the version prepared(TEXT)│ +│--only-available-in-distOnly consider package ids with packages prepared in the dist folder│ +│--excluded-pr-listComa-separated list of PRs to exclude from the issue.(TEXT)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/images/breeze/output_setup.svg b/images/breeze/output_setup.svg index 206ba399382960..91df1eb737dd25 100644 --- a/images/breeze/output_setup.svg +++ b/images/breeze/output_setup.svg @@ -35,8 +35,8 @@ .breeze-setup-r1 { fill: #c5c8c6;font-weight: bold } .breeze-setup-r2 { fill: #c5c8c6 } .breeze-setup-r3 { fill: #d0b344;font-weight: bold } -.breeze-setup-r4 { fill: #68a0b3;font-weight: bold } -.breeze-setup-r5 { fill: #868887 } +.breeze-setup-r4 { fill: #868887 } +.breeze-setup-r5 { fill: #68a0b3;font-weight: bold } .breeze-setup-r6 { fill: #98a84b;font-weight: bold } @@ -99,20 +99,20 @@ -Usage: breeze setup [OPTIONS] COMMAND [ARGS]... +Usage: breeze setup [OPTIONS] COMMAND [ARGS]... Tools that developers can use to configure Breeze -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│autocomplete                    Enables autocompletion of breeze commands.                                        │ -│config                          Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                │ -│regenerate-command-images       Regenerate breeze command images.                                                 │ -│self-upgrade                    Self upgrade Breeze.                                                              │ -│version                         Print information about version of apache-airflow-breeze.                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│autocomplete                    Enables autocompletion of breeze commands.                                        │ +│config                          Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                │ +│regenerate-command-images       Regenerate breeze command images.                                                 │ +│self-upgrade                    Self upgrade Breeze.                                                              │ +│version                         Print information about version of apache-airflow-breeze.                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_setup_regenerate-command-images.svg b/images/breeze/output_setup_regenerate-command-images.svg index dc74dc78529d49..3ecbb459e5d44b 100644 --- a/images/breeze/output_setup_regenerate-command-images.svg +++ b/images/breeze/output_setup_regenerate-command-images.svg @@ -35,8 +35,8 @@ .breeze-setup-regenerate-command-images-r1 { fill: #c5c8c6;font-weight: bold } .breeze-setup-regenerate-command-images-r2 { fill: #c5c8c6 } .breeze-setup-regenerate-command-images-r3 { fill: #d0b344;font-weight: bold } -.breeze-setup-regenerate-command-images-r4 { fill: #68a0b3;font-weight: bold } -.breeze-setup-regenerate-command-images-r5 { fill: #868887 } +.breeze-setup-regenerate-command-images-r4 { fill: #868887 } +.breeze-setup-regenerate-command-images-r5 { fill: #68a0b3;font-weight: bold } .breeze-setup-regenerate-command-images-r6 { fill: #8d7b39 } .breeze-setup-regenerate-command-images-r7 { fill: #98a84b;font-weight: bold } @@ -142,34 +142,34 @@ -Usage: breeze setup regenerate-command-images [OPTIONS] +Usage: breeze setup regenerate-command-images [OPTIONS] Regenerate breeze command images. -╭─ Image regeneration option ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--forceForces regeneration of all images│ -│--commandCommand(s) to regenerate images for (optional, might be repeated)                                   │ -│(main | build-docs | ci:find-newer-dependencies | ci:fix-ownership | ci:free-space |                │ -│ci:get-workflow-info | ci:resource-check | ci:selective-check | ci | ci-image:build | ci-image:pull │ -│| ci-image:verify | ci-image | cleanup | compile-www-assets | exec | k8s:build-k8s-image |          │ -│k8s:configure-cluster | k8s:create-cluster | k8s:delete-cluster | k8s:deploy-airflow | k8s:k9s |    │ -│k8s:logs | k8s:run-complete-tests | k8s:setup-env | k8s:shell | k8s:status | k8s:tests |            │ -│k8s:upload-k8s-image | k8s | prod-image:build | prod-image:pull | prod-image:verify | prod-image |  │ -│release-management:generate-constraints | release-management:prepare-airflow-package |              │ -│release-management:prepare-provider-documentation | release-management:prepare-provider-packages |  │ -│release-management:release-prod-images | release-management:verify-provider-packages |              │ -│release-management | setup:autocomplete | setup:config | setup:regenerate-command-images |          │ -│setup:self-upgrade | setup:version | setup | shell | start-airflow | static-checks | stop |         │ -│testing:docker-compose-tests | testing:helm-tests | testing:integration-tests | testing:tests |     │ -│testing)                                                                                            │ -│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ -│together with --command flag or --force.                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Image regeneration option ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--forceForces regeneration of all images│ +│--commandCommand(s) to regenerate images for (optional, might be repeated)                                   │ +│(main | build-docs | ci:find-newer-dependencies | ci:fix-ownership | ci:free-space |                │ +│ci:get-workflow-info | ci:resource-check | ci:selective-check | ci | ci-image:build | ci-image:pull │ +│| ci-image:verify | ci-image | cleanup | compile-www-assets | exec | k8s:build-k8s-image |          │ +│k8s:configure-cluster | k8s:create-cluster | k8s:delete-cluster | k8s:deploy-airflow | k8s:k9s |    │ +│k8s:logs | k8s:run-complete-tests | k8s:setup-env | k8s:shell | k8s:status | k8s:tests |            │ +│k8s:upload-k8s-image | k8s | prod-image:build | prod-image:pull | prod-image:verify | prod-image |  │ +│release-management:generate-constraints | release-management:generate-issue-content |               │ +│release-management:prepare-airflow-package | release-management:prepare-provider-documentation |    │ +│release-management:prepare-provider-packages | release-management:release-prod-images |             │ +│release-management:verify-provider-packages | release-management | setup:autocomplete | setup:config│ +│| setup:regenerate-command-images | setup:self-upgrade | setup:version | setup | shell |            │ +│start-airflow | static-checks | stop | testing:docker-compose-tests | testing:helm-tests |          │ +│testing:integration-tests | testing:tests | testing)                                                │ +│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used│ +│together with --command flag or --force.                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/in_container/run_prepare_provider_documentation.sh b/scripts/in_container/run_prepare_provider_documentation.sh index 9b1e11f11127c3..37044bdbbe5bb0 100755 --- a/scripts/in_container/run_prepare_provider_documentation.sh +++ b/scripts/in_container/run_prepare_provider_documentation.sh @@ -107,11 +107,6 @@ function run_prepare_documentation() { echo "${COLOR_RED}There were errors when preparing documentation. Exiting! ${COLOR_RESET}" exit 1 else - if [[ ${GENERATE_PROVIDERS_ISSUE=} == "true" || ${GENERATE_PROVIDERS_ISSUE} == "True" ]]; then - echo - python3 dev/provider_packages/prepare_provider_packages.py generate-issue-content "${prepared_documentation[@]}" - echo - fi echo echo "${COLOR_YELLOW}Please review the updated files, classify the changelog entries and commit the changes!${COLOR_RESET}" echo