From c8a91a5197338b0349cb95ea775eaf8bb2d342c4 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 24 Dec 2022 14:48:14 +0100 Subject: [PATCH] Wait for asset compilation to finish before starting airflow in Breeze (#28575) Asset compilation is performed in background and especially if it is run for the first time it might take some time. At the same time database intialization is done so usually it is not a problem but it some cases database might be initialized faster and webserver would start without assets compiled leading to nasty output. This change adds waiting for the compilation - tmux will not split screens and run webserver before it is done. (cherry picked from commit 979a72a7a3bff4e6cb8132360408584f65f2b203) --- Dockerfile.ci | 30 +++++++++++ dev/breeze/README.md | 2 +- dev/breeze/setup.cfg | 1 + .../utils/docker_command_utils.py | 1 + .../src/airflow_breeze/utils/path_utils.py | 3 ++ .../src/airflow_breeze/utils/run_utils.py | 53 ++++++++++++++----- scripts/ci/docker-compose/local.yml | 3 ++ .../pre_commit_compile_www_assets.py | 4 +- .../pre_commit_compile_www_assets_dev.py | 2 +- scripts/docker/entrypoint_ci.sh | 30 +++++++++++ scripts/in_container/bin/run_tmux | 1 - 11 files changed, 113 insertions(+), 17 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 74123f9427a0a8..96a0c9ba4b45c7 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -598,6 +598,35 @@ export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}} : "${AIRFLOW_SOURCES:?"ERROR: AIRFLOW_SOURCES not set !!!!"}" +function wait_for_asset_compilation() { + if [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; then + echo + echo "${COLOR_YELLOW}Waiting for asset compilation to complete in the background.${COLOR_RESET}" + echo + local counter=0 + while [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; do + echo "${COLOR_BLUE}Still waiting .....${COLOR_RESET}" + sleep 1 + ((counter=counter+1)) + if [[ ${counter} == "30" ]]; then + echo + echo "${COLOR_YELLOW}The asset compilation is taking too long.${COLOR_YELLOW}" + echo """ +If it does not complete soon, you might want to stop it and remove file lock: + * press Ctrl-C + * run 'rm ${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock' +""" + fi + if [[ ${counter} == "60" ]]; then + echo + echo "${COLOR_RED}The asset compilation is taking too long. Exiting.${COLOR_RED}" + echo + exit 1 + fi + done + fi +} + if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then @@ -785,6 +814,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then export AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} + wait_for_asset_compilation # shellcheck source=scripts/in_container/bin/run_tmux exec run_tmux fi diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 6872f4378abaaf..7dd12fbe682616 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: 99e484ad56c10cbba1755bb3a994f55d4acacc477ea73e23bbddd1e2f95f477f7fe873b0f8f20873a1b3d78e173d713660e59ab3c6f292ef836489043740c061 +Package config hash: 670c60fceb07f18c6fabce5e1382039bc9cc5773d339b54b6957088c484d548ee99e66d11b8eb6cf6872d7467147bcf8661249dc9e64824350edc3eddd57ed5d --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg index eaddc76e172613..69d420c5ff56f3 100644 --- a/dev/breeze/setup.cfg +++ b/dev/breeze/setup.cfg @@ -55,6 +55,7 @@ packages = find: install_requires = cached_property>=1.5.0;python_version<="3.7" click + filelock inputimeout importlib-metadata>=4.4; python_version < "3.8" pendulum diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 7a6a9afa6b7d4a..55cb94d7d88216 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -74,6 +74,7 @@ VOLUMES_FOR_SELECTED_MOUNTS = [ (".bash_aliases", "/root/.bash_aliases"), (".bash_history", "/root/.bash_history"), + (".build", "/opt/airflow/.build"), (".coveragerc", "/opt/airflow/.coveragerc"), (".dockerignore", "/opt/airflow/.dockerignore"), (".flake8", "/opt/airflow/.flake8"), diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 7138d24c369743..33142072dcaff3 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -263,6 +263,9 @@ def find_airflow_sources_root_to_operate_on() -> Path: AIRFLOW_SOURCES_ROOT = find_airflow_sources_root_to_operate_on().resolve() BUILD_CACHE_DIR = AIRFLOW_SOURCES_ROOT / ".build" +WWW_CACHE_DIR = BUILD_CACHE_DIR / "www" +WWW_ASSET_COMPILE_LOCK = WWW_CACHE_DIR / ".asset_compile.lock" +WWW_ASSET_OUT_FILE = WWW_CACHE_DIR / "asset_compile.out" DAGS_DIR = AIRFLOW_SOURCES_ROOT / "dags" FILES_DIR = AIRFLOW_SOURCES_ROOT / "files" HOOKS_DIR = AIRFLOW_SOURCES_ROOT / "hooks" diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index 506aff693fe711..70cf89687eb8a9 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -36,7 +36,7 @@ from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.console import Output, get_console -from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, WWW_ASSET_COMPILE_LOCK, WWW_ASSET_OUT_FILE from airflow_breeze.utils.shared_options import get_dry_run, get_verbose RunCommandResult = Union[subprocess.CompletedProcess, subprocess.CalledProcessError] @@ -393,16 +393,43 @@ def get_ci_image_for_pre_commits() -> str: return airflow_image -def _run_compile_internally(command_to_execute: list[str]): +def _run_compile_internally(command_to_execute: list[str], dev: bool) -> RunCommandResult: + from filelock import SoftFileLock, Timeout + env = os.environ.copy() - compile_www_assets_result = run_command( - command_to_execute, - check=False, - no_output_dump_on_exception=True, - text=True, - env=env, - ) - return compile_www_assets_result + if dev: + return run_command( + command_to_execute, + check=False, + no_output_dump_on_exception=True, + text=True, + env=env, + ) + else: + WWW_ASSET_COMPILE_LOCK.parent.mkdir(parents=True, exist_ok=True) + try: + WWW_ASSET_COMPILE_LOCK.unlink() + except FileNotFoundError: + pass + try: + with SoftFileLock(WWW_ASSET_COMPILE_LOCK, timeout=5): + with open(WWW_ASSET_OUT_FILE, "w") as f: + return run_command( + command_to_execute, + check=False, + no_output_dump_on_exception=True, + text=True, + env=env, + stderr=subprocess.STDOUT, + stdout=f, + ) + except Timeout: + get_console().print("[error]Another asset compilation is running. Exiting[/]\n") + get_console().print("[warning]If you are sure there is no other compilation,[/]") + get_console().print("[warning]Remove the lock file and re-run compilation:[/]") + get_console().print(WWW_ASSET_COMPILE_LOCK) + get_console().print() + sys.exit(1) def run_compile_www_assets( @@ -425,9 +452,11 @@ def run_compile_www_assets( "manual", "compile-www-assets-dev" if dev else "compile-www-assets", "--all-files", + "--verbose", ] + get_console().print(f"[info] The output of the asset compilation is stored in: [/]{WWW_ASSET_OUT_FILE}\n") if run_in_background: - thread = Thread(daemon=True, target=_run_compile_internally, args=(command_to_execute,)) + thread = Thread(daemon=True, target=_run_compile_internally, args=(command_to_execute, dev)) thread.start() else: - return _run_compile_internally(command_to_execute) + return _run_compile_internally(command_to_execute, dev) diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index b56b44af0aea62..c03b7bcc96c4f2 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -33,6 +33,9 @@ services: - type: bind source: ../../../.bash_history target: /root/.bash_history + - type: bind + source: ../../../.build + target: /opt/airflow/.build - type: bind source: ../../../.coveragerc target: /opt/airflow/.coveragerc diff --git a/scripts/ci/pre_commit/pre_commit_compile_www_assets.py b/scripts/ci/pre_commit/pre_commit_compile_www_assets.py index 26f0fb8dc3c2a5..a2b4723b93a3c0 100755 --- a/scripts/ci/pre_commit/pre_commit_compile_www_assets.py +++ b/scripts/ci/pre_commit/pre_commit_compile_www_assets.py @@ -26,7 +26,7 @@ from common_precommit_utils import get_directory_hash # isort: skip # noqa AIRFLOW_SOURCES_PATH = Path(__file__).parents[3].resolve() -WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www_dir_hash.txt" +WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www" / "hash.txt" if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( @@ -42,8 +42,8 @@ if new_hash == old_hash: print("The WWW directory has not changed! Skip regeneration.") sys.exit(0) - WWW_HASH_FILE.write_text(new_hash) env = os.environ.copy() env["FORCE_COLOR"] = "true" subprocess.check_call(["yarn", "install", "--frozen-lockfile"], cwd=str(www_directory)) subprocess.check_call(["yarn", "run", "build"], cwd=str(www_directory), env=env) + WWW_HASH_FILE.write_text(new_hash) diff --git a/scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py b/scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py index bcd906eb158f20..778e8d67d1253d 100755 --- a/scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py +++ b/scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py @@ -28,7 +28,7 @@ ) AIRFLOW_SOURCES_PATH = Path(__file__).parents[3].resolve() -WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www_dir_hash.txt" +WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www" / "hash.txt" if __name__ == "__main__": www_directory = Path("airflow") / "www" diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index ca147c61c031df..0301cf4538416d 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -45,6 +45,35 @@ export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}} : "${AIRFLOW_SOURCES:?"ERROR: AIRFLOW_SOURCES not set !!!!"}" +function wait_for_asset_compilation() { + if [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; then + echo + echo "${COLOR_YELLOW}Waiting for asset compilation to complete in the background.${COLOR_RESET}" + echo + local counter=0 + while [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; do + echo "${COLOR_BLUE}Still waiting .....${COLOR_RESET}" + sleep 1 + ((counter=counter+1)) + if [[ ${counter} == "30" ]]; then + echo + echo "${COLOR_YELLOW}The asset compilation is taking too long.${COLOR_YELLOW}" + echo """ +If it does not complete soon, you might want to stop it and remove file lock: + * press Ctrl-C + * run 'rm ${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock' +""" + fi + if [[ ${counter} == "60" ]]; then + echo + echo "${COLOR_RED}The asset compilation is taking too long. Exiting.${COLOR_RED}" + echo + exit 1 + fi + done + fi +} + if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then @@ -232,6 +261,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then export AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES} + wait_for_asset_compilation # shellcheck source=scripts/in_container/bin/run_tmux exec run_tmux fi diff --git a/scripts/in_container/bin/run_tmux b/scripts/in_container/bin/run_tmux index 8811863b4c1890..a877411d28a353 100755 --- a/scripts/in_container/bin/run_tmux +++ b/scripts/in_container/bin/run_tmux @@ -15,7 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - if [ ! -e /usr/local/bin/stop_airflow ]; then ln -s "/opt/airflow/scripts/in_container/stop_tmux_airflow.sh" /usr/local/bin/stop_airflow || true fi